iOS Audio Crackling issue when send audio data to UDP server and Play

//
//  ViewController.swift
//  RTPAudioDemo
//
//  Created by Jignesh Patel on 18/11/24.
//

import UIKit
import RTP
import Network
import AudioUnit
import AVFAudio
import YbridOpus

class ViewController: UIViewController {

    @IBOutlet weak var txtHost : UITextField!
    @IBOutlet weak var txtPort : UITextField!
    
    @IBOutlet weak var btnConnect : UIButton!
    @IBOutlet weak var btnRecord : UIButton!
    
    var connection: Connection?
    var keepAliveTimer: Timer?

    var recordAudio = RecordAudio_v2()
    
    var sNumber = 0
    var ssrc = 0
    
    private let audioEngine = AVAudioEngine()
    private let playerNode = AVAudioPlayerNode()
    private var audioFormat: AVAudioFormat!

    // A thread-safe queue for received audio data
    private var audioDataQueue = DispatchQueue(label: "AudioDataQueue")
    private var receivedData = Data()
    
    let encoder = OpusEncoder(sampleRate: 48000, channels: 1)
    let decoder = OpusDecoder(sampleRate: 48000, channels: 1)
    
    var frameSize : Int = 960

    var isRecording = false
    
    override func viewDidLoad() {
        super.viewDidLoad()
        self.txtHost.text = "35.91.126.251"
        self.txtPort.text = "16604"
        // Initialize the connection
        // Use the input node's format for compatibility
        let inputNode = audioEngine.inputNode
        let inputFormat = inputNode.inputFormat(forBus: 0)
        
        // Validate the format
        let sampleRate = inputFormat.sampleRate
        let channelCount = inputFormat.channelCount
        
        guard let audioFormat = AVAudioFormat(standardFormatWithSampleRate: sampleRate, channels: channelCount) else {
            fatalError("Invalid audio format")
        }
        self.audioFormat = audioFormat
        setupSession()
        setupAudioEngine()

        recordAudio.audioDataInputCallBack = { data in
            print("Data Count \(data.count)")
            let time = Date().timeIntervalSince1970
            let keepAlivePacket = try! Packet(payloadType: .opus, payload: "".data(using: .utf8)!, ssrc: UInt32(exactly: self.ssrc)!, sequenceNumber: SequenceNumber(self.sNumber), timestamp: Timestamp(time))
            self.connection?.send(keepAlivePacket)
            self.ssrc += 1
            self.sNumber += 1
        }
    }
    
    fileprivate func setupSession() {
        let desiredSampleRate: Double = Double(48000)
        let audioSession = AVAudioSession.sharedInstance()
        do {
            try audioSession.setCategory(.playAndRecord, mode: .default, options: [.defaultToSpeaker, .allowBluetooth, .allowBluetoothA2DP])
            try audioSession.setPreferredSampleRate(desiredSampleRate) // Match the sample rate of your converter
            try audioSession.setActive(true)
        } catch {
            print("Error setting up audio session: \(error)")
        }
    }
    private func setupAudioEngine() {
        audioEngine.attach(playerNode)
        audioEngine.connect(playerNode, to: audioEngine.mainMixerNode, format: audioFormat)
        do {
            try audioEngine.start()
            print("Audio engine started")
        } catch {
            print("Failed to start audio engine: \(error)")
        }
    }

    func setupConnection(ipStr : String, port : UInt16) {
        // Replace with your actual host and port
        let host = NWEndpoint.Host(ipStr)
        let port = NWEndpoint.Port(integerLiteral: port)
        
        connection = Connection(host: host, port: port) { [weak self] packet in
            // Handle received packets
            self?.handleReceivedPacket(packet)
        }
        connection?.start()
        print("Connection started.")
        // Start sending KEEP_ALIVE packets
        self.btnConnect.setTitle("Connected", for: .normal)
        startKeepAlive()
    }
    
    func startKeepAlive() {
        // Schedule the timer to send KEEP_ALIVE every 10 seconds
        keepAliveTimer = Timer.scheduledTimer(withTimeInterval: 10.0, repeats: true) { [weak self] _ in
            self?.sendKeepAlivePacket()
        }
    }
    
    func sendKeepAlivePacket() {
        guard let connection = connection else { return }
        let dataObj = "KEEP_ALIVE".data(using: .utf8)
        let keepAlivePacket = try! Packet(payloadType: .marker, payload: dataObj!, ssrc: 1, sequenceNumber: 0, timestamp: 0)
        connection.send(keepAlivePacket)
        print("Sent KEEP_ALIVE packet.")
    }
    
    func stopKeepAlive() {
        // Invalidate the timer when not needed
        keepAliveTimer?.invalidate()
        keepAliveTimer = nil
    }
    
    @IBAction func connectButtonPressed(_ sender: UIButton) {
        guard let ipAddress = txtHost.text,
              let port = UInt16(txtPort.text ?? "") else { return }
        if self.connection == nil {
            self.setupConnection(ipStr: ipAddress, port: port)
        }else{
            connection?.stop()
            connection = nil
            self.btnConnect.setTitle("Connect", for: .normal)
        }
    }
    
    @IBAction func startRecord(_ sender: UIButton) {
        if isRecording {
            isRecording = false
            self.btnRecord.setTitle("Start Recording", for: .normal)
            self.stopSendingAudio()
        } else {
            isRecording = true
            self.btnRecord.setTitle("Stop Recording", for: .normal)
            self.startSendingAudio()
        }
    }
    
    func startSendingAudio() {
        let inputNode = audioEngine.inputNode
        let audioInputFormat = inputNode.inputFormat(forBus: 0)
        
        print("Audio format sample rate: \(audioInputFormat.sampleRate)")
        print("Audio format channel count: \(audioInputFormat.channelCount)")
        
        print("Audio format sample rate: \(audioInputFormat.sampleRate)")
        print("Audio format channel count: \(audioInputFormat.channelCount)")
        
        inputNode.installTap(onBus: 0, bufferSize: 1024, format: audioInputFormat) { buffer, _ in
            guard let channelData = buffer.floatChannelData else { return }
            let channels = Int(buffer.format.channelCount)
            let frames = Int(buffer.frameLength)
            
            // Flatten the multi-channel audio data into a single data array
            var audioData = Data()
            for frame in 0...size))
                }
            }
            
            if self.isRecording {
                // Send the audio data via UDP
                do {
                    // Example PCM data (Int16 array) - replace with actual PCM data
                    if let pcmData = self.getPcmData(from: buffer) {
                        let time = Date().timeIntervalSince1970
                        let keepAlivePacket = try! Packet(payloadType: .marker, payload: pcmData, ssrc: UInt32(exactly: self.ssrc)!, sequenceNumber: SequenceNumber(self.sNumber), timestamp: Timestamp(time))
                        self.connection?.send(keepAlivePacket)
                        self.ssrc += 1
                        self.sNumber += 1
                        print("Encoded data: \(pcmData)")
                    }
                } catch {
                    print("Failed to encode: \(error)")
                }
                
            }
        }
        
        do {
            try audioEngine.start()
            print("Audio Engine started")
        } catch {
            print("Failed to start audio engine: \(error)")
        }
    }
        
    func stopSendingAudio() {
//        audioEngine.stop()
//        playerNode.stop()
    }
    
    func handleReceivedPacket(_ packet: Packet) {
        // Update UI or process the received packet
        print("Received packet: SSRC=\(packet.ssrc), Sequence=\(packet.sequenceNumber)")
        //print("Received packet DATA: payload=\(String(describing: String.init(data: packet.payload, encoding: .utf8)))")
        // Example: Update a label or log the packet info
        print("Data Count \(packet.payload.count)")
        if packet.payload.count > 10 {
            self.audioDataQueue.sync {
                self.receivedData.append(packet.payload)
            }
            self.processAudioData()
        }
    }
   
    private func processAudioData() {
        audioDataQueue.sync {
            while receivedData.count >= Int(audioFormat.streamDescription.pointee.mBytesPerFrame) {
                // Extract one frame of audio data
                let frameSize = Int(audioFormat.streamDescription.pointee.mBytesPerFrame)
                let frameData = receivedData.subdata(in: 0.. AVAudioPCMBuffer? {
        let frameLength = AVAudioFrameCount(int16Array.count / Int(format.channelCount))
        
        // Create an AVAudioPCMBuffer
        guard let pcmBuffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frameLength) else {
            print("Failed to create AVAudioPCMBuffer.")
            return nil
        }
        
        pcmBuffer.frameLength = frameLength
        
        // Fill the buffer's channel data
        let channelData = pcmBuffer.floatChannelData?[0]
        
        for frame in 0.. AVAudioPCMBuffer? {
        guard let audioFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: 48000, channels: 1, interleaved: false) else {
            print("Failed to create audio format.")
            return nil
        }
        
        let bytesPerFrame = audioFormat.streamDescription.pointee.mBytesPerFrame
        let frameLength = UInt32(data.count) / bytesPerFrame
        
        guard let buffer = AVAudioPCMBuffer(pcmFormat: audioFormat, frameCapacity: frameLength) else {
            print("Failed to create AVAudioPCMBuffer.")
            return nil
        }
        buffer.frameLength = frameLength
        
        // Copy raw audio data into the buffer
        data.withUnsafeBytes { rawBufferPointer in
            if let baseAddress = rawBufferPointer.baseAddress {
                memcpy(buffer.int16ChannelData![0], baseAddress, data.count) // Ensure format supports this!
            }
        }
        return buffer
    }
    
    func sendPacket() {
        // Replace with your actual packet creation logic
        //let packet = Packet(ssrc: 1234, sequenceNumber: 5678, payload: Data([0x01, 0x02, 0x03]))
        
        //connection?.send(packet)
        //print("Packet sent: SSRC=\(packet.ssrc), Sequence=\(packet.sequenceNumber)")
    }
        
    deinit {
        connection?.stop()
        print("Connection stopped.")
    }
    
    func getPcmData(from buffer: AVAudioPCMBuffer) -> Data? {
        guard let channelData = buffer.floatChannelData else {
            print("Failed to retrieve channel data.")
            return nil
        }
        
        let channelCount = Int(buffer.format.channelCount)
        let frameLength = Int(buffer.frameLength)
        
        // Assuming we are working with the first channel
        let channelDataPointer = channelData[0]
        
        // Convert Float32 PCM data to Int16
        var pcmDataArray = [Int16]()
        for frame in 0.. [UInt8]? {
        guard let encoder = opusEncoder else { return nil }
        var encodedData = [UInt8](repeating: 0, count: maxPacketSize)
        let encodedBytes = opus_encode(encoder, pcmBuffer, Int32(frameSize), &encodedData, opus_int32(maxPacketSize))
        
        if encodedBytes < 0 {
            print("Failed to encode PCM buffer with error: \(encodedBytes)")
            return nil
        }

        return Array(encodedData.prefix(Int(encodedBytes)))
    }

    deinit {
        if let encoder = opusEncoder {
            opus_encoder_destroy(encoder)
        }
    }
}

class OpusDecoder {
    var opusDecoder: OpaquePointer?

    init(sampleRate: Int32, channels: Int32) {
        var error: Int32 = 0
        opusDecoder = opus_decoder_create(sampleRate, channels, &error)
        if error != OPUS_OK {
            print("Failed to create Opus decoder with error: \(error)")
        }
    }

    func decode(opusData: [UInt8], frameSize: Int32) -> [Int16]? {
        guard let decoder = opusDecoder else { return nil }
        var pcmBuffer = [Int16](repeating: 0, count: Int(frameSize) * 2) // Adjust size as needed
        let decodedSamples = opus_decode(decoder, opusData, opus_int32(opusData.count), &pcmBuffer, frameSize, 0)
        
        if decodedSamples < 0 {
            print("Failed to decode Opus data with error: \(decodedSamples)")
            return nil
        }

        return Array(pcmBuffer.prefix(Int(decodedSamples) * 2))
    }

    deinit {
        if let decoder = opusDecoder {
            opus_decoder_destroy(decoder)
        }
    }
}