unity 使用FFmpeg.Auto库 解析RTSP流视频 并在unity实时播放视频和声音。

使用FFmpeg.Auto 在unity解析RTSP流视频,目前视频可以正常解析和播放。

声音的播放和解析出现问题,以下声音解析部分代码

if (aStream != null) if (pPacket->stream_index == aStream->index)
        {
          int ret = ffmpeg.avcodec_decode_audio4(pCodecContext_A, pAudioFrame, &got_picture, pPacket);
          if (ret < 0)
          {
            return;
          }

          // double timeset = ffmpeg.av_frame_get_best_effort_timestamp(pAudioFrame) * ffmpeg.av_q2d(aStream->time_base);
          if (got_picture > 0)
          {
            ffmpeg.swr_convert(au_convert_ctx, &out_buffer, 44100, (byte**)&pAudioFrame->data, pAudioFrame->nb_samples);
            convertedFrameVoice = Marshal.AllocHGlobal(Out_buffer_size);            
            RtlMoveMemory((void*)convertedFrameVoice, out_buffer, Out_buffer_size);
            Marshal.Copy(convertedFrameVoice, voiceByte, 0, voiceByte.Length);  

            //解析后的byte[]数据放入队列
            voiceQ.Enqueue(voiceByte);                        
          }
          ffmpeg.av_packet_unref(pPacket);//释放数据包对象引用
          ffmpeg.av_frame_unref(pAudioFrame);//释放解码帧对象引用 }

unity读取队列中的声音数据,并播放
    private AudioClip audioClip;
    private AudioSource audioSource;
    public UpdateVideoAndVoice updateVideo;
    void Start()
    {

        audioClip = AudioClip.Create("MySinusoid", 2048, 2, 44100, false);
        audioSource = GetComponent<AudioSource>();
        audioSource.clip = audioClip;      
    }

     void Update()
    {
        time += Time.deltaTime;

        if (time >= 0.02)
        {
            if (updateVideo.convertedFrameBufferPtr != IntPtr.Zero)
            {
                SetIntPtr(texture, updateVideo.convertedFrameBufferPtr);

            }

            SetAudio(updateVideo.voiceQ);
            time = 0f;
        }        
    }
    //设置clip,播放声音
    private void SetAudio(Queue<byte[]> queueB)
    {
        Debug.Log(queueB.Count);
        if (queueB.Count > 0)
        {
            audioSource.clip.SetData(PCM2Floats(queueB.Dequeue()), 0);
            audioSource.Play();
        }
    }

    //byte[] 转 flaot[]
    float[] PCM2Floats(byte[] bytes)
    {
        float max = -(float)System.Int16.MinValue;
        float[] samples = new float[bytes.Length / 2];

        for (int i = 0; i < samples.Length; i++)
        {
            short int16sample = System.BitConverter.ToInt16(bytes, i * 2);
            samples[i] = (float)int16sample / max;
            if (samples[i] <= -1.0f || samples[i] >= 1.0f)
            {
                Debug.Log("数组值 超界");
            }
        }

        return samples;
    }

1播放有声音数据,但是声音很不清楚,明显的卡顿现象,不知道是解析后的声音数据不对,还是unity播放这样的一帧一帧的声音数据 思路方法不对