使用FFmpeg.Auto 在unity解析RTSP流视频,目前视频可以正常解析和播放。
声音的播放和解析出现问题,以下声音解析部分代码 if (aStream != null) if (pPacket->stream_index == aStream->index) { int ret = ffmpeg.avcodec_decode_audio4(pCodecContext_A, pAudioFrame, &got_picture, pPacket); if (ret < 0) { return; } // double timeset = ffmpeg.av_frame_get_best_effort_timestamp(pAudioFrame) * ffmpeg.av_q2d(aStream->time_base); if (got_picture > 0) { ffmpeg.swr_convert(au_convert_ctx, &out_buffer, 44100, (byte**)&pAudioFrame->data, pAudioFrame->nb_samples); convertedFrameVoice = Marshal.AllocHGlobal(Out_buffer_size); RtlMoveMemory((void*)convertedFrameVoice, out_buffer, Out_buffer_size); Marshal.Copy(convertedFrameVoice, voiceByte, 0, voiceByte.Length); //解析后的byte[]数据放入队列 voiceQ.Enqueue(voiceByte); } ffmpeg.av_packet_unref(pPacket);//释放数据包对象引用 ffmpeg.av_frame_unref(pAudioFrame);//释放解码帧对象引用 } unity读取队列中的声音数据,并播放 private AudioClip audioClip; private AudioSource audioSource; public UpdateVideoAndVoice updateVideo; void Start() { audioClip = AudioClip.Create("MySinusoid", 2048, 2, 44100, false); audioSource = GetComponent<AudioSource>(); audioSource.clip = audioClip; } void Update() { time += Time.deltaTime; if (time >= 0.02) { if (updateVideo.convertedFrameBufferPtr != IntPtr.Zero) { SetIntPtr(texture, updateVideo.convertedFrameBufferPtr); } SetAudio(updateVideo.voiceQ); time = 0f; } } //设置clip,播放声音 private void SetAudio(Queue<byte[]> queueB) { Debug.Log(queueB.Count); if (queueB.Count > 0) { audioSource.clip.SetData(PCM2Floats(queueB.Dequeue()), 0); audioSource.Play(); } } //byte[] 转 flaot[] float[] PCM2Floats(byte[] bytes) { float max = -(float)System.Int16.MinValue; float[] samples = new float[bytes.Length / 2]; for (int i = 0; i < samples.Length; i++) { short int16sample = System.BitConverter.ToInt16(bytes, i * 2); samples[i] = (float)int16sample / max; if (samples[i] <= -1.0f || samples[i] >= 1.0f) { Debug.Log("数组值 超界"); } } return samples; }
1播放有声音数据,但是声音很不清楚,明显的卡顿现象,不知道是解析后的声音数据不对,还是unity播放这样的一帧一帧的声音数据 思路方法不对