----------------------------------------------------------------------------------------------------------------------------------------
一分钟快速搭建 rtmpd 服务器: https://blog.csdn.net/freeabc/article/details/102880984
软件下载地址: http://www.qiyicc.com/download/rtmpd.rar
github 地址:https://github.com/superconvert/smart_rtmpd
-----------------------------------------------------------------------------------------------------------------------------------------
WebRTC 接收到 offer 指令后流程分析与 jitterbuffer 数据到解码器的流程分析
//*************************************************************************************************
//
// 接收端收到对方的 offer 指令后的流程分析
//
//*************************************************************************************************
1. 接收到对方的 offer 指令 ( websocket 信令 )
WebSocketRTCClient.java
1 2 3 4 5 6 7 8 9 10 11 | void WebSocketRTCClient::onWebSocketMessage(final String msg) // 接收到 offer 指令 if (type.equals("offer")) { if (!initiator) { SessionDescription sdp = new SessionDescription( SessionDescription.Type.fromCanonicalForm(type), json.getString("sdp")); events.onRemoteDescription(sdp); } else { reportError("Received offer for call receiver: " + msg); } } |
2. 回调到此函数
CallActivity.java
1 2 3 4 5 6 7 | void CallActivity::onRemoteDescription(final SessionDescription sdp) // 这个函数的流程分析参见下面的 3 peerConnectionClient.setRemoteDescription(sdp); if (!signalingParameters.initiator) { // 这个函数的流程分析参见下面的 4 peerConnectionClient.createAnswer(); } |
3. peerConnectionClient.setRemoteDescription 流程分析,主要是建立传输对象,以及
pipeline ( socket --> jitterbuffer --> encoder --> render ) 上所有的组件,并管理这些组件的关系
// Java
1 2 | void PeerConnectionClient::setRemoteDescription(final SessionDescription sdp) peerConnection.setRemoteDescription(sdpObserver, sdpRemote); |
// Java
1 2 | PeerConnection::setRemoteDescription(SdpObserver observer, SessionDescription sdp) nativeSetRemoteDescription(observer, sdp); |
// JNI C++
1 2 3 4 5 6 7 8 9 | JNI_GENERATOR_EXPORT void Java_org_webrtc_PeerConnection_nativeSetRemoteDescription( JNIEnv* env, jobject jcaller, jobject observer, jobject sdp) { return JNI_PeerConnection_SetRemoteDescription(env, base::android::JavaParamRef<jobject>(env, jcaller), base::android::JavaParamRef<jobject>(env, observer), base::android::JavaParamRef<jobject>(env, sdp)); } |
3.1
./sdk/android/src/jni/pc/peer_connection.cc
1 2 3 4 5 6 7 8 9 10 | static void JNI_PeerConnection_SetRemoteDescription( JNIEnv* jni, const JavaParamRef<jobject>& j_pc, const JavaParamRef<jobject>& j_observer, const JavaParamRef<jobject>& j_sdp) { rtc::scoped_refptr<SetSdpObserverJni> observer( new rtc::RefCountedObject<SetSdpObserverJni>(jni, j_observer, nullptr)); ExtractNativePC(jni, j_pc)->SetRemoteDescription( observer, JavaToNativeSessionDescription(jni, j_sdp).release()); } |
3.2
./pc/peer_connection.cc
1 2 3 4 5 6 7 | void PeerConnection::SetRemoteDescription( SetSessionDescriptionObserver* observer, SessionDescriptionInterface* desc_ptr) this_weak_ptr->DoSetRemoteDescription( std::move(desc), rtc::scoped_refptr<SetRemoteDescriptionObserverInterface>( new SetRemoteDescriptionObserverAdapter(this_weak_ptr.get(), std::move(observer_refptr)))); |
3.3
./pc/peer_connection.cc
1 2 3 4 5 6 | void PeerConnection::DoSetRemoteDescription( std::unique_ptr<SessionDescriptionInterface> desc, rtc::scoped_refptr<SetRemoteDescriptionObserverInterface> observer) error = ApplyRemoteDescription(std::move(desc)); observer->OnSetRemoteDescriptionComplete(RTCError::OK()); |
3.4
./pc/peer_connection.cc
1 2 3 4 5 6 7 8 9 | RTCError PeerConnection::ApplyRemoteDescription( std::unique_ptr<SessionDescriptionInterface> desc) // 参见流程 3.4.1 RTCError error = PushdownTransportDescription(cricket::CS_REMOTE, type); // 参见流程 3.4.2 RTCError error = CreateChannels(*remote_description()->description()); // 参见流程 3.4.3 error = UpdateSessionState(type, cricket::CS_REMOTE, remote_description()->description()); |
3.4.1 PushdownTransportDescription
1 2 3 4 5 6 7 8 9 10 11 12 13 | RTCError PeerConnection::PushdownTransportDescription( cricket::ContentSource source, SdpType type) { if (source == cricket::CS_LOCAL) { const SessionDescriptionInterface* sdesc = local_description(); RTC_DCHECK(sdesc); return transport_controller_->SetLocalDescription(type, sdesc->description()); } else { // 调用这个流程 const SessionDescriptionInterface* sdesc = remote_description(); RTC_DCHECK(sdesc); return transport_controller_->SetRemoteDescription(type, sdesc->description()); } } |
./pc/jsep_transport_controller.cc
1 2 3 | RTCError JsepTransportController::SetRemoteDescription( SdpType type, const cricket::SessionDescription* description) return ApplyDescription_n(/*local=*/false, type, description); |
./pc/jsep_transport_controller.cc
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 | RTCError JsepTransportController::ApplyDescription_n( bool local, SdpType type, const cricket::SessionDescription* description) for (const cricket::ContentInfo& content_info : description->contents()) { // Don't create transports for rejected m-lines and bundled m-lines." if (content_info.rejected || (IsBundled(content_info.name) && content_info.name != *bundled_mid())) { continue; } // 参见博文 https://blog.csdn.net/freeabc/article/details/106287318 // 内,有关 JsepTransportController::MaybeCreateJsepTransport 的分析 // 其实就是创建一个底层的传输对象,并绑定之间的传输关系 error = MaybeCreateJsepTransport(local, content_info, *description); if (!error.ok()) { return error; } } for (size_t i = 0; i < description->contents().size(); ++i) { SetIceRole_n(DetermineIceRole(transport, transport_info, type, local)); transport->SetRemoteJsepTransportDescription(jsep_description, type); } |
3.4.2 CreateChannels 创建音视频通道
// 参见博文 https://blog.csdn.net/freeabc/article/details/106287318
1 2 3 4 5 6 7 8 | RTCError PeerConnection::CreateChannels(const SessionDescription& desc) // 就是创建一个 VideoChannel ,而 VideoChannel 的 media_channel 就是 WebRtcVideoChannel cricket::VideoChannel* video_channel = CreateVideoChannel(video->name); if (!video_channel) { LOG_AND_RETURN_ERROR(RTCErrorType::INTERNAL_ERROR, "Failed to create video channel."); } // 绑定 RtpTransceiver 与 VideoChannel GetVideoTransceiver()->internal()->SetChannel(video_channel); |
3.4.3 UpdateSessionState 主要是更新通道并建立各个 pipeline 的链接
// 参见博文 https://blog.csdn.net/freeabc/article/details/106287318
1 2 3 | RTCError PeerConnection::UpdateSessionState(SdpType type, cricket::ContentSource source, const cricket::SessionDescription* description) error = PushdownMediaDescription(type, source); |
1 2 3 4 5 6 7 8 | RTCError PeerConnection::PushdownMediaDescription(SdpType type, cricket::ContentSource source) for (const auto& transceiver : transceivers_) { const ContentInfo* content_info = FindMediaSectionForTransceiver(transceiver, sdesc); cricket::ChannelInterface* channel = transceiver->internal()->channel(); const MediaContentDescription* content_desc = content_info->media_description(); bool success = (source == cricket::CS_LOCAL) ? channel->SetLocalContent(content_desc, type, &error) : channel->SetRemoteContent(content_desc, type, &error); |
./pc/channel.cc
1 2 3 4 5 6 | bool BaseChannel::SetRemoteContent(const MediaContentDescription* content, SdpType type, std::string* error_desc) { TRACE_EVENT0("webrtc", "BaseChannel::SetRemoteContent"); return InvokeOnWorker<bool>(RTC_FROM_HERE, Bind(&BaseChannel::SetRemoteContent_w, this, content, type, error_desc)); } |
./pc/channel.cc
1 2 3 4 5 6 | bool VideoChannel::SetRemoteContent_w(const MediaContentDescription* content, SdpType type, std::string* error_desc) if (!UpdateRemoteStreams_w(video->streams(), type, error_desc)) { SafeSetError("Failed to set remote video description streams.", error_desc); return false; } |
./pc/channel.cc
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 | bool BaseChannel::UpdateRemoteStreams_w(const std::vector<StreamParams>& streams, SdpType type, std::string* error_desc) for (const StreamParams& new_stream : streams) { // We allow a StreamParams with an empty list of SSRCs, in which case the // MediaChannel will cache the parameters and use them for any unsignaled // stream received later. if ((!new_stream.has_ssrcs() && !HasStreamWithNoSsrcs(remote_streams_)) || !GetStreamBySsrc(remote_streams_, new_stream.first_ssrc())) { // 参见流程 3.4.3.1 if (AddRecvStream_w(new_stream)) { } } } // 参见流程 3.4.3.2,就是建立底层的接收到的 RTP 数据到 channel 里面来 RegisterRtpDemuxerSink(); remote_streams_ = streams; |
3.4.3.1
./pc/channel.cc
1 2 3 4 5 6 7 | bool BaseChannel::AddRecvStream_w(const StreamParams& sp) { RTC_DCHECK(worker_thread() == rtc::Thread::Current()); // media_channel 就是 WebRtcVideoChannel 对象 // 参看博客 https://blog.csdn.net/freeabc/article/details/106287318 // 流程 22 return media_channel()->AddRecvStream(sp); } |
./media/engine/webrtc_video_engine.cc
1 2 3 | bool WebRtcVideoChannel::AddRecvStream(const StreamParams& sp) { return AddRecvStream(sp, false); } |
./media/engine/webrtc_video_engine.cc
1 2 3 4 5 6 | bool WebRtcVideoChannel::AddRecvStream(const StreamParams& sp, bool default_stream) receive_streams_[ssrc] = new WebRtcVideoReceiveStream( this, call_, sp, std::move(config), decoder_factory_, default_stream, recv_codecs_, flexfec_config); |
有关 WebRtcVideoReceiveStream 的深入分析,参见下面的
<<视频数据接收到后,从 jitterbuffer 中到解码的过程分析>>
至此 WebRtcVideoChannel 中的 receive_streams_ 对象产生, 就是 WebRtcVideoReceiveStream 流
3.4.3.2
参见博文 https://blog.csdn.net/freeabc/article/details/106142951
流程 10.3.3
主要目的是:我们看到 VideoChannel 做为 Sink 加到 RtpTransport 里的 rtp_demuxer_, 所以 RtpDemuxer::OnRtpPacket
会调用 VideoChannel::OnRtpPacket,建立 pipeline ( socket ---> jitterbuffer ---> decoder ---> render ) 之间部件的关联
4. peerConnectionClient.createAnswer 流程分析
1 2 3 | void PeerConnectionClient::createAnswer() { peerConnection.createAnswer(sdpObserver, sdpMediaConstraints); } |
1 2 3 | void PeerConnection::createAnswer(SdpObserver observer, MediaConstraints constraints) { nativeCreateAnswer(observer, constraints); } |
1 2 3 4 5 6 7 8 | JNI_GENERATOR_EXPORT void Java_org_webrtc_PeerConnection_nativeCreateAnswer( JNIEnv* env, jobject jcaller, jobject observer, jobject constraints) { return JNI_PeerConnection_CreateAnswer(env, base::android::JavaParamRef<jobject>(env, jcaller), base::android::JavaParamRef<jobject>(env, observer), base::android::JavaParamRef<jobject>(env, constraints)); } |
./sdk/android/src/jni/pc/peer_connection.cc
1 2 3 4 5 6 7 8 9 10 11 12 13 | void JNI_PeerConnection_CreateAnswer( JNIEnv* jni, const JavaParamRef<jobject>& j_pc, const JavaParamRef<jobject>& j_observer, const JavaParamRef<jobject>& j_constraints) { std::unique_ptr<MediaConstraints> constraints = JavaToNativeMediaConstraints(jni, j_constraints); rtc::scoped_refptr<CreateSdpObserverJni> observer( new rtc::RefCountedObject<CreateSdpObserverJni>(jni, j_observer, std::move(constraints))); PeerConnectionInterface::RTCOfferAnswerOptions options; CopyConstraintsIntoOfferAnswerOptions(observer->constraints(), &options); ExtractNativePC(jni, j_pc)->CreateAnswer(observer, options); } |
./pc/peer_connection.cc
1 2 3 | void PeerConnection::CreateAnswer(CreateSessionDescriptionObserver* observer, const RTCOfferAnswerOptions& options) this_weak_ptr->DoCreateAnswer(options, observer_wrapper); |
./pc/peer_connection.cc
1 2 3 4 5 | void PeerConnection::DoCreateAnswer( const RTCOfferAnswerOptions& options, rtc::scoped_refptr<CreateSessionDescriptionObserver> observer) webrtc_session_desc_factory_->CreateAnswer(observer, session_options); |
./pc/webrtc_session_description_factory.cc
1 2 3 4 | void WebRtcSessionDescriptionFactory::CreateAnswer( CreateSessionDescriptionObserver* observer, const cricket::MediaSessionOptions& session_options) InternalCreateAnswer(request); |
./pc/webrtc_session_description_factory.cc
1 2 3 | void WebRtcSessionDescriptionFactory::InternalCreateAnswer( CreateSessionDescriptionRequest request) PostCreateSessionDescriptionSucceeded(request.observer, std::move(answer)); |
// 产生相应的 SDP 并通知上层,做相应的准备工作。
// 后续流程参考 https://blog.csdn.net/freeabc/article/details/106287318
// 这个流程主要是初始化操作,包括网络,中间层的 channel, receive, sender 对象的创建等。
//*************************************************************************************************
//
// 视频数据接收到后,从 jitterbuffer 中到解码的过程分析
//
//*************************************************************************************************
1. 我们分析 WebRtcVideoReceiveStream 的构造函数
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 | WebRtcVideoChannel::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream( WebRtcVideoChannel* channel, webrtc::Call* call, const StreamParams& sp, webrtc::VideoReceiveStream::Config config, webrtc::VideoDecoderFactory* decoder_factory, bool default_stream, const std::vector<VideoCodecSettings>& recv_codecs, const webrtc::FlexfecReceiveStream::Config& flexfec_config) : channel_(channel), call_(call), stream_params_(sp), stream_(NULL), default_stream_(default_stream), config_(std::move(config)), flexfec_config_(flexfec_config), flexfec_stream_(nullptr), decoder_factory_(decoder_factory), sink_(NULL), first_frame_timestamp_(-1), estimated_remote_start_ntp_time_ms_(0) { config_.renderer = this; ConfigureCodecs(recv_codecs); ConfigureFlexfecCodec(flexfec_config.payload_type); MaybeRecreateWebRtcFlexfecStream(); // 参见下面的分析 2 RecreateWebRtcVideoStream(); } |
2.
./media/engine/webrtc_video_engine.cc
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 | void WebRtcVideoChannel::WebRtcVideoReceiveStream::RecreateWebRtcVideoStream() { absl::optional<int> base_minimum_playout_delay_ms; if (stream_) { base_minimum_playout_delay_ms = stream_->GetBaseMinimumPlayoutDelayMs(); MaybeDissociateFlexfecFromVideo(); call_->DestroyVideoReceiveStream(stream_); stream_ = nullptr; } webrtc::VideoReceiveStream::Config config = config_.Copy(); config.rtp.protected_by_flexfec = (flexfec_stream_ != nullptr); config.stream_id = stream_params_.id; // 这个地方产生了一个 VideoReceiveStream 对象,参见下面的流程 3 stream_ = call_->CreateVideoReceiveStream(std::move(config)); if (base_minimum_playout_delay_ms) { stream_->SetBaseMinimumPlayoutDelayMs(base_minimum_playout_delay_ms.value()); } MaybeAssociateFlexfecWithVideo(); // 这个地方参见下面的流程 4 stream_->Start(); if (webrtc::field_trial::IsEnabled( "WebRTC-Video-BufferPacketsWithUnknownSsrc")) { channel_->BackfillBufferedPackets(stream_params_.ssrcs); } } |
3. 最终产生一个 VideoReceiveStream 对象
./call/call.cc
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 | webrtc::VideoReceiveStream* Call::CreateVideoReceiveStream( webrtc::VideoReceiveStream::Config configuration) { TRACE_EVENT0("webrtc", "Call::CreateVideoReceiveStream"); RTC_DCHECK_RUN_ON(&configuration_sequence_checker_); receive_side_cc_.SetSendPeriodicFeedback( SendPeriodicFeedback(configuration.rtp.extensions)); RegisterRateObserver(); VideoReceiveStream* receive_stream = new VideoReceiveStream( task_queue_factory_, &video_receiver_controller_, num_cpu_cores_, transport_send_ptr_->packet_router(), std::move(configuration), module_process_thread_.get(), call_stats_.get(), clock_); const webrtc::VideoReceiveStream::Config& config = receive_stream->config(); { WriteLockScoped write_lock(*receive_crit_); if (config.rtp.rtx_ssrc) { // We record identical config for the rtx stream as for the main // stream. Since the transport_send_cc negotiation is per payload // type, we may get an incorrect value for the rtx stream, but // that is unlikely to matter in practice. receive_rtp_config_.emplace(config.rtp.rtx_ssrc, ReceiveRtpConfig(config)); } receive_rtp_config_.emplace(config.rtp.remote_ssrc, ReceiveRtpConfig(config)); video_receive_streams_.insert(receive_stream); ConfigureSync(config.sync_group); } receive_stream->SignalNetworkState(video_network_state_); UpdateAggregateNetworkState(); event_log_->Log(std::make_unique<RtcEventVideoReceiveStreamConfig>( CreateRtcLogStreamConfig(config))); return receive_stream; } |
4. VideoReceiveStream 的 start 其实就是 jitterbuffer 的初始化工作
./video/video_receive_stream.cc
1 2 3 4 5 6 7 8 9 10 | void VideoReceiveStream::Start() // jitterbuffer 的初始化见流程 4.1 frame_buffer_->Start(); // 这个地方开启一个 jitterbuffer 的处理 decode_queue_.PostTask([this] { RTC_DCHECK_RUN_ON(&decode_queue_); decoder_stopped_ = false; // 参见下面的流程 5,投递一个从 jitterbuffer 获取帧的任务 StartNextDecode(); }); |
4.1 Jitter buffer 的产生是 VideoReceiveStream 构造函数中进行的
./video/video_receive_stream.cc
1 2 3 4 5 6 7 8 9 10 11 12 | VideoReceiveStream::VideoReceiveStream( TaskQueueFactory* task_queue_factory, RtpStreamReceiverControllerInterface* receiver_controller, int num_cpu_cores, PacketRouter* packet_router, VideoReceiveStream::Config config, ProcessThread* process_thread, CallStats* call_stats, Clock* clock, VCMTiming* timing) // ./modules/video_coding/frame_buffer2.cc frame_buffer_.reset(new video_coding::FrameBuffer(clock_, timing_.get(), &stats_proxy_)); |
5. StartNextDecode 的处理流程分析
./video/video_receive_stream.cc
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 | void VideoReceiveStream::StartNextDecode() { TRACE_EVENT0("webrtc", "VideoReceiveStream::StartNextDecode"); // 参见流程 6 frame_buffer_->NextFrame( GetWaitMs(), keyframe_required_, &decode_queue_, /* encoded frame handler */ [this](std::unique_ptr<EncodedFrame> frame, ReturnReason res) { RTC_DCHECK_EQ(frame == nullptr, res == ReturnReason::kTimeout); RTC_DCHECK_EQ(frame != nullptr, res == ReturnReason::kFrameFound); decode_queue_.PostTask([this, frame = std::move(frame)]() mutable { RTC_DCHECK_RUN_ON(&decode_queue_); if (decoder_stopped_) return; if (frame) { // 开始进行解码处理,参见下面的流程 10 开始准备解码数据了 HandleEncodedFrame(std::move(frame)); } else { HandleFrameBufferTimeout(); } StartNextDecode(); }); } ); } |
6. jitterbuffer 的 NextFrame 函数的定义
./modules/video_coding/frame_buffer2.cc
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 | void FrameBuffer::NextFrame( int64_t max_wait_time_ms, bool keyframe_required, rtc::TaskQueue* callback_queue, std::function<void(std::unique_ptr<EncodedFrame>, ReturnReason)> handler) { RTC_DCHECK_RUN_ON(callback_queue); TRACE_EVENT0("webrtc", "FrameBuffer::NextFrame"); int64_t latest_return_time_ms = clock_->TimeInMilliseconds() + max_wait_time_ms; rtc::CritScope lock(&crit_); if (stopped_) { return; } latest_return_time_ms_ = latest_return_time_ms; keyframe_required_ = keyframe_required; // 上述的 lambada 就是这个 frame_handler_ = handler; callback_queue_ = callback_queue; // 参见下面流程 7 StartWaitForNextFrameOnQueue(); } |
7.
./modules/video_coding/frame_buffer2.cc
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 | void FrameBuffer::StartWaitForNextFrameOnQueue() { RTC_DCHECK(callback_queue_); RTC_DCHECK(!callback_task_.Running()); // 这个里面从接收的队列里 frames_ 取出一帧放到待解码队列 参见流程 8 int64_t wait_ms = FindNextFrame(clock_->TimeInMilliseconds()); callback_task_ = RepeatingTaskHandle::DelayedStart( callback_queue_->Get(), TimeDelta::ms(wait_ms), [this] { // If this task has not been cancelled, we did not get any new frames // while waiting. Continue with frame delivery. rtc::CritScope lock(&crit_); if (!frames_to_decode_.empty()) { // 这个首先调用 GetNextFrame 流程 9 ,然后再调用上面的 lambada 流程 10 // We have frames, deliver! frame_handler_(absl::WrapUnique(GetNextFrame()), kFrameFound); CancelCallback(); return TimeDelta::Zero(); // Ignored. } else if (clock_->TimeInMilliseconds() >= latest_return_time_ms_) { // We have timed out, signal this and stop repeating. frame_handler_(nullptr, kTimeout); CancelCallback(); return TimeDelta::Zero(); // Ignored. } else { // If there's no frames to decode and there is still time left, it // means that the frame buffer was cleared between creation and // execution of this task. Continue waiting for the remaining time. int64_t wait_ms = FindNextFrame(clock_->TimeInMilliseconds()); return TimeDelta::ms(wait_ms); } } ); } |
./rtc_base/task_utils/repeating_task.h
1 2 3 4 5 6 7 8 9 10 | template <class Closure> static RepeatingTaskHandle DelayedStart(TaskQueueBase* task_queue, TimeDelta first_delay, Closure&& closure) { auto repeating_task = std::make_unique<webrtc_repeating_task_impl::RepeatingTaskImpl<Closure>>( task_queue, first_delay, std::forward<Closure>(closure)); auto* repeating_task_ptr = repeating_task.get(); task_queue->PostDelayedTask(std::move(repeating_task), first_delay.ms()); return RepeatingTaskHandle(repeating_task_ptr); } |
8.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 | int64_t FrameBuffer::FindNextFrame(int64_t now_ms) { // frames_ 接收队列 for (auto frame_it = frames_.begin(); frame_it != frames_.end() && frame_it->first <= last_continuous_frame_; ++frame_it) { EncodedFrame* frame = frame_it->second.frame.get(); std::vector<FrameMap::iterator> current_superframe; current_superframe.push_back(frame_it); bool last_layer_completed = frame_it->second.frame->is_last_spatial_layer; FrameMap::iterator next_frame_it = frame_it; while (true) { ++next_frame_it; if (next_frame_it == frames_.end() || next_frame_it->first.picture_id != frame->id.picture_id || !next_frame_it->second.continuous) { break; } // Check if the next frame has some undecoded references other than // the previous frame in the same superframe. size_t num_allowed_undecoded_refs = (next_frame_it->second.frame->inter_layer_predicted) ? 1 : 0; if (next_frame_it->second.num_missing_decodable > num_allowed_undecoded_refs) { break; } // All frames in the superframe should have the same timestamp. if (frame->Timestamp() != next_frame_it->second.frame->Timestamp()) { RTC_LOG(LS_WARNING) << "Frames in a single superframe have different" " timestamps. Skipping undecodable superframe."; break; } // 获取一帧 current_superframe.push_back(next_frame_it); last_layer_completed = next_frame_it->second.frame->is_last_spatial_layer; } } // 解码队列 frames_to_decode_ = std::move(current_superframe); } |
// 我们讲一下 frames_ 数据的由来
// 参见博文 https://blog.csdn.net/freeabc/article/details/106142951 知道 视频流接收最后都调用
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 | int64_t FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) // 把数据加到这个队列 auto info = frames_.emplace(id, FrameInfo()).first; // 接收到的视频数据 info->second.frame = std::move(frame); // 这个激发 new_continuous_frame_event_.Set(); // 继续投递一个任务继续执行 StartWaitForNextFrameOnQueue if (callback_queue_) { callback_queue_->PostTask([this] { rtc::CritScope lock(&crit_); if (!callback_task_.Running()) return; RTC_CHECK(frame_handler_); callback_task_.Stop(); StartWaitForNextFrameOnQueue(); }); } |
9.
1 2 3 4 5 6 7 8 9 10 11 12 | EncodedFrame* FrameBuffer::GetNextFrame() { std::vector<EncodedFrame*> frames_out; for (FrameMap::iterator& frame_it : frames_to_decode_) { EncodedFrame* frame = frame_it->second.frame.release(); frames_out.push_back(frame); } UpdateJitterDelay(); UpdateTimingFrameInfo(); return CombineAndDeleteFrames(frames_out); } |
10.
./video/video_receive_stream.cc
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 | void VideoReceiveStream::HandleEncodedFrame(std::unique_ptr<EncodedFrame> frame) { int64_t now_ms = clock_->TimeInMilliseconds(); // Current OnPreDecode only cares about QP for VP8. int qp = -1; if (frame->CodecSpecific()->codecType == kVideoCodecVP8) { if (!vp8::GetQp(frame->data(), frame->size(), &qp)) { RTC_LOG(LS_WARNING) << "Failed to extract QP from VP8 video frame"; } } stats_proxy_.OnPreDecode(frame->CodecSpecific()->codecType, qp); HandleKeyFrameGeneration(frame->FrameType() == VideoFrameType::kVideoFrameKey, now_ms); // 参见 10.1 的流程分析 int decode_result = video_receiver_.Decode(frame.get()); if (decode_result == WEBRTC_VIDEO_CODEC_OK || decode_result == WEBRTC_VIDEO_CODEC_OK_REQUEST_KEYFRAME) { keyframe_required_ = false; frame_decoded_ = true; rtp_video_stream_receiver_.FrameDecoded(frame->id.picture_id); if (decode_result == WEBRTC_VIDEO_CODEC_OK_REQUEST_KEYFRAME) RequestKeyFrame(now_ms); } else if (!frame_decoded_ || !keyframe_required_ || (last_keyframe_request_ms_ + max_wait_for_keyframe_ms_ < now_ms)) { keyframe_required_ = true; // TODO(philipel): Remove this keyframe request when downstream project // has been fixed. RequestKeyFrame(now_ms); } if (encoded_frame_buffer_function_) { frame->Retain(); encoded_frame_buffer_function_(WebRtcRecordableEncodedFrame(*frame)); } } |
10.1 VideoReceiver2 video_receiver_;
./modules/video_coding/video_receiver2.cc
1 2 3 4 5 6 7 8 9 10 11 12 | int32_t VideoReceiver2::Decode(const VCMEncodedFrame* frame) { RTC_DCHECK_RUN_ON(&decoder_thread_checker_); TRACE_EVENT0("webrtc", "VideoReceiver2::Decode"); // Change decoder if payload type has changed VCMGenericDecoder* decoder = codecDataBase_.GetDecoder(*frame, &decodedFrameCallback_); if (decoder == nullptr) { return VCM_NO_CODEC_REGISTERED; } // 解码器流程分析完毕,具体解码具体流程参见 10.2 return decoder->Decode(*frame, clock_->TimeInMilliseconds()); } |
我们分析 codecDataBase_ 的由来,参见这个函数,我们看到就是 VideoReceiveStream 的 video_decoders_
./video/video_receive_stream.cc
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 | void VideoReceiveStream::Start() // 这个地方一般把三种编码都注册进去 VP8, VP9, h264(avc) for (const Decoder& decoder : config_.decoders) { // 参见下面的 LegacyCreateVideoDecoder 分析 std::unique_ptr<VideoDecoder> video_decoder = decoder.decoder_factory->LegacyCreateVideoDecoder(decoder.video_format, config_.stream_id); if (!decoded_output_file.empty()) { char filename_buffer[256]; rtc::SimpleStringBuilder ssb(filename_buffer); ssb << decoded_output_file << "/webrtc_receive_stream_" << this->config_.rtp.remote_ssrc << "-" << rtc::TimeMicros() << ".ivf"; video_decoder = CreateFrameDumpingDecoderWrapper(std::move(video_decoder), FileWrapper::OpenWriteOnly(ssb.str())); } video_decoders_.push_back(std::move(video_decoder)); video_receiver_.RegisterExternalDecoder(video_decoders_.back().get(), decoder.payload_type); } |
// 我们看看 RegisterExternalDecoder 函数
./modules/video_coding/decoder_database.h
1 2 3 4 5 6 | void VCMDecoderDataBase::RegisterExternalDecoder(VideoDecoder* external_decoder, uint8_t payload_type) { // If payload value already exists, erase old and insert new. VCMExtDecoderMapItem* ext_decoder = new VCMExtDecoderMapItem(external_decoder, payload_type); DeregisterExternalDecoder(payload_type); dec_external_map_[payload_type] = ext_decoder; } |
// LegacyCreateVideoDecoder 分析
我们继续分析 LegacyCreateVideoDecoder 解码器的创建过程,首先分析解码类厂,然后分析 CreateVideoDecoder 函数
./api/video_codecs/video_decoder_factory.cc
1 2 3 4 5 | std::unique_ptr<VideoDecoder> VideoDecoderFactory::LegacyCreateVideoDecoder( const SdpVideoFormat& format, const std::string& receive_stream_id) { return CreateVideoDecoder(format); } |
//------------------------------------------------
// 解码类厂的创建
//------------------------------------------------
./sdk/android/src/jni/pc/peer_connection_factory.cc
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 | ScopedJavaLocalRef<jobject> CreatePeerConnectionFactoryForJava( JNIEnv* jni, const JavaParamRef<jobject>& jcontext, const JavaParamRef<jobject>& joptions, rtc::scoped_refptr<AudioDeviceModule> audio_device_module, rtc::scoped_refptr<AudioEncoderFactory> audio_encoder_factory, rtc::scoped_refptr<AudioDecoderFactory> audio_decoder_factory, const JavaParamRef<jobject>& jencoder_factory, const JavaParamRef<jobject>& jdecoder_factory, rtc::scoped_refptr<AudioProcessing> audio_processor, std::unique_ptr<FecControllerFactoryInterface> fec_controller_factory, std::unique_ptr<NetworkControllerFactoryInterface> network_controller_factory, std::unique_ptr<NetworkStatePredictorFactoryInterface> network_state_predictor_factory, std::unique_ptr<MediaTransportFactory> media_transport_factory, std::unique_ptr<NetEqFactory> neteq_factory) media_dependencies.video_decoder_factory = absl::WrapUnique(CreateVideoDecoderFactory(jni, jdecoder_factory)); |
./sdk/android/src/jni/pc/video.cc
1 2 3 4 5 6 7 | VideoDecoderFactory* CreateVideoDecoderFactory( JNIEnv* jni, const JavaRef<jobject>& j_decoder_factory) { return IsNull(jni, j_decoder_factory) ? nullptr : new VideoDecoderFactoryWrapper(jni, j_decoder_factory); } |
// 类厂创建完毕,就是一个 VideoDecoderFactoryWrapper 对象
// CreateVideoDecoder 函数如下:
./sdk/android/src/jni/video_decoder_factory_wrapper.cc
1 2 3 4 5 6 7 8 9 10 11 12 13 | std::unique_ptr<VideoDecoder> VideoDecoderFactoryWrapper::CreateVideoDecoder( const SdpVideoFormat& format) { JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedJavaLocalRef<jobject> j_codec_info = SdpVideoFormatToVideoCodecInfo(jni, format); // 参见流程 1 ScopedJavaLocalRef<jobject> decoder = Java_VideoDecoderFactory_createDecoder( jni, decoder_factory_, j_codec_info); if (!decoder.obj()) return nullptr; // JNI 层的对应 Java 层的 video decoder 对象,参见流程 2 return JavaToNativeVideoDecoder(jni, decoder); } |
1.
// 这个就是调用 Java 层的产生硬解码器了
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 | static base::android::ScopedJavaLocalRef<jobject> Java_VideoDecoderFactory_createDecoder(JNIEnv* env, const base::android::JavaRef<jobject>& obj, const base::android::JavaRef<jobject>& info) { jclass clazz = org_webrtc_VideoDecoderFactory_clazz(env); CHECK_CLAZZ(env, obj.obj(), org_webrtc_VideoDecoderFactory_clazz(env), NULL); jni_generator::JniJavaCallContextChecked call_context; call_context.Init<base::android::MethodID::TYPE_INSTANCE>( env, clazz, "createDecoder", "(Lorg/webrtc/VideoCodecInfo;)Lorg/webrtc/VideoDecoder;", &g_org_webrtc_VideoDecoderFactory_createDecoder); jobject ret = env->CallObjectMethod(obj.obj(), call_context.base.method_id, info.obj()); return base::android::ScopedJavaLocalRef<jobject>(env, ret); } |
// Android Java
1 2 3 4 5 6 7 8 9 10 11 12 13 14 | VideoDecoder DefaultVideoDecoderFactorypublic::createDecoder(VideoCodecInfo codecType) { VideoDecoder softwareDecoder = softwareVideoDecoderFactory.createDecoder(codecType); final VideoDecoder hardwareDecoder = hardwareVideoDecoderFactory.createDecoder(codecType); if (softwareDecoder == null && platformSoftwareVideoDecoderFactory != null) { softwareDecoder = platformSoftwareVideoDecoderFactory.createDecoder(codecType); } if (hardwareDecoder != null && softwareDecoder != null) { // 一般都是这个返回给底层 // Both hardware and software supported, wrap it in a software fallback return new VideoDecoderFallback( /* fallback= */ softwareDecoder, /* primary= */ hardwareDecoder); } return hardwareDecoder != null ? hardwareDecoder : softwareDecoder; } |
// Android Java --- HardwareVideoDecoderFactory 的基类 MediaCodecVideoDecoderFactory
1 2 3 4 5 6 7 8 9 10 11 12 13 | public VideoDecoder MediaCodecVideoDecoderFactory::createDecoder(VideoCodecInfo codecType) { VideoCodecType type = VideoCodecType.valueOf(codecType.getName()); MediaCodecInfo info = findCodecForType(type); if (info == null) { return null; } CodecCapabilities capabilities = info.getCapabilitiesForType(type.mimeType()); return new AndroidVideoDecoder(new MediaCodecWrapperFactoryImpl(), info.getName(), type, MediaCodecUtils.selectColorFormat(MediaCodecUtils.DECODER_COLOR_FORMATS, capabilities), sharedContext); } |
2.
./sdk/android/src/jni/video_decoder_wrapper.cc
1 2 3 4 5 6 7 8 9 10 11 12 13 14 | std::unique_ptr<VideoDecoder> JavaToNativeVideoDecoder( JNIEnv* jni, const JavaRef<jobject>& j_decoder) { // 通知 Java 产生解码器 const jlong native_decoder = Java_VideoDecoder_createNativeVideoDecoder(jni, j_decoder); VideoDecoder* decoder; if (native_decoder == 0) { decoder = new VideoDecoderWrapper(jni, j_decoder); } else { decoder = reinterpret_cast<VideoDecoder*>(native_decoder); } return std::unique_ptr<VideoDecoder>(decoder); } |
Java_VideoDecoder_createNativeVideoDecoder 这个会调用 Java 层的
1 2 3 | public long VideoDecoderFallback::createNativeVideoDecoder() { return nativeCreateDecoder(fallback, primary); } |
1 2 3 4 5 6 7 8 | JNI_GENERATOR_EXPORT jlong Java_org_webrtc_VideoDecoderFallback_nativeCreateDecoder( JNIEnv* env, jclass jcaller, jobject fallback, jobject primary) { return JNI_VideoDecoderFallback_CreateDecoder(env, base::android::JavaParamRef<jobject>(env, fallback), base::android::JavaParamRef<jobject>(env, primary)); } |
./sdk/android/src/jni/video_decoder_fallback.cc
1 2 3 4 5 6 | static jlong JNI_VideoDecoderFallback_CreateDecoder( JNIEnv* jni, const JavaParamRef<jobject>& j_fallback_decoder, const JavaParamRef<jobject>& j_primary_decoder) VideoDecoder* nativeWrapper = CreateVideoDecoderSoftwareFallbackWrapper(std::move(fallback_decoder), std::move(primary_decoder)) |
./api/video_codecs/video_decoder_software_fallback_wrapper.cc
1 2 3 4 5 6 | std::unique_ptr<VideoDecoder> CreateVideoDecoderSoftwareFallbackWrapper( std::unique_ptr<VideoDecoder> sw_fallback_decoder, std::unique_ptr<VideoDecoder> hw_decoder) { return std::make_unique<VideoDecoderSoftwareFallbackWrapper>( std::move(sw_fallback_decoder), std::move(hw_decoder)); } |
// VideoDecoderSoftwareFallbackWrapper 这个就是 VCMGenericDecoder 里的 decoder_
10.2
./modules/video_coding/generic_decoder.cc
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, int64_t nowMs) { TRACE_EVENT1("webrtc", "VCMGenericDecoder::Decode", "timestamp", frame.Timestamp()); _frameInfos[_nextFrameInfoIdx].decodeStartTimeMs = nowMs; _frameInfos[_nextFrameInfoIdx].renderTimeMs = frame.RenderTimeMs(); _frameInfos[_nextFrameInfoIdx].rotation = frame.rotation(); _frameInfos[_nextFrameInfoIdx].timing = frame.video_timing(); _frameInfos[_nextFrameInfoIdx].ntp_time_ms = frame.EncodedImage().ntp_time_ms_; _frameInfos[_nextFrameInfoIdx].packet_infos = frame.PacketInfos(); // Set correctly only for key frames. Thus, use latest key frame // content type. If the corresponding key frame was lost, decode will fail // and content type will be ignored. if (frame.FrameType() == VideoFrameType::kVideoFrameKey) { _frameInfos[_nextFrameInfoIdx].content_type = frame.contentType(); _last_keyframe_content_type = frame.contentType(); } else { _frameInfos[_nextFrameInfoIdx].content_type = _last_keyframe_content_type; } _callback->Map(frame.Timestamp(), &_frameInfos[_nextFrameInfoIdx]); _nextFrameInfoIdx = (_nextFrameInfoIdx + 1) % kDecoderFrameMemoryLength; // 根据 10.1 的分析,我们知道 decoder_ 就是 VideoDecoderSoftwareFallbackWrapper int32_t ret = decoder_->Decode(frame.EncodedImage(), frame.MissingFrame(), frame.RenderTimeMs()); _callback->OnDecoderImplementationName(decoder_->ImplementationName()); if (ret < WEBRTC_VIDEO_CODEC_OK) { RTC_LOG(LS_WARNING) << "Failed to decode frame with timestamp " << frame.Timestamp() << ", error code: " << ret; _callback->Pop(frame.Timestamp()); return ret; } else if (ret == WEBRTC_VIDEO_CODEC_NO_OUTPUT) { // No output _callback->Pop(frame.Timestamp()); } return ret; } |
./api/video_codecs/video_decoder_software_fallback_wrapper.cc
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 | int32_t VideoDecoderSoftwareFallbackWrapper::Decode( const EncodedImage& input_image, bool missing_frames, int64_t render_time_ms) { switch (decoder_type_) { case DecoderType::kNone: return WEBRTC_VIDEO_CODEC_UNINITIALIZED; case DecoderType::kHardware: { int32_t ret = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; ret = hw_decoder_->Decode(input_image, missing_frames, render_time_ms); if (ret != WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE) { return ret; } // HW decoder returned WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE or // initialization failed, fallback to software. if (!InitFallbackDecoder()) { return ret; } // Fallback decoder initialized, fall-through. RTC_FALLTHROUGH(); } case DecoderType::kFallback: return fallback_decoder_->Decode(input_image, missing_frames, render_time_ms); default: RTC_NOTREACHED(); return WEBRTC_VIDEO_CODEC_ERROR; } } |
// Java 层的 org.webrtc 硬解码函数
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 | public VideoCodecStatus AndroidVideoDecoder::decode(EncodedImage frame, DecodeInfo info) int index; try { index = codec.dequeueInputBuffer(DEQUEUE_INPUT_TIMEOUT_US); } catch (IllegalStateException e) { Logging.e(TAG, "dequeueInputBuffer failed", e); return VideoCodecStatus.ERROR; } if (index < 0) { // Decoder is falling behind. No input buffers available. // The decoder can't simply drop frames; it might lose a key frame. Logging.e(TAG, "decode() - no HW buffers available; decoder falling behind"); return VideoCodecStatus.ERROR; } ByteBuffer buffer; try { buffer = codec.getInputBuffers()[index]; } catch (IllegalStateException e) { Logging.e(TAG, "getInputBuffers failed", e); return VideoCodecStatus.ERROR; } if (buffer.capacity() < size) { Logging.e(TAG, "decode() - HW buffer too small"); return VideoCodecStatus.ERROR; } buffer.put(frame.buffer); frameInfos.offer(new FrameInfo(SystemClock.elapsedRealtime(), frame.rotation)); try { codec.queueInputBuffer(index, 0 /* offset */, size, TimeUnit.NANOSECONDS.toMicros(frame.captureTimeNs), 0 /* flags */); } catch (IllegalStateException e) { Logging.e(TAG, "queueInputBuffer failed", e); frameInfos.pollLast(); return VideoCodecStatus.ERROR; } if (keyFrameRequired) { keyFrameRequired = false; } return VideoCodecStatus.OK; |
//-----------------------------------------------------------------------
// 致此,我们分析的数据从 jitterbuffer 到解码器进行解码的流程基本完毕!
//-----------------------------------------------------------------------