1.前言
本文介紹webrtc的API層整體結構和通話的api流程， 適合作為對webrtc有一定認知但是想研究源碼的第一篇文章，推薦piasy的webrtc源碼導讀13以及webrtc源碼導讀10，本文實際就是在它們的基礎上結合了webrtc源碼上的example分析, 如果想要進一步分析整個呼叫過程，可以參考webrtc呼叫建立過程系列寫的真的很用心
2.正文
2.1 關鍵類介紹
2.1.1 PeerConnection
最核心的類，沒有之一，所有的功能接口都由它提供，圍繞它轉，提供了下列類型的接口:
-
流相關: stream 和 track
stream接口現已被標准棄用，mdn的上說addstream會導致后續的track變化感知不敏感
track代表媒體軌，比如麥克風音軌，攝像頭畫軌, 提供媒體源的載體，將流廣播給訂閱者
local_streams()
remote_streams()
AddStream()
RemoveStream()
AddTrack()
RemoveTrack()
RemoveTrackNew()
-
傳輸相關: transceiver, sender, receiver
一個tranceiver包含了一個sender(編碼，發送), receiver(解碼接收)。addtrack的時候就是將track添加到transceiver的過程， 一個 sender 至多有一個要發送的 track一個要接收的 track 有一個 receiver，tranceiver使用mid作為標識。
AddTransceiver()
GetTransceivers()
CreateSender()
GetSenders()
GetReceivers()
- SDP和ICE相關
local_description()
remote_description()
current_local_description()
current_remote_description()
pending_local_description()
pending_remote_description()
RestartIce()
CreateOffer()
CreateAnswer()
SetLocalDescription()
SetRemoteDescription()
AddIceCandidate()
RemoveIceCandidates()
- 其它
SetBitrate()
SetAudioPlayout()
SetAudioRecording()
LookupDtlsTransportByMid()
GetSctpTransport()
signaling_state()
ice_connection_state()
standardized_ice_connection_state()
peer_connection_state()
ice_gathering_state()
2.1.2 PeerConnectionFactory
提供peerconnection，track，audio/video source的創建接口
CreatePeerConnection()
CreateLocalMediaStream()
CreateAudioSource()
CreateVideoTrack()
CreateAudioTrack()
2.2 通話的流程
2.2.1 流程概覽
PeerConnectionFactory::CreatePeerConnection() // 創建Pc
PeerConnection::CreateAudioTrack() // 創建音軌
PeerConnection::AddTrack() // 添加音軌
PeerConnection::CreateVideoTrack() // 創建視軌
PeerConnection::AddTrack() // 添加視軌
PeerConnection::CreateOffer() // 創建offer sdp
PeerConnection::SetLocalDescription() // 設置local sdp
send sdp to peer... // 發送給對端
-----------------------------------------------------------
receive remote sdp... // 收到對端的sdp
PeerConnection::SetRemoteDescription() // 設置remote sdp
OnIceCandidate() // ice candidate收集到了
send candidate to peer... // 發送給對端
------------------------------------------------------------
receive remote candidate... // 收到對端的ice candidate
PeerConnection::AddIceCandidate() // 設置到pc中
2.2.2 創建PeerConnection
在InitializePeerConnection()函數中首先調用 webrtc::CreatePeerConnectionFactory()創建PeerConnectionFactory, 注意，創建的參數中需要提供[Audio|Video] [Encoder|Decoder]Factory()以提供視頻編解碼器
bool Conductor::InitializePeerConnection() {
RTC_DCHECK(!peer_connection_factory_);
RTC_DCHECK(!peer_connection_);
// 創建PC factory
peer_connection_factory_ = webrtc::CreatePeerConnectionFactory(
nullptr /* network_thread */, nullptr /* worker_thread */,
nullptr /* signaling_thread */, nullptr /* default_adm */,
webrtc::CreateBuiltinAudioEncoderFactory(),
webrtc::CreateBuiltinAudioDecoderFactory(),
webrtc::CreateBuiltinVideoEncoderFactory(),
webrtc::CreateBuiltinVideoDecoderFactory(), nullptr /* audio_mixer */,
nullptr /* audio_processing */);
if (!peer_connection_factory_) {
main_wnd_->MessageBox("Error", "Failed to initialize PeerConnectionFactory",
true);
DeletePeerConnection();
return false;
}
// 創建PC
if (!CreatePeerConnection(/*dtls=*/true)) {
main_wnd_->MessageBox("Error", "CreatePeerConnection failed", true);
DeletePeerConnection();
}
AddTracks();
return peer_connection_ != nullptr;
}
隨后在CreatePeerConnection()中創建PeerConnection, 創建的時候可以放RTCConfiguration 用來做當前PC進行一些選項配置： dtls, audio, video, jitter, ice等等（FEC的配置不在此處，在創建PeerConncetionFactory中）
bool Conductor::CreatePeerConnection(bool dtls) {
RTC_DCHECK(peer_connection_factory_);
RTC_DCHECK(!peer_connection_);
webrtc::PeerConnectionInterface::RTCConfiguration config;
config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan;
config.enable_dtls_srtp = dtls;
webrtc::PeerConnectionInterface::IceServer server;
server.uri = GetPeerConnectionString();
config.servers.push_back(server);
// 創建pc
peer_connection_ = peer_connection_factory_->CreatePeerConnection(
config, nullptr, nullptr, this);
return peer_connection_ != nullptr;
}
如此，PC就創建完畢了
2.2.3 設置track
創建完PC后就開始添加track，在函數AddTracks()中
void Conductor::AddTracks() {
if (!peer_connection_->GetSenders().empty()) {
return; // Already added tracks.
}
// 創建音軌
rtc::scoped_refptr<webrtc::AudioTrackInterface> audio_track(
peer_connection_factory_->CreateAudioTrack(
kAudioLabel, peer_connection_factory_->CreateAudioSource(
cricket::AudioOptions())));
// 添加音軌
auto result_or_error = peer_connection_->AddTrack(audio_track, {kStreamId});
if (!result_or_error.ok()) {
RTC_LOG(LS_ERROR) << "Failed to add audio track to PeerConnection: "
<< result_or_error.error().message();
}
// 創建視頻源
rtc::scoped_refptr<CapturerTrackSource> video_device =
CapturerTrackSource::Create();
if (video_device) {
// 創建視軌
rtc::scoped_refptr<webrtc::VideoTrackInterface> video_track_(
peer_connection_factory_->CreateVideoTrack(kVideoLabel, video_device));
main_wnd_->StartLocalRenderer(video_track_);
// 添加視軌
result_or_error = peer_connection_->AddTrack(video_track_, {kStreamId});
if (!result_or_error.ok()) {
RTC_LOG(LS_ERROR) << "Failed to add video track to PeerConnection: "
<< result_or_error.error().message();
}
} else {
RTC_LOG(LS_ERROR) << "OpenVideoCaptureDevice failed";
}
main_wnd_->SwitchToStreamingUI();
}
audio_track(音軌)的創建使用的audio_source(音頻源)來自於peer_connection_factory_->CreateAudioSource(options) ，options可以配置是否做增益，噪聲消除等，但是audio_source的這種創建方式就導致我們似乎無法自定義一個音頻源？ 創建完的````audio_track通過AddTrack()``的方式添加到pc中
而video_track(視軌)的創建使用的video_source來自於自定義的CapturerTrackSource，不妨來看看，這個類的彎彎繞繞比較多, 如下，在Create() 中調用VideoCaptureFactory::CreateDeviceInfo()獲取所有的視頻捕獲設備的信息，然后創建了一個VcmCapturer, 最后使用VcmCapturer創建CapturerTrackSource；CapturerTrackSourceoverride了source()函數， 調用者調用source()獲取到video_source源也就是VcmCapturer, VcmCapturer override了OnFrame() 用來提供幀
class VcmCapturer : public TestVideoCapturer,
public rtc::VideoSinkInterface<VideoFrame> {
public:
static VcmCapturer* Create(size_t width,
size_t height,
size_t target_fps,
size_t capture_device_index);
virtual ~VcmCapturer();
// 提供幀
void OnFrame(const VideoFrame& frame) override;
private:
VcmCapturer();
bool Init(size_t width,
size_t height,
size_t target_fps,
size_t capture_device_index);
void Destroy();
rtc::scoped_refptr<VideoCaptureModule> vcm_;
VideoCaptureCapability capability_;
};
class CapturerTrackSource : public webrtc::VideoTrackSource {
public:
static rtc::scoped_refptr<CapturerTrackSource> Create() {
const size_t kWidth = 640;
const size_t kHeight = 480;
const size_t kFps = 30;
std::unique_ptr<webrtc::test::VcmCapturer> capturer;
// 獲取所有的視頻捕獲設備
std::unique_ptr<webrtc::VideoCaptureModule::DeviceInfo> info(
webrtc::VideoCaptureFactory::CreateDeviceInfo());
if (!info) {
return nullptr;
}
int num_devices = info->NumberOfDevices();
for (int i = 0; i < num_devices; ++i) {
// 創建一個VcmCapturer， VcmCapturer繼承了VideoSinkInterface
capturer = absl::WrapUnique(
webrtc::test::VcmCapturer::Create(kWidth, kHeight, kFps, i));
if (capturer) {
// 使用video capturer創建一個CapturerTrackSource
return new rtc::RefCountedObject<CapturerTrackSource>(
std::move(capturer));
}
}
return nullptr;
}
protected:
explicit CapturerTrackSource(
std::unique_ptr<webrtc::test::VcmCapturer> capturer)
: VideoTrackSource(/*remote=*/false), capturer_(std::move(capturer)) {}
private:
// 提供源
rtc::VideoSourceInterface<webrtc::VideoFrame>* source() override {
return capturer_.get();
}
std::unique_ptr<webrtc::test::VcmCapturer> capturer_;
};
然后通過video_source創建的video_track也通過AddTrack() 的方式添加到peerconnection中
2.2.4 設置SDP
SDP的流程由於涉及到雙方，所以放一個圖直接說明
作為發起端， PeerConnection執行 CreateOffer()
void Conductor::ConnectToPeer(int peer_id) {
RTC_DCHECK(peer_id_ == -1);
RTC_DCHECK(peer_id != -1);
if (peer_connection_.get()) {
main_wnd_->MessageBox(
"Error", "We only support connecting to one peer at a time", true);
return;
}
if (InitializePeerConnection()) {
peer_id_ = peer_id;
// 創建offer
peer_connection_->CreateOffer(
this, webrtc::PeerConnectionInterface::RTCOfferAnswerOptions());
} else {
main_wnd_->MessageBox("Error", "Failed to initialize PeerConnection", true);
}
}
offer創建完成后通過異步回調的方式調用Conductor::OnSuccess()；
調用SetLocalDescription()把本地SDP設置到當前的PeerConnection中，根據SDP進行底層相關組件的創建和初始化 ，同時把創建的SDP轉成json發送給遠端
void Conductor::OnSuccess(webrtc::SessionDescriptionInterface* desc) {
//設置本地SDP
peer_connection_->SetLocalDescription(
DummySetSessionDescriptionObserver::Create(), desc);
std::string sdp;
desc->ToString(&sdp);
// For loopback test. To save some connecting delay.
if (loopback_) {
//如果是PeerConnection測試，整個SDP都是一樣的，只需要
// 吧offer改成answer即可
// Replace message type from "offer" to "answer"
std::unique_ptr<webrtc::SessionDescriptionInterface> session_description =
webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp);
peer_connection_->SetRemoteDescription(
DummySetSessionDescriptionObserver::Create(),
session_description.release());
return;
}
Json::StyledWriter writer;
Json::Value jmessage;
jmessage[kSessionDescriptionTypeName] =
webrtc::SdpTypeToString(desc->GetType());
jmessage[kSessionDescriptionSdpName] = sdp;
SendMessage(writer.write(jmessage));
}
過一段時間后，對端會發來remote sdp， 通過SetRemoteDescriptioin()設置好遠端的sdp，對於發起端而言，SDP就設置完畢了，這個設置會根據對端發送來的SDP，去創建相應的track和別的東西。
void Conductor::OnMessageFromPeer(int peer_id, const std::string& message) {
.....
Json::Reader reader;
Json::Value jmessage;
if (!reader.parse(message, jmessage)) {
RTC_LOG(WARNING) << "Received unknown message. " << message;
return;
}
std::string type_str;
std::string json_object;
rtc::GetStringFromJsonObject(jmessage, kSessionDescriptionTypeName,
&type_str);
if (!type_str.empty()) {
// remote sdp come
if (type_str == "offer-loopback") {
// This is a loopback call.
// Recreate the peerconnection with DTLS disabled.
if (!ReinitializePeerConnectionForLoopback()) {
RTC_LOG(LS_ERROR) << "Failed to initialize our PeerConnection instance";
DeletePeerConnection();
client_->SignOut();
}
return;
}
absl::optional<webrtc::SdpType> type_maybe =
webrtc::SdpTypeFromString(type_str);
if (!type_maybe) {
RTC_LOG(LS_ERROR) << "Unknown SDP type: " << type_str;
return;
}
webrtc::SdpType type = *type_maybe;
std::string sdp;
if (!rtc::GetStringFromJsonObject(jmessage, kSessionDescriptionSdpName,
&sdp)) {
RTC_LOG(WARNING) << "Can't parse received session description message.";
return;
}
webrtc::SdpParseError error;
std::unique_ptr<webrtc::SessionDescriptionInterface> session_description =
webrtc::CreateSessionDescription(type, sdp, &error);
if (!session_description) {
RTC_LOG(WARNING) << "Can't parse received session description message. "
"SdpParseError was: "
<< error.description;
return;
}
RTC_LOG(INFO) << " Received session description :" << message;
// 設置remote sdp
peer_connection_->SetRemoteDescription(
DummySetSessionDescriptionObserver::Create(),
session_description.release());
if (type == webrtc::SdpType::kOffer) {
// 如果是offer類型， 創建一個answer作為回應
peer_connection_->CreateAnswer(
this, webrtc::PeerConnectionInterface::RTCOfferAnswerOptions());
}
}
.....
}
而對於被發起端而言，仍然是通過OnMessageFromPeer( )接收到遠端的offer sdp，如上，然后通過SetRemoteDescription()設置好該SDP后，使用CreateAnswer()創建answer，answer創建完畢之后會觸發Conductor::OnSuccess()，需要將本地生成的SDP進行SetLocalDescription()，然后發送給對端。
2.2.5 設置 ICE
調用了接口SetLocalDescription()后，底層開始收集ICE Candidate，收集完成后回調Conductor::OnIceCandidate(), 解析之后寫成json的形式發送給對方
void Conductor::OnIceCandidate(const webrtc::IceCandidateInterface* candidate) {
RTC_LOG(INFO) << __FUNCTION__ << " " << candidate->sdp_mline_index();
// For loopback test. To save some connecting delay.
if (loopback_) {
if (!peer_connection_->AddIceCandidate(candidate)) {
RTC_LOG(WARNING) << "Failed to apply the received candidate";
}
return;
}
Json::StyledWriter writer;
Json::Value jmessage;
// 寫成json 發送給對方
jmessage[kCandidateSdpMidName] = candidate->sdp_mid();
jmessage[kCandidateSdpMlineIndexName] = candidate->sdp_mline_index();
std::string sdp;
if (!candidate->ToString(&sdp)) {
RTC_LOG(LS_ERROR) << "Failed to serialize candidate";
return;
}
jmessage[kCandidateSdpName] = sdp;
SendMessage(writer.write(jmessage));
}
對端在收到candidate之后，就執行AddCandidate()將candidate添加到PeerConnection中
void Conductor::OnMessageFromPeer(int peer_id, const std::string& message) {
....
std::string type_str;
std::string json_object;
rtc::GetStringFromJsonObject(jmessage, kSessionDescriptionTypeName,
&type_str);
if (!type_str.empty()) {
// handle sdp
.....
} else {
// handle candidate
std::string sdp_mid;
int sdp_mlineindex = 0;
std::string sdp;
if (!rtc::GetStringFromJsonObject(jmessage, kCandidateSdpMidName,
&sdp_mid) ||
!rtc::GetIntFromJsonObject(jmessage, kCandidateSdpMlineIndexName,
&sdp_mlineindex) ||
!rtc::GetStringFromJsonObject(jmessage, kCandidateSdpName, &sdp)) {
RTC_LOG(WARNING) << "Can't parse received message.";
return;
}
webrtc::SdpParseError error;
// 創建candidate
std::unique_ptr<webrtc::IceCandidateInterface> candidate(
webrtc::CreateIceCandidate(sdp_mid, sdp_mlineindex, sdp, &error));
if (!candidate.get()) {
RTC_LOG(WARNING) << "Can't parse received candidate message. "
"SdpParseError was: "
<< error.description;
return;
}
// 添加candidate
if (!peer_connection_->AddIceCandidate(candidate.get())) {
RTC_LOG(WARNING) << "Failed to apply the received candidate";
return;
}
RTC_LOG(INFO) << " Received candidate :" << message;
}
}