webrtc-SVC+simulcast改造
1.simulcast+SVC打开
std::string str[] = {"low", "medium", "high"};
std::string msid[] = {"l", "m", "h"};
//double pri = 0.8;
//添加初始化参数，在此处设置时域层数，push 多少webrtc::RtpEncodingParameters就是多少层simulcast
webrtc::RtpTransceiverInit rtpTI;
for (int i = 3; i >= 1; i--) {
webrtc::RtpEncodingParameters videoEncoding;
videoEncoding.rid = str[i-1];
//videoEncoding.max_bitrate_bps = 3 * i * 100 * 1000;
//videoEncoding.bitrate_priority = pri;
videoEncoding.num_temporal_layers = 3;
rtpTI.send_encodings.push_back(videoEncoding);
//pri -= 0.2;
rtpTI.stream_ids.push_back(msid[i-1]); //这个是SDP中msid参数的名字
}
//单层simulcast的时候设置时域层数
//rtpTI.stream_ids.push_back("cam");
//webrtc::RtpEncodingParameters videoEncoding;
//videoEncoding.rid = str[2];
//videoEncoding.num_temporal_layers = 3;
//rtpTI.send_encodings.push_back(videoEncoding);
auto ret = peer_connection_->AddTransceiver(video_track_,rtpTI);
//这个可以获取当前设置的参数
webrtc::RtpParameters para =
peer_connection_->GetSenders()[1]->GetParameters();
2.设置编码优先顺序（编码选择）
src/media/engine/internal_encoder_factory.cc
std::vector<SdpVideoFormat> InternalEncoderFactory::GetSupportedFormats()
const {
std::vector<SdpVideoFormat> supported_codecs;
//for (const webrtc::SdpVideoFormat& format : webrtc::SupportedH264Codecs())
// supported_codecs.push_back(format);
// for (const webrtc::SdpVideoFormat& format : webrtc::SupportedVP9Codecs())
// supported_codecs.push_back(format);
supported_codecs.push_back(SdpVideoFormat(cricket::kVp8CodecName));
return supported_codecs;
}
3.设置H264可通过AddTransceiver参数初始化支持SVC
增加对H264的支持：
src/media/engine/webrtc_video_engine.cc
bool IsTemporalLayersSupported(const std::string& codec_name) {
return absl::EqualsIgnoreCase(codec_name, kVp8CodecName) ||
absl::EqualsIgnoreCase(codec_name, kVp9CodecName) ||
absl::EqualsIgnoreCase(codec_name, kH264CodecName);
}
4.关于simulcast和svc的常量参数设置
src/api/video/video_codec_constans.h
enum : int { kMaxEncoderBuffers = 8 };
enum : int { kMaxSimulcastStreams = 3 };
enum : int { kMaxSpatialLayers = 5 };
enum : int { kMaxTemporalStreams = 4 };
5.simulcast
media_session.cc
static bool AddStreamParams(){
...
//这是对应的流的信息
StreamParams stream_param =
sender.rids.empty()
?
// Signal SSRCs and legacy simulcast (if requested).
//老版本planb. rids为空，使用num_simulcast_layer来创建，内部调用GenerateSsrcs
CreateStreamParamsForNewSenderWithSsrcs(
sender, rtcp_cname, include_rtx_streams,
include_flexfec_stream, ssrc_generator)
:
// Signal RIDs and spec-compliant simulcast (if requested).
CreateStreamParamsForNewSenderWithRids(sender, rtcp_cname);
...
}
AddTransceiver只能在webrtc::SdpSemantics::kUnifiedPlan模式下，这个在CreatePeerConnection时设置进去，目前设置完simulcast参数后，数据未推上去，应该是服务端暂未支持。
webrtc::SdpSemantics::kPlanB模式下对应的是AddTrack,但是设置simulcast层数是在CreateOffer设置进去，webrtc::PeerConnectionInterface::RTCOfferAnswerOptions.num_simulcast_layers,但是此时根据抓包和断点看到的是只有2层，RTCOfferAnswerOptions里面默认是两层。
注：webrtc会根据当前视频的分辨率，以及预设的常量来决定实际的simulcast层数，例如640x480-位于(960,540)和(640,320)，所以参数是设置为(640,320)，最多2层simulcast
void Conductor::ConnectToPeer(int peer_id) {
//RTC_DCHECK(peer_id_ == -1);
//RTC_DCHECK(peer_id != -1);
if (peer_connection_.get()) {
main_wnd_->MessageBox(
"Error", "We only support connecting to one peer at a time", true);
return;
}
if (InitializePeerConnection()) {
webrtc::PeerConnectionInterface::RTCOfferAnswerOptions options =
webrtc::PeerConnectionInterface::RTCOfferAnswerOptions();
options.num_simulcast_layers = 3;
peer_id_ = peer_id;
peer_connection_->CreateOffer(
this, options);
} else {
main_wnd_->MessageBox("Error", "Failed to initialize PeerConnection", true);
}
}
src/media/engine/simulcast.cc
// These tables describe from which resolution we can use how many
// simulcast layers at what bitrates (maximum, target, and minimum).
// Important!! Keep this table from high resolution to low resolution.
// clang-format off
const SimulcastFormat kSimulcastFormats[] = {
{1920, 1080, 3, 5000, 4000, 800},
{1280, 720, 3, 2500, 2500, 600},
{960, 540, 3, 1200, 1200, 350},
{640, 360, 2, 700, 500, 150},
{480, 270, 2, 450, 350, 150},
{320, 180, 1, 200, 150, 30},
{0, 0, 1, 200, 150, 30}
};
FindSimulcastFormatIndex:
{
...
for (uint32_t i = 0; i < arraysize(kSimulcastFormats); ++i) {
if (width * height >=
kSimulcastFormats[i].width * kSimulcastFormats[i].height) {
return i;
}
}
...
}
ReconfigureEncoder
↓
CreateEncoderStreams
↓
GetSimulcastConfig
↓
LimitSimulcastLayerCount
↓
FindSimulcastFormatIndex
6.simulcast码率设置
编码相关的设置在src/video/video_stream_encoder.cc-ReconfigureEncoder
ReconfigureEncoder
↓
CreateEncoderStreams
↓
GetSimulcastConfig
↓
GetNormalSimulcastLayers 设置宽高码率，同时会设置默认时域层
DefaultNumberOfTemporalLayers
FindSimulcastMaxBitrateBps
FindSimulcastTargetBitrateBps
FindSimulcastMinBitrateBps
kDefaultVideoMaxFramerate = 60
计算simulcast码率的时候使用双线性插值：
const int total_pixels_up =
kSimulcastFormats[index - 1].width * kSimulcastFormats[index - 1].height;
const int total_pixels_down =
kSimulcastFormats[index].width * kSimulcastFormats[index].height;
const int total_pixels = width * height;
const float rate = (total_pixels_up - total_pixels) /
static_cast<float>(total_pixels_up - total_pixels_down);
SimulcastFormat res;
res.width = width;
res.height = height;
res.max_layers = kSimulcastFormats[index].max_layers;
res.max_bitrate_kbps =
kSimulcastFormats[index - 1].max_bitrate_kbps * (1.0 - rate) +
kSimulcastFormats[index].max_bitrate_kbps * rate;
在设置0层simulcast时，如果打开了kUseBaseHeavyVP8TL3RateAllocationFieldTrial（"WebRTC-UseBaseHeavyVP8TL3RateAllocation"），最大码率和目标码率将会乘以系数以适应0时域层
// If alternative temporal rate allocation is selected, adjust the
// bitrate of the lowest simulcast stream so that absolute bitrate for
// the base temporal layer matches the bitrate for the base temporal
// layer with the default 3 simulcast streams. Otherwise we risk a
// higher threshold for receiving a feed at all.
if (num_temporal_layers == 3) {
if (webrtc::field_trial::IsEnabled(
kUseBaseHeavyVP8TL3RateAllocationFieldTrial)) {
// Base heavy allocation increases TL0 bitrate from 40% to 60%.
rate_factor = 0.4 / 0.6;
}
} else {
rate_factor =
webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(3, 0) /
webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
num_temporal_layers, 0);
}
static const float
kLayerRateAllocation[kMaxTemporalStreams][kMaxTemporalStreams] = {
{1.0f, 1.0f, 1.0f, 1.0f}, // 1 layer
{0.6f, 1.0f, 1.0f, 1.0f}, // 2 layers {60%, 40%}
{0.4f, 0.6f, 1.0f, 1.0f}, // 3 layers {40%, 20%, 40%}
{0.25f, 0.4f, 0.6f, 1.0f} // 4 layers {25%, 15%, 20%, 40%}
};
static const float kBaseHeavy3TlRateAllocation[kMaxTemporalStreams] = {
0.6f, 0.8f, 1.0f, 1.0f // 3 layers {60%, 20%, 20%}
};
7.时域层码率设置
ReconfigureEncoder
↓
EncoderSimulcastProxy::InitEncode
↓
LibvpxVp8Encoder::InitEncode
↓
SimulcastRateAllocator::Allocate
↓
SimulcastRateAllocator::GetTemporalRateAllocation 获取对应simulcast层数的时域层比率乘以目标码率
时域层的所有码率总和是当前simulcast层的目标码率
std::vector<uint32_t> SimulcastRateAllocator::DefaultTemporalLayerAllocation(
int bitrate_kbps,
int max_bitrate_kbps,
int simulcast_id) const {
const size_t num_temporal_layers = NumTemporalStreams(simulcast_id);
std::vector<uint32_t> bitrates;
for (size_t i = 0; i < num_temporal_layers; ++i) {
float layer_bitrate =
bitrate_kbps * GetTemporalRateAllocation(num_temporal_layers, i);
bitrates.push_back(static_cast<uint32_t>(layer_bitrate + 0.5));
}
// Allocation table is of aggregates, transform to individual rates.
uint32_t sum = 0;
for (size_t i = 0; i < num_temporal_layers; ++i) {
uint32_t layer_bitrate = bitrates[i];
RTC_DCHECK_LE(sum, bitrates[i]);
bitrates[i] -= sum;
sum = layer_bitrate;
if (sum >= static_cast<uint32_t>(bitrate_kbps)) {
// Sum adds up; any subsequent layers will be 0.
bitrates.resize(i + 1);
break;
}
}
return bitrates;
}
