diff --git a/common_types.cc b/common_types.cc index 0b262a6ab9..6c9a59554a 100644 --- a/common_types.cc +++ b/common_types.cc @@ -16,6 +16,7 @@ #include #include "rtc_base/checks.h" +#include "rtc_base/strings/string_builder.h" #include "rtc_base/stringutils.h" namespace webrtc { @@ -232,13 +233,37 @@ uint32_t BitrateAllocation::GetSpatialLayerSum(size_t spatial_index) const { return sum; } +std::vector BitrateAllocation::GetTemporalLayerAllocation( + size_t spatial_index) const { + RTC_CHECK_LT(spatial_index, kMaxSpatialLayers); + std::vector temporal_rates; + + // Find the highest temporal layer with a defined bitrate in order to + // determine the size of the temporal layer allocation. + for (size_t i = kMaxTemporalStreams; i > 0; --i) { + if (has_bitrate_[spatial_index][i - 1]) { + temporal_rates.resize(i); + break; + } + } + + for (size_t i = 0; i < temporal_rates.size(); ++i) { + temporal_rates[i] = bitrates_[spatial_index][i]; + } + + return temporal_rates; +} + std::string BitrateAllocation::ToString() const { if (sum_ == 0) return "BitrateAllocation [ [] ]"; - // TODO(sprang): Replace this stringstream with something cheaper. - std::ostringstream oss; - oss << "BitrateAllocation ["; + // Max string length in practice is 260, but let's have some overhead and + // round up to nearest power of two. + char string_buf[512]; + rtc::SimpleStringBuilder ssb(string_buf); + + ssb << "BitrateAllocation ["; uint32_t spatial_cumulator = 0; for (int si = 0; si < kMaxSpatialLayers; ++si) { RTC_DCHECK_LE(spatial_cumulator, sum_); @@ -247,11 +272,11 @@ std::string BitrateAllocation::ToString() const { const uint32_t layer_sum = GetSpatialLayerSum(si); if (layer_sum == sum_) { - oss << " ["; + ssb << " ["; } else { if (si > 0) - oss << ","; - oss << std::endl << " ["; + ssb << ","; + ssb << '\n' << " ["; } spatial_cumulator += layer_sum; @@ -262,23 +287,18 @@ std::string BitrateAllocation::ToString() const { break; if (ti > 0) - oss << ", "; + ssb << ", "; uint32_t bitrate = bitrates_[si][ti]; - oss << bitrate; + ssb << bitrate; temporal_cumulator += bitrate; } - oss << "]"; + ssb << "]"; } RTC_DCHECK_EQ(spatial_cumulator, sum_); - oss << " ]"; - return oss.str(); -} - -std::ostream& BitrateAllocation::operator<<(std::ostream& os) const { - os << ToString(); - return os; + ssb << " ]"; + return ssb.str(); } } // namespace webrtc diff --git a/common_types.h b/common_types.h index 482cb0475c..312bc2427c 100644 --- a/common_types.h +++ b/common_types.h @@ -611,6 +611,11 @@ class BitrateAllocation { // Get the sum of all the temporal layer for a specific spatial layer. uint32_t GetSpatialLayerSum(size_t spatial_index) const; + // Returns a vector of the temporal layer bitrates for the specific spatial + // layer. Length of the returned vector is cropped to the highest temporal + // layer with a defined bitrate. + std::vector GetTemporalLayerAllocation(size_t spatial_index) const; + uint32_t get_sum_bps() const { return sum_; } // Sum of all bitrates. uint32_t get_sum_kbps() const { return (sum_ + 500) / 1000; } @@ -623,7 +628,6 @@ class BitrateAllocation { // Expensive, please use only in tests. std::string ToString() const; - std::ostream& operator<<(std::ostream& os) const; private: uint32_t sum_; diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers.cc b/modules/video_coding/codecs/vp8/default_temporal_layers.cc index 1767c5863f..02b3a3708a 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers.cc +++ b/modules/video_coding/codecs/vp8/default_temporal_layers.cc @@ -321,42 +321,27 @@ uint8_t DefaultTemporalLayers::Tl0PicIdx() const { return tl0_pic_idx_; } -std::vector DefaultTemporalLayers::OnRatesUpdated( - int bitrate_kbps, - int max_bitrate_kbps, - int framerate) { - std::vector bitrates; - for (size_t i = 0; i < num_layers_; ++i) { - float layer_bitrate = - bitrate_kbps * kVp8LayerRateAlloction[num_layers_ - 1][i]; - bitrates.push_back(static_cast(layer_bitrate + 0.5)); +void DefaultTemporalLayers::OnRatesUpdated( + const std::vector& bitrates_bps, + int framerate_fps) { + RTC_DCHECK_GT(bitrates_bps.size(), 0); + RTC_DCHECK_LE(bitrates_bps.size(), num_layers_); + // |bitrates_bps| uses individual rate per layer, but Vp8EncoderConfig wants + // the accumulated rate, so sum them up. + new_bitrates_bps_ = bitrates_bps; + new_bitrates_bps_->resize(num_layers_); + for (size_t i = 1; i < num_layers_; ++i) { + (*new_bitrates_bps_)[i] += (*new_bitrates_bps_)[i - 1]; } - new_bitrates_kbps_ = rtc::Optional>(bitrates); - - // Allocation table is of aggregates, transform to individual rates. - uint32_t sum = 0; - for (size_t i = 0; i < num_layers_; ++i) { - uint32_t layer_bitrate = bitrates[i]; - RTC_DCHECK_LE(sum, bitrates[i]); - bitrates[i] -= sum; - sum = layer_bitrate; - - if (sum >= static_cast(bitrate_kbps)) { - // Sum adds up; any subsequent layers will be 0. - bitrates.resize(i + 1); - break; - } - } - - return bitrates; } bool DefaultTemporalLayers::UpdateConfiguration(Vp8EncoderConfig* cfg) { - if (!new_bitrates_kbps_) + if (!new_bitrates_bps_) { return false; + } for (size_t i = 0; i < num_layers_; ++i) { - cfg->ts_target_bitrate[i] = (*new_bitrates_kbps_)[i]; + cfg->ts_target_bitrate[i] = (*new_bitrates_bps_)[i] / 1000; // ..., 4, 2, 1 cfg->ts_rate_decimator[i] = 1 << (num_layers_ - i - 1); } @@ -366,7 +351,7 @@ bool DefaultTemporalLayers::UpdateConfiguration(Vp8EncoderConfig* cfg) { memcpy(cfg->ts_layer_id, &temporal_ids_[0], sizeof(unsigned int) * temporal_ids_.size()); - new_bitrates_kbps_ = rtc::Optional>(); + new_bitrates_bps_.reset(); return true; } diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers.h b/modules/video_coding/codecs/vp8/default_temporal_layers.h index 0865fc8bfd..2abc41588d 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers.h +++ b/modules/video_coding/codecs/vp8/default_temporal_layers.h @@ -31,11 +31,9 @@ class DefaultTemporalLayers : public TemporalLayers { // and/or update the reference buffers. TemporalLayers::FrameConfig UpdateLayerConfig(uint32_t timestamp) override; - // Update state based on new bitrate target and incoming framerate. - // Returns the bitrate allocation for the active temporal layers. - std::vector OnRatesUpdated(int bitrate_kbps, - int max_bitrate_kbps, - int framerate) override; + // New target bitrate, per temporal layer. + void OnRatesUpdated(const std::vector& bitrates_bps, + int framerate_fps) override; bool UpdateConfiguration(Vp8EncoderConfig* cfg) override; @@ -57,7 +55,8 @@ class DefaultTemporalLayers : public TemporalLayers { uint8_t tl0_pic_idx_; uint8_t pattern_idx_; bool last_base_layer_sync_; - rtc::Optional> new_bitrates_kbps_; + // Updated cumulative bitrates, per temporal layer. + rtc::Optional> new_bitrates_bps_; }; class DefaultTemporalLayersChecker : public TemporalLayersChecker { diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc b/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc index c09d01ed83..8314e2e47b 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc +++ b/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc @@ -10,13 +10,14 @@ #include "modules/video_coding/codecs/vp8/default_temporal_layers.h" #include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h" +#include "modules/video_coding/codecs/vp8/simulcast_rate_allocator.h" #include "modules/video_coding/include/video_codec_interface.h" #include "test/field_trial.h" #include "test/gtest.h" namespace webrtc { namespace test { - +namespace { enum { kTemporalUpdateLast = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF, @@ -49,12 +50,32 @@ enum { VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF, }; +std::vector GetTemporalLayerRates(int target_bitrate_kbps, + int framerate_fps, + int num_temporal_layers) { + VideoCodec codec; + codec.codecType = VideoCodecType::kVideoCodecVP8; + codec.numberOfSimulcastStreams = 1; + codec.targetBitrate = target_bitrate_kbps; + codec.maxBitrate = target_bitrate_kbps; + codec.maxFramerate = framerate_fps; + codec.simulcastStream[0].targetBitrate = target_bitrate_kbps; + codec.simulcastStream[0].maxBitrate = target_bitrate_kbps; + codec.simulcastStream[0].numberOfTemporalLayers = num_temporal_layers; + codec.simulcastStream[0].active = true; + SimulcastRateAllocator allocator(codec, nullptr); + return allocator.GetAllocation(target_bitrate_kbps, framerate_fps) + .GetTemporalLayerAllocation(0); +} + +} // namespace + TEST(TemporalLayersTest, 2Layers) { DefaultTemporalLayers tl(2, 0); DefaultTemporalLayersChecker checker(2, 0); Vp8EncoderConfig cfg; CodecSpecificInfoVP8 vp8_info; - tl.OnRatesUpdated(500, 500, 30); + tl.OnRatesUpdated(GetTemporalLayerRates(500, 30, 1), 30); tl.UpdateConfiguration(&cfg); int expected_flags[16] = { @@ -102,7 +123,7 @@ TEST(TemporalLayersTest, 3Layers) { DefaultTemporalLayersChecker checker(3, 0); Vp8EncoderConfig cfg; CodecSpecificInfoVP8 vp8_info; - tl.OnRatesUpdated(500, 500, 30); + tl.OnRatesUpdated(GetTemporalLayerRates(500, 30, 1), 30); tl.UpdateConfiguration(&cfg); int expected_flags[16] = { @@ -151,7 +172,7 @@ TEST(TemporalLayersTest, Alternative3Layers) { DefaultTemporalLayersChecker checker(3, 0); Vp8EncoderConfig cfg; CodecSpecificInfoVP8 vp8_info; - tl.OnRatesUpdated(500, 500, 30); + tl.OnRatesUpdated(GetTemporalLayerRates(500, 30, 1), 30); tl.UpdateConfiguration(&cfg); int expected_flags[8] = {kTemporalUpdateLast, @@ -187,7 +208,7 @@ TEST(TemporalLayersTest, 4Layers) { DefaultTemporalLayersChecker checker(4, 0); Vp8EncoderConfig cfg; CodecSpecificInfoVP8 vp8_info; - tl.OnRatesUpdated(500, 500, 30); + tl.OnRatesUpdated(GetTemporalLayerRates(500, 30, 1), 30); tl.UpdateConfiguration(&cfg); int expected_flags[16] = { kTemporalUpdateLast, @@ -234,7 +255,7 @@ TEST(TemporalLayersTest, KeyFrame) { DefaultTemporalLayersChecker checker(3, 0); Vp8EncoderConfig cfg; CodecSpecificInfoVP8 vp8_info; - tl.OnRatesUpdated(500, 500, 30); + tl.OnRatesUpdated(GetTemporalLayerRates(500, 30, 1), 30); tl.UpdateConfiguration(&cfg); int expected_flags[8] = { @@ -347,7 +368,7 @@ TEST_P(TemporalLayersReferenceTest, ValidFrameConfigs) { const int num_layers = GetParam(); DefaultTemporalLayers tl(num_layers, 0); Vp8EncoderConfig cfg; - tl.OnRatesUpdated(500, 500, 30); + tl.OnRatesUpdated(GetTemporalLayerRates(500, 30, 1), 30); tl.UpdateConfiguration(&cfg); // Run through the pattern and store the frame dependencies, plus keep track diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc index 8593613ba6..f1effc7624 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc @@ -314,6 +314,10 @@ int LibvpxVp8Encoder::SetRateAllocation(const BitrateAllocation& bitrate, SetStreamState(send_stream, stream_idx); configurations_[i].rc_target_bitrate = target_bitrate_kbps; + if (send_stream) { + temporal_layers_[stream_idx]->OnRatesUpdated( + bitrate.GetTemporalLayerAllocation(stream_idx), new_framerate); + } UpdateVpxConfiguration(temporal_layers_[stream_idx].get(), &configurations_[i]); @@ -342,6 +346,7 @@ void LibvpxVp8Encoder::SetupTemporalLayers(int num_streams, const VideoCodec& codec) { RTC_DCHECK(codec.VP8().tl_factory != nullptr); const TemporalLayersFactory* tl_factory = codec.VP8().tl_factory; + RTC_DCHECK(temporal_layers_.empty()); if (num_streams == 1) { temporal_layers_.emplace_back( tl_factory->Create(0, num_temporal_layers, tl0_pic_idx_[0])); @@ -546,8 +551,10 @@ int LibvpxVp8Encoder::InitEncode(const VideoCodec* inst, } configurations_[0].rc_target_bitrate = stream_bitrates[stream_idx]; - temporal_layers_[stream_idx]->OnRatesUpdated( - stream_bitrates[stream_idx], inst->maxBitrate, inst->maxFramerate); + if (stream_bitrates[stream_idx] > 0) { + temporal_layers_[stream_idx]->OnRatesUpdated( + allocation.GetTemporalLayerAllocation(stream_idx), inst->maxFramerate); + } UpdateVpxConfiguration(temporal_layers_[stream_idx].get(), &configurations_[0]); @@ -570,8 +577,11 @@ int LibvpxVp8Encoder::InitEncode(const VideoCodec* inst, inst->simulcastStream[stream_idx].height, kVp832ByteAlign); SetStreamState(stream_bitrates[stream_idx] > 0, stream_idx); configurations_[i].rc_target_bitrate = stream_bitrates[stream_idx]; - temporal_layers_[stream_idx]->OnRatesUpdated( - stream_bitrates[stream_idx], inst->maxBitrate, inst->maxFramerate); + if (stream_bitrates[stream_idx] > 0) { + temporal_layers_[stream_idx]->OnRatesUpdated( + allocation.GetTemporalLayerAllocation(stream_idx), + inst->maxFramerate); + } UpdateVpxConfiguration(temporal_layers_[stream_idx].get(), &configurations_[i]); } diff --git a/modules/video_coding/codecs/vp8/screenshare_layers.cc b/modules/video_coding/codecs/vp8/screenshare_layers.cc index a857cc165c..1e8427b7e1 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers.cc +++ b/modules/video_coding/codecs/vp8/screenshare_layers.cc @@ -239,37 +239,42 @@ TemporalLayers::FrameConfig ScreenshareLayers::UpdateLayerConfig( return tl_config; } -std::vector ScreenshareLayers::OnRatesUpdated(int bitrate_kbps, - int max_bitrate_kbps, - int framerate) { - RTC_DCHECK_GT(framerate, 0); +void ScreenshareLayers::OnRatesUpdated( + const std::vector& bitrates_bps, + int framerate_fps) { + RTC_DCHECK_GT(framerate_fps, 0); + RTC_DCHECK_GE(bitrates_bps.size(), 1); + RTC_DCHECK_LE(bitrates_bps.size(), 2); + + // |bitrates_bps| uses individual rates per layer, but we want to use the + // accumulated rate here. + uint32_t tl0_kbps = bitrates_bps[0] / 1000; + uint32_t tl1_kbps = tl0_kbps; + if (bitrates_bps.size() > 1) { + tl1_kbps += bitrates_bps[1] / 1000; + } + if (!target_framerate_) { - // First OnRatesUpdated() is called during construction, with the configured - // targets as parameters. - target_framerate_.emplace(framerate); + // First OnRatesUpdated() is called during construction, with the + // configured targets as parameters. + target_framerate_ = framerate_fps; capture_framerate_ = target_framerate_; bitrate_updated_ = true; } else { - bitrate_updated_ = - bitrate_kbps != static_cast(layers_[0].target_rate_kbps_) || - max_bitrate_kbps != static_cast(layers_[1].target_rate_kbps_) || - (capture_framerate_ && - framerate != static_cast(*capture_framerate_)); - if (framerate < 0) { + bitrate_updated_ = capture_framerate_ && + framerate_fps != static_cast(*capture_framerate_); + bitrate_updated_ |= tl0_kbps != layers_[0].target_rate_kbps_; + bitrate_updated_ |= tl1_kbps != layers_[1].target_rate_kbps_; + + if (framerate_fps < 0) { capture_framerate_.reset(); } else { - capture_framerate_.emplace(framerate); + capture_framerate_ = framerate_fps; } } - layers_[0].target_rate_kbps_ = bitrate_kbps; - layers_[1].target_rate_kbps_ = max_bitrate_kbps; - - std::vector allocation; - allocation.push_back(bitrate_kbps); - if (max_bitrate_kbps > bitrate_kbps) - allocation.push_back(max_bitrate_kbps - bitrate_kbps); - return allocation; + layers_[0].target_rate_kbps_ = tl0_kbps; + layers_[1].target_rate_kbps_ = tl1_kbps; } void ScreenshareLayers::FrameEncoded(unsigned int size, int qp) { diff --git a/modules/video_coding/codecs/vp8/screenshare_layers.h b/modules/video_coding/codecs/vp8/screenshare_layers.h index 794b02b852..63c97e04fb 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers.h +++ b/modules/video_coding/codecs/vp8/screenshare_layers.h @@ -37,11 +37,9 @@ class ScreenshareLayers : public TemporalLayers { // and/or update the reference buffers. TemporalLayers::FrameConfig UpdateLayerConfig(uint32_t timestamp) override; - // Update state based on new bitrate target and incoming framerate. - // Returns the bitrate allocation for the active temporal layers. - std::vector OnRatesUpdated(int bitrate_kbps, - int max_bitrate_kbps, - int framerate) override; + // New target bitrate, per temporal layer. + void OnRatesUpdated(const std::vector& bitrates_bps, + int framerate_fps) override; // Update the encoder configuration with target bitrates or other parameters. // Returns true iff the configuration was actually modified. diff --git a/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc b/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc index 85508a470b..2b9fb0dbb7 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc +++ b/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc @@ -26,7 +26,7 @@ using ::testing::NiceMock; using ::testing::Return; namespace webrtc { - +namespace { // 5 frames per second at 90 kHz. const uint32_t kTimestampDelta5Fps = 90000 / 5; const int kDefaultQp = 54; @@ -43,6 +43,11 @@ const int kTl1Flags = VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST; const int kTl1SyncFlags = VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST; +const std::vector kDefault2TlBitratesBps = { + kDefaultTl0BitrateKbps * 1000, + (kDefaultTl1BitrateKbps - kDefaultTl0BitrateKbps) * 1000}; + +} // namespace class ScreenshareLayerTest : public ::testing::Test { protected: @@ -99,10 +104,7 @@ class ScreenshareLayerTest : public ::testing::Test { memset(&vp8_cfg, 0, sizeof(Vp8EncoderConfig)); vp8_cfg.rc_min_quantizer = min_qp_; vp8_cfg.rc_max_quantizer = max_qp_; - EXPECT_THAT(layers_->OnRatesUpdated(kDefaultTl0BitrateKbps, - kDefaultTl1BitrateKbps, kFrameRate), - ElementsAre(kDefaultTl0BitrateKbps, - kDefaultTl1BitrateKbps - kDefaultTl0BitrateKbps)); + layers_->OnRatesUpdated(kDefault2TlBitratesBps, kFrameRate); EXPECT_TRUE(layers_->UpdateConfiguration(&vp8_cfg)); frame_size_ = FrameSizeForBitrate(vp8_cfg.rc_target_bitrate); return vp8_cfg; @@ -344,10 +346,9 @@ TEST_F(ScreenshareLayerTest, TooHighBitrate) { TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL0) { const int kTl0_kbps = 100; const int kTl1_kbps = 1000; - layers_->OnRatesUpdated(kTl0_kbps, kTl1_kbps, 5); - - EXPECT_THAT(layers_->OnRatesUpdated(kTl0_kbps, kTl1_kbps, 5), - ElementsAre(kTl0_kbps, kTl1_kbps - kTl0_kbps)); + const std::vector layer_rates = {kTl0_kbps * 1000, + (kTl1_kbps - kTl0_kbps) * 1000}; + layers_->OnRatesUpdated(layer_rates, kFrameRate); EXPECT_TRUE(layers_->UpdateConfiguration(&cfg_)); EXPECT_EQ(static_cast( @@ -358,8 +359,9 @@ TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL0) { TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL1) { const int kTl0_kbps = 100; const int kTl1_kbps = 450; - EXPECT_THAT(layers_->OnRatesUpdated(kTl0_kbps, kTl1_kbps, 5), - ElementsAre(kTl0_kbps, kTl1_kbps - kTl0_kbps)); + const std::vector layer_rates = {kTl0_kbps * 1000, + (kTl1_kbps - kTl0_kbps) * 1000}; + layers_->OnRatesUpdated(layer_rates, kFrameRate); EXPECT_TRUE(layers_->UpdateConfiguration(&cfg_)); EXPECT_EQ(static_cast( @@ -369,12 +371,11 @@ TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL1) { TEST_F(ScreenshareLayerTest, TargetBitrateBelowTL0) { const int kTl0_kbps = 100; - const int kTl1_kbps = 100; - EXPECT_THAT(layers_->OnRatesUpdated(kTl0_kbps, kTl1_kbps, 5), - ElementsAre(kTl0_kbps)); + const std::vector layer_rates = {kTl0_kbps * 1000}; + layers_->OnRatesUpdated(layer_rates, kFrameRate); EXPECT_TRUE(layers_->UpdateConfiguration(&cfg_)); - EXPECT_EQ(static_cast(kTl1_kbps), cfg_.rc_target_bitrate); + EXPECT_EQ(static_cast(kTl0_kbps), cfg_.rc_target_bitrate); } TEST_F(ScreenshareLayerTest, EncoderDrop) { @@ -432,7 +433,8 @@ TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) { const int kLargeFrameSizeBytes = 100000; const uint32_t kStartTimestamp = 1234; - layers_->OnRatesUpdated(kLowBitrateKbps, kLowBitrateKbps, 5); + const std::vector layer_rates = {kLowBitrateKbps * 1000}; + layers_->OnRatesUpdated(layer_rates, kFrameRate); layers_->UpdateConfiguration(&cfg_); EXPECT_EQ(kTl0Flags, @@ -628,7 +630,7 @@ TEST_F(ScreenshareLayerTest, AdjustsBitrateWhenDroppingFrames) { const uint32_t kTimestampDelta10Fps = kTimestampDelta5Fps / 2; const int kNumFrames = 30; uint32_t default_bitrate = cfg_.rc_target_bitrate; - layers_->OnRatesUpdated(kDefaultTl0BitrateKbps, kDefaultTl1BitrateKbps, 10); + layers_->OnRatesUpdated(kDefault2TlBitratesBps, 10); int num_dropped_frames = 0; for (int i = 0; i < kNumFrames; ++i) { diff --git a/modules/video_coding/codecs/vp8/simulcast_rate_allocator.cc b/modules/video_coding/codecs/vp8/simulcast_rate_allocator.cc index b865f5eff6..483a36e92e 100644 --- a/modules/video_coding/codecs/vp8/simulcast_rate_allocator.cc +++ b/modules/video_coding/codecs/vp8/simulcast_rate_allocator.cc @@ -15,6 +15,7 @@ #include #include +#include "modules/video_coding/codecs/vp8/include/vp8_common_types.h" #include "rtc_base/checks.h" namespace webrtc { @@ -46,7 +47,7 @@ BitrateAllocation SimulcastRateAllocator::GetAllocation( void SimulcastRateAllocator::DistributeAllocationToSimulcastLayers( uint32_t total_bitrate_bps, - BitrateAllocation* allocated_bitrates_bps) { + BitrateAllocation* allocated_bitrates_bps) const { uint32_t left_to_allocate = total_bitrate_bps; if (codec_.maxBitrate && codec_.maxBitrate * 1000 < left_to_allocate) left_to_allocate = codec_.maxBitrate * 1000; @@ -122,7 +123,7 @@ void SimulcastRateAllocator::DistributeAllocationToSimulcastLayers( void SimulcastRateAllocator::DistributeAllocationToTemporalLayers( uint32_t framerate, - BitrateAllocation* allocated_bitrates_bps) { + BitrateAllocation* allocated_bitrates_bps) const { const int num_spatial_streams = std::max(1, static_cast(codec_.numberOfSimulcastStreams)); @@ -130,32 +131,25 @@ void SimulcastRateAllocator::DistributeAllocationToTemporalLayers( // available temporal layers. for (int simulcast_id = 0; simulcast_id < num_spatial_streams; ++simulcast_id) { - // TODO(shampson): Consider adding a continue here if the simulcast stream - // is inactive. Currently this is not added because the call - // below to OnRatesUpdated changes the TemporalLayer's - // state. - auto tl_it = temporal_layers_.find(simulcast_id); - if (tl_it == temporal_layers_.end()) - continue; // TODO(sprang): If > 1 SS, assume default TL alloc? - uint32_t target_bitrate_kbps = allocated_bitrates_bps->GetBitrate(simulcast_id, 0) / 1000; + if (target_bitrate_kbps == 0) { + continue; + } const uint32_t expected_allocated_bitrate_kbps = target_bitrate_kbps; RTC_DCHECK_EQ( target_bitrate_kbps, allocated_bitrates_bps->GetSpatialLayerSum(simulcast_id) / 1000); - const int num_temporal_streams = std::max( - 1, codec_.numberOfSimulcastStreams == 0 - ? codec_.VP8().numberOfTemporalLayers - : codec_.simulcastStream[simulcast_id].numberOfTemporalLayers); - + const int num_temporal_streams = NumTemporalStreams(simulcast_id); uint32_t max_bitrate_kbps; // Legacy temporal-layered only screenshare, or simulcast screenshare // with legacy mode for simulcast stream 0. - if (codec_.mode == kScreensharing && codec_.targetBitrate > 0 && + const bool conference_screenshare_mode = + codec_.mode == kScreensharing && codec_.targetBitrate > 0 && ((num_spatial_streams == 1 && num_temporal_streams == 2) || // Legacy. - (num_spatial_streams > 1 && simulcast_id == 0))) { // Simulcast. + (num_spatial_streams > 1 && simulcast_id == 0)); // Simulcast. + if (conference_screenshare_mode) { // TODO(holmer): This is a "temporary" hack for screensharing, where we // interpret the startBitrate as the encoder target bitrate. This is // to allow for a different max bitrate, so if the codec can't meet @@ -170,8 +164,18 @@ void SimulcastRateAllocator::DistributeAllocationToTemporalLayers( max_bitrate_kbps = codec_.simulcastStream[simulcast_id].maxBitrate; } - std::vector tl_allocation = tl_it->second->OnRatesUpdated( - target_bitrate_kbps, max_bitrate_kbps, framerate); + std::vector tl_allocation; + if (num_temporal_streams == 1) { + tl_allocation.push_back(target_bitrate_kbps); + } else { + if (conference_screenshare_mode) { + tl_allocation = ScreenshareTemporalLayerAllocation( + target_bitrate_kbps, max_bitrate_kbps, framerate, simulcast_id); + } else { + tl_allocation = DefaultTemporalLayerAllocation( + target_bitrate_kbps, max_bitrate_kbps, framerate, simulcast_id); + } + } RTC_DCHECK_GT(tl_allocation.size(), 0); RTC_DCHECK_LE(tl_allocation.size(), num_temporal_streams); @@ -188,6 +192,54 @@ void SimulcastRateAllocator::DistributeAllocationToTemporalLayers( } } +std::vector SimulcastRateAllocator::DefaultTemporalLayerAllocation( + int bitrate_kbps, + int max_bitrate_kbps, + int framerate, + int simulcast_id) const { + const size_t num_temporal_layers = NumTemporalStreams(simulcast_id); + std::vector bitrates; + for (size_t i = 0; i < num_temporal_layers; ++i) { + float layer_bitrate = + bitrate_kbps * kVp8LayerRateAlloction[num_temporal_layers - 1][i]; + bitrates.push_back(static_cast(layer_bitrate + 0.5)); + } + + // Allocation table is of aggregates, transform to individual rates. + uint32_t sum = 0; + for (size_t i = 0; i < num_temporal_layers; ++i) { + uint32_t layer_bitrate = bitrates[i]; + RTC_DCHECK_LE(sum, bitrates[i]); + bitrates[i] -= sum; + sum = layer_bitrate; + + if (sum >= static_cast(bitrate_kbps)) { + // Sum adds up; any subsequent layers will be 0. + bitrates.resize(i + 1); + break; + } + } + + return bitrates; +} + +std::vector +SimulcastRateAllocator::ScreenshareTemporalLayerAllocation( + int bitrate_kbps, + int max_bitrate_kbps, + int framerate, + int simulcast_id) const { + if (simulcast_id > 0) { + return DefaultTemporalLayerAllocation(bitrate_kbps, max_bitrate_kbps, + framerate, simulcast_id); + } + std::vector allocation; + allocation.push_back(bitrate_kbps); + if (max_bitrate_kbps > bitrate_kbps) + allocation.push_back(max_bitrate_kbps - bitrate_kbps); + return allocation; +} + uint32_t SimulcastRateAllocator::GetPreferredBitrateBps(uint32_t framerate) { // Create a temporary instance without temporal layers, as they may be // stateful, and updating the bitrate to max here can cause side effects. @@ -201,4 +253,11 @@ const VideoCodec& webrtc::SimulcastRateAllocator::GetCodec() const { return codec_; } +int SimulcastRateAllocator::NumTemporalStreams(size_t simulcast_id) const { + return std::max( + 1, codec_.numberOfSimulcastStreams == 0 + ? codec_.VP8().numberOfTemporalLayers + : codec_.simulcastStream[simulcast_id].numberOfTemporalLayers); +} + } // namespace webrtc diff --git a/modules/video_coding/codecs/vp8/simulcast_rate_allocator.h b/modules/video_coding/codecs/vp8/simulcast_rate_allocator.h index b4b6f315e0..958fdbc293 100644 --- a/modules/video_coding/codecs/vp8/simulcast_rate_allocator.h +++ b/modules/video_coding/codecs/vp8/simulcast_rate_allocator.h @@ -15,6 +15,7 @@ #include #include +#include #include "api/video_codecs/video_encoder.h" #include "common_video/include/video_bitrate_allocator.h" @@ -41,10 +42,21 @@ class SimulcastRateAllocator : public VideoBitrateAllocator, private: void DistributeAllocationToSimulcastLayers( uint32_t total_bitrate_bps, - BitrateAllocation* allocated_bitrates_bps); + BitrateAllocation* allocated_bitrates_bps) const; void DistributeAllocationToTemporalLayers( uint32_t framerate, - BitrateAllocation* allocated_bitrates_bps); + BitrateAllocation* allocated_bitrates_bps) const; + std::vector DefaultTemporalLayerAllocation(int bitrate_kbps, + int max_bitrate_kbps, + int framerate, + int simulcast_id) const; + std::vector ScreenshareTemporalLayerAllocation( + int bitrate_kbps, + int max_bitrate_kbps, + int framerate, + int simulcast_id) const; + int NumTemporalStreams(size_t simulcast_id) const; + const VideoCodec codec_; std::map temporal_layers_; std::unique_ptr tl_factory_; diff --git a/modules/video_coding/codecs/vp8/simulcast_test_utility.h b/modules/video_coding/codecs/vp8/simulcast_test_utility.h index b5e46fce5e..231d36e1b5 100644 --- a/modules/video_coding/codecs/vp8/simulcast_test_utility.h +++ b/modules/video_coding/codecs/vp8/simulcast_test_utility.h @@ -575,6 +575,7 @@ class TestVp8Simulcast : public ::testing::Test { settings_.simulcastStream[i].maxBitrate = 0; settings_.simulcastStream[i].width = settings_.width; settings_.simulcastStream[i].height = settings_.height; + settings_.simulcastStream[i].numberOfTemporalLayers = 1; } // Setting input image to new resolution. input_buffer_ = I420Buffer::Create(settings_.width, settings_.height); diff --git a/modules/video_coding/codecs/vp8/temporal_layers.h b/modules/video_coding/codecs/vp8/temporal_layers.h index cd47bf9f6e..e9af0124d2 100644 --- a/modules/video_coding/codecs/vp8/temporal_layers.h +++ b/modules/video_coding/codecs/vp8/temporal_layers.h @@ -103,11 +103,9 @@ class TemporalLayers { // and/or update the reference buffers. virtual FrameConfig UpdateLayerConfig(uint32_t timestamp) = 0; - // Update state based on new bitrate target and incoming framerate. - // Returns the bitrate allocation for the active temporal layers. - virtual std::vector OnRatesUpdated(int bitrate_kbps, - int max_bitrate_kbps, - int framerate) = 0; + // New target bitrate, per temporal layer. + virtual void OnRatesUpdated(const std::vector& bitrates_bps, + int framerate_fps) = 0; // Update the encoder configuration with target bitrates or other parameters. // Returns true iff the configuration was actually modified. diff --git a/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc b/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc index dad1e8bded..880ed79047 100644 --- a/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc +++ b/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc @@ -30,7 +30,7 @@ constexpr uint32_t kFramerateFps = 5; class MockTemporalLayers : public TemporalLayers { public: MOCK_METHOD1(UpdateLayerConfig, TemporalLayers::FrameConfig(uint32_t)); - MOCK_METHOD3(OnRatesUpdated, std::vector(int, int, int)); + MOCK_METHOD2(OnRatesUpdated, void(const std::vector&, int)); MOCK_METHOD1(UpdateConfiguration, bool(Vp8EncoderConfig*)); MOCK_METHOD4(PopulateCodecSpecific, void(bool, @@ -487,7 +487,7 @@ TEST_F(SimulcastRateAllocatorTest, GetPreferredBitrateBps) { MockTemporalLayers mock_layers; allocator_.reset(new SimulcastRateAllocator(codec_, nullptr)); allocator_->OnTemporalLayersCreated(0, &mock_layers); - EXPECT_CALL(mock_layers, OnRatesUpdated(_, _, _)).Times(0); + EXPECT_CALL(mock_layers, OnRatesUpdated(_, _)).Times(0); EXPECT_EQ(codec_.maxBitrate * 1000, allocator_->GetPreferredBitrateBps(codec_.maxFramerate)); } diff --git a/video/screenshare_loopback.cc b/video/screenshare_loopback.cc index 0dcd69fcf6..e36fcf9ce9 100644 --- a/video/screenshare_loopback.cc +++ b/video/screenshare_loopback.cc @@ -52,7 +52,7 @@ int TargetBitrateKbps() { return static_cast(FLAG_target_bitrate); } -DEFINE_int(max_bitrate, 2000, "Call and stream max bitrate in kbps."); +DEFINE_int(max_bitrate, 1000, "Call and stream max bitrate in kbps."); int MaxBitrateKbps() { return static_cast(FLAG_max_bitrate); }