From ce4523839856a380b5c9cd5f51770b86d791f156 Mon Sep 17 00:00:00 2001 From: Ilya Nikolaevskiy Date: Wed, 23 Oct 2024 15:18:17 +0200 Subject: [PATCH] Dont use SimulcastToSvcConverter if the middle stream is inactive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: chromium:375048794 Change-Id: I0acc3b0096c81e00d60c9339b86f30fbe8f92212 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/366523 Reviewed-by: Henrik Boström Commit-Queue: Ilya Nikolaevskiy Cr-Commit-Position: refs/heads/main@{#43296} --- .../codecs/vp9/libvpx_vp9_encoder.cc | 4 +- modules/video_coding/svc/BUILD.gn | 1 + .../svc/simulcast_to_svc_converter.cc | 35 ++++ .../svc/simulcast_to_svc_converter.h | 2 + .../simulcast_to_svc_converter_unittest.cc | 150 +++++++++++++++++- 5 files changed, 187 insertions(+), 5 deletions(-) diff --git a/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc b/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc index 2da0918062..65906a324b 100644 --- a/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc +++ b/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc @@ -58,7 +58,6 @@ #include "modules/video_coding/svc/scalable_video_controller_no_layering.h" #include "modules/video_coding/svc/svc_rate_allocator.h" #include "modules/video_coding/utility/framerate_controller_deprecated.h" -#include "modules/video_coding/utility/simulcast_utility.h" #include "rtc_base/checks.h" #include "rtc_base/containers/flat_map.h" #include "rtc_base/experiments/field_trial_list.h" @@ -553,8 +552,7 @@ int LibvpxVp9Encoder::InitEncode(const VideoCodec* inst, } if (enable_svc_for_simulcast_ && codec_.numberOfSimulcastStreams > 1) { - if (!SimulcastUtility::ValidSimulcastParameters( - codec_, codec_.numberOfSimulcastStreams)) { + if (!SimulcastToSvcConverter::IsConfigSupported(codec_)) { return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED; } RTC_LOG(LS_INFO) << "Rewriting simulcast config to SVC."; diff --git a/modules/video_coding/svc/BUILD.gn b/modules/video_coding/svc/BUILD.gn index 603b02993a..d029d13868 100644 --- a/modules/video_coding/svc/BUILD.gn +++ b/modules/video_coding/svc/BUILD.gn @@ -98,6 +98,7 @@ rtc_source_set("simulcast_to_svc_converter") { "../../../api/video:encoded_image", "../../../api/video_codecs:video_codecs_api", "../../../modules/video_coding:video_codec_interface", + "../../../modules/video_coding:video_coding_utility", "../../../rtc_base:checks", "../../../rtc_base/system:rtc_export", ] diff --git a/modules/video_coding/svc/simulcast_to_svc_converter.cc b/modules/video_coding/svc/simulcast_to_svc_converter.cc index 77d5b294b5..6863d9b905 100644 --- a/modules/video_coding/svc/simulcast_to_svc_converter.cc +++ b/modules/video_coding/svc/simulcast_to_svc_converter.cc @@ -12,6 +12,7 @@ #include "modules/video_coding/svc/create_scalability_structure.h" #include "modules/video_coding/svc/scalability_mode_util.h" +#include "modules/video_coding/utility/simulcast_utility.h" #include "rtc_base/checks.h" namespace webrtc { @@ -138,4 +139,38 @@ SimulcastToSvcConverter::LayerState::LayerState( video_controller->OnRatesUpdated(dummy_bitrates); } +// static +bool SimulcastToSvcConverter::IsConfigSupported(const VideoCodec& codec) { + if (codec.numberOfSimulcastStreams <= 1 || + !SimulcastUtility::ValidSimulcastParameters( + codec, codec.numberOfSimulcastStreams)) { + return false; + } + // Ensure there's 4:2:1 scaling. + for (int i = 1; i < codec.numberOfSimulcastStreams; ++i) { + if (codec.simulcastStream[i].active && + codec.simulcastStream[i - 1].active && + (codec.simulcastStream[i].width != + codec.simulcastStream[i - 1].width * 2 || + codec.simulcastStream[i].height != + codec.simulcastStream[i - 1].height * 2)) { + return false; + } + } + int first_active_layer = -1; + int last_active_layer = -1; + int num_active_layers = 0; + for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) { + if (codec.simulcastStream[i].active) { + if (first_active_layer < 0) + first_active_layer = i; + last_active_layer = i; + ++num_active_layers; + } + } + // Active layers must form a continuous segment. Can't have holes, because + // most SVC encoders can't process that. + return num_active_layers == last_active_layer - first_active_layer + 1; +} + } // namespace webrtc diff --git a/modules/video_coding/svc/simulcast_to_svc_converter.h b/modules/video_coding/svc/simulcast_to_svc_converter.h index 5d03d4aa58..8472fe7511 100644 --- a/modules/video_coding/svc/simulcast_to_svc_converter.h +++ b/modules/video_coding/svc/simulcast_to_svc_converter.h @@ -35,6 +35,8 @@ class RTC_EXPORT SimulcastToSvcConverter { ~SimulcastToSvcConverter() = default; + static bool IsConfigSupported(const VideoCodec& codec); + VideoCodec GetConfig() const; void EncodeStarted(bool force_keyframe); diff --git a/modules/video_coding/svc/simulcast_to_svc_converter_unittest.cc b/modules/video_coding/svc/simulcast_to_svc_converter_unittest.cc index fcf8c48362..6af35b5df9 100644 --- a/modules/video_coding/svc/simulcast_to_svc_converter_unittest.cc +++ b/modules/video_coding/svc/simulcast_to_svc_converter_unittest.cc @@ -99,7 +99,7 @@ TEST(SimulcastToSvc, ConvertsEncodedImage) { .minBitrate = 100, .qpMax = 150, .active = true}; - codec.simulcastStream[2] = {.width = 12800, + codec.simulcastStream[2] = {.width = 1280, .height = 720, .maxFramerate = 30, .numberOfTemporalLayers = 3, @@ -168,7 +168,7 @@ TEST(SimulcastToSvc, PredictsInternalStateCorrectlyOnFrameDrops) { .minBitrate = 100, .qpMax = 150, .active = true}; - codec.simulcastStream[2] = {.width = 12800, + codec.simulcastStream[2] = {.width = 1280, .height = 720, .maxFramerate = 30, .numberOfTemporalLayers = 3, @@ -235,4 +235,150 @@ TEST(SimulcastToSvc, PredictsInternalStateCorrectlyOnFrameDrops) { } } +TEST(SimulcastToSvc, SupportsOnlyContinuousActiveStreams) { + VideoCodec codec; + codec.codecType = kVideoCodecVP9; + codec.SetScalabilityMode(ScalabilityMode::kL1T3); + codec.width = 1280; + codec.height = 720; + codec.minBitrate = 10; + codec.maxBitrate = 2500; + codec.numberOfSimulcastStreams = 3; + codec.VP9()->numberOfSpatialLayers = 1; + codec.VP9()->interLayerPred = InterLayerPredMode::kOff; + + codec.simulcastStream[0] = {.width = 320, + .height = 180, + .maxFramerate = 30, + .numberOfTemporalLayers = 3, + .maxBitrate = 100, + .targetBitrate = 70, + .minBitrate = 50, + .qpMax = 150, + .active = true}; + codec.simulcastStream[1] = {.width = 640, + .height = 360, + .maxFramerate = 30, + .numberOfTemporalLayers = 3, + .maxBitrate = 250, + .targetBitrate = 150, + .minBitrate = 100, + .qpMax = 150, + .active = true}; + codec.simulcastStream[2] = {.width = 1280, + .height = 720, + .maxFramerate = 30, + .numberOfTemporalLayers = 3, + .maxBitrate = 1500, + .targetBitrate = 1200, + .minBitrate = 800, + .qpMax = 150, + .active = true}; + EXPECT_TRUE(SimulcastToSvcConverter::IsConfigSupported(codec)); + + codec.simulcastStream[0].active = false; + codec.simulcastStream[1].active = true; + codec.simulcastStream[2].active = true; + EXPECT_TRUE(SimulcastToSvcConverter::IsConfigSupported(codec)); + + codec.simulcastStream[0].active = true; + codec.simulcastStream[1].active = true; + codec.simulcastStream[2].active = false; + EXPECT_TRUE(SimulcastToSvcConverter::IsConfigSupported(codec)); + + codec.simulcastStream[0].active = true; + codec.simulcastStream[1].active = false; + codec.simulcastStream[2].active = true; + EXPECT_FALSE(SimulcastToSvcConverter::IsConfigSupported(codec)); +} + +TEST(SimulcastToSvc, SupportsOnlySameTemporalStructure) { + VideoCodec codec; + codec.codecType = kVideoCodecVP9; + codec.width = 1280; + codec.height = 720; + codec.minBitrate = 10; + codec.maxBitrate = 2500; + codec.numberOfSimulcastStreams = 3; + codec.VP9()->numberOfSpatialLayers = 1; + codec.VP9()->interLayerPred = InterLayerPredMode::kOff; + + codec.simulcastStream[0] = {.width = 320, + .height = 180, + .maxFramerate = 30, + .numberOfTemporalLayers = 3, + .maxBitrate = 100, + .targetBitrate = 70, + .minBitrate = 50, + .qpMax = 150, + .active = true}; + codec.simulcastStream[1] = {.width = 640, + .height = 360, + .maxFramerate = 30, + .numberOfTemporalLayers = 3, + .maxBitrate = 250, + .targetBitrate = 150, + .minBitrate = 100, + .qpMax = 150, + .active = true}; + codec.simulcastStream[2] = {.width = 1280, + .height = 720, + .maxFramerate = 30, + .numberOfTemporalLayers = 3, + .maxBitrate = 1500, + .targetBitrate = 1200, + .minBitrate = 800, + .qpMax = 150, + .active = true}; + EXPECT_TRUE(SimulcastToSvcConverter::IsConfigSupported(codec)); + + codec.simulcastStream[0].numberOfTemporalLayers = 1; + EXPECT_FALSE(SimulcastToSvcConverter::IsConfigSupported(codec)); +} + +TEST(SimulcastToSvc, SupportsOnly421Scaling) { + VideoCodec codec; + codec.codecType = kVideoCodecVP9; + codec.width = 1280; + codec.height = 720; + codec.minBitrate = 10; + codec.maxBitrate = 2500; + codec.numberOfSimulcastStreams = 3; + codec.VP9()->numberOfSpatialLayers = 1; + codec.VP9()->interLayerPred = InterLayerPredMode::kOff; + + codec.simulcastStream[0] = {.width = 320, + .height = 180, + .maxFramerate = 30, + .numberOfTemporalLayers = 3, + .maxBitrate = 100, + .targetBitrate = 70, + .minBitrate = 50, + .qpMax = 150, + .active = true}; + codec.simulcastStream[1] = {.width = 640, + .height = 360, + .maxFramerate = 30, + .numberOfTemporalLayers = 3, + .maxBitrate = 250, + .targetBitrate = 150, + .minBitrate = 100, + .qpMax = 150, + .active = true}; + codec.simulcastStream[2] = {.width = 1280, + .height = 720, + .maxFramerate = 30, + .numberOfTemporalLayers = 3, + .maxBitrate = 1500, + .targetBitrate = 1200, + .minBitrate = 800, + .qpMax = 150, + .active = true}; + EXPECT_TRUE(SimulcastToSvcConverter::IsConfigSupported(codec)); + + codec.simulcastStream[0].width = 160; + codec.simulcastStream[0].height = 90; + EXPECT_FALSE(SimulcastToSvcConverter::IsConfigSupported(codec)); +} + } // namespace webrtc