diff --git a/api/audio_options.h b/api/audio_options.h index b714998c6b..1b0d1ad0bd 100644 --- a/api/audio_options.h +++ b/api/audio_options.h @@ -75,6 +75,8 @@ struct RTC_EXPORT AudioOptions { // and check if any other AudioOptions members are unused. absl::optional combined_audio_video_bwe; // Enable audio network adaptor. + // TODO(webrtc:11717): Remove this API in favor of adaptivePtime in + // RtpEncodingParameters. absl::optional audio_network_adaptor; // Config string for audio network adaptor. absl::optional audio_network_adaptor_config; diff --git a/api/rtp_parameters.h b/api/rtp_parameters.h index 11335e92ed..b667bf812c 100644 --- a/api/rtp_parameters.h +++ b/api/rtp_parameters.h @@ -473,6 +473,10 @@ struct RTC_EXPORT RtpEncodingParameters { // Called "encodingId" in ORTC. std::string rid; + // Allow dynamic frame length changes for audio: + // https://w3c.github.io/webrtc-extensions/#dom-rtcrtpencodingparameters-adaptiveptime + bool adaptive_ptime = false; + bool operator==(const RtpEncodingParameters& o) const { return ssrc == o.ssrc && bitrate_priority == o.bitrate_priority && network_priority == o.network_priority && @@ -481,7 +485,8 @@ struct RTC_EXPORT RtpEncodingParameters { max_framerate == o.max_framerate && num_temporal_layers == o.num_temporal_layers && scale_resolution_down_by == o.scale_resolution_down_by && - active == o.active && rid == o.rid; + active == o.active && rid == o.rid && + adaptive_ptime == o.adaptive_ptime; } bool operator!=(const RtpEncodingParameters& o) const { return !(*this == o); diff --git a/media/BUILD.gn b/media/BUILD.gn index b0d64c834e..b6c78fdb39 100644 --- a/media/BUILD.gn +++ b/media/BUILD.gn @@ -317,6 +317,7 @@ rtc_library("rtc_audio_video") { "../rtc_base", "../rtc_base:audio_format_to_string", "../rtc_base:checks", + "../rtc_base:ignore_wundef", "../rtc_base:rtc_task_queue", "../rtc_base:stringutils", "../rtc_base/experiments:field_trial_parser", @@ -359,7 +360,10 @@ rtc_library("rtc_audio_video") { deps += [ "../modules/video_capture:video_capture_internal_impl" ] } if (rtc_enable_protobuf) { - deps += [ "../modules/audio_processing/aec_dump:aec_dump_impl" ] + deps += [ + "../modules/audio_coding:ana_config_proto", + "../modules/audio_processing/aec_dump:aec_dump_impl", + ] } else { deps += [ "../modules/audio_processing/aec_dump:null_aec_dump_factory" ] } diff --git a/media/engine/webrtc_voice_engine.cc b/media/engine/webrtc_voice_engine.cc index 661bfd64dc..38dc3462ac 100644 --- a/media/engine/webrtc_voice_engine.cc +++ b/media/engine/webrtc_voice_engine.cc @@ -36,7 +36,9 @@ #include "rtc_base/constructor_magic.h" #include "rtc_base/experiments/field_trial_parser.h" #include "rtc_base/experiments/field_trial_units.h" +#include "rtc_base/experiments/struct_parameters_parser.h" #include "rtc_base/helpers.h" +#include "rtc_base/ignore_wundef.h" #include "rtc_base/logging.h" #include "rtc_base/race_checker.h" #include "rtc_base/strings/audio_format_to_string.h" @@ -46,6 +48,16 @@ #include "system_wrappers/include/field_trial.h" #include "system_wrappers/include/metrics.h" +#if WEBRTC_ENABLE_PROTOBUF +RTC_PUSH_IGNORING_WUNDEF() +#ifdef WEBRTC_ANDROID_PLATFORM_BUILD +#include "external/webrtc/webrtc/modules/audio_coding/audio_network_adaptor/config.pb.h" +#else +#include "modules/audio_coding/audio_network_adaptor/config.pb.h" +#endif +RTC_POP_IGNORING_WUNDEF() +#endif + namespace cricket { namespace { @@ -191,6 +203,38 @@ absl::optional ComputeSendBitrate(int max_send_bitrate_bps, } } +struct AdaptivePtimeConfig { + bool enabled = false; + webrtc::DataRate min_payload_bitrate = webrtc::DataRate::KilobitsPerSec(16); + webrtc::DataRate min_encoder_bitrate = webrtc::DataRate::KilobitsPerSec(12); + bool use_slow_adaptation = true; + + absl::optional audio_network_adaptor_config; + + std::unique_ptr Parser() { + return webrtc::StructParametersParser::Create( // + "enabled", &enabled, // + "min_payload_bitrate", &min_payload_bitrate, // + "min_encoder_bitrate", &min_encoder_bitrate, // + "use_slow_adaptation", &use_slow_adaptation); + } + + AdaptivePtimeConfig() { + Parser()->Parse( + webrtc::field_trial::FindFullName("WebRTC-Audio-AdaptivePtime")); +#if WEBRTC_ENABLE_PROTOBUF + webrtc::audio_network_adaptor::config::ControllerManager config; + auto* frame_length_controller = + config.add_controllers()->mutable_frame_length_controller_v2(); + frame_length_controller->set_min_payload_bitrate_bps( + min_payload_bitrate.bps()); + frame_length_controller->set_use_slow_adaptation(use_slow_adaptation); + config.add_controllers()->mutable_bitrate_controller(); + audio_network_adaptor_config = config.SerializeAsString(); +#endif + } +}; + } // namespace WebRtcVoiceEngine::WebRtcVoiceEngine( @@ -737,7 +781,6 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream config_.rtp.extensions = extensions; config_.has_dscp = rtp_parameters_.encodings[0].network_priority != webrtc::Priority::kLow; - config_.audio_network_adaptor_config = audio_network_adaptor_config; config_.encoder_factory = encoder_factory; config_.codec_pair_id = codec_pair_id; config_.track_id = track_id; @@ -748,6 +791,9 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream rtp_parameters_.rtcp.cname = c_name; rtp_parameters_.header_extensions = extensions; + audio_network_adaptor_config_from_options_ = audio_network_adaptor_config; + UpdateAudioNetworkAdaptorConfig(); + if (send_codec_spec) { UpdateSendCodecSpec(*send_codec_spec); } @@ -798,10 +844,12 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream void SetAudioNetworkAdaptorConfig( const absl::optional& audio_network_adaptor_config) { RTC_DCHECK(worker_thread_checker_.IsCurrent()); - if (config_.audio_network_adaptor_config == audio_network_adaptor_config) { + if (audio_network_adaptor_config_from_options_ == + audio_network_adaptor_config) { return; } - config_.audio_network_adaptor_config = audio_network_adaptor_config; + audio_network_adaptor_config_from_options_ = audio_network_adaptor_config; + UpdateAudioNetworkAdaptorConfig(); UpdateAllowedBitrateRange(); ReconfigureAudioSendStream(); } @@ -948,6 +996,7 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream rtp_parameters_.encodings[0].max_bitrate_bps; double old_priority = rtp_parameters_.encodings[0].bitrate_priority; webrtc::Priority old_dscp = rtp_parameters_.encodings[0].network_priority; + bool old_adaptive_ptime = rtp_parameters_.encodings[0].adaptive_ptime; rtp_parameters_ = parameters; config_.bitrate_priority = rtp_parameters_.encodings[0].bitrate_priority; config_.has_dscp = (rtp_parameters_.encodings[0].network_priority != @@ -956,15 +1005,19 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream bool reconfigure_send_stream = (rtp_parameters_.encodings[0].max_bitrate_bps != old_rtp_max_bitrate) || (rtp_parameters_.encodings[0].bitrate_priority != old_priority) || - (rtp_parameters_.encodings[0].network_priority != old_dscp); + (rtp_parameters_.encodings[0].network_priority != old_dscp) || + (rtp_parameters_.encodings[0].adaptive_ptime != old_adaptive_ptime); if (rtp_parameters_.encodings[0].max_bitrate_bps != old_rtp_max_bitrate) { // Update the bitrate range. if (send_rate) { config_.send_codec_spec->target_bitrate_bps = send_rate; } - UpdateAllowedBitrateRange(); } if (reconfigure_send_stream) { + // Changing adaptive_ptime may update the audio network adaptor config + // used. + UpdateAudioNetworkAdaptorConfig(); + UpdateAllowedBitrateRange(); ReconfigureAudioSendStream(); } @@ -1000,6 +1053,7 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream // The order of precedence, from lowest to highest is: // - a reasonable default of 32kbps min/max // - fixed target bitrate from codec spec + // - lower min bitrate if adaptive ptime is enabled // - bitrate configured in the rtp_parameter encodings settings const int kDefaultBitrateBps = 32000; config_.min_bitrate_bps = kDefaultBitrateBps; @@ -1011,6 +1065,12 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream config_.max_bitrate_bps = *config_.send_codec_spec->target_bitrate_bps; } + if (rtp_parameters_.encodings[0].adaptive_ptime) { + config_.min_bitrate_bps = std::min( + config_.min_bitrate_bps, + static_cast(adaptive_ptime_config_.min_encoder_bitrate.bps())); + } + if (rtp_parameters_.encodings[0].min_bitrate_bps) { config_.min_bitrate_bps = *rtp_parameters_.encodings[0].min_bitrate_bps; } @@ -1044,12 +1104,24 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream UpdateAllowedBitrateRange(); } + void UpdateAudioNetworkAdaptorConfig() { + if (adaptive_ptime_config_.enabled || + rtp_parameters_.encodings[0].adaptive_ptime) { + config_.audio_network_adaptor_config = + adaptive_ptime_config_.audio_network_adaptor_config; + return; + } + config_.audio_network_adaptor_config = + audio_network_adaptor_config_from_options_; + } + void ReconfigureAudioSendStream() { RTC_DCHECK(worker_thread_checker_.IsCurrent()); RTC_DCHECK(stream_); stream_->Reconfigure(config_); } + const AdaptivePtimeConfig adaptive_ptime_config_; rtc::ThreadChecker worker_thread_checker_; rtc::RaceChecker audio_capture_race_checker_; webrtc::Call* call_ = nullptr; @@ -1067,6 +1139,9 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream int max_send_bitrate_bps_; webrtc::RtpParameters rtp_parameters_; absl::optional audio_codec_spec_; + // TODO(webrtc:11717): Remove this once audio_network_adaptor in AudioOptions + // has been removed. + absl::optional audio_network_adaptor_config_from_options_; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(WebRtcAudioSendStream); }; diff --git a/media/engine/webrtc_voice_engine_unittest.cc b/media/engine/webrtc_voice_engine_unittest.cc index d1556014ab..d70019e9f3 100644 --- a/media/engine/webrtc_voice_engine_unittest.cc +++ b/media/engine/webrtc_voice_engine_unittest.cc @@ -1219,6 +1219,46 @@ TEST_P(WebRtcVoiceEngineTestFake, SetRtpParametersEncodingsActive) { EXPECT_TRUE(GetSendStream(kSsrcX).IsSending()); } +TEST_P(WebRtcVoiceEngineTestFake, SetRtpParametersAdaptivePtime) { + EXPECT_TRUE(SetupSendStream()); + // Get current parameters and change "adaptive_ptime" to true. + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(kSsrcX); + ASSERT_EQ(1u, parameters.encodings.size()); + ASSERT_FALSE(parameters.encodings[0].adaptive_ptime); + parameters.encodings[0].adaptive_ptime = true; + EXPECT_TRUE(channel_->SetRtpSendParameters(kSsrcX, parameters).ok()); + EXPECT_TRUE(GetAudioNetworkAdaptorConfig(kSsrcX)); + EXPECT_EQ(12000, GetSendStreamConfig(kSsrcX).min_bitrate_bps); + + parameters.encodings[0].adaptive_ptime = false; + EXPECT_TRUE(channel_->SetRtpSendParameters(kSsrcX, parameters).ok()); + EXPECT_FALSE(GetAudioNetworkAdaptorConfig(kSsrcX)); + EXPECT_EQ(32000, GetSendStreamConfig(kSsrcX).min_bitrate_bps); +} + +TEST_P(WebRtcVoiceEngineTestFake, + DisablingAdaptivePtimeDoesNotRemoveAudioNetworkAdaptorFromOptions) { + EXPECT_TRUE(SetupSendStream()); + send_parameters_.options.audio_network_adaptor = true; + send_parameters_.options.audio_network_adaptor_config = {"1234"}; + SetSendParameters(send_parameters_); + EXPECT_EQ(send_parameters_.options.audio_network_adaptor_config, + GetAudioNetworkAdaptorConfig(kSsrcX)); + + webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(kSsrcX); + parameters.encodings[0].adaptive_ptime = false; + EXPECT_TRUE(channel_->SetRtpSendParameters(kSsrcX, parameters).ok()); + EXPECT_EQ(send_parameters_.options.audio_network_adaptor_config, + GetAudioNetworkAdaptorConfig(kSsrcX)); +} + +TEST_P(WebRtcVoiceEngineTestFake, AdaptivePtimeFieldTrial) { + webrtc::test::ScopedFieldTrials override_field_trials( + "WebRTC-Audio-AdaptivePtime/enabled:true/"); + EXPECT_TRUE(SetupSendStream()); + EXPECT_TRUE(GetAudioNetworkAdaptorConfig(kSsrcX)); +} + // Test that SetRtpSendParameters configures the correct encoding channel for // each SSRC. TEST_P(WebRtcVoiceEngineTestFake, RtpParametersArePerStream) {