diff --git a/call/adaptation/resource.cc b/call/adaptation/resource.cc index 917d1b5c82..bb3e63deb5 100644 --- a/call/adaptation/resource.cc +++ b/call/adaptation/resource.cc @@ -41,6 +41,14 @@ ResourceUsageState Resource::usage_state() const { return usage_state_; } +bool Resource::IsAdaptationUpAllowed( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after, + const Resource& reason_resource) const { + return true; +} + ResourceListenerResponse Resource::OnResourceUsageStateMeasured( ResourceUsageState usage_state) { ResourceListenerResponse response = ResourceListenerResponse::kNothing; diff --git a/call/adaptation/resource.h b/call/adaptation/resource.h index 935746d1da..7c802eb411 100644 --- a/call/adaptation/resource.h +++ b/call/adaptation/resource.h @@ -15,6 +15,8 @@ #include #include "absl/types/optional.h" +#include "call/adaptation/video_source_restrictions.h" +#include "call/adaptation/video_stream_input_state.h" namespace webrtc { @@ -83,6 +85,14 @@ class Resource { void UnregisterListener(ResourceListener* listener); ResourceUsageState usage_state() const; + // This method allows the Resource to reject a proposed adaptation in the "up" + // direction if it predicts this would cause overuse of this resource. The + // default implementation unconditionally returns true (= allowed). + virtual bool IsAdaptationUpAllowed( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after, + const Resource& reason_resource) const; virtual std::string name() const = 0; diff --git a/call/adaptation/video_source_restrictions.cc b/call/adaptation/video_source_restrictions.cc index b848bf80bc..e9ae45e79e 100644 --- a/call/adaptation/video_source_restrictions.cc +++ b/call/adaptation/video_source_restrictions.cc @@ -65,4 +65,15 @@ void VideoSourceRestrictions::set_max_frame_rate( max_frame_rate_ = std::move(max_frame_rate); } +bool DidIncreaseResolution(VideoSourceRestrictions restrictions_before, + VideoSourceRestrictions restrictions_after) { + if (!restrictions_before.max_pixels_per_frame().has_value()) { + return false; + } + if (!restrictions_after.max_pixels_per_frame().has_value()) + return true; + return restrictions_after.max_pixels_per_frame().value() > + restrictions_before.max_pixels_per_frame().value(); +} + } // namespace webrtc diff --git a/call/adaptation/video_source_restrictions.h b/call/adaptation/video_source_restrictions.h index 48266fa900..a889657072 100644 --- a/call/adaptation/video_source_restrictions.h +++ b/call/adaptation/video_source_restrictions.h @@ -65,6 +65,9 @@ class VideoSourceRestrictions { absl::optional max_frame_rate_; }; +bool DidIncreaseResolution(VideoSourceRestrictions restrictions_before, + VideoSourceRestrictions restrictions_after); + } // namespace webrtc #endif // CALL_ADAPTATION_VIDEO_SOURCE_RESTRICTIONS_H_ diff --git a/video/adaptation/resource_adaptation_processor.cc b/video/adaptation/resource_adaptation_processor.cc index f0b827c7de..388a82a92e 100644 --- a/video/adaptation/resource_adaptation_processor.cc +++ b/video/adaptation/resource_adaptation_processor.cc @@ -45,55 +45,6 @@ bool IsFramerateScalingEnabled(DegradationPreference degradation_preference) { degradation_preference == DegradationPreference::BALANCED; } -// Returns modified restrictions where any constraints that don't apply to the -// degradation preference are cleared. -VideoSourceRestrictions ApplyDegradationPreference( - VideoSourceRestrictions source_restrictions, - DegradationPreference degradation_preference) { - switch (degradation_preference) { - case DegradationPreference::BALANCED: - break; - case DegradationPreference::MAINTAIN_FRAMERATE: - source_restrictions.set_max_frame_rate(absl::nullopt); - break; - case DegradationPreference::MAINTAIN_RESOLUTION: - source_restrictions.set_max_pixels_per_frame(absl::nullopt); - source_restrictions.set_target_pixels_per_frame(absl::nullopt); - break; - case DegradationPreference::DISABLED: - source_restrictions.set_max_pixels_per_frame(absl::nullopt); - source_restrictions.set_target_pixels_per_frame(absl::nullopt); - source_restrictions.set_max_frame_rate(absl::nullopt); - } - return source_restrictions; -} - -// Returns VideoAdaptationCounters where constraints that don't apply to the -// degredation preference are cleared. This behaviour must reflect that of -// ApplyDegredationPreference for SourceRestrictions. Any to that method must -// also change this one. -VideoAdaptationCounters ApplyDegradationPreference( - VideoAdaptationCounters counters, - DegradationPreference degradation_preference) { - switch (degradation_preference) { - case DegradationPreference::BALANCED: - break; - case DegradationPreference::MAINTAIN_FRAMERATE: - counters.fps_adaptations = 0; - break; - case DegradationPreference::MAINTAIN_RESOLUTION: - counters.resolution_adaptations = 0; - break; - case DegradationPreference::DISABLED: - counters.resolution_adaptations = 0; - counters.fps_adaptations = 0; - break; - default: - RTC_NOTREACHED(); - } - return counters; -} - std::string ToString(VideoAdaptationReason reason) { switch (reason) { case VideoAdaptationReason::kQuality: @@ -185,6 +136,106 @@ class ResourceAdaptationProcessor::InitialFrameDropper { int initial_framedrop_; }; +ResourceAdaptationProcessor::PreventAdaptUpDueToActiveCounts:: + PreventAdaptUpDueToActiveCounts(ResourceAdaptationProcessor* processor) + : processor_(processor) {} + +bool ResourceAdaptationProcessor::PreventAdaptUpDueToActiveCounts:: + IsAdaptationUpAllowed(const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after, + const Resource& reason_resource) const { + VideoAdaptationReason reason = + processor_->GetReasonFromResource(reason_resource); + // We can't adapt up if we're already at the highest setting. + // Note that this only includes counts relevant to the current degradation + // preference. e.g. we previously adapted resolution, now prefer adpating fps, + // only count the fps adaptations and not the previous resolution adaptations. + // TODO(hbos): Why would the reason matter? If a particular resource doesn't + // want us to go up it should prevent us from doing so itself rather than to + // have this catch-all reason- and stats-based approach. + int num_downgrades = FilterVideoAdaptationCountersByDegradationPreference( + processor_->active_counts_[reason], + processor_->effective_degradation_preference()) + .Total(); + RTC_DCHECK_GE(num_downgrades, 0); + return num_downgrades > 0; +} + +ResourceAdaptationProcessor::PreventIncreaseResolutionDueToBitrateResource:: + PreventIncreaseResolutionDueToBitrateResource( + ResourceAdaptationProcessor* processor) + : processor_(processor) {} + +bool ResourceAdaptationProcessor:: + PreventIncreaseResolutionDueToBitrateResource::IsAdaptationUpAllowed( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after, + const Resource& reason_resource) const { + VideoAdaptationReason reason = + processor_->GetReasonFromResource(reason_resource); + // If increasing resolution due to kQuality, make sure bitrate limits are not + // violated. + // TODO(hbos): Why are we allowing violating bitrate constraints if adapting + // due to CPU? Shouldn't this condition be checked regardless of reason? + if (reason == VideoAdaptationReason::kQuality && + DidIncreaseResolution(restrictions_before, restrictions_after)) { + uint32_t bitrate_bps = processor_->encoder_target_bitrate_bps_.value_or(0); + absl::optional bitrate_limits = + processor_->encoder_settings_.has_value() + ? processor_->encoder_settings_->encoder_info() + .GetEncoderBitrateLimitsForResolution( + // Need some sort of expected resulting pixels to be used + // instead of unrestricted. + GetHigherResolutionThan( + input_state.frame_size_pixels().value())) + : absl::nullopt; + if (bitrate_limits.has_value() && bitrate_bps != 0) { + RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, + input_state.frame_size_pixels().value()); + return bitrate_bps >= + static_cast(bitrate_limits->min_start_bitrate_bps); + } + } + return true; +} + +ResourceAdaptationProcessor::PreventAdaptUpInBalancedResource:: + PreventAdaptUpInBalancedResource(ResourceAdaptationProcessor* processor) + : processor_(processor) {} + +bool ResourceAdaptationProcessor::PreventAdaptUpInBalancedResource:: + IsAdaptationUpAllowed(const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after, + const Resource& reason_resource) const { + VideoAdaptationReason reason = + processor_->GetReasonFromResource(reason_resource); + // Don't adapt if BalancedDegradationSettings applies and determines this will + // exceed bitrate constraints. + // TODO(hbos): Why are we allowing violating balanced settings if adapting due + // CPU? Shouldn't this condition be checked regardless of reason? + if (reason == VideoAdaptationReason::kQuality && + processor_->effective_degradation_preference() == + DegradationPreference::BALANCED && + !processor_->stream_adapter_->balanced_settings().CanAdaptUp( + input_state.video_codec_type(), + input_state.frame_size_pixels().value(), + processor_->encoder_target_bitrate_bps_.value_or(0))) { + return false; + } + if (reason == VideoAdaptationReason::kQuality && + DidIncreaseResolution(restrictions_before, restrictions_after) && + !processor_->stream_adapter_->balanced_settings().CanAdaptUpResolution( + input_state.video_codec_type(), + input_state.frame_size_pixels().value(), + processor_->encoder_target_bitrate_bps_.value_or(0))) { + return false; + } + return true; +} + ResourceAdaptationProcessor::ResourceAdaptationProcessor( VideoStreamInputStateProvider* input_state_provider, Clock* clock, @@ -192,7 +243,12 @@ ResourceAdaptationProcessor::ResourceAdaptationProcessor( std::unique_ptr overuse_detector, VideoStreamEncoderObserver* encoder_stats_observer, ResourceAdaptationProcessorListener* adaptation_listener) - : input_state_provider_(input_state_provider), + : prevent_adapt_up_due_to_active_counts_(this), + prevent_increase_resolution_due_to_bitrate_resource_(this), + prevent_adapt_up_in_balanced_resource_(this), + encode_usage_resource_(std::move(overuse_detector)), + quality_scaler_resource_(), + input_state_provider_(input_state_provider), adaptation_listener_(adaptation_listener), clock_(clock), state_(State::kStopped), @@ -200,11 +256,8 @@ ResourceAdaptationProcessor::ResourceAdaptationProcessor( degradation_preference_(DegradationPreference::DISABLED), effective_degradation_preference_(DegradationPreference::DISABLED), stream_adapter_(std::make_unique()), - encode_usage_resource_( - std::make_unique(std::move(overuse_detector))), - quality_scaler_resource_(std::make_unique()), - initial_frame_dropper_(std::make_unique( - quality_scaler_resource_.get())), + initial_frame_dropper_( + std::make_unique(&quality_scaler_resource_)), quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()), encoder_target_bitrate_bps_(absl::nullopt), quality_rampup_done_(false), @@ -214,8 +267,14 @@ ResourceAdaptationProcessor::ResourceAdaptationProcessor( active_counts_() { RTC_DCHECK(adaptation_listener_); RTC_DCHECK(encoder_stats_observer_); - AddResource(encode_usage_resource_.get(), VideoAdaptationReason::kCpu); - AddResource(quality_scaler_resource_.get(), VideoAdaptationReason::kQuality); + AddResource(&prevent_adapt_up_due_to_active_counts_, + VideoAdaptationReason::kQuality); + AddResource(&prevent_increase_resolution_due_to_bitrate_resource_, + VideoAdaptationReason::kQuality); + AddResource(&prevent_adapt_up_in_balanced_resource_, + VideoAdaptationReason::kQuality); + AddResource(&encode_usage_resource_, VideoAdaptationReason::kCpu); + AddResource(&quality_scaler_resource_, VideoAdaptationReason::kQuality); } ResourceAdaptationProcessor::~ResourceAdaptationProcessor() { @@ -230,7 +289,7 @@ void ResourceAdaptationProcessor::StartResourceAdaptation( // should be passed in and why. If resources are separated from modules then // those resources may be started or stopped separately from the module. RTC_DCHECK_EQ(adaptation_listener, adaptation_listener_); - encode_usage_resource_->StartCheckForOveruse(GetCpuOveruseOptions()); + encode_usage_resource_.StartCheckForOveruse(GetCpuOveruseOptions()); for (auto& resource_and_reason : resources_) { resource_and_reason.resource->RegisterListener(this); } @@ -238,8 +297,8 @@ void ResourceAdaptationProcessor::StartResourceAdaptation( } void ResourceAdaptationProcessor::StopResourceAdaptation() { - encode_usage_resource_->StopCheckForOveruse(); - quality_scaler_resource_->StopCheckForOveruse(); + encode_usage_resource_.StopCheckForOveruse(); + quality_scaler_resource_.StopCheckForOveruse(); for (auto& resource_and_reason : resources_) { resource_and_reason.resource->UnregisterListener(this); } @@ -307,12 +366,12 @@ void ResourceAdaptationProcessor::ResetVideoSourceRestrictions() { void ResourceAdaptationProcessor::OnFrameDroppedDueToSize() { VideoAdaptationCounters counters_before = stream_adapter_->adaptation_counters(); - OnResourceOveruse(VideoAdaptationReason::kQuality); - if (degradation_preference() == DegradationPreference::BALANCED && + OnResourceOveruse(quality_scaler_resource_); + if (degradation_preference_ == DegradationPreference::BALANCED && stream_adapter_->adaptation_counters().fps_adaptations > counters_before.fps_adaptations) { // Adapt framerate in same step as resolution. - OnResourceOveruse(VideoAdaptationReason::kQuality); + OnResourceOveruse(quality_scaler_resource_); } if (stream_adapter_->adaptation_counters().resolution_adaptations > counters_before.resolution_adaptations) { @@ -324,8 +383,8 @@ void ResourceAdaptationProcessor::OnFrameDroppedDueToSize() { void ResourceAdaptationProcessor::OnEncodeStarted( const VideoFrame& cropped_frame, int64_t time_when_first_seen_us) { - encode_usage_resource_->OnEncodeStarted(cropped_frame, - time_when_first_seen_us); + encode_usage_resource_.OnEncodeStarted(cropped_frame, + time_when_first_seen_us); } void ResourceAdaptationProcessor::OnEncodeCompleted( @@ -336,15 +395,15 @@ void ResourceAdaptationProcessor::OnEncodeCompleted( uint32_t timestamp = encoded_image.Timestamp(); int64_t capture_time_us = encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec; - encode_usage_resource_->OnEncodeCompleted( - timestamp, time_sent_in_us, capture_time_us, encode_duration_us); + encode_usage_resource_.OnEncodeCompleted(timestamp, time_sent_in_us, + capture_time_us, encode_duration_us); // Inform |quality_scaler_resource_| of the encode completed event. - quality_scaler_resource_->OnEncodeCompleted(encoded_image, time_sent_in_us); + quality_scaler_resource_.OnEncodeCompleted(encoded_image, time_sent_in_us); } void ResourceAdaptationProcessor::OnFrameDropped( EncodedImageCallback::DropReason reason) { - quality_scaler_resource_->OnFrameDropped(reason); + quality_scaler_resource_.OnFrameDropped(reason); } bool ResourceAdaptationProcessor::DropInitialFrames() const { @@ -359,10 +418,10 @@ void ResourceAdaptationProcessor::OnMaybeEncodeFrame() { void ResourceAdaptationProcessor::UpdateQualityScalerSettings( absl::optional qp_thresholds) { if (qp_thresholds.has_value()) { - quality_scaler_resource_->StopCheckForOveruse(); - quality_scaler_resource_->StartCheckForOveruse(qp_thresholds.value()); + quality_scaler_resource_.StopCheckForOveruse(); + quality_scaler_resource_.StartCheckForOveruse(qp_thresholds.value()); } else { - quality_scaler_resource_->StopCheckForOveruse(); + quality_scaler_resource_.StopCheckForOveruse(); } initial_frame_dropper_->OnQualityScalerSettingsUpdated(); } @@ -377,7 +436,7 @@ void ResourceAdaptationProcessor::ConfigureQualityScaler( // TODO(https://crbug.com/webrtc/11222): Should this move to // QualityScalerResource? if (quality_scaling_allowed) { - if (!quality_scaler_resource_->is_started()) { + if (!quality_scaler_resource_.is_started()) { // Quality scaler has not already been configured. // Use experimental thresholds if available. @@ -396,13 +455,13 @@ void ResourceAdaptationProcessor::ConfigureQualityScaler( // Set the qp-thresholds to the balanced settings if balanced mode. if (degradation_preference_ == DegradationPreference::BALANCED && - quality_scaler_resource_->is_started()) { + quality_scaler_resource_.is_started()) { absl::optional thresholds = stream_adapter_->balanced_settings().GetQpThresholds( GetVideoCodecTypeOrGeneric(encoder_settings_), LastInputFrameSizeOrDefault()); if (thresholds) { - quality_scaler_resource_->SetQpThresholds(*thresholds); + quality_scaler_resource_.SetQpThresholds(*thresholds); } } UpdateStatsAdaptationSettings(); @@ -411,28 +470,15 @@ void ResourceAdaptationProcessor::ConfigureQualityScaler( ResourceListenerResponse ResourceAdaptationProcessor::OnResourceUsageStateMeasured( const Resource& resource) { - const auto& registered_resource = - absl::c_find_if(resources_, [&resource](const ResourceAndReason& r) { - return r.resource == &resource; - }); - RTC_DCHECK(registered_resource != resources_.end()) - << resource.name() << " not found."; - - const VideoAdaptationReason reason = registered_resource->reason; switch (resource.usage_state()) { case ResourceUsageState::kOveruse: - return OnResourceOveruse(reason); + return OnResourceOveruse(resource); case ResourceUsageState::kStable: // Do nothing. - // - // This module has two resources: |encoude_usage_resource_| and - // |quality_scaler_resource_|. A smarter adaptation module might not - // attempt to adapt up unless ALL resources were underused, but this - // module acts on each resource's measurement in isolation - without - // taking the current usage of any other resource into account. + // TODO(https://crbug.com/webrtc/11172): Delete kStable in favor of null. return ResourceListenerResponse::kNothing; case ResourceUsageState::kUnderuse: - OnResourceUnderuse(reason); + OnResourceUnderuse(resource); return ResourceListenerResponse::kNothing; } } @@ -445,52 +491,56 @@ bool ResourceAdaptationProcessor::HasSufficientInputForAdaptation( input_state.frames_per_second() >= kMinFrameRateFps); } +VideoAdaptationReason ResourceAdaptationProcessor::GetReasonFromResource( + const Resource& resource) const { + const auto& registered_resource = + absl::c_find_if(resources_, [&resource](const ResourceAndReason& r) { + return r.resource == &resource; + }); + RTC_DCHECK(registered_resource != resources_.end()) + << resource.name() << " not found."; + return registered_resource->reason; +} + void ResourceAdaptationProcessor::OnResourceUnderuse( - VideoAdaptationReason reason) { + const Resource& reason_resource) { VideoStreamInputState input_state = input_state_provider_->InputState(); if (effective_degradation_preference_ == DegradationPreference::DISABLED || !HasSufficientInputForAdaptation(input_state)) { return; } - // We can't adapt up if we're already at the highest setting. - // Note that this only includes counts relevant to the current degradation - // preference. e.g. we previously adapted resolution, now prefer adpating fps, - // only count the fps adaptations and not the previous resolution adaptations. - // - // TODO(https://crbug.com/webrtc/11394): Checking the counts for reason should - // be replaced with checking the overuse state of all resources. This is - // effectively trying to infer if the the Resource specified by |reason| is OK - // with adapting up by looking at active counters. If the relevant Resources - // simply told us this directly we wouldn't have to depend on stats counters - // to abort VideoStreamAdapter::GetAdaptationUp(). This may be possible by - // peeking the next restrictions (VideoStreamAdapter::PeekNextRestrictions()), - // and asking the Resource: "Can we apply these restrictions without - // overusing?" or if there is a ResourceUsageState::kStable. - int num_downgrades = ApplyDegradationPreference(active_counts_[reason], - degradation_preference_) - .Total(); - RTC_DCHECK_GE(num_downgrades, 0); - if (num_downgrades == 0) - return; // Update video input states and encoder settings for accurate adaptation. - stream_adapter_->SetInput(input_state, encoder_settings_, - encoder_target_bitrate_bps_); - // Should we adapt, and if so: how? - Adaptation adaptation = stream_adapter_->GetAdaptationUp(reason); + stream_adapter_->SetInput(input_state); + // How can this stream be adapted up? + Adaptation adaptation = stream_adapter_->GetAdaptationUp(); if (adaptation.status() != Adaptation::Status::kValid) return; + // Are all resources OK with this adaptation being applied? + VideoSourceRestrictions restrictions_before = + stream_adapter_->source_restrictions(); + VideoSourceRestrictions restrictions_after = + stream_adapter_->PeekNextRestrictions(adaptation); + if (!absl::c_all_of(resources_, [&input_state, &restrictions_before, + &restrictions_after, &reason_resource]( + ResourceAndReason resource_and_reason) { + return resource_and_reason.resource->IsAdaptationUpAllowed( + input_state, restrictions_before, restrictions_after, + reason_resource); + })) { + return; + } // Apply adaptation. stream_adapter_->ApplyAdaptation(adaptation); // Update VideoSourceRestrictions based on adaptation. This also informs the // |adaptation_listener_|. MaybeUpdateVideoSourceRestrictions(); // Stats and logging. - UpdateAdaptationStats(reason); + UpdateAdaptationStats(GetReasonFromResource(reason_resource)); RTC_LOG(LS_INFO) << ActiveCountsToString(); } ResourceListenerResponse ResourceAdaptationProcessor::OnResourceOveruse( - VideoAdaptationReason reason) { + const Resource& reason_resource) { VideoStreamInputState input_state = input_state_provider_->InputState(); if (!input_state.has_input()) { return ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency; @@ -500,9 +550,8 @@ ResourceListenerResponse ResourceAdaptationProcessor::OnResourceOveruse( return ResourceListenerResponse::kNothing; } // Update video input states and encoder settings for accurate adaptation. - stream_adapter_->SetInput(input_state, encoder_settings_, - encoder_target_bitrate_bps_); - // Should we adapt, and if so: how? + stream_adapter_->SetInput(input_state); + // How can this stream be adapted down? Adaptation adaptation = stream_adapter_->GetAdaptationDown(); if (adaptation.min_pixel_limit_reached()) encoder_stats_observer_->OnMinPixelLimitReached(); @@ -515,7 +564,7 @@ ResourceListenerResponse ResourceAdaptationProcessor::OnResourceOveruse( // |adaptation_listener_|. MaybeUpdateVideoSourceRestrictions(); // Stats and logging. - UpdateAdaptationStats(reason); + UpdateAdaptationStats(GetReasonFromResource(reason_resource)); RTC_LOG(INFO) << ActiveCountsToString(); return response; } @@ -566,8 +615,9 @@ void ResourceAdaptationProcessor::MaybeUpdateEffectiveDegradationPreference() { } void ResourceAdaptationProcessor::MaybeUpdateVideoSourceRestrictions() { - VideoSourceRestrictions new_restrictions = ApplyDegradationPreference( - stream_adapter_->source_restrictions(), degradation_preference_); + VideoSourceRestrictions new_restrictions = + FilterRestrictionsByDegradationPreference( + stream_adapter_->source_restrictions(), degradation_preference_); if (video_source_restrictions_ != new_restrictions) { video_source_restrictions_ = std::move(new_restrictions); adaptation_listener_->OnVideoSourceRestrictionsUpdated( @@ -587,15 +637,13 @@ void ResourceAdaptationProcessor::MaybeUpdateTargetFrameRate() { // module. This is used to make sure overuse detection doesn't needlessly // trigger in low and/or variable framerate scenarios. absl::optional target_frame_rate = - ApplyDegradationPreference(stream_adapter_->source_restrictions(), - degradation_preference_) - .max_frame_rate(); + video_source_restrictions_.max_frame_rate(); if (!target_frame_rate.has_value() || (codec_max_frame_rate.has_value() && codec_max_frame_rate.value() < target_frame_rate.value())) { target_frame_rate = codec_max_frame_rate; } - encode_usage_resource_->SetTargetFrameRate(target_frame_rate); + encode_usage_resource_.SetTargetFrameRate(target_frame_rate); } void ResourceAdaptationProcessor::OnAdaptationCountChanged( @@ -682,7 +730,7 @@ void ResourceAdaptationProcessor::UpdateStatsAdaptationSettings() const { IsFramerateScalingEnabled(degradation_preference_)); VideoStreamEncoderObserver::AdaptationSettings quality_settings = - quality_scaler_resource_->is_started() + quality_scaler_resource_.is_started() ? cpu_settings : VideoStreamEncoderObserver::AdaptationSettings(); encoder_stats_observer_->UpdateAdaptationSettings(cpu_settings, @@ -690,7 +738,7 @@ void ResourceAdaptationProcessor::UpdateStatsAdaptationSettings() const { } void ResourceAdaptationProcessor::MaybePerformQualityRampupExperiment() { - if (!quality_scaler_resource_->is_started()) + if (!quality_scaler_resource_.is_started()) return; if (quality_rampup_done_) @@ -707,7 +755,7 @@ void ResourceAdaptationProcessor::MaybePerformQualityRampupExperiment() { if (encoder_settings_ && encoder_target_bitrate_bps_.value_or(0) == encoder_settings_->video_codec().maxBitrate * 1000 && - quality_scaler_resource_->QpFastFilterLow()) { + quality_scaler_resource_.QpFastFilterLow()) { try_quality_rampup = true; } } diff --git a/video/adaptation/resource_adaptation_processor.h b/video/adaptation/resource_adaptation_processor.h index b417f8b53f..c276ad7810 100644 --- a/video/adaptation/resource_adaptation_processor.h +++ b/video/adaptation/resource_adaptation_processor.h @@ -135,11 +135,12 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, bool HasSufficientInputForAdaptation( const VideoStreamInputState& input_state) const; + VideoAdaptationReason GetReasonFromResource(const Resource& resource) const; // Performs the adaptation by getting the next target, applying it and // informing listeners of the new VideoSourceRestriction and adapt counters. - void OnResourceUnderuse(VideoAdaptationReason reason); - ResourceListenerResponse OnResourceOveruse(VideoAdaptationReason reason); + void OnResourceUnderuse(const Resource& reason_resource); + ResourceListenerResponse OnResourceOveruse(const Resource& reason_resource); CpuOveruseOptions GetCpuOveruseOptions() const; int LastInputFrameSizeOrDefault() const; @@ -179,6 +180,73 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, void ResetActiveCounts(); std::string ActiveCountsToString() const; + // Does not trigger adaptations, only prevents adapting up based on + // |active_counts_|. + class PreventAdaptUpDueToActiveCounts final : public Resource { + public: + explicit PreventAdaptUpDueToActiveCounts( + ResourceAdaptationProcessor* processor); + ~PreventAdaptUpDueToActiveCounts() override = default; + + std::string name() const override { + return "PreventAdaptUpDueToActiveCounts"; + } + + bool IsAdaptationUpAllowed( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after, + const Resource& reason_resource) const override; + + private: + ResourceAdaptationProcessor* processor_; + } prevent_adapt_up_due_to_active_counts_; + + // Does not trigger adaptations, only prevents adapting up resolution. + class PreventIncreaseResolutionDueToBitrateResource final : public Resource { + public: + explicit PreventIncreaseResolutionDueToBitrateResource( + ResourceAdaptationProcessor* processor); + ~PreventIncreaseResolutionDueToBitrateResource() override = default; + + std::string name() const override { + return "PreventIncreaseResolutionDueToBitrateResource"; + } + + bool IsAdaptationUpAllowed( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after, + const Resource& reason_resource) const override; + + private: + ResourceAdaptationProcessor* processor_; + } prevent_increase_resolution_due_to_bitrate_resource_; + + // Does not trigger adaptations, only prevents adapting up in BALANCED. + class PreventAdaptUpInBalancedResource final : public Resource { + public: + explicit PreventAdaptUpInBalancedResource( + ResourceAdaptationProcessor* processor); + ~PreventAdaptUpInBalancedResource() override = default; + + std::string name() const override { + return "PreventAdaptUpInBalancedResource"; + } + + bool IsAdaptationUpAllowed( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after, + const Resource& reason_resource) const override; + + private: + ResourceAdaptationProcessor* processor_; + } prevent_adapt_up_in_balanced_resource_; + + EncodeUsageResource encode_usage_resource_; + QualityScalerResource quality_scaler_resource_; + VideoStreamInputStateProvider* const input_state_provider_; ResourceAdaptationProcessorListener* const adaptation_listener_; Clock* clock_; @@ -190,8 +258,6 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, DegradationPreference effective_degradation_preference_; // Keeps track of source restrictions that this adaptation processor outputs. const std::unique_ptr stream_adapter_; - const std::unique_ptr encode_usage_resource_; - const std::unique_ptr quality_scaler_resource_; const std::unique_ptr initial_frame_dropper_; const bool quality_scaling_experiment_enabled_; // This is the last non-zero target bitrate for the encoder. diff --git a/video/adaptation/video_stream_adapter.cc b/video/adaptation/video_stream_adapter.cc index 064ef0a459..28fdcfe279 100644 --- a/video/adaptation/video_stream_adapter.cc +++ b/video/adaptation/video_stream_adapter.cc @@ -58,6 +58,52 @@ int GetLowerResolutionThan(int pixel_count) { RTC_DCHECK(pixel_count != std::numeric_limits::max()); return (pixel_count * 3) / 5; } + +} // namespace + +VideoSourceRestrictions FilterRestrictionsByDegradationPreference( + VideoSourceRestrictions source_restrictions, + DegradationPreference degradation_preference) { + switch (degradation_preference) { + case DegradationPreference::BALANCED: + break; + case DegradationPreference::MAINTAIN_FRAMERATE: + source_restrictions.set_max_frame_rate(absl::nullopt); + break; + case DegradationPreference::MAINTAIN_RESOLUTION: + source_restrictions.set_max_pixels_per_frame(absl::nullopt); + source_restrictions.set_target_pixels_per_frame(absl::nullopt); + break; + case DegradationPreference::DISABLED: + source_restrictions.set_max_pixels_per_frame(absl::nullopt); + source_restrictions.set_target_pixels_per_frame(absl::nullopt); + source_restrictions.set_max_frame_rate(absl::nullopt); + } + return source_restrictions; +} + +VideoAdaptationCounters FilterVideoAdaptationCountersByDegradationPreference( + VideoAdaptationCounters counters, + DegradationPreference degradation_preference) { + switch (degradation_preference) { + case DegradationPreference::BALANCED: + break; + case DegradationPreference::MAINTAIN_FRAMERATE: + counters.fps_adaptations = 0; + break; + case DegradationPreference::MAINTAIN_RESOLUTION: + counters.resolution_adaptations = 0; + break; + case DegradationPreference::DISABLED: + counters.resolution_adaptations = 0; + counters.fps_adaptations = 0; + break; + default: + RTC_NOTREACHED(); + } + return counters; +} + // TODO(hbos): Use absl::optional<> instead? int GetHigherResolutionThan(int pixel_count) { return pixel_count != std::numeric_limits::max() @@ -65,30 +111,6 @@ int GetHigherResolutionThan(int pixel_count) { : std::numeric_limits::max(); } -// One of the conditions used in VideoStreamAdapter::GetAdaptationUp(). -// TODO(hbos): Whether or not we can adapt up due to encoder settings and -// bitrate should be expressed as a bandwidth-related Resource. -bool CanAdaptUpResolution( - const absl::optional& encoder_settings, - absl::optional encoder_target_bitrate_bps, - int input_pixels) { - uint32_t bitrate_bps = encoder_target_bitrate_bps.value_or(0); - absl::optional bitrate_limits = - encoder_settings.has_value() - ? encoder_settings->encoder_info() - .GetEncoderBitrateLimitsForResolution( - GetHigherResolutionThan(input_pixels)) - : absl::nullopt; - if (!bitrate_limits.has_value() || bitrate_bps == 0) { - return true; // No limit configured or bitrate provided. - } - RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, input_pixels); - return bitrate_bps >= - static_cast(bitrate_limits->min_start_bitrate_bps); -} - -} // namespace - Adaptation::Step::Step(StepType type, int target) : type(type), target(target) {} @@ -315,8 +337,6 @@ VideoStreamAdapter::VideoStreamAdapter() adaptation_validation_id_(0), degradation_preference_(DegradationPreference::DISABLED), input_state_(), - encoder_settings_(absl::nullopt), - encoder_target_bitrate_bps_(absl::nullopt), last_adaptation_request_(absl::nullopt) {} VideoStreamAdapter::~VideoStreamAdapter() {} @@ -359,21 +379,15 @@ VideoStreamAdapter::SetDegradationPreference( : SetDegradationPreferenceResult::kRestrictionsNotCleared; } -void VideoStreamAdapter::SetInput( - VideoStreamInputState input_state, - absl::optional encoder_settings, - absl::optional encoder_target_bitrate_bps) { +void VideoStreamAdapter::SetInput(VideoStreamInputState input_state) { // Invalidate any previously returned Adaptation. ++adaptation_validation_id_; input_state_ = input_state; source_restrictor_->set_min_pixels_per_frame( input_state_.min_pixels_per_frame()); - encoder_settings_ = encoder_settings; - encoder_target_bitrate_bps_ = encoder_target_bitrate_bps; } -Adaptation VideoStreamAdapter::GetAdaptationUp( - VideoAdaptationReason reason) const { +Adaptation VideoStreamAdapter::GetAdaptationUp() const { RTC_DCHECK_NE(degradation_preference_, DegradationPreference::DISABLED); RTC_DCHECK(input_state_.HasInputFrameSizeAndFramesPerSecond()); // Don't adapt if we're awaiting a previous adaptation to have an effect. @@ -387,16 +401,6 @@ Adaptation VideoStreamAdapter::GetAdaptationUp( return Adaptation(adaptation_validation_id_, Adaptation::Status::kAwaitingPreviousAdaptation); } - // Don't adapt if BalancedDegradationSettings applies and determines this will - // exceed bitrate constraints. - if (reason == VideoAdaptationReason::kQuality && - degradation_preference_ == DegradationPreference::BALANCED && - !balanced_settings_.CanAdaptUp(input_state_.video_codec_type(), - input_state_.frame_size_pixels().value(), - encoder_target_bitrate_bps_.value_or(0))) { - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kIsBitrateConstrained); - } // Maybe propose targets based on degradation preference. switch (degradation_preference_) { @@ -411,28 +415,10 @@ Adaptation VideoStreamAdapter::GetAdaptationUp( Adaptation::Step(Adaptation::StepType::kIncreaseFrameRate, target_fps)); } - // Fall-through to maybe-adapting resolution, unless |balanced_settings_| - // forbids it based on bitrate. - if (reason == VideoAdaptationReason::kQuality && - !balanced_settings_.CanAdaptUpResolution( - input_state_.video_codec_type(), - input_state_.frame_size_pixels().value(), - encoder_target_bitrate_bps_.value_or(0))) { - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kIsBitrateConstrained); - } // Scale up resolution. ABSL_FALLTHROUGH_INTENDED; } case DegradationPreference::MAINTAIN_FRAMERATE: { - // Don't adapt resolution if CanAdaptUpResolution() forbids it based on - // bitrate and limits specified by encoder capabilities. - if (reason == VideoAdaptationReason::kQuality && - !CanAdaptUpResolution(encoder_settings_, encoder_target_bitrate_bps_, - input_state_.frame_size_pixels().value())) { - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kIsBitrateConstrained); - } // Attempt to increase pixel count. int target_pixels = input_state_.frame_size_pixels().value(); if (source_restrictor_->adaptation_counters().resolution_adaptations == diff --git a/video/adaptation/video_stream_adapter.h b/video/adaptation/video_stream_adapter.h index 481d4fac5f..a73394d085 100644 --- a/video/adaptation/video_stream_adapter.h +++ b/video/adaptation/video_stream_adapter.h @@ -16,8 +16,6 @@ #include "absl/types/optional.h" #include "api/rtp_parameters.h" #include "api/video/video_adaptation_counters.h" -#include "api/video/video_adaptation_reason.h" -#include "call/adaptation/encoder_settings.h" #include "call/adaptation/resource.h" #include "call/adaptation/video_source_restrictions.h" #include "call/adaptation/video_stream_input_state.h" @@ -26,9 +24,19 @@ namespace webrtc { +class VideoStreamAdapter; + extern const int kMinFrameRateFps; -class VideoStreamAdapter; +VideoSourceRestrictions FilterRestrictionsByDegradationPreference( + VideoSourceRestrictions source_restrictions, + DegradationPreference degradation_preference); + +VideoAdaptationCounters FilterVideoAdaptationCountersByDegradationPreference( + VideoAdaptationCounters counters, + DegradationPreference degradation_preference); + +int GetHigherResolutionThan(int pixel_count); // Represents one step that the VideoStreamAdapter can take when adapting the // VideoSourceRestrictions up or down. Or, if adaptation is not valid, provides @@ -45,23 +53,7 @@ class Adaptation final { // Cannot adapt. The resolution or frame rate requested by a recent // adaptation has not yet been reflected in the input resolution or frame // rate; adaptation is refused to avoid "double-adapting". - // TODO(hbos): Can this be rephrased as a resource usage measurement - // cooldown mechanism? In a multi-stream setup, we need to wait before - // adapting again across streams. The best way to achieve this is probably - // to not act on racy resource usage measurements, regardless of individual - // adapters. When this logic is moved or replaced then remove this enum - // value. kAwaitingPreviousAdaptation, - // Cannot adapt. The adaptation that would have been proposed by the adapter - // violates bitrate constraints and is therefore rejected. - // TODO(hbos): This is a version of being resource limited, except in order - // to know if we are constrained we need to have a proposed adaptation in - // mind, thus the resource alone cannot determine this in isolation. - // Proposal: ask resources for permission to apply a proposed adaptation. - // This allows rejecting a given resolution or frame rate based on bitrate - // limits without coupling it with the adapter's proposal logic. When this - // is done, remove this enum value. - kIsBitrateConstrained, }; // The status of this Adaptation. To find out how this Adaptation affects @@ -140,13 +132,11 @@ class VideoStreamAdapter { SetDegradationPreferenceResult SetDegradationPreference( DegradationPreference degradation_preference); // The adaptaiton logic depends on these inputs. - void SetInput(VideoStreamInputState input_state, - absl::optional encoder_settings, - absl::optional encoder_target_bitrate_bps); + void SetInput(VideoStreamInputState input_state); // Returns an adaptation that we are guaranteed to be able to apply, or a // status code indicating the reason why we cannot adapt. - Adaptation GetAdaptationUp(VideoAdaptationReason reason) const; + Adaptation GetAdaptationUp() const; Adaptation GetAdaptationDown() const; // Returns the restrictions that result from applying the adaptation, without // actually applying it. If the adaptation is not valid, current restrictions @@ -189,8 +179,6 @@ class VideoStreamAdapter { // https://w3c.github.io/mst-content-hint/#dom-rtcdegradationpreference DegradationPreference degradation_preference_; VideoStreamInputState input_state_; - absl::optional encoder_settings_; - absl::optional encoder_target_bitrate_bps_; // The input frame rate, resolution and adaptation direction of the last // ApplyAdaptationTarget(). Used to avoid adapting twice if a recent // adaptation has not had an effect on the input frame rate or resolution yet. diff --git a/video/adaptation/video_stream_adapter_unittest.cc b/video/adaptation/video_stream_adapter_unittest.cc index 3c371685eb..a0f1603693 100644 --- a/video/adaptation/video_stream_adapter_unittest.cc +++ b/video/adaptation/video_stream_adapter_unittest.cc @@ -30,10 +30,6 @@ namespace webrtc { namespace { -// GetAdaptationUp() requires an AdaptReason. This is only used in edge cases, -// so most tests don't care what reason is used. -const auto kReasonDontCare = VideoAdaptationReason::kQuality; - const int kBalancedHighResolutionPixels = 1280 * 720; const int kBalancedHighFrameRateFps = 30; @@ -53,20 +49,14 @@ std::string BalancedFieldTrialConfig() { rtc::ToString(kBalancedHighFrameRateFps) + "/"; } -VideoStreamInputState InputState( - int input_pixels, - int input_fps, - absl::optional encoder_settings) { +VideoStreamInputState InputState(int input_pixels, + int input_fps, + int min_pixels_per_frame) { VideoStreamInputState input_state; input_state.set_has_input(true); input_state.set_frame_size_pixels(input_pixels); input_state.set_frames_per_second(input_fps); - if (encoder_settings.has_value()) { - input_state.set_video_codec_type( - encoder_settings->encoder_config().codec_type); - input_state.set_min_pixels_per_frame( - encoder_settings->encoder_info().scaling_settings.min_pixels_per_frame); - } + input_state.set_min_pixels_per_frame(min_pixels_per_frame); return input_state; } @@ -80,15 +70,13 @@ class FakeVideoStream { FakeVideoStream(VideoStreamAdapter* adapter, int input_pixels, int input_fps, - absl::optional encoder_settings, - absl::optional encoder_target_bitrate_bps) + int min_pixels_per_frame) : adapter_(adapter), input_pixels_(input_pixels), input_fps_(input_fps), - encoder_settings_(std::move(encoder_settings)), - encoder_target_bitrate_bps_(std::move(encoder_target_bitrate_bps)) { - adapter_->SetInput(InputState(input_pixels_, input_fps_, encoder_settings_), - encoder_settings_, encoder_target_bitrate_bps_); + min_pixels_per_frame_(min_pixels_per_frame) { + adapter_->SetInput( + InputState(input_pixels_, input_fps_, min_pixels_per_frame_)); } int input_pixels() const { return input_pixels_; } @@ -111,40 +99,17 @@ class FakeVideoStream { if (restrictions.max_frame_rate().has_value()) { input_fps_ = restrictions.max_frame_rate().value(); } - adapter_->SetInput(InputState(input_pixels_, input_fps_, encoder_settings_), - encoder_settings_, encoder_target_bitrate_bps_); + adapter_->SetInput( + InputState(input_pixels_, input_fps_, min_pixels_per_frame_)); } private: VideoStreamAdapter* adapter_; int input_pixels_; int input_fps_; - absl::optional encoder_settings_; - absl::optional encoder_target_bitrate_bps_; + int min_pixels_per_frame_; }; -EncoderSettings EncoderSettingsWithMinPixelsPerFrame(int min_pixels_per_frame) { - VideoEncoder::EncoderInfo encoder_info; - encoder_info.scaling_settings.min_pixels_per_frame = min_pixels_per_frame; - return EncoderSettings(std::move(encoder_info), VideoEncoderConfig(), - VideoCodec()); -} - -EncoderSettings EncoderSettingsWithBitrateLimits(int resolution_pixels, - int min_start_bitrate_bps) { - VideoEncoder::EncoderInfo encoder_info; - // For bitrate limits, we only care about the next resolution up's - // min_start_bitrate_bps. (...Why do we look at start bitrate and not min - // bitrate?) - encoder_info.resolution_bitrate_limits.emplace_back( - resolution_pixels, - /* min_start_bitrate_bps */ min_start_bitrate_bps, - /* min_bitrate_bps */ 0, - /* max_bitrate_bps */ 0); - return EncoderSettings(std::move(encoder_info), VideoEncoderConfig(), - VideoCodec()); -} - } // namespace TEST(VideoStreamAdapterTest, NoRestrictionsByDefault) { @@ -157,8 +122,7 @@ TEST(VideoStreamAdapterTest, MaintainFramerate_DecreasesPixelsToThreeFifths) { const int kInputPixels = 1280 * 720; VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); - adapter.SetInput(InputState(kInputPixels, 30, absl::nullopt), absl::nullopt, - absl::nullopt); + adapter.SetInput(InputState(kInputPixels, 30, kDefaultMinPixelsPerFrame)); Adaptation adaptation = adapter.GetAdaptationDown(); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); EXPECT_FALSE(adaptation.min_pixel_limit_reached()); @@ -175,10 +139,7 @@ TEST(VideoStreamAdapterTest, MaintainFramerate_DecreasesPixelsToLimitReached) { const int kMinPixelsPerFrame = 640 * 480; VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); - auto encoder_settings = - EncoderSettingsWithMinPixelsPerFrame(kMinPixelsPerFrame); - adapter.SetInput(InputState(kMinPixelsPerFrame + 1, 30, encoder_settings), - encoder_settings, absl::nullopt); + adapter.SetInput(InputState(kMinPixelsPerFrame + 1, 30, kMinPixelsPerFrame)); // Even though we are above kMinPixelsPerFrame, because adapting down would // have exceeded the limit, we are said to have reached the limit already. // This differs from the frame rate adaptation logic, which would have clamped @@ -192,8 +153,8 @@ TEST(VideoStreamAdapterTest, MaintainFramerate_DecreasesPixelsToLimitReached) { TEST(VideoStreamAdapterTest, MaintainFramerate_IncreasePixelsToFiveThirds) { VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); - FakeVideoStream fake_stream(&adapter, - 1280 * 720, 30, absl::nullopt, absl::nullopt); + FakeVideoStream fake_stream(&adapter, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); // Go down twice, ensuring going back up is still a restricted resolution. fake_stream.ApplyAdaptation(adapter.GetAdaptationDown()); fake_stream.ApplyAdaptation(adapter.GetAdaptationDown()); @@ -201,7 +162,7 @@ TEST(VideoStreamAdapterTest, MaintainFramerate_IncreasePixelsToFiveThirds) { int input_pixels = fake_stream.input_pixels(); // Go up once. The target is 5/3 and the max is 12/5 of the target. const int target = (input_pixels * 5) / 3; - fake_stream.ApplyAdaptation(adapter.GetAdaptationUp(kReasonDontCare)); + fake_stream.ApplyAdaptation(adapter.GetAdaptationUp()); EXPECT_EQ(static_cast((target * 12) / 5), adapter.source_restrictions().max_pixels_per_frame()); EXPECT_EQ(static_cast(target), @@ -213,15 +174,15 @@ TEST(VideoStreamAdapterTest, MaintainFramerate_IncreasePixelsToFiveThirds) { TEST(VideoStreamAdapterTest, MaintainFramerate_IncreasePixelsToUnrestricted) { VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); - FakeVideoStream fake_stream(&adapter, - 1280 * 720, 30, absl::nullopt, absl::nullopt); + FakeVideoStream fake_stream(&adapter, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); // We are unrestricted by default and should not be able to adapt up. EXPECT_EQ(Adaptation::Status::kLimitReached, - adapter.GetAdaptationUp(kReasonDontCare).status()); + adapter.GetAdaptationUp().status()); // If we go down once and then back up we should not have any restrictions. fake_stream.ApplyAdaptation(adapter.GetAdaptationDown()); EXPECT_EQ(1, adapter.adaptation_counters().resolution_adaptations); - fake_stream.ApplyAdaptation(adapter.GetAdaptationUp(kReasonDontCare)); + fake_stream.ApplyAdaptation(adapter.GetAdaptationUp()); EXPECT_EQ(VideoSourceRestrictions(), adapter.source_restrictions()); EXPECT_EQ(0, adapter.adaptation_counters().Total()); } @@ -230,8 +191,8 @@ TEST(VideoStreamAdapterTest, MaintainResolution_DecreasesFpsToTwoThirds) { const int kInputFps = 30; VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); - adapter.SetInput(InputState(1280 * 720, kInputFps, absl::nullopt), - absl::nullopt, absl::nullopt); + adapter.SetInput( + InputState(1280 * 720, kInputFps, kDefaultMinPixelsPerFrame)); Adaptation adaptation = adapter.GetAdaptationDown(); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); adapter.ApplyAdaptation(adaptation); @@ -248,7 +209,7 @@ TEST(VideoStreamAdapterTest, MaintainResolution_DecreasesFpsToLimitReached) { VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); FakeVideoStream fake_stream(&adapter, 1280 * 720, kMinFrameRateFps + 1, - absl::nullopt, absl::nullopt); + kDefaultMinPixelsPerFrame); // If we are not yet at the limit and the next step would exceed it, the step // is clamped such that we end up exactly on the limit. Adaptation adaptation = adapter.GetAdaptationDown(); @@ -265,15 +226,15 @@ TEST(VideoStreamAdapterTest, MaintainResolution_DecreasesFpsToLimitReached) { TEST(VideoStreamAdapterTest, MaintainResolution_IncreaseFpsToThreeHalves) { VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); - FakeVideoStream fake_stream(&adapter, - 1280 * 720, 30, absl::nullopt, absl::nullopt); + FakeVideoStream fake_stream(&adapter, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); // Go down twice, ensuring going back up is still a restricted frame rate. fake_stream.ApplyAdaptation(adapter.GetAdaptationDown()); fake_stream.ApplyAdaptation(adapter.GetAdaptationDown()); EXPECT_EQ(2, adapter.adaptation_counters().fps_adaptations); int input_fps = fake_stream.input_fps(); // Go up once. The target is 3/2 of the input. - Adaptation adaptation = adapter.GetAdaptationUp(kReasonDontCare); + Adaptation adaptation = adapter.GetAdaptationUp(); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); fake_stream.ApplyAdaptation(adaptation); EXPECT_EQ(absl::nullopt, @@ -288,15 +249,15 @@ TEST(VideoStreamAdapterTest, MaintainResolution_IncreaseFpsToThreeHalves) { TEST(VideoStreamAdapterTest, MaintainResolution_IncreaseFpsToUnrestricted) { VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); - FakeVideoStream fake_stream(&adapter, - 1280 * 720, 30, absl::nullopt, absl::nullopt); + FakeVideoStream fake_stream(&adapter, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); // We are unrestricted by default and should not be able to adapt up. EXPECT_EQ(Adaptation::Status::kLimitReached, - adapter.GetAdaptationUp(kReasonDontCare).status()); + adapter.GetAdaptationUp().status()); // If we go down once and then back up we should not have any restrictions. fake_stream.ApplyAdaptation(adapter.GetAdaptationDown()); EXPECT_EQ(1, adapter.adaptation_counters().fps_adaptations); - fake_stream.ApplyAdaptation(adapter.GetAdaptationUp(kReasonDontCare)); + fake_stream.ApplyAdaptation(adapter.GetAdaptationUp()); EXPECT_EQ(VideoSourceRestrictions(), adapter.source_restrictions()); EXPECT_EQ(0, adapter.adaptation_counters().Total()); } @@ -307,8 +268,8 @@ TEST(VideoStreamAdapterTest, Balanced_DecreaseFrameRate) { VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::BALANCED); adapter.SetInput(InputState(kBalancedMediumResolutionPixels, - kBalancedHighFrameRateFps, absl::nullopt), - absl::nullopt, absl::nullopt); + kBalancedHighFrameRateFps, + kDefaultMinPixelsPerFrame)); // If our frame rate is higher than the frame rate associated with our // resolution we should try to adapt to the frame rate associated with our // resolution: kBalancedMediumFrameRateFps. @@ -331,8 +292,8 @@ TEST(VideoStreamAdapterTest, Balanced_DecreaseResolution) { VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::BALANCED); FakeVideoStream fake_stream(&adapter, kBalancedHighResolutionPixels, - kBalancedHighFrameRateFps, absl::nullopt, - absl::nullopt); + kBalancedHighFrameRateFps, + kDefaultMinPixelsPerFrame); // If we are not below the current resolution's frame rate limit, we should // adapt resolution according to "maintain-framerate" logic (three fifths). // @@ -405,8 +366,8 @@ TEST(VideoStreamAdapterTest, Balanced_IncreaseFrameRateAndResolution) { VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::BALANCED); FakeVideoStream fake_stream(&adapter, kBalancedHighResolutionPixels, - kBalancedHighFrameRateFps, absl::nullopt, - absl::nullopt); + kBalancedHighFrameRateFps, + kDefaultMinPixelsPerFrame); // The desired starting point of this test is having adapted frame rate twice. // This requires performing a number of adaptations. constexpr size_t kReducedPixelsFirstStep = @@ -452,7 +413,7 @@ TEST(VideoStreamAdapterTest, Balanced_IncreaseFrameRateAndResolution) { // the next resolution configuration up ("high") is kBalancedHighFrameRateFps // and "balanced" prefers adapting frame rate if not already applied. { - Adaptation adaptation = adapter.GetAdaptationUp(kReasonDontCare); + Adaptation adaptation = adapter.GetAdaptationUp(); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); fake_stream.ApplyAdaptation(adaptation); EXPECT_EQ(static_cast(kBalancedHighFrameRateFps), @@ -468,7 +429,7 @@ TEST(VideoStreamAdapterTest, Balanced_IncreaseFrameRateAndResolution) { constexpr size_t kReducedPixelsSecondStepUp = (kReducedPixelsThirdStep * 5) / 3; { - Adaptation adaptation = adapter.GetAdaptationUp(kReasonDontCare); + Adaptation adaptation = adapter.GetAdaptationUp(); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); fake_stream.ApplyAdaptation(adaptation); EXPECT_EQ(kReducedPixelsSecondStepUp, @@ -479,7 +440,7 @@ TEST(VideoStreamAdapterTest, Balanced_IncreaseFrameRateAndResolution) { // Now that our resolution is back in the high-range, the next frame rate to // try out is "unlimited". { - Adaptation adaptation = adapter.GetAdaptationUp(kReasonDontCare); + Adaptation adaptation = adapter.GetAdaptationUp(); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); fake_stream.ApplyAdaptation(adaptation); EXPECT_EQ(absl::nullopt, adapter.source_restrictions().max_frame_rate()); @@ -490,7 +451,7 @@ TEST(VideoStreamAdapterTest, Balanced_IncreaseFrameRateAndResolution) { constexpr size_t kReducedPixelsFirstStepUp = (kReducedPixelsSecondStepUp * 5) / 3; { - Adaptation adaptation = adapter.GetAdaptationUp(kReasonDontCare); + Adaptation adaptation = adapter.GetAdaptationUp(); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); fake_stream.ApplyAdaptation(adaptation); EXPECT_EQ(kReducedPixelsFirstStepUp, @@ -500,7 +461,7 @@ TEST(VideoStreamAdapterTest, Balanced_IncreaseFrameRateAndResolution) { } // The last step up should make us entirely unrestricted. { - Adaptation adaptation = adapter.GetAdaptationUp(kReasonDontCare); + Adaptation adaptation = adapter.GetAdaptationUp(); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); fake_stream.ApplyAdaptation(adaptation); EXPECT_EQ(VideoSourceRestrictions(), adapter.source_restrictions()); @@ -514,11 +475,11 @@ TEST(VideoStreamAdapterTest, Balanced_LimitReached) { VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::BALANCED); FakeVideoStream fake_stream(&adapter, kBalancedLowResolutionPixels, - kBalancedLowFrameRateFps, absl::nullopt, - absl::nullopt); + kBalancedLowFrameRateFps, + kDefaultMinPixelsPerFrame); // Attempting to adapt up while unrestricted should result in kLimitReached. EXPECT_EQ(Adaptation::Status::kLimitReached, - adapter.GetAdaptationUp(kReasonDontCare).status()); + adapter.GetAdaptationUp().status()); // Adapting down once result in restricted frame rate, in this case we reach // the lowest possible frame rate immediately: kBalancedLowFrameRateFps. fake_stream.ApplyAdaptation(adapter.GetAdaptationDown()); @@ -556,8 +517,7 @@ TEST(VideoStreamAdapterTest, Balanced_LimitReached) { TEST(VideoStreamAdapterTest, MaintainFramerate_AwaitingPreviousAdaptationDown) { VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); - adapter.SetInput(InputState(1280 * 720, 30, absl::nullopt), absl::nullopt, - absl::nullopt); + adapter.SetInput(InputState(1280 * 720, 30, kDefaultMinPixelsPerFrame)); // Adapt down once, but don't update the input. adapter.ApplyAdaptation(adapter.GetAdaptationDown()); EXPECT_EQ(1, adapter.adaptation_counters().resolution_adaptations); @@ -574,59 +534,33 @@ TEST(VideoStreamAdapterTest, MaintainFramerate_AwaitingPreviousAdaptationDown) { TEST(VideoStreamAdapterTest, MaintainFramerate_AwaitingPreviousAdaptationUp) { VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); - FakeVideoStream fake_stream(&adapter, - 1280 * 720, 30, absl::nullopt, absl::nullopt); + FakeVideoStream fake_stream(&adapter, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); // Perform two adaptation down so that adapting up twice is possible. fake_stream.ApplyAdaptation(adapter.GetAdaptationDown()); fake_stream.ApplyAdaptation(adapter.GetAdaptationDown()); EXPECT_EQ(2, adapter.adaptation_counters().resolution_adaptations); // Adapt up once, but don't update the input. - adapter.ApplyAdaptation(adapter.GetAdaptationUp(kReasonDontCare)); + adapter.ApplyAdaptation(adapter.GetAdaptationUp()); EXPECT_EQ(1, adapter.adaptation_counters().resolution_adaptations); { // Having performed the adaptation, but not updated the input based on the // new restrictions, adapting again in the same direction will not work. - Adaptation adaptation = adapter.GetAdaptationUp(kReasonDontCare); + Adaptation adaptation = adapter.GetAdaptationUp(); EXPECT_EQ(Adaptation::Status::kAwaitingPreviousAdaptation, adaptation.status()); } } -// TODO(hbos): Also add BitrateConstrained test coverage for the BALANCED -// degradation preference. -TEST(VideoStreamAdapterTest, BitrateConstrained_MaintainFramerate) { - const int kInputPixels = 1280 * 720; - const int kBitrateLimit = 1000; - VideoStreamAdapter adapter; - adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); - FakeVideoStream fake_stream( - &adapter, kInputPixels, 30, - EncoderSettingsWithBitrateLimits(kInputPixels, kBitrateLimit), - // The target bitrate is one less than necessary - // to adapt up. - kBitrateLimit - 1); - // Adapt down so that it would be possible to adapt up if we weren't bitrate - // constrainted. - fake_stream.ApplyAdaptation(adapter.GetAdaptationDown()); - EXPECT_EQ(1, adapter.adaptation_counters().resolution_adaptations); - // Adapting up for reason kQuality should not work because this exceeds the - // bitrate limit. - // TODO(hbos): Why would the reason matter? If the signal was kCpu then the - // current code allows us to violate this bitrate constraint. This does not - // make any sense: either we are limited or we are not, end of story. - EXPECT_EQ(Adaptation::Status::kIsBitrateConstrained, - adapter.GetAdaptationUp(VideoAdaptationReason::kQuality).status()); -} - TEST(VideoStreamAdapterTest, PeekNextRestrictions) { VideoStreamAdapter adapter; // Any non-disabled DegradationPreference will do. adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); - FakeVideoStream fake_stream(&adapter, - 1280 * 720, 30, absl::nullopt, absl::nullopt); + FakeVideoStream fake_stream(&adapter, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); // When adaptation is not possible. { - Adaptation adaptation = adapter.GetAdaptationUp(kReasonDontCare); + Adaptation adaptation = adapter.GetAdaptationUp(); EXPECT_EQ(Adaptation::Status::kLimitReached, adaptation.status()); EXPECT_EQ(adapter.PeekNextRestrictions(adaptation), adapter.source_restrictions()); @@ -642,7 +576,7 @@ TEST(VideoStreamAdapterTest, PeekNextRestrictions) { } // When we adapt up. { - Adaptation adaptation = adapter.GetAdaptationUp(kReasonDontCare); + Adaptation adaptation = adapter.GetAdaptationUp(); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); VideoSourceRestrictions next_restrictions = adapter.PeekNextRestrictions(adaptation); @@ -658,8 +592,7 @@ TEST(VideoStreamAdapterTest, kRestrictionsNotCleared, adapter.SetDegradationPreference( DegradationPreference::MAINTAIN_FRAMERATE)); - adapter.SetInput(InputState(1280 * 720, 30, absl::nullopt), absl::nullopt, - absl::nullopt); + adapter.SetInput(InputState(1280 * 720, 30, kDefaultMinPixelsPerFrame)); adapter.ApplyAdaptation(adapter.GetAdaptationDown()); EXPECT_NE(VideoSourceRestrictions(), adapter.source_restrictions()); EXPECT_NE(0, adapter.adaptation_counters().Total()); @@ -691,8 +624,7 @@ TEST(VideoStreamAdapterDeathTest, SetDegradationPreferenceInvalidatesAdaptations) { VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); - adapter.SetInput(InputState(1280 * 720, 30, absl::nullopt), absl::nullopt, - absl::nullopt); + adapter.SetInput(InputState(1280 * 720, 30, kDefaultMinPixelsPerFrame)); Adaptation adaptation = adapter.GetAdaptationDown(); adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); EXPECT_DEATH(adapter.ApplyAdaptation(adaptation), ""); @@ -701,11 +633,9 @@ TEST(VideoStreamAdapterDeathTest, TEST(VideoStreamAdapterDeathTest, SetInputInvalidatesAdaptations) { VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); - adapter.SetInput(InputState(1280 * 720, 30, absl::nullopt), absl::nullopt, - absl::nullopt); + adapter.SetInput(InputState(1280 * 720, 30, kDefaultMinPixelsPerFrame)); Adaptation adaptation = adapter.GetAdaptationDown(); - adapter.SetInput(InputState(1280 * 720, 31, absl::nullopt), absl::nullopt, - absl::nullopt); + adapter.SetInput(InputState(1280 * 720, 31, kDefaultMinPixelsPerFrame)); EXPECT_DEATH(adapter.PeekNextRestrictions(adaptation), ""); }