From 784af1f42e89735587c442855fa01fc90475c449 Mon Sep 17 00:00:00 2001 From: Per K Date: Tue, 26 Mar 2024 10:11:04 +0100 Subject: [PATCH] FrameCadenceAdapter keep track of Input framerate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor FrameCandenceAdapter to keep track of input frame rate. Also a field trial WebRTC-FrameCadenceAdapter-UseVideoFrameTimestamp is added to control if VideoFrame timestamp should be used or local clock when calculating frame rate. Uma is recorded to tell if input frame timestamp is monotonically increasing. Bug: webrtc:10481, webrtc:15887 Change-Id: I6d698e9f9dcfe8c023d2d35371435c47f70102b0 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/342760 Commit-Queue: Per Kjellander Reviewed-by: Erik Språng Cr-Commit-Position: refs/heads/main@{#41967} --- experiments/field_trials.py | 3 + .../pcc/pcc_network_controller_unittest.cc | 3 +- video/frame_cadence_adapter.cc | 71 ++++++++++++++----- video/frame_cadence_adapter.h | 4 -- video/frame_cadence_adapter_unittest.cc | 62 ++++++++++++---- video/video_stream_encoder.cc | 6 -- video/video_stream_encoder_unittest.cc | 71 +++++++++++-------- 7 files changed, 148 insertions(+), 72 deletions(-) diff --git a/experiments/field_trials.py b/experiments/field_trials.py index 85b073bb86..6580ac9009 100755 --- a/experiments/field_trials.py +++ b/experiments/field_trials.py @@ -80,6 +80,9 @@ ACTIVE_FIELD_TRIALS: FrozenSet[FieldTrial] = frozenset([ FieldTrial('WebRTC-EncoderDataDumpDirectory', 'b/296242528', date(2024, 4, 1)), + FieldTrial('WebRTC-FrameCadenceAdapter-UseVideoFrameTimestamp', + 'webrtc:15887', + date(2024, 10, 1)), FieldTrial('WebRTC-IPv6NetworkResolutionFixes', 'webrtc:14334', date(2024, 4, 1)), diff --git a/modules/congestion_controller/pcc/pcc_network_controller_unittest.cc b/modules/congestion_controller/pcc/pcc_network_controller_unittest.cc index 31bc13f824..f7cf2c45c9 100644 --- a/modules/congestion_controller/pcc/pcc_network_controller_unittest.cc +++ b/modules/congestion_controller/pcc/pcc_network_controller_unittest.cc @@ -112,7 +112,8 @@ TEST(PccNetworkControllerTest, UpdatesTargetSendRate) { ret_net->UpdateConfig( [](NetworkSimulationConfig* c) { c->delay = TimeDelta::Millis(200); }); s.RunFor(TimeDelta::Seconds(35)); - EXPECT_NEAR(client->target_rate().kbps(), 170, 50); + EXPECT_LE(client->target_rate().kbps(), 200); + EXPECT_GT(client->target_rate().kbps(), 90); } } // namespace test diff --git a/video/frame_cadence_adapter.cc b/video/frame_cadence_adapter.cc index 4aea1acec6..77f4ba9c74 100644 --- a/video/frame_cadence_adapter.cc +++ b/video/frame_cadence_adapter.cc @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -58,15 +59,15 @@ class AdapterMode { virtual absl::optional GetInputFrameRateFps() = 0; // Updates the frame rate. - virtual void UpdateFrameRate() = 0; + virtual void UpdateFrameRate(Timestamp frame_timestamp) = 0; }; // Implements a pass-through adapter. Single-threaded. class PassthroughAdapterMode : public AdapterMode { public: - PassthroughAdapterMode(Clock* clock, - FrameCadenceAdapterInterface::Callback* callback) - : clock_(clock), callback_(callback) { + explicit PassthroughAdapterMode( + FrameCadenceAdapterInterface::Callback* callback) + : callback_(callback) { sequence_checker_.Detach(); } @@ -80,16 +81,18 @@ class PassthroughAdapterMode : public AdapterMode { absl::optional GetInputFrameRateFps() override { RTC_DCHECK_RUN_ON(&sequence_checker_); - return input_framerate_.Rate(clock_->TimeInMilliseconds()); + return last_frame_rate_; } - void UpdateFrameRate() override { + void UpdateFrameRate(Timestamp frame_timestamp) override { RTC_DCHECK_RUN_ON(&sequence_checker_); - input_framerate_.Update(1, clock_->TimeInMilliseconds()); + // RateStatistics will calculate a too high rate immediately after Update. + last_frame_rate_ = input_framerate_.Rate(frame_timestamp.ms()); + input_framerate_.Update(1, frame_timestamp.ms()); } private: - Clock* const clock_; + absl::optional last_frame_rate_; FrameCadenceAdapterInterface::Callback* const callback_; RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_; // Input frame rate statistics for use when not in zero-hertz mode. @@ -125,7 +128,7 @@ class ZeroHertzAdapterMode : public AdapterMode { bool queue_overload, const VideoFrame& frame) override; absl::optional GetInputFrameRateFps() override; - void UpdateFrameRate() override {} + void UpdateFrameRate(Timestamp frame_timestamp) override {} // Notified on dropped frames. void OnDiscardedFrame(); @@ -283,12 +286,14 @@ class VSyncEncodeAdapterMode : public AdapterMode { absl::optional GetInputFrameRateFps() override { RTC_DCHECK_RUN_ON(&queue_sequence_checker_); - return input_framerate_.Rate(clock_->TimeInMilliseconds()); + return last_frame_rate_; } - void UpdateFrameRate() override { + void UpdateFrameRate(Timestamp frame_timestamp) override { RTC_DCHECK_RUN_ON(&queue_sequence_checker_); - input_framerate_.Update(1, clock_->TimeInMilliseconds()); + // RateStatistics will calculate a too high rate immediately after Update. + last_frame_rate_ = input_framerate_.Rate(frame_timestamp.ms()); + input_framerate_.Update(1, frame_timestamp.ms()); } void EncodeAllEnqueuedFrames(); @@ -308,6 +313,8 @@ class VSyncEncodeAdapterMode : public AdapterMode { RTC_NO_UNIQUE_ADDRESS SequenceChecker queue_sequence_checker_; rtc::scoped_refptr queue_safety_flag_; // Input frame rate statistics for use when not in zero-hertz mode. + absl::optional last_frame_rate_ + RTC_GUARDED_BY(queue_sequence_checker_); RateStatistics input_framerate_ RTC_GUARDED_BY(queue_sequence_checker_){ FrameCadenceAdapterInterface::kFrameRateAveragingWindowSizeMs, 1000}; FrameCadenceAdapterInterface::Callback* const callback_; @@ -315,8 +322,8 @@ class VSyncEncodeAdapterMode : public AdapterMode { Metronome* metronome_; TaskQueueBase* const worker_queue_; RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_sequence_checker_; - // `worker_safety_` protects tasks on the worker queue related to `metronome_` - // since metronome usage must happen on worker thread. + // `worker_safety_` protects tasks on the worker queue related to + // `metronome_` since metronome usage must happen on worker thread. ScopedTaskSafetyDetached worker_safety_; Timestamp expected_next_tick_ RTC_GUARDED_BY(worker_sequence_checker_) = Timestamp::PlusInfinity(); @@ -339,7 +346,6 @@ class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface { void SetZeroHertzModeEnabled( absl::optional params) override; absl::optional GetInputFrameRateFps() override; - void UpdateFrameRate() override; void UpdateLayerQualityConvergence(size_t spatial_index, bool quality_converged) override; void UpdateLayerStatus(size_t spatial_index, bool enabled) override; @@ -354,6 +360,7 @@ class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface { const VideoTrackSourceConstraints& constraints) override; private: + void UpdateFrameRate(Timestamp frame_timestamp); // Called from OnFrame in both pass-through and zero-hertz mode. void OnFrameOnMainQueue(Timestamp post_time, bool queue_overload, @@ -383,6 +390,13 @@ class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface { // Kill-switch for the queue overload mechanism in zero-hertz mode. const bool frame_cadence_adapter_zero_hertz_queue_overload_enabled_; + // Field trial for using timestamp from video frames, rather than clock when + // calculating input frame rate. + const bool use_video_frame_timestamp_; + // Used for verifying that timestamps are monotonically increasing. + absl::optional last_incoming_frame_timestamp_; + bool incoming_frame_timestamp_monotonically_increasing_ = true; + // The three possible modes we're under. absl::optional passthrough_adapter_; absl::optional zero_hertz_adapter_; @@ -837,6 +851,8 @@ FrameCadenceAdapterImpl::FrameCadenceAdapterImpl( !field_trials.IsDisabled("WebRTC-ZeroHertzScreenshare")), frame_cadence_adapter_zero_hertz_queue_overload_enabled_( !field_trials.IsDisabled("WebRTC-ZeroHertzQueueOverload")), + use_video_frame_timestamp_(field_trials.IsEnabled( + "WebRTC-FrameCadenceAdapter-UseVideoFrameTimestamp")), metronome_(metronome), worker_queue_(worker_queue) {} @@ -849,6 +865,10 @@ FrameCadenceAdapterImpl::~FrameCadenceAdapterImpl() { absl::Cleanup cleanup = [adapter = std::move(vsync_encode_adapter_)] {}; worker_queue_->PostTask([cleanup = std::move(cleanup)] {}); } + + RTC_HISTOGRAM_BOOLEAN( + "WebRTC.Video.InputFrameTimestampMonotonicallyIncreasing", + incoming_frame_timestamp_monotonically_increasing_); } void FrameCadenceAdapterImpl::Initialize(Callback* callback) { @@ -859,7 +879,7 @@ void FrameCadenceAdapterImpl::Initialize(Callback* callback) { vsync_encode_adapter_ = std::make_unique( clock_, queue_, safety_.flag(), metronome_, worker_queue_, callback_); } else { - passthrough_adapter_.emplace(clock_, callback); + passthrough_adapter_.emplace(callback); } ConfigureCurrentAdapterWithoutZeroHertz(); } @@ -877,7 +897,7 @@ absl::optional FrameCadenceAdapterImpl::GetInputFrameRateFps() { return current_adapter_mode_->GetInputFrameRateFps(); } -void FrameCadenceAdapterImpl::UpdateFrameRate() { +void FrameCadenceAdapterImpl::UpdateFrameRate(Timestamp frame_timestamp) { RTC_DCHECK_RUN_ON(queue_); // The frame rate need not be updated for the zero-hertz adapter. The // vsync encode and passthrough adapter however uses it. Always pass frames @@ -885,10 +905,10 @@ void FrameCadenceAdapterImpl::UpdateFrameRate() { // there be an adapter switch. if (metronome_) { RTC_CHECK(vsync_encode_adapter_); - vsync_encode_adapter_->UpdateFrameRate(); + vsync_encode_adapter_->UpdateFrameRate(frame_timestamp); } else { RTC_CHECK(passthrough_adapter_); - passthrough_adapter_->UpdateFrameRate(); + passthrough_adapter_->UpdateFrameRate(frame_timestamp); } } @@ -979,6 +999,19 @@ void FrameCadenceAdapterImpl::OnFrameOnMainQueue(Timestamp post_time, const VideoFrame& frame) { RTC_DCHECK_RUN_ON(queue_); current_adapter_mode_->OnFrame(post_time, queue_overload, frame); + if (last_incoming_frame_timestamp_ && + last_incoming_frame_timestamp_ >= + Timestamp::Micros(frame.timestamp_us())) { + RTC_LOG(LS_ERROR) + << "Incoming frame timestamp is not monotonically increasing" + << " current: " << frame.timestamp_us() + << " last: " << last_incoming_frame_timestamp_.value().us(); + incoming_frame_timestamp_monotonically_increasing_ = false; + } + last_incoming_frame_timestamp_ = Timestamp::Micros(frame.timestamp_us()); + Timestamp update_frame_rate_timestamp = + use_video_frame_timestamp_ ? *last_incoming_frame_timestamp_ : post_time; + UpdateFrameRate(update_frame_rate_timestamp); } bool FrameCadenceAdapterImpl::IsZeroHertzScreenshareEnabled() const { diff --git a/video/frame_cadence_adapter.h b/video/frame_cadence_adapter.h index ec8e667b04..0dc2e5ab7e 100644 --- a/video/frame_cadence_adapter.h +++ b/video/frame_cadence_adapter.h @@ -99,10 +99,6 @@ class FrameCadenceAdapterInterface // zero-hertz mode is off, and returns the max framerate in zero-hertz mode. virtual absl::optional GetInputFrameRateFps() = 0; - // Updates frame rate. This is done unconditionally irrespective of adapter - // mode. - virtual void UpdateFrameRate() = 0; - // Updates quality convergence status for an enabled spatial layer. // Convergence means QP has dropped to a low-enough level to warrant ceasing // to send identical frames at high frequency. diff --git a/video/frame_cadence_adapter_unittest.cc b/video/frame_cadence_adapter_unittest.cc index 54548de9bb..94dfecc9c3 100644 --- a/video/frame_cadence_adapter_unittest.cc +++ b/video/frame_cadence_adapter_unittest.cc @@ -10,6 +10,8 @@ #include "video/frame_cadence_adapter.h" +#include +#include #include #include @@ -134,7 +136,8 @@ TEST(FrameCadenceAdapterTest, FrameRateFollowsRateStatisticsByDefault) { test::ScopedKeyValueConfig no_field_trials; GlobalSimulatedTimeController time_controller(Timestamp::Zero()); auto adapter = CreateAdapter(no_field_trials, time_controller.GetClock()); - adapter->Initialize(nullptr); + MockCallback callback; + adapter->Initialize(&callback); // Create an "oracle" rate statistics which should be followed on a sequence // of frames. @@ -143,10 +146,13 @@ TEST(FrameCadenceAdapterTest, FrameRateFollowsRateStatisticsByDefault) { for (int frame = 0; frame != 10; ++frame) { time_controller.AdvanceTime(TimeDelta::Millis(10)); + absl::optional expected_fps = + rate.Rate(time_controller.GetClock()->TimeInMilliseconds()); rate.Update(1, time_controller.GetClock()->TimeInMilliseconds()); - adapter->UpdateFrameRate(); - EXPECT_EQ(rate.Rate(time_controller.GetClock()->TimeInMilliseconds()), - adapter->GetInputFrameRateFps()) + // FrameCadanceAdapter::OnFrame post the frame to another sequence. + adapter->OnFrame(CreateFrameWithTimestamps(&time_controller)); + time_controller.AdvanceTime(TimeDelta::Millis(0)); + EXPECT_EQ(expected_fps, adapter->GetInputFrameRateFps()) << " failed for frame " << frame; } } @@ -156,7 +162,8 @@ TEST(FrameCadenceAdapterTest, ZeroHertzFieldTrialDisabler feature_disabler; GlobalSimulatedTimeController time_controller(Timestamp::Zero()); auto adapter = CreateAdapter(feature_disabler, time_controller.GetClock()); - adapter->Initialize(nullptr); + MockCallback callback; + adapter->Initialize(&callback); // Create an "oracle" rate statistics which should be followed on a sequence // of frames. @@ -165,10 +172,14 @@ TEST(FrameCadenceAdapterTest, for (int frame = 0; frame != 10; ++frame) { time_controller.AdvanceTime(TimeDelta::Millis(10)); + absl::optional expected_fps = + rate.Rate(time_controller.GetClock()->TimeInMilliseconds()); + rate.Update(1, time_controller.GetClock()->TimeInMilliseconds()); - adapter->UpdateFrameRate(); - EXPECT_EQ(rate.Rate(time_controller.GetClock()->TimeInMilliseconds()), - adapter->GetInputFrameRateFps()) + // FrameCadanceAdapter::OnFrame post the frame to another sequence. + adapter->OnFrame(CreateFrameWithTimestamps(&time_controller)); + time_controller.AdvanceTime(TimeDelta::Millis(0)); + EXPECT_EQ(adapter->GetInputFrameRateFps(), expected_fps) << " failed for frame " << frame; } } @@ -177,13 +188,16 @@ TEST(FrameCadenceAdapterTest, FrameRateFollowsMaxFpsWhenZeroHertzActivated) { ZeroHertzFieldTrialEnabler enabler; GlobalSimulatedTimeController time_controller(Timestamp::Zero()); auto adapter = CreateAdapter(enabler, time_controller.GetClock()); - adapter->Initialize(nullptr); + MockCallback callback; + adapter->Initialize(&callback); adapter->SetZeroHertzModeEnabled( FrameCadenceAdapterInterface::ZeroHertzModeParams{}); adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1}); for (int frame = 0; frame != 10; ++frame) { time_controller.AdvanceTime(TimeDelta::Millis(10)); - adapter->UpdateFrameRate(); + // FrameCadanceAdapter::OnFrame post the frame to another sequence. + adapter->OnFrame(CreateFrameWithTimestamps(&time_controller)); + time_controller.AdvanceTime(TimeDelta::Millis(0)); EXPECT_EQ(adapter->GetInputFrameRateFps(), 1u); } } @@ -216,7 +230,8 @@ TEST(FrameCadenceAdapterTest, ZeroHertzFieldTrialEnabler enabler; GlobalSimulatedTimeController time_controller(Timestamp::Zero()); auto adapter = CreateAdapter(enabler, time_controller.GetClock()); - adapter->Initialize(nullptr); + MockCallback callback; + adapter->Initialize(&callback); adapter->SetZeroHertzModeEnabled( FrameCadenceAdapterInterface::ZeroHertzModeParams{}); adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1}); @@ -226,15 +241,16 @@ TEST(FrameCadenceAdapterTest, for (int frame = 0; frame != MAX; ++frame) { time_controller.AdvanceTime(TimeDelta::Millis(10)); rate.Update(1, time_controller.GetClock()->TimeInMilliseconds()); - adapter->UpdateFrameRate(); + adapter->OnFrame(CreateFrameWithTimestamps(&time_controller)); + time_controller.AdvanceTime(TimeDelta::Millis(0)); } // Turn off zero hertz on the next-last frame; after the last frame we // should see a value that tracks the rate oracle. adapter->SetZeroHertzModeEnabled(absl::nullopt); // Last frame. time_controller.AdvanceTime(TimeDelta::Millis(10)); - rate.Update(1, time_controller.GetClock()->TimeInMilliseconds()); - adapter->UpdateFrameRate(); + adapter->OnFrame(CreateFrameWithTimestamps(&time_controller)); + time_controller.AdvanceTime(TimeDelta::Millis(0)); EXPECT_EQ(rate.Rate(time_controller.GetClock()->TimeInMilliseconds()), adapter->GetInputFrameRateFps()); @@ -1038,6 +1054,24 @@ TEST_F(FrameCadenceAdapterMetricsTest, RecordsTimeUntilFirstFrame) { ElementsAre(Pair(666, 1))); } +TEST_F(FrameCadenceAdapterMetricsTest, + RecordsFrameTimestampMonotonicallyIncreasing) { + MockCallback callback; + test::ScopedKeyValueConfig no_field_trials; + std::unique_ptr adapter = + CreateAdapter(no_field_trials, time_controller_.GetClock()); + adapter->Initialize(&callback); + time_controller_.AdvanceTime(TimeDelta::Millis(666)); + adapter->OnFrame(CreateFrameWithTimestamps(&time_controller_)); + adapter->OnFrame(CreateFrameWithTimestamps(&time_controller_)); + time_controller_.AdvanceTime(TimeDelta::Zero()); + adapter = nullptr; + DepleteTaskQueues(); + EXPECT_THAT(metrics::Samples( + "WebRTC.Video.InputFrameTimestampMonotonicallyIncreasing"), + ElementsAre(Pair(false, 1))); +} + TEST(FrameCadenceAdapterRealTimeTest, TimestampsDoNotDrift) { // This regression test must be performed in realtime because of limitations // in GlobalSimulatedTimeController. diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 3b104ef265..2f4c6e927d 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -1784,13 +1784,7 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, // According to the testcase // InitialFrameDropOffWhenEncoderDisabledScaling, the return value // from GetScalingSettings should enable or disable the frame drop. - - // Update input frame rate before we start using it. If we update it after - // any potential frame drop we are going to artificially increase frame sizes. - // Poll the rate before updating, otherwise we risk the rate being estimated - // a little too high at the start of the call when then window is small. uint32_t framerate_fps = GetInputFramerateFps(); - frame_cadence_adapter_->UpdateFrameRate(); int64_t now_ms = env_.clock().TimeInMilliseconds(); if (pending_encoder_reconfiguration_) { diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 7b21f20e29..6769a2806e 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -574,7 +574,7 @@ class AdaptingFrameForwarder : public test::FrameForwarder { .set_video_frame_buffer(rtc::make_ref_counted( nullptr, out_width, out_height)) .set_ntp_time_ms(video_frame.ntp_time_ms()) - .set_timestamp_ms(99) + .set_timestamp_ms(video_frame.timestamp_us() * 1000) .set_rotation(kVideoRotation_0) .build(); if (video_frame.has_update_rect()) { @@ -791,7 +791,6 @@ class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface { (override)); MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override)); MOCK_METHOD(absl::optional, GetInputFrameRateFps, (), (override)); - MOCK_METHOD(void, UpdateFrameRate, (), (override)); MOCK_METHOD(void, UpdateLayerQualityConvergence, (size_t spatial_index, bool converged), @@ -952,7 +951,7 @@ class VideoStreamEncoderTest : public ::testing::Test { .set_video_frame_buffer(rtc::make_ref_counted( destruction_event, codec_width_, codec_height_)) .set_ntp_time_ms(ntp_time_ms) - .set_timestamp_ms(99) + .set_timestamp_ms(ntp_time_ms) .set_rotation(kVideoRotation_0) .build(); } @@ -964,7 +963,7 @@ class VideoStreamEncoderTest : public ::testing::Test { .set_video_frame_buffer(rtc::make_ref_counted( destruction_event, codec_width_, codec_height_)) .set_ntp_time_ms(ntp_time_ms) - .set_timestamp_ms(99) + .set_timestamp_ms(ntp_time_ms) .set_rotation(kVideoRotation_0) .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1}) .build(); @@ -998,7 +997,7 @@ class VideoStreamEncoderTest : public ::testing::Test { .set_video_frame_buffer(rtc::make_ref_counted( destruction_event, width, height)) .set_ntp_time_ms(ntp_time_ms) - .set_timestamp_ms(99) + .set_timestamp_ms(ntp_time_ms) .set_rotation(kVideoRotation_0) .build(); } @@ -1011,7 +1010,7 @@ class VideoStreamEncoderTest : public ::testing::Test { .set_video_frame_buffer(rtc::make_ref_counted( destruction_event, width, height)) .set_ntp_time_ms(ntp_time_ms) - .set_timestamp_ms(99) + .set_timestamp_ms(ntp_time_ms) .set_rotation(kVideoRotation_0) .build(); } @@ -1533,6 +1532,7 @@ class VideoStreamEncoderTest : public ::testing::Test { if (num_received_layers_ == num_expected_layers_) { encoded_frame_event_.Set(); } + return Result(Result::OK, last_timestamp_); } @@ -1663,28 +1663,34 @@ TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) { // The encoder will cache up to one frame for a short duration. Adding two // frames means that the first frame will be dropped and the second frame will // be sent when the encoder is enabled. - video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event)); + const int64_t kFrame1TimestampMs = CurrentTimeMs(); + video_source_.IncomingCapturedFrame( + CreateFrame(kFrame1TimestampMs, &frame_destroyed_event)); AdvanceTime(TimeDelta::Millis(10)); - video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr)); + const int64_t kFrame2TimestampMs = CurrentTimeMs(); + video_source_.IncomingCapturedFrame(CreateFrame(kFrame2TimestampMs, nullptr)); AdvanceTime(TimeDelta::Zero()); EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout)); video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0); - // The pending frame should be received. - WaitForEncodedFrame(2); - video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr)); + // The pending frame should be encoded. + WaitForEncodedFrame(kFrame2TimestampMs); - WaitForEncodedFrame(3); + const int64_t kFrame3TimestampMs = CurrentTimeMs(); + video_source_.IncomingCapturedFrame(CreateFrame(kFrame3TimestampMs, nullptr)); + + WaitForEncodedFrame(kFrame3TimestampMs); video_stream_encoder_->Stop(); } TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) { + int64_t time_ms = 123; video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0); - video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr)); - WaitForEncodedFrame(1); + video_source_.IncomingCapturedFrame(CreateFrame(time_ms, nullptr)); + WaitForEncodedFrame(time_ms); video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0); @@ -1692,14 +1698,17 @@ TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) { // The encoder will cache up to one frame for a short duration. Adding two // frames means that the first frame will be dropped and the second frame will // be sent when the encoder is resumed. - video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr)); - video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr)); + time_ms += 30; + video_source_.IncomingCapturedFrame(CreateFrame(time_ms, nullptr)); + time_ms += 30; + video_source_.IncomingCapturedFrame(CreateFrame(time_ms, nullptr)); video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0); - WaitForEncodedFrame(3); - video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr)); - WaitForEncodedFrame(4); + WaitForEncodedFrame(time_ms); + time_ms += 20; + video_source_.IncomingCapturedFrame(CreateFrame(time_ms, nullptr)); + WaitForEncodedFrame(time_ms); video_stream_encoder_->Stop(); } @@ -4707,20 +4716,24 @@ TEST_F(VideoStreamEncoderTest, TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) { const int kWidth = 640; const int kHeight = 360; + int64_t ntp_timestamp_ms = 123; video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0); for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) { - video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight)); - WaitForEncodedFrame(i); + video_source_.IncomingCapturedFrame( + CreateFrame(ntp_timestamp_ms, kWidth, kHeight)); + WaitForEncodedFrame(ntp_timestamp_ms); + ntp_timestamp_ms += 20; } video_stream_encoder_->TriggerCpuOveruse(); for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) { - video_source_.IncomingCapturedFrame(CreateFrame( - SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight)); - WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i); + video_source_.IncomingCapturedFrame( + CreateFrame(ntp_timestamp_ms, kWidth, kHeight)); + WaitForEncodedFrame(ntp_timestamp_ms); + ntp_timestamp_ms += 20; } video_stream_encoder_->Stop(); @@ -4739,13 +4752,16 @@ TEST_F(VideoStreamEncoderTest, kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0); const int kWidth = 640; const int kHeight = 360; + int64_t ntp_timestamp_ms = 123; video_stream_encoder_->SetSource(&video_source_, webrtc::DegradationPreference::DISABLED); for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) { - video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight)); - WaitForEncodedFrame(i); + video_source_.IncomingCapturedFrame( + CreateFrame(ntp_timestamp_ms, kWidth, kHeight)); + WaitForEncodedFrame(ntp_timestamp_ms); + ntp_timestamp_ms += 20; } video_stream_encoder_->Stop(); @@ -5516,7 +5532,7 @@ TEST_F(VideoStreamEncoderTest, // Reconfigure the encoder with a new (higher max framerate), max fps should // still respect the adaptation. video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate; - source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight)); + source.IncomingCapturedFrame(CreateFrame(2, kFrameWidth, kFrameHeight)); video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config), kMaxPayloadLength); video_stream_encoder_->WaitUntilTaskQueueIsIdle(); @@ -9513,7 +9529,6 @@ TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) { /*max_data_payload_length=*/1000); EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps); - EXPECT_CALL(*adapter_ptr, UpdateFrameRate); PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1); factory.DepleteTaskQueues(); }