diff --git a/api/BUILD.gn b/api/BUILD.gn index e969fecbb8..0352f993f3 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -98,6 +98,12 @@ rtc_library("rtp_packet_info") { absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } +rtc_source_set("video_track_source_constraints") { + visibility = [ "*" ] + sources = [ "video_track_source_constraints.h" ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + rtc_library("media_stream_interface") { visibility = [ "*" ] sources = [ @@ -110,6 +116,7 @@ rtc_library("media_stream_interface") { ":audio_options_api", ":rtp_parameters", ":scoped_refptr", + ":video_track_source_constraints", "../modules/audio_processing:audio_processing_statistics", "../rtc_base:checks", "../rtc_base:refcount", diff --git a/api/media_stream_interface.h b/api/media_stream_interface.h index 874b4dbf70..ad497d943b 100644 --- a/api/media_stream_interface.h +++ b/api/media_stream_interface.h @@ -28,6 +28,7 @@ #include "api/video/video_frame.h" #include "api/video/video_sink_interface.h" #include "api/video/video_source_interface.h" +#include "api/video_track_source_constraints.h" #include "modules/audio_processing/include/audio_processing_statistics.h" #include "rtc_base/ref_count.h" #include "rtc_base/system/rtc_export.h" @@ -146,8 +147,6 @@ class VideoTrackSourceInterface : public MediaSourceInterface, // Add an encoded video sink to the source and additionally cause // a key frame to be generated from the source. The sink will be // invoked from a decoder queue. - // TODO(bugs.webrtc.org/11114): make pure virtual once downstream project - // adapts. virtual void AddEncodedSink( rtc::VideoSinkInterface* sink) = 0; @@ -155,6 +154,13 @@ class VideoTrackSourceInterface : public MediaSourceInterface, virtual void RemoveEncodedSink( rtc::VideoSinkInterface* sink) = 0; + // Notify about constraints set on the source. The information eventually gets + // routed to attached sinks via VideoSinkInterface<>::OnConstraintsChanged. + // The call is expected to happen on the network thread. + // TODO(crbug/1255737): make pure virtual once downstream project adapts. + virtual void ProcessConstraints( + const webrtc::VideoTrackSourceConstraints& constraints) {} + protected: ~VideoTrackSourceInterface() override = default; }; diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn index ec74869962..e6052fe763 100644 --- a/api/video/BUILD.gn +++ b/api/video/BUILD.gn @@ -60,6 +60,7 @@ rtc_library("video_frame") { "..:array_view", "..:rtp_packet_info", "..:scoped_refptr", + "..:video_track_source_constraints", "../../rtc_base:checks", "../../rtc_base:rtc_base_approved", "../../rtc_base/memory:aligned_malloc", diff --git a/api/video/video_sink_interface.h b/api/video/video_sink_interface.h index 88cf9d924f..9c1f5f3214 100644 --- a/api/video/video_sink_interface.h +++ b/api/video/video_sink_interface.h @@ -11,6 +11,8 @@ #ifndef API_VIDEO_VIDEO_SINK_INTERFACE_H_ #define API_VIDEO_VIDEO_SINK_INTERFACE_H_ +#include "absl/types/optional.h" +#include "api/video_track_source_constraints.h" #include "rtc_base/checks.h" namespace rtc { @@ -25,6 +27,11 @@ class VideoSinkInterface { // Should be called by the source when it discards the frame due to rate // limiting. virtual void OnDiscardedFrame() {} + + // Called on the network thread when video constraints change. + // TODO(crbug/1255737): make pure virtual once downstream project adapts. + virtual void OnConstraintsChanged( + const webrtc::VideoTrackSourceConstraints& constraints) {} }; } // namespace rtc diff --git a/api/video_track_source_constraints.h b/api/video_track_source_constraints.h new file mode 100644 index 0000000000..55e5396d62 --- /dev/null +++ b/api/video_track_source_constraints.h @@ -0,0 +1,32 @@ +/* + * Copyright 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This file contains interfaces for MediaStream, MediaTrack and MediaSource. +// These interfaces are used for implementing MediaStream and MediaTrack as +// defined in http://dev.w3.org/2011/webrtc/editor/webrtc.html#stream-api. These +// interfaces must be used only with PeerConnection. + +#ifndef API_VIDEO_TRACK_SOURCE_CONSTRAINTS_H_ +#define API_VIDEO_TRACK_SOURCE_CONSTRAINTS_H_ + +#include "absl/types/optional.h" + +namespace webrtc { + +// This struct definition describes constraints on the video source that may be +// set with VideoTrackSourceInterface::ProcessConstraints. +struct VideoTrackSourceConstraints { + absl::optional min_fps; + absl::optional max_fps; +}; + +} // namespace webrtc + +#endif // API_VIDEO_TRACK_SOURCE_CONSTRAINTS_H_ diff --git a/media/base/adapted_video_track_source.cc b/media/base/adapted_video_track_source.cc index 2fce973f68..f8f8f2dad1 100644 --- a/media/base/adapted_video_track_source.cc +++ b/media/base/adapted_video_track_source.cc @@ -114,4 +114,9 @@ bool AdaptedVideoTrackSource::AdaptFrame(int width, return true; } +void AdaptedVideoTrackSource::ProcessConstraints( + const webrtc::VideoTrackSourceConstraints& constraints) { + broadcaster_.ProcessConstraints(constraints); +} + } // namespace rtc diff --git a/media/base/adapted_video_track_source.h b/media/base/adapted_video_track_source.h index d40baeff6d..1386fbd9db 100644 --- a/media/base/adapted_video_track_source.h +++ b/media/base/adapted_video_track_source.h @@ -86,6 +86,8 @@ class RTC_EXPORT AdaptedVideoTrackSource rtc::VideoSinkInterface* sink) override {} void RemoveEncodedSink( rtc::VideoSinkInterface* sink) override {} + void ProcessConstraints( + const webrtc::VideoTrackSourceConstraints& constraints) override; cricket::VideoAdapter video_adapter_; diff --git a/media/base/video_broadcaster.cc b/media/base/video_broadcaster.cc index 1b55786338..1167d7fb32 100644 --- a/media/base/video_broadcaster.cc +++ b/media/base/video_broadcaster.cc @@ -32,6 +32,13 @@ void VideoBroadcaster::AddOrUpdateSink( if (!FindSinkPair(sink)) { // `Sink` is a new sink, which didn't receive previous frame. previous_frame_sent_to_all_sinks_ = false; + + if (last_constraints_.has_value()) { + RTC_LOG(LS_INFO) << __func__ << " forwarding stored constraints min_fps " + << last_constraints_->min_fps.value_or(-1) << " max_fps " + << last_constraints_->max_fps.value_or(-1); + sink->OnConstraintsChanged(*last_constraints_); + } } VideoSourceBase::AddOrUpdateSink(sink, wants); UpdateWants(); @@ -100,6 +107,18 @@ void VideoBroadcaster::OnDiscardedFrame() { } } +void VideoBroadcaster::ProcessConstraints( + const webrtc::VideoTrackSourceConstraints& constraints) { + webrtc::MutexLock lock(&sinks_and_wants_lock_); + RTC_LOG(LS_INFO) << __func__ << " min_fps " + << constraints.min_fps.value_or(-1) << " max_fps " + << constraints.max_fps.value_or(-1) << " broadcasting to " + << sink_pairs().size() << " sinks."; + last_constraints_ = constraints; + for (auto& sink_pair : sink_pairs()) + sink_pair.sink->OnConstraintsChanged(constraints); +} + void VideoBroadcaster::UpdateWants() { VideoSinkWants wants; wants.rotation_applied = false; diff --git a/media/base/video_broadcaster.h b/media/base/video_broadcaster.h index 2f4e578224..c253d44b09 100644 --- a/media/base/video_broadcaster.h +++ b/media/base/video_broadcaster.h @@ -11,6 +11,7 @@ #ifndef MEDIA_BASE_VIDEO_BROADCASTER_H_ #define MEDIA_BASE_VIDEO_BROADCASTER_H_ +#include "api/media_stream_interface.h" #include "api/scoped_refptr.h" #include "api/sequence_checker.h" #include "api/video/video_frame_buffer.h" @@ -31,6 +32,11 @@ class VideoBroadcaster : public VideoSourceBase, public: VideoBroadcaster(); ~VideoBroadcaster() override; + + // Adds a new, or updates an already existing sink. If the sink is new and + // ProcessConstraints has been called previously, the new sink's + // OnConstraintsCalled method will be invoked with the most recent + // constraints. void AddOrUpdateSink(VideoSinkInterface* sink, const VideoSinkWants& wants) override; void RemoveSink(VideoSinkInterface* sink) override; @@ -50,6 +56,11 @@ class VideoBroadcaster : public VideoSourceBase, void OnDiscardedFrame() override; + // Called on the network thread when constraints change. Forwards the + // constraints to sinks added with AddOrUpdateSink via OnConstraintsChanged. + void ProcessConstraints( + const webrtc::VideoTrackSourceConstraints& constraints); + protected: void UpdateWants() RTC_EXCLUSIVE_LOCKS_REQUIRED(sinks_and_wants_lock_); const rtc::scoped_refptr& GetBlackFrameBuffer( @@ -62,6 +73,8 @@ class VideoBroadcaster : public VideoSourceBase, rtc::scoped_refptr black_frame_buffer_; bool previous_frame_sent_to_all_sinks_ RTC_GUARDED_BY(sinks_and_wants_lock_) = true; + absl::optional last_constraints_ + RTC_GUARDED_BY(sinks_and_wants_lock_); }; } // namespace rtc diff --git a/media/base/video_broadcaster_unittest.cc b/media/base/video_broadcaster_unittest.cc index b007278547..b9672375a7 100644 --- a/media/base/video_broadcaster_unittest.cc +++ b/media/base/video_broadcaster_unittest.cc @@ -16,13 +16,31 @@ #include "api/video/i420_buffer.h" #include "api/video/video_frame.h" #include "api/video/video_rotation.h" +#include "api/video/video_source_interface.h" #include "media/base/fake_video_renderer.h" +#include "test/gmock.h" #include "test/gtest.h" using cricket::FakeVideoRenderer; using rtc::VideoBroadcaster; using rtc::VideoSinkWants; +using ::testing::AllOf; +using ::testing::Eq; +using ::testing::Field; +using ::testing::Mock; +using ::testing::Optional; + +class MockSink : public rtc::VideoSinkInterface { + public: + void OnFrame(const webrtc::VideoFrame&) override {} + + MOCK_METHOD(void, + OnConstraintsChanged, + (const webrtc::VideoTrackSourceConstraints& constraints), + (override)); +}; + TEST(VideoBroadcasterTest, frame_wanted) { VideoBroadcaster broadcaster; EXPECT_FALSE(broadcaster.frame_wanted()); @@ -232,3 +250,83 @@ TEST(VideoBroadcasterTest, SinkWantsBlackFrames) { EXPECT_TRUE(sink2.black_frame()); EXPECT_EQ(30, sink2.timestamp_us()); } + +TEST(VideoBroadcasterTest, ConstraintsChangedNotCalledOnSinkAddition) { + MockSink sink; + VideoBroadcaster broadcaster; + EXPECT_CALL(sink, OnConstraintsChanged).Times(0); + broadcaster.AddOrUpdateSink(&sink, VideoSinkWants()); +} + +TEST(VideoBroadcasterTest, ForwardsLastConstraintsOnAdd) { + MockSink sink; + VideoBroadcaster broadcaster; + broadcaster.ProcessConstraints(webrtc::VideoTrackSourceConstraints{2, 3}); + broadcaster.ProcessConstraints(webrtc::VideoTrackSourceConstraints{1, 4}); + EXPECT_CALL( + sink, + OnConstraintsChanged(AllOf( + Field(&webrtc::VideoTrackSourceConstraints::min_fps, Optional(1)), + Field(&webrtc::VideoTrackSourceConstraints::max_fps, Optional(4))))); + broadcaster.AddOrUpdateSink(&sink, VideoSinkWants()); +} + +TEST(VideoBroadcasterTest, UpdatesOnlyNewSinksWithConstraints) { + MockSink sink1; + VideoBroadcaster broadcaster; + broadcaster.AddOrUpdateSink(&sink1, VideoSinkWants()); + broadcaster.ProcessConstraints(webrtc::VideoTrackSourceConstraints{1, 4}); + Mock::VerifyAndClearExpectations(&sink1); + EXPECT_CALL(sink1, OnConstraintsChanged).Times(0); + MockSink sink2; + EXPECT_CALL( + sink2, + OnConstraintsChanged(AllOf( + Field(&webrtc::VideoTrackSourceConstraints::min_fps, Optional(1)), + Field(&webrtc::VideoTrackSourceConstraints::max_fps, Optional(4))))); + broadcaster.AddOrUpdateSink(&sink2, VideoSinkWants()); +} + +TEST(VideoBroadcasterTest, ForwardsConstraintsToSink) { + MockSink sink; + VideoBroadcaster broadcaster; + EXPECT_CALL(sink, OnConstraintsChanged).Times(0); + broadcaster.AddOrUpdateSink(&sink, VideoSinkWants()); + Mock::VerifyAndClearExpectations(&sink); + + EXPECT_CALL(sink, OnConstraintsChanged(AllOf( + Field(&webrtc::VideoTrackSourceConstraints::min_fps, + Eq(absl::nullopt)), + Field(&webrtc::VideoTrackSourceConstraints::max_fps, + Eq(absl::nullopt))))); + broadcaster.ProcessConstraints( + webrtc::VideoTrackSourceConstraints{absl::nullopt, absl::nullopt}); + Mock::VerifyAndClearExpectations(&sink); + + EXPECT_CALL( + sink, + OnConstraintsChanged(AllOf( + Field(&webrtc::VideoTrackSourceConstraints::min_fps, + Eq(absl::nullopt)), + Field(&webrtc::VideoTrackSourceConstraints::max_fps, Optional(3))))); + broadcaster.ProcessConstraints( + webrtc::VideoTrackSourceConstraints{absl::nullopt, 3}); + Mock::VerifyAndClearExpectations(&sink); + + EXPECT_CALL( + sink, + OnConstraintsChanged(AllOf( + Field(&webrtc::VideoTrackSourceConstraints::min_fps, Optional(2)), + Field(&webrtc::VideoTrackSourceConstraints::max_fps, + Eq(absl::nullopt))))); + broadcaster.ProcessConstraints( + webrtc::VideoTrackSourceConstraints{2, absl::nullopt}); + Mock::VerifyAndClearExpectations(&sink); + + EXPECT_CALL( + sink, + OnConstraintsChanged(AllOf( + Field(&webrtc::VideoTrackSourceConstraints::min_fps, Optional(2)), + Field(&webrtc::VideoTrackSourceConstraints::max_fps, Optional(3))))); + broadcaster.ProcessConstraints(webrtc::VideoTrackSourceConstraints{2, 3}); +} diff --git a/pc/video_track_source_proxy.h b/pc/video_track_source_proxy.h index 8914dd0525..6e71bb1615 100644 --- a/pc/video_track_source_proxy.h +++ b/pc/video_track_source_proxy.h @@ -42,6 +42,9 @@ PROXY_SECONDARY_METHOD1(void, PROXY_SECONDARY_METHOD1(void, RemoveEncodedSink, rtc::VideoSinkInterface*) +PROXY_SECONDARY_METHOD1(void, + ProcessConstraints, + const webrtc::VideoTrackSourceConstraints&) END_PROXY_MAP(VideoTrackSource) } // namespace webrtc diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 9180bbb917..f3872cce5b 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -1372,6 +1372,18 @@ void VideoStreamEncoder::OnDiscardedFrame() { VideoStreamEncoderObserver::DropReason::kSource); } +void VideoStreamEncoder::OnConstraintsChanged( + const webrtc::VideoTrackSourceConstraints& constraints) { + // This method is called on the network thread. + RTC_LOG(LS_INFO) << __func__ << " min_fps " + << constraints.min_fps.value_or(-1) << " max_fps " + << constraints.max_fps.value_or(-1); + main_queue_->PostTask(ToQueuedTask(task_safety_, [this, constraints] { + RTC_DCHECK_RUN_ON(main_queue_); + source_constraints_ = constraints; + })); +} + bool VideoStreamEncoder::EncoderPaused() const { RTC_DCHECK_RUN_ON(&encoder_queue_); // Pause video if paused by caller or as long as the network is down or the diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 2b181551b2..93c79030e3 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -176,6 +176,8 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, // Implements VideoSinkInterface. void OnFrame(const VideoFrame& video_frame) override; void OnDiscardedFrame() override; + void OnConstraintsChanged( + const webrtc::VideoTrackSourceConstraints& constraints) override; void MaybeEncodeVideoFrame(const VideoFrame& frame, int64_t time_when_posted_in_ms); @@ -239,6 +241,10 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, encoder_selector_; VideoStreamEncoderObserver* const encoder_stats_observer_; + // The source's constraints. + absl::optional source_constraints_ + RTC_GUARDED_BY(main_queue_); + VideoEncoderConfig encoder_config_ RTC_GUARDED_BY(&encoder_queue_); std::unique_ptr encoder_ RTC_GUARDED_BY(&encoder_queue_) RTC_PT_GUARDED_BY(&encoder_queue_);