diff --git a/api/BUILD.gn b/api/BUILD.gn index c661dc56ad..798fa13a17 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -1466,3 +1466,19 @@ rtc_library("field_trials") { ] absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } + +rtc_library("frame_transformer_factory") { + visibility = [ "*" ] + sources = [ + "frame_transformer_factory.cc", + "frame_transformer_factory.h", + ] + deps = [ + ":frame_transformer_interface", + ":scoped_refptr", + "../modules/rtp_rtcp", + "../rtc_base:refcount", + "video:encoded_frame", + "video:video_frame_metadata", + ] +} diff --git a/api/frame_transformer_factory.cc b/api/frame_transformer_factory.cc new file mode 100644 index 0000000000..af08372e37 --- /dev/null +++ b/api/frame_transformer_factory.cc @@ -0,0 +1,33 @@ +/* + * Copyright 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/frame_transformer_factory.h" + +#include "modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h" + +namespace webrtc { + +std::unique_ptr CreateVideoSenderFrame() { + RTC_CHECK_NOTREACHED(); + return nullptr; +} + +std::unique_ptr CreateVideoReceiverFrame() { + RTC_CHECK_NOTREACHED(); + return nullptr; +} + +std::unique_ptr CloneVideoFrame( + TransformableVideoFrameInterface* original) { + // At the moment, only making sender frames from receiver frames is supported. + return CloneSenderVideoFrame(original); +} + +} // namespace webrtc diff --git a/api/frame_transformer_factory.h b/api/frame_transformer_factory.h new file mode 100644 index 0000000000..8ba9c292d5 --- /dev/null +++ b/api/frame_transformer_factory.h @@ -0,0 +1,39 @@ +/* + * Copyright 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_FRAME_TRANSFORMER_FACTORY_H_ +#define API_FRAME_TRANSFORMER_FACTORY_H_ + +#include +#include + +#include "api/frame_transformer_interface.h" +#include "api/scoped_refptr.h" +#include "api/video/encoded_frame.h" +#include "api/video/video_frame_metadata.h" + +// This file contains EXPERIMENTAL functions to create video frames from +// either an old video frame or directly from parameters. +// These functions will be used in Chrome functionality to manipulate +// encoded frames from Javascript. +namespace webrtc { + +// TODO(bugs.webrtc.org/14708): Add the required parameters to these APIs. +std::unique_ptr CreateVideoSenderFrame(); +// TODO(bugs.webrtc.org/14708): Consider whether Receiver frames ever make sense +// to create. +std::unique_ptr CreateVideoReceiverFrame(); +// Creates a new frame with the same metadata as the original. +// The original can be a sender or receiver frame. +RTC_EXPORT std::unique_ptr CloneVideoFrame( + TransformableVideoFrameInterface* original); +} // namespace webrtc + +#endif // API_FRAME_TRANSFORMER_FACTORY_H_ diff --git a/modules/rtp_rtcp/BUILD.gn b/modules/rtp_rtcp/BUILD.gn index ead114bc91..e046882357 100644 --- a/modules/rtp_rtcp/BUILD.gn +++ b/modules/rtp_rtcp/BUILD.gn @@ -596,6 +596,7 @@ if (rtc_include_tests) { ] deps = [ ":fec_test_helper", + ":frame_transformer_factory_unittest", ":mock_rtp_rtcp", ":rtcp_transceiver", ":rtp_packetizer_av1_test_helper", @@ -605,6 +606,7 @@ if (rtc_include_tests) { "../../api:array_view", "../../api:create_time_controller", "../../api:field_trials_registry", + "../../api:frame_transformer_factory", "../../api:libjingle_peerconnection_api", "../../api:mock_frame_encryptor", "../../api:rtp_headers", @@ -669,3 +671,19 @@ if (rtc_include_tests) { ] } } + +rtc_source_set("frame_transformer_factory_unittest") { + testonly = true + sources = [ "source/frame_transformer_factory_unittest.cc" ] + deps = [ + "../../api:frame_transformer_factory", + "../../api:transport_api", + "../../call:video_stream_api", + "../../modules/rtp_rtcp", + "../../rtc_base:rtc_event", + "../../test:mock_frame_transformer", + "../../test:test_support", + "../../video", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/memory" ] +} diff --git a/modules/rtp_rtcp/source/frame_transformer_factory_unittest.cc b/modules/rtp_rtcp/source/frame_transformer_factory_unittest.cc new file mode 100644 index 0000000000..e011a76ed5 --- /dev/null +++ b/modules/rtp_rtcp/source/frame_transformer_factory_unittest.cc @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/frame_transformer_factory.h" + +#include +#include +#include +#include + +#include "absl/memory/memory.h" +#include "api/call/transport.h" +#include "call/video_receive_stream.h" +#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" +#include "rtc_base/event.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/mock_frame_transformer.h" + +namespace webrtc { +namespace { + +using testing::NiceMock; +using testing::Return; + +class MockTransformableVideoFrame + : public webrtc::TransformableVideoFrameInterface { + public: + MOCK_METHOD(rtc::ArrayView, GetData, (), (const override)); + MOCK_METHOD(void, SetData, (rtc::ArrayView data), (override)); + MOCK_METHOD(uint8_t, GetPayloadType, (), (const, override)); + MOCK_METHOD(uint32_t, GetSsrc, (), (const, override)); + MOCK_METHOD(uint32_t, GetTimestamp, (), (const, override)); + MOCK_METHOD(TransformableFrameInterface::Direction, + GetDirection, + (), + (const, override)); + MOCK_METHOD(bool, IsKeyFrame, (), (const, override)); + MOCK_METHOD(std::vector, GetAdditionalData, (), (const, override)); + MOCK_METHOD(const webrtc::VideoFrameMetadata&, + GetMetadata, + (), + (const, override)); +}; + +TEST(FrameTransformerFactory, CloneVideoFrame) { + NiceMock original_frame; + uint8_t data[10]; + std::fill_n(data, 10, 5); + rtc::ArrayView data_view(data); + EXPECT_CALL(original_frame, GetData()).WillRepeatedly(Return(data_view)); + auto cloned_frame = CloneVideoFrame(&original_frame); + EXPECT_EQ(cloned_frame->GetData().size(), 10u); + EXPECT_THAT(cloned_frame->GetData(), testing::Each(5u)); +} + +} // namespace +} // namespace webrtc diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc index 3e42781c3e..a60c39395a 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc @@ -18,6 +18,7 @@ #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" #include "modules/rtp_rtcp/source/rtp_sender_video.h" #include "rtc_base/checks.h" +#include "rtc_base/logging.h" namespace webrtc { namespace { @@ -161,8 +162,9 @@ void RTPSenderVideoFrameTransformerDelegate::OnTransformedFrame( EnsureEncoderQueueCreated(); - if (!sender_) + if (!sender_) { return; + } rtc::scoped_refptr delegate(this); encoder_queue_->PostTask( [delegate = std::move(delegate), frame = std::move(frame)]() mutable { @@ -212,4 +214,37 @@ void RTPSenderVideoFrameTransformerDelegate::Reset() { sender_ = nullptr; } } + +std::unique_ptr CloneSenderVideoFrame( + TransformableVideoFrameInterface* original) { + auto encoded_image_buffer = EncodedImageBuffer::Create( + original->GetData().data(), original->GetData().size()); + EncodedImage encoded_image; + encoded_image.SetEncodedData(encoded_image_buffer); + RTPVideoHeader new_header; + absl::optional new_codec_type; + // TODO(bugs.webrtc.org/14708): Figure out a way to get the header information + // without casting to TransformableVideoSenderFrame. + if (original->GetDirection() == + TransformableFrameInterface::Direction::kSender) { + // TODO(bugs.webrtc.org/14708): Figure out a way to bulletproof this cast. + auto original_as_sender = + static_cast(original); + new_header = original_as_sender->GetHeader(); + new_codec_type = original_as_sender->GetCodecType(); + } else { + // TODO(bugs.webrtc.org/14708): Make this codec dependent + new_header.video_type_header.emplace(); + new_codec_type = kVideoCodecVP8; + // TODO(bugs.webrtc.org/14708): Fill in the new_header when it's not + // `Direction::kSender` + } + // TODO(bugs.webrtc.org/14708): Fill in other EncodedImage parameters + return std::make_unique( + encoded_image, new_header, original->GetPayloadType(), new_codec_type, + original->GetTimestamp(), + absl::nullopt, // expected_retransmission_time_ms + original->GetSsrc()); +} + } // namespace webrtc diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h index 0c2a64372c..04cdfd3c6a 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h +++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h @@ -87,6 +87,10 @@ class RTPSenderVideoFrameTransformerDelegate : public TransformedFrameCallback { std::unique_ptr owned_encoder_queue_; }; +// Method to support cloning a Sender frame from another frame +std::unique_ptr CloneSenderVideoFrame( + TransformableVideoFrameInterface* original); + } // namespace webrtc #endif // MODULES_RTP_RTCP_SOURCE_RTP_SENDER_VIDEO_FRAME_TRANSFORMER_DELEGATE_H_ diff --git a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc index 13527128c9..3eacda86a2 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc @@ -17,6 +17,7 @@ #include "absl/memory/memory.h" #include "api/field_trials_registry.h" +#include "api/frame_transformer_factory.h" #include "api/rtp_headers.h" #include "api/task_queue/task_queue_base.h" #include "api/task_queue/task_queue_factory.h" @@ -38,6 +39,7 @@ #include "modules/rtp_rtcp/source/rtp_video_layers_allocation_extension.h" #include "rtc_base/arraysize.h" #include "rtc_base/checks.h" +#include "rtc_base/logging.h" #include "rtc_base/rate_limiter.h" #include "rtc_base/thread.h" #include "test/gmock.h" @@ -1645,5 +1647,45 @@ TEST_F(RtpSenderVideoWithFrameTransformerTest, kDefaultExpectedRetransmissionTimeMs); } +TEST_F(RtpSenderVideoWithFrameTransformerTest, + OnTransformedFrameSendsVideoWhenCloned) { + auto mock_frame_transformer = + rtc::make_ref_counted>(); + rtc::scoped_refptr callback; + EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameSinkCallback) + .WillOnce(SaveArg<0>(&callback)); + std::unique_ptr rtp_sender_video = + CreateSenderWithFrameTransformer(mock_frame_transformer); + ASSERT_TRUE(callback); + + auto encoded_image = CreateDefaultEncodedImage(); + RTPVideoHeader video_header; + video_header.frame_type = VideoFrameType::kVideoFrameKey; + ON_CALL(*mock_frame_transformer, Transform) + .WillByDefault( + [&callback](std::unique_ptr frame) { + auto clone = CloneVideoFrame( + static_cast(frame.get())); + EXPECT_TRUE(clone); + callback->OnTransformedFrame(std::move(clone)); + }); + auto encoder_queue = time_controller_.GetTaskQueueFactory()->CreateTaskQueue( + "encoder_queue", TaskQueueFactory::Priority::NORMAL); + encoder_queue->PostTask([&] { + rtp_sender_video->SendEncodedImage(kPayload, kType, kTimestamp, + *encoded_image, video_header, + kDefaultExpectedRetransmissionTimeMs); + }); + time_controller_.AdvanceTime(TimeDelta::Zero()); + EXPECT_EQ(transport_.packets_sent(), 1); + encoder_queue->PostTask([&] { + rtp_sender_video->SendEncodedImage(kPayload, kType, kTimestamp, + *encoded_image, video_header, + kDefaultExpectedRetransmissionTimeMs); + }); + time_controller_.AdvanceTime(TimeDelta::Zero()); + EXPECT_EQ(transport_.packets_sent(), 2); +} + } // namespace } // namespace webrtc