From fe3654d5dceb3d931cd773afae511e90180d39ba Mon Sep 17 00:00:00 2001 From: noahric Date: Fri, 1 Jul 2016 09:05:54 -0700 Subject: [PATCH] Add native handle support to SimulcastEncoderAdapter. If all subencoders support textures, the adapter will claim support. Texture frames will be passed on directly to subencoders, without any attempt at scaling, and subencoders will be expected to sample/scale correctly from source textures. BUG= NOTRY=true Review-Url: https://codereview.webrtc.org/2099483002 Cr-Commit-Position: refs/heads/master@{#13365} --- .../codecs/vp8/simulcast_encoder_adapter.cc | 21 ++++--- .../vp8/simulcast_encoder_adapter_unittest.cc | 63 ++++++++++++++----- 2 files changed, 62 insertions(+), 22 deletions(-) diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc index 4f389b88ae..7b1e9d9d96 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc @@ -289,10 +289,16 @@ int SimulcastEncoderAdapter::Encode( // If scaling isn't required, because the input resolution // matches the destination or the input image is empty (e.g. // a keyframe request for encoders with internal camera - // sources), pass the image on directly. Otherwise, we'll - // scale it to match what the encoder expects (below). + // sources) or the source image has a native handle, pass the image on + // directly. Otherwise, we'll scale it to match what the encoder expects + // (below). + // For texture frames, the underlying encoder is expected to be able to + // correctly sample/scale the source texture. + // TODO(perkj): ensure that works going forward, and figure out how this + // affects webrtc:5683. if ((dst_width == src_width && dst_height == src_height) || - input_image.IsZeroSize()) { + input_image.IsZeroSize() || + input_image.video_frame_buffer()->native_handle()) { int ret = streaminfos_[stream_idx].encoder->Encode( input_image, codec_specific_info, &stream_frame_types); if (ret != WEBRTC_VIDEO_CODEC_OK) { @@ -510,10 +516,11 @@ void SimulcastEncoderAdapter::OnDroppedFrame() { bool SimulcastEncoderAdapter::SupportsNativeHandle() const { // We should not be calling this method before streaminfos_ are configured. RTC_DCHECK(!streaminfos_.empty()); - // TODO(pbos): Support textures when using more than one encoder. - if (streaminfos_.size() != 1) - return false; - return streaminfos_[0].encoder->SupportsNativeHandle(); + for (const auto& streaminfo : streaminfos_) { + if (!streaminfo.encoder->SupportsNativeHandle()) + return false; + } + return true; } const char* SimulcastEncoderAdapter::ImplementationName() const { diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter_unittest.cc index 89c61cec3a..efddb7213d 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter_unittest.cc +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter_unittest.cc @@ -118,11 +118,11 @@ class MockVideoEncoder : public VideoEncoder { return 0; } - int32_t Encode(const VideoFrame& inputImage, - const CodecSpecificInfo* codecSpecificInfo, - const std::vector* frame_types) /* override */ { - return encode_return_value_; - } + MOCK_METHOD3( + Encode, + int32_t(const VideoFrame& inputImage, + const CodecSpecificInfo* codecSpecificInfo, + const std::vector* frame_types) /* override */); int32_t RegisterEncodeCompleteCallback( EncodedImageCallback* callback) /* override */ { @@ -160,15 +160,10 @@ class MockVideoEncoder : public VideoEncoder { supports_native_handle_ = enabled; } - void set_encode_return_value(int value) { - encode_return_value_ = value; - } - MOCK_CONST_METHOD0(ImplementationName, const char*()); private: bool supports_native_handle_ = false; - int encode_return_value_ = WEBRTC_VIDEO_CODEC_OK; VideoCodec codec_; EncodedImageCallback* callback_; }; @@ -452,9 +447,7 @@ TEST_F(TestSimulcastEncoderAdapterFake, SupportsImplementationName) { } TEST_F(TestSimulcastEncoderAdapterFake, - SupportsNativeHandleDisabledForMultipleStreams) { - // TODO(pbos): Implement actual test (verify that it works) when implemented - // for multiple streams. + SupportsNativeHandleForMultipleStreams) { TestVp8Simulcast::DefaultSettings( &codec_, static_cast(kTestTemporalLayerProfile)); codec_.numberOfSimulcastStreams = 3; @@ -463,7 +456,47 @@ TEST_F(TestSimulcastEncoderAdapterFake, ASSERT_EQ(3u, helper_->factory()->encoders().size()); for (MockVideoEncoder* encoder : helper_->factory()->encoders()) encoder->set_supports_native_handle(true); + // If one encoder doesn't support it, then overall support is disabled. + helper_->factory()->encoders()[0]->set_supports_native_handle(false); EXPECT_FALSE(adapter_->SupportsNativeHandle()); + // Once all do, then the adapter claims support. + helper_->factory()->encoders()[0]->set_supports_native_handle(true); + EXPECT_TRUE(adapter_->SupportsNativeHandle()); +} + +class FakeNativeHandleBuffer : public NativeHandleBuffer { + public: + FakeNativeHandleBuffer(void* native_handle, int width, int height) + : NativeHandleBuffer(native_handle, width, height) {} + rtc::scoped_refptr NativeToI420Buffer() override { + RTC_NOTREACHED(); + return nullptr; + } +}; + +TEST_F(TestSimulcastEncoderAdapterFake, + NativeHandleForwardingForMultipleStreams) { + TestVp8Simulcast::DefaultSettings( + &codec_, static_cast(kTestTemporalLayerProfile)); + codec_.numberOfSimulcastStreams = 3; + // High start bitrate, so all streams are enabled. + codec_.startBitrate = 3000; + EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200)); + adapter_->RegisterEncodeCompleteCallback(this); + ASSERT_EQ(3u, helper_->factory()->encoders().size()); + for (MockVideoEncoder* encoder : helper_->factory()->encoders()) + encoder->set_supports_native_handle(true); + EXPECT_TRUE(adapter_->SupportsNativeHandle()); + + rtc::scoped_refptr buffer( + new rtc::RefCountedObject(this, 1280, 720)); + VideoFrame input_frame(buffer, 100, 1000, kVideoRotation_180); + // Expect calls with the given video frame verbatim, since it's a texture + // frame and can't otherwise be modified/resized. + for (MockVideoEncoder* encoder : helper_->factory()->encoders()) + EXPECT_CALL(*encoder, Encode(::testing::Ref(input_frame), _, _)).Times(1); + std::vector frame_types(3, kVideoFrameKey); + EXPECT_EQ(0, adapter_->Encode(input_frame, NULL, &frame_types)); } TEST_F(TestSimulcastEncoderAdapterFake, TestFailureReturnCodesFromEncodeCalls) { @@ -474,8 +507,8 @@ TEST_F(TestSimulcastEncoderAdapterFake, TestFailureReturnCodesFromEncodeCalls) { adapter_->RegisterEncodeCompleteCallback(this); ASSERT_EQ(3u, helper_->factory()->encoders().size()); // Tell the 2nd encoder to request software fallback. - helper_->factory()->encoders()[1]->set_encode_return_value( - WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE); + EXPECT_CALL(*helper_->factory()->encoders()[1], Encode(_, _, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE)); // Send a fake frame and assert the return is software fallback. VideoFrame input_frame;