From 7cbee84610a8d4f2bbc86c55d9ee02d25be19f72 Mon Sep 17 00:00:00 2001 From: Sebastian Jansson Date: Tue, 6 Aug 2019 17:19:38 +0200 Subject: [PATCH] Reland "Adds PeerConnection scenario test framework." MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a reland of ad5c4accad00e04de08e2b62d366cc1f8e0320a5 It was flaky due to starting ICE signaling before SDP negotiation finished. This was solved by adding an helper for adding ice candidates which will wait until the peer connection is ready if needed. Original change's description: > Adds PeerConnection scenario test framework. > > Bug: webrtc:10839 > Change-Id: If67eeb680d016d66c69d8e761a88c240e4931a5d > Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/147276 > Commit-Queue: Sebastian Jansson > Reviewed-by: Steve Anton > Reviewed-by: Erik Språng > Cr-Commit-Position: refs/heads/master@{#28754} Bug: webrtc:10839 Change-Id: I6eb8f482561c87e7b0f20d2431d21a41b26c91d5 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/147877 Reviewed-by: Steve Anton Commit-Queue: Sebastian Jansson Cr-Commit-Position: refs/heads/master@{#28777} --- test/BUILD.gn | 1 + test/network/BUILD.gn | 5 +- test/peer_scenario/BUILD.gn | 44 +++ test/peer_scenario/DEPS | 5 + test/peer_scenario/peer_scenario.cc | 75 +++++ test/peer_scenario/peer_scenario.h | 104 ++++++ test/peer_scenario/peer_scenario_client.cc | 299 ++++++++++++++++++ test/peer_scenario/peer_scenario_client.h | 159 ++++++++++ test/peer_scenario/sdp_callbacks.cc | 54 ++++ test/peer_scenario/sdp_callbacks.h | 43 +++ test/peer_scenario/signaling_route.cc | 107 +++++++ test/peer_scenario/signaling_route.h | 55 ++++ test/peer_scenario/tests/BUILD.gn | 24 ++ .../tests/peer_scenario_quality_test.cc | 39 +++ .../tests/remote_estimate_test.cc | 49 +++ test/scenario/stats_collection.cc | 22 +- test/scenario/stats_collection.h | 6 +- 17 files changed, 1085 insertions(+), 6 deletions(-) create mode 100644 test/peer_scenario/BUILD.gn create mode 100644 test/peer_scenario/DEPS create mode 100644 test/peer_scenario/peer_scenario.cc create mode 100644 test/peer_scenario/peer_scenario.h create mode 100644 test/peer_scenario/peer_scenario_client.cc create mode 100644 test/peer_scenario/peer_scenario_client.h create mode 100644 test/peer_scenario/sdp_callbacks.cc create mode 100644 test/peer_scenario/sdp_callbacks.h create mode 100644 test/peer_scenario/signaling_route.cc create mode 100644 test/peer_scenario/signaling_route.h create mode 100644 test/peer_scenario/tests/BUILD.gn create mode 100644 test/peer_scenario/tests/peer_scenario_quality_test.cc create mode 100644 test/peer_scenario/tests/remote_estimate_test.cc diff --git a/test/BUILD.gn b/test/BUILD.gn index 5af4766810..a16c465c1f 100644 --- a/test/BUILD.gn +++ b/test/BUILD.gn @@ -390,6 +390,7 @@ if (rtc_include_tests) { "../rtc_base/system:file_wrapper", "../test:single_threaded_task_queue", "pc/e2e:e2e_unittests", + "peer_scenario/tests", "scenario:scenario_unittests", "time_controller", "time_controller:time_controller_unittests", diff --git a/test/network/BUILD.gn b/test/network/BUILD.gn index be372f1654..2470c008ab 100644 --- a/test/network/BUILD.gn +++ b/test/network/BUILD.gn @@ -14,7 +14,10 @@ rtc_source_set("emulated_network") { ":*", ] if (rtc_include_tests) { - visibility += [ "../scenario" ] + visibility += [ + "../scenario:*", + "../peer_scenario:*", + ] } testonly = true sources = [ diff --git a/test/peer_scenario/BUILD.gn b/test/peer_scenario/BUILD.gn new file mode 100644 index 0000000000..85a0c71ed9 --- /dev/null +++ b/test/peer_scenario/BUILD.gn @@ -0,0 +1,44 @@ +# Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../webrtc.gni") + +if (rtc_include_tests) { + rtc_source_set("peer_scenario") { + testonly = true + sources = [ + "peer_scenario.cc", + "peer_scenario.h", + "peer_scenario_client.cc", + "peer_scenario_client.h", + "sdp_callbacks.cc", + "sdp_callbacks.h", + "signaling_route.cc", + "signaling_route.h", + ] + deps = [ + "../:video_test_common", + "../../api:libjingle_peerconnection_api", + "../../api:network_emulation_manager_api", + "../../api:rtc_stats_api", + "../../api/audio_codecs:builtin_audio_decoder_factory", + "../../api/audio_codecs:builtin_audio_encoder_factory", + "../../api/rtc_event_log:rtc_event_log_factory", + "../../api/task_queue:default_task_queue_factory", + "../../api/video_codecs:builtin_video_decoder_factory", + "../../api/video_codecs:builtin_video_encoder_factory", + "../../media:rtc_audio_video", + "../../modules/audio_device:audio_device_impl", + "../../p2p:rtc_p2p", + "../../pc:pc_test_utils", + "..//network:emulated_network", + "../scenario", + "//third_party/abseil-cpp/absl/memory:memory", + ] + } +} diff --git a/test/peer_scenario/DEPS b/test/peer_scenario/DEPS new file mode 100644 index 0000000000..68e9f46087 --- /dev/null +++ b/test/peer_scenario/DEPS @@ -0,0 +1,5 @@ +include_rules = [ + "+pc", + "+p2p", +] + diff --git a/test/peer_scenario/peer_scenario.cc b/test/peer_scenario/peer_scenario.cc new file mode 100644 index 0000000000..fae3c78677 --- /dev/null +++ b/test/peer_scenario/peer_scenario.cc @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/peer_scenario/peer_scenario.h" + +#include "absl/memory/memory.h" + +namespace webrtc { +namespace test { + +PeerScenario::PeerScenario() : signaling_thread_(rtc::Thread::Current()) {} + +PeerScenarioClient* PeerScenario::CreateClient( + PeerScenarioClient::Config config) { + peer_clients_.emplace_back(net(), thread(), config); + return &peer_clients_.back(); +} + +SignalingRoute PeerScenario::ConnectSignaling( + PeerScenarioClient* caller, + PeerScenarioClient* callee, + std::vector send_link, + std::vector ret_link) { + return SignalingRoute(caller, callee, net_.CreateTrafficRoute(send_link), + net_.CreateTrafficRoute(ret_link)); +} + +void PeerScenario::SimpleConnection( + PeerScenarioClient* caller, + PeerScenarioClient* callee, + std::vector send_link, + std::vector ret_link) { + net()->CreateRoute(caller->endpoint(), send_link, callee->endpoint()); + net()->CreateRoute(callee->endpoint(), ret_link, caller->endpoint()); + auto signaling = ConnectSignaling(caller, callee, send_link, ret_link); + signaling.StartIceSignaling(); + rtc::Event done; + signaling.NegotiateSdp( + [&](const SessionDescriptionInterface&) { done.Set(); }); + RTC_CHECK(WaitAndProcess(&done)); +} + +void PeerScenario::AttachVideoQualityAnalyzer(VideoQualityAnalyzer* analyzer, + VideoTrackInterface* send_track, + PeerScenarioClient* receiver) { + video_quality_pairs_.emplace_back(clock(), analyzer); + auto pair = &video_quality_pairs_.back(); + send_track->AddOrUpdateSink(&pair->capture_tap_, rtc::VideoSinkWants()); + receiver->AddVideoReceiveSink(send_track->id(), &pair->decode_tap_); +} + +bool PeerScenario::WaitAndProcess(rtc::Event* event, TimeDelta max_duration) { + constexpr int kStepMs = 5; + if (event->Wait(0)) + return true; + for (int elapsed = 0; elapsed < max_duration.ms(); elapsed += kStepMs) { + thread()->ProcessMessages(kStepMs); + if (event->Wait(0)) + return true; + } + return false; +} + +void PeerScenario::ProcessMessages(TimeDelta duration) { + thread()->ProcessMessages(duration.ms()); +} + +} // namespace test +} // namespace webrtc diff --git a/test/peer_scenario/peer_scenario.h b/test/peer_scenario/peer_scenario.h new file mode 100644 index 0000000000..f945fb46fa --- /dev/null +++ b/test/peer_scenario/peer_scenario.h @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef TEST_PEER_SCENARIO_PEER_SCENARIO_H_ +#define TEST_PEER_SCENARIO_PEER_SCENARIO_H_ + +// The peer connection scenario test framework enables writing end to end unit +// tests on the peer connection level. It's similar to the Scenario test but +// uses the full stack, including SDP and ICE negotiation. This ensures that +// features work end to end. It's also diffferent from the other tests on peer +// connection level in that it does not rely on any mocks or fakes other than +// for media input and networking. Additionally it provides direct access to the +// underlying peer connection class. + +#include +#include + +#include "test/network/network_emulation_manager.h" +#include "test/peer_scenario/peer_scenario_client.h" +#include "test/peer_scenario/signaling_route.h" +#include "test/scenario/stats_collection.h" +#include "test/scenario/video_frame_matcher.h" + +namespace webrtc { +namespace test { + +// The PeerScenario class represents a PeerConnection simulation scenario. The +// main purpose is to maintain ownership and ensure safe destruction order of +// clients and network emulation. Additionally it reduces the amount of bolier +// plate requited for some actions. For example usage see the existing tests +// using this class. Note that it should be used from a single calling thread. +// This thread will also be assigned as the signaling thread for all peer +// connections that are created. This means that the process methods must be +// used when waiting to ensure that messages are processed on the signaling +// thread. +class PeerScenario { + public: + PeerScenario(); + NetworkEmulationManagerImpl* net() { return &net_; } + rtc::Thread* thread() { return signaling_thread_; } + + // Creates a client wrapping a peer connection conforming to the given config. + // The client will share the signaling thread with the scenario. To maintain + // control of destruction order, ownership is kept within the scenario. + PeerScenarioClient* CreateClient(PeerScenarioClient::Config config); + + // Sets up a signaling route that can be used for SDP and ICE. + SignalingRoute ConnectSignaling(PeerScenarioClient* caller, + PeerScenarioClient* callee, + std::vector send_link, + std::vector ret_link); + + // Connects two clients over given links. This will also start ICE signaling + // and SDP negotiation with default behavior. For customized behavior, + // ConnectSignaling should be used to allow more detailed control, for + // instance to allow different signaling and media routes. + void SimpleConnection(PeerScenarioClient* caller, + PeerScenarioClient* callee, + std::vector send_link, + std::vector ret_link); + + // Starts feeding the results of comparing captured frames from |send_track| + // with decoded frames on |receiver| to |analyzer|. + // TODO(srte): Provide a way to detach to allow removal of tracks. + void AttachVideoQualityAnalyzer(VideoQualityAnalyzer* analyzer, + VideoTrackInterface* send_track, + PeerScenarioClient* receiver); + + // Waits on |event| while processing messages on the signaling thread. + bool WaitAndProcess(rtc::Event* event, + TimeDelta max_duration = TimeDelta::seconds(5)); + + // Process messages on the signaling thread for the given duration. + void ProcessMessages(TimeDelta duration); + + private: + // Helper struct to maintain ownership of the matcher and taps. + struct PeerVideoQualityPair { + public: + PeerVideoQualityPair(Clock* capture_clock, VideoQualityAnalyzer* analyzer) + : matcher_({analyzer->Handler()}), + capture_tap_(capture_clock, &matcher_), + decode_tap_(capture_clock, &matcher_, 0) {} + VideoFrameMatcher matcher_; + CapturedFrameTap capture_tap_; + DecodedFrameTap decode_tap_; + }; + Clock* clock() { return Clock::GetRealTimeClock(); } + + rtc::Thread* const signaling_thread_; + std::list video_quality_pairs_; + NetworkEmulationManagerImpl net_; + std::list peer_clients_; +}; + +} // namespace test +} // namespace webrtc +#endif // TEST_PEER_SCENARIO_PEER_SCENARIO_H_ diff --git a/test/peer_scenario/peer_scenario_client.cc b/test/peer_scenario/peer_scenario_client.cc new file mode 100644 index 0000000000..45091971cb --- /dev/null +++ b/test/peer_scenario/peer_scenario_client.cc @@ -0,0 +1,299 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/peer_scenario/peer_scenario_client.h" + +#include +#include + +#include "absl/memory/memory.h" +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "api/rtc_event_log/rtc_event_log_factory.h" +#include "api/task_queue/default_task_queue_factory.h" +#include "api/video_codecs/builtin_video_decoder_factory.h" +#include "api/video_codecs/builtin_video_encoder_factory.h" +#include "media/engine/webrtc_media_engine.h" +#include "modules/audio_device/include/test_audio_device.h" +#include "p2p/client/basic_port_allocator.h" +#include "test/frame_generator_capturer.h" +#include "test/peer_scenario/sdp_callbacks.h" + +namespace webrtc { +namespace test { + +namespace { + +constexpr char kCommonStreamId[] = "stream_id"; + +std::map CreateEndpoints( + NetworkEmulationManager* net, + std::map endpoint_configs) { + std::map endpoints; + for (const auto& kv : endpoint_configs) + endpoints[kv.first] = net->CreateEndpoint(kv.second); + return endpoints; +} + +class LambdaPeerConnectionObserver final : public PeerConnectionObserver { + public: + explicit LambdaPeerConnectionObserver( + PeerScenarioClient::CallbackHandlers* handlers) + : handlers_(handlers) {} + void OnSignalingChange( + PeerConnectionInterface::SignalingState new_state) override { + for (const auto& handler : handlers_->on_signaling_change) + handler(new_state); + } + void OnDataChannel( + rtc::scoped_refptr data_channel) override { + for (const auto& handler : handlers_->on_data_channel) + handler(data_channel); + } + void OnRenegotiationNeeded() override { + for (const auto& handler : handlers_->on_renegotiation_needed) + handler(); + } + void OnStandardizedIceConnectionChange( + PeerConnectionInterface::IceConnectionState new_state) override { + for (const auto& handler : handlers_->on_standardized_ice_connection_change) + handler(new_state); + } + void OnConnectionChange( + PeerConnectionInterface::PeerConnectionState new_state) override { + for (const auto& handler : handlers_->on_connection_change) + handler(new_state); + } + void OnIceGatheringChange( + PeerConnectionInterface::IceGatheringState new_state) override { + for (const auto& handler : handlers_->on_ice_gathering_change) + handler(new_state); + } + void OnIceCandidate(const IceCandidateInterface* candidate) override { + for (const auto& handler : handlers_->on_ice_candidate) + handler(candidate); + } + void OnIceCandidateError(const std::string& host_candidate, + const std::string& url, + int error_code, + const std::string& error_text) override { + for (const auto& handler : handlers_->on_ice_candidate_error) + handler(host_candidate, url, error_code, error_text); + } + void OnIceCandidatesRemoved( + const std::vector& candidates) override { + for (const auto& handler : handlers_->on_ice_candidates_removed) + handler(candidates); + } + void OnAddTrack(rtc::scoped_refptr receiver, + const std::vector >& + streams) override { + for (const auto& handler : handlers_->on_add_track) + handler(receiver, streams); + } + void OnTrack( + rtc::scoped_refptr transceiver) override { + for (const auto& handler : handlers_->on_track) + handler(transceiver); + } + void OnRemoveTrack( + rtc::scoped_refptr receiver) override { + for (const auto& handler : handlers_->on_remove_track) + handler(receiver); + } + + private: + PeerScenarioClient::CallbackHandlers* handlers_; +}; +} // namespace + +PeerScenarioClient::PeerScenarioClient(NetworkEmulationManager* net, + rtc::Thread* signaling_thread, + PeerScenarioClient::Config config) + : endpoints_(CreateEndpoints(net, config.endpoints)), + signaling_thread_(signaling_thread), + worker_thread_(rtc::Thread::Create()), + handlers_(config.handlers), + observer_(new LambdaPeerConnectionObserver(&handlers_)) { + worker_thread_->SetName("worker", this); + worker_thread_->Start(); + + handlers_.on_track.push_back( + [this](rtc::scoped_refptr transceiver) { + auto track = transceiver->receiver()->track().get(); + if (track->kind() == MediaStreamTrackInterface::kVideoKind) { + auto* video = static_cast(track); + RTC_DCHECK_RUN_ON(signaling_thread_); + for (auto* sink : track_id_to_video_sinks_[track->id()]) { + video->AddOrUpdateSink(sink, rtc::VideoSinkWants()); + } + } + }); + handlers_.on_signaling_change.push_back( + [this](PeerConnectionInterface::SignalingState state) { + if (state == PeerConnectionInterface::SignalingState::kStable && + peer_connection_->current_remote_description()) { + RTC_DCHECK_RUN_ON(signaling_thread_); + for (const auto& candidate : pending_ice_candidates_) { + RTC_CHECK(peer_connection_->AddIceCandidate(candidate.get())); + } + pending_ice_candidates_.clear(); + } + }); + + std::vector endpoints_vector; + for (const auto& kv : endpoints_) + endpoints_vector.push_back(kv.second); + auto* manager = net->CreateEmulatedNetworkManagerInterface(endpoints_vector); + + PeerConnectionFactoryDependencies pcf_deps; + pcf_deps.network_thread = manager->network_thread(); + pcf_deps.signaling_thread = signaling_thread_; + pcf_deps.worker_thread = worker_thread_.get(); + pcf_deps.call_factory = CreateCallFactory(); + pcf_deps.task_queue_factory = CreateDefaultTaskQueueFactory(); + task_queue_factory_ = pcf_deps.task_queue_factory.get(); + pcf_deps.event_log_factory = + absl::make_unique(task_queue_factory_); + + cricket::MediaEngineDependencies media_deps; + media_deps.task_queue_factory = task_queue_factory_; + media_deps.adm = TestAudioDeviceModule::Create( + task_queue_factory_, + TestAudioDeviceModule::CreatePulsedNoiseCapturer( + config.audio.pulsed_noise->amplitude * + std::numeric_limits::max(), + config.audio.sample_rate, config.audio.channels), + TestAudioDeviceModule::CreateDiscardRenderer(config.audio.sample_rate)); + + media_deps.audio_processing = AudioProcessingBuilder().Create(); + media_deps.video_encoder_factory = CreateBuiltinVideoEncoderFactory(); + media_deps.video_decoder_factory = CreateBuiltinVideoDecoderFactory(); + media_deps.audio_encoder_factory = CreateBuiltinAudioEncoderFactory(); + media_deps.audio_decoder_factory = CreateBuiltinAudioDecoderFactory(); + + pcf_deps.media_engine = cricket::CreateMediaEngine(std::move(media_deps)); + pcf_deps.fec_controller_factory = nullptr; + pcf_deps.network_controller_factory = nullptr; + pcf_deps.network_state_predictor_factory = nullptr; + pcf_deps.media_transport_factory = nullptr; + + pc_factory_ = CreateModularPeerConnectionFactory(std::move(pcf_deps)); + + PeerConnectionDependencies pc_deps(observer_.get()); + pc_deps.allocator = absl::make_unique( + manager->network_manager()); + pc_deps.allocator->set_flags(pc_deps.allocator->flags() | + cricket::PORTALLOCATOR_DISABLE_TCP); + peer_connection_ = + pc_factory_->CreatePeerConnection(config.rtc_config, std::move(pc_deps)); +} + +EmulatedEndpoint* PeerScenarioClient::endpoint(int index) { + RTC_CHECK_GT(endpoints_.size(), index); + return endpoints_.at(index); +} + +PeerScenarioClient::AudioSendTrack PeerScenarioClient::CreateAudio( + std::string track_id, + cricket::AudioOptions options) { + AudioSendTrack res; + auto source = pc_factory_->CreateAudioSource(options); + auto track = pc_factory_->CreateAudioTrack(track_id, source); + res.track = track; + res.sender = peer_connection_->AddTrack(track, {kCommonStreamId}).value(); + return res; +} + +PeerScenarioClient::VideoSendTrack PeerScenarioClient::CreateVideo( + std::string track_id, + VideoSendTrackConfig config) { + VideoSendTrack res; + auto capturer = FrameGeneratorCapturer::Create(clock(), *task_queue_factory_, + config.generator); + res.capturer = capturer.get(); + capturer->Init(); + res.source = + new rtc::RefCountedObject( + std::move(capturer), config.screencast); + auto track = pc_factory_->CreateVideoTrack(track_id, res.source); + res.track = track; + res.sender = peer_connection_->AddTrack(track, {kCommonStreamId}).MoveValue(); + return res; +} + +void PeerScenarioClient::AddVideoReceiveSink( + std::string track_id, + rtc::VideoSinkInterface* video_sink) { + RTC_DCHECK_RUN_ON(signaling_thread_); + track_id_to_video_sinks_[track_id].push_back(video_sink); +} + +void PeerScenarioClient::CreateAndSetSdp( + std::function offer_handler) { + peer_connection_->CreateOffer( + SdpCreateObserver([=](SessionDescriptionInterface* offer) { + std::string sdp_offer; + offer->ToString(&sdp_offer); + printf("%s\n", sdp_offer.c_str()); + peer_connection_->SetLocalDescription( + SdpSetObserver([sdp_offer, offer_handler]() { + offer_handler(std::move(sdp_offer)); + }), + offer); + }), + PeerConnectionInterface::RTCOfferAnswerOptions()); +} + +void PeerScenarioClient::SetSdpOfferAndGetAnswer( + std::string remote_offer, + std::function answer_handler) { + peer_connection_->SetRemoteDescription( + CreateSessionDescription(SdpType::kOffer, remote_offer), + SdpSetObserver([=]() { + peer_connection_->CreateAnswer( + SdpCreateObserver([=](SessionDescriptionInterface* answer) { + std::string sdp_answer; + answer->ToString(&sdp_answer); + printf("%s\n", sdp_answer.c_str()); + peer_connection_->SetLocalDescription( + SdpSetObserver([answer_handler, sdp_answer]() { + answer_handler(sdp_answer); + }), + answer); + }), + PeerConnectionInterface::RTCOfferAnswerOptions()); + })); +} + +void PeerScenarioClient::SetSdpAnswer( + std::string remote_answer, + std::function done_handler) { + peer_connection_->SetRemoteDescription( + CreateSessionDescription(SdpType::kAnswer, remote_answer), + SdpSetObserver([remote_answer, done_handler] { + auto answer = CreateSessionDescription(SdpType::kAnswer, remote_answer); + done_handler(*answer); + })); +} + +void PeerScenarioClient::AddIceCandidate( + std::unique_ptr candidate) { + if (peer_connection_->signaling_state() == + PeerConnectionInterface::SignalingState::kStable && + peer_connection_->current_remote_description()) { + RTC_CHECK(peer_connection_->AddIceCandidate(candidate.get())); + } else { + RTC_DCHECK_RUN_ON(signaling_thread_); + pending_ice_candidates_.push_back(std::move(candidate)); + } +} + +} // namespace test +} // namespace webrtc diff --git a/test/peer_scenario/peer_scenario_client.h b/test/peer_scenario/peer_scenario_client.h new file mode 100644 index 0000000000..d87ec5e279 --- /dev/null +++ b/test/peer_scenario/peer_scenario_client.h @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef TEST_PEER_SCENARIO_PEER_SCENARIO_CLIENT_H_ +#define TEST_PEER_SCENARIO_PEER_SCENARIO_CLIENT_H_ + +#include +#include +#include +#include +#include +#include + +#include "absl/memory/memory.h" +#include "api/peer_connection_interface.h" +#include "api/test/network_emulation_manager.h" +#include "pc/test/frame_generator_capturer_video_track_source.h" + +namespace webrtc { +namespace test { + +// Wrapper for a PeerConnection for use in PeerScenario tests. It's intended to +// be a minimal wrapper for a peer connection that's simple to use in testing. +// In particular the constructor hides a lot of the required setup for a peer +// connection. +class PeerScenarioClient { + public: + struct CallbackHandlers { + std::vector> + on_signaling_change; + std::vector)>> + on_data_channel; + std::vector> on_renegotiation_needed; + std::vector< + std::function> + on_standardized_ice_connection_change; + std::vector< + std::function> + on_connection_change; + std::vector> + on_ice_gathering_change; + std::vector> + on_ice_candidate; + std::vector> + on_ice_candidate_error; + std::vector&)>> + on_ice_candidates_removed; + std::vector, + const std::vector>&)>> + on_add_track; + std::vector< + std::function)>> + on_track; + std::vector)>> + on_remove_track; + }; + struct Config { + // WebRTC only support one audio device that is setup up on construction, so + // we provide the audio generator configuration here rather than on creation + // of the tracks. This is unlike video, where multiple capture sources can + // be used at the same time. + struct AudioSource { + int sample_rate = 48000; + int channels = 1; + struct PulsedNoise { + double amplitude = 0.1; + }; + absl::optional pulsed_noise = PulsedNoise(); + } audio; + std::string client_name; + // The created endpoints can be accessed using the map key as |index| in + // PeerScenarioClient::endpoint(index). + std::map endpoints = { + {0, EmulatedEndpointConfig()}}; + CallbackHandlers handlers; + PeerConnectionInterface::RTCConfiguration rtc_config; + Config() { rtc_config.sdp_semantics = SdpSemantics::kUnifiedPlan; } + }; + + struct VideoSendTrackConfig { + FrameGeneratorCapturerConfig generator; + bool screencast = false; + }; + + struct AudioSendTrack { + AudioTrackInterface* track; + RtpSenderInterface* sender; + }; + + struct VideoSendTrack { + FrameGeneratorCapturer* capturer; + FrameGeneratorCapturerVideoTrackSource* source; + VideoTrackInterface* track; + RtpSenderInterface* sender; + }; + + PeerScenarioClient(NetworkEmulationManager* net, + rtc::Thread* signaling_thread, + Config config); + + PeerConnectionFactoryInterface* factory() { return pc_factory_.get(); } + PeerConnectionInterface* pc() { return peer_connection_.get(); } + rtc::Thread* thread() { return signaling_thread_; } + Clock* clock() { return Clock::GetRealTimeClock(); } + + // Returns the endpoint created from the EmulatedEndpointConfig with the same + // index in PeerScenarioClient::config. + EmulatedEndpoint* endpoint(int index = 0); + + AudioSendTrack CreateAudio(std::string track_id, + cricket::AudioOptions options); + VideoSendTrack CreateVideo(std::string track_id, VideoSendTrackConfig config); + + void AddVideoReceiveSink(std::string track_id, + rtc::VideoSinkInterface* video_sink); + + CallbackHandlers* handlers() { return &handlers_; } + + // Note that there's no provision for munging SDP as that is deprecated + // behavior. + void CreateAndSetSdp(std::function offer_handler); + void SetSdpOfferAndGetAnswer(std::string remote_offer, + std::function answer_handler); + void SetSdpAnswer( + std::string remote_answer, + std::function + done_handler); + + // Adds the given ice candidate when the peer connection is ready. + void AddIceCandidate(std::unique_ptr candidate); + + private: + const std::map endpoints_; + rtc::Thread* const signaling_thread_; + const std::unique_ptr worker_thread_; + CallbackHandlers handlers_ RTC_GUARDED_BY(signaling_thread_); + const std::unique_ptr observer_; + TaskQueueFactory* task_queue_factory_; + std::map*>> + track_id_to_video_sinks_ RTC_GUARDED_BY(signaling_thread_); + std::list> pending_ice_candidates_ + RTC_GUARDED_BY(signaling_thread_); + + rtc::scoped_refptr pc_factory_; + rtc::scoped_refptr peer_connection_; +}; + +} // namespace test +} // namespace webrtc + +#endif // TEST_PEER_SCENARIO_PEER_SCENARIO_CLIENT_H_ diff --git a/test/peer_scenario/sdp_callbacks.cc b/test/peer_scenario/sdp_callbacks.cc new file mode 100644 index 0000000000..0208c6403f --- /dev/null +++ b/test/peer_scenario/sdp_callbacks.cc @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/peer_scenario/sdp_callbacks.h" + +#include + +namespace webrtc { +namespace test { + +webrtc_sdp_obs_impl::SdpSetObserversInterface* SdpSetObserver( + std::function callback) { + class SdpSetObserver : public webrtc_sdp_obs_impl::SdpSetObserversInterface { + public: + explicit SdpSetObserver(std::function callback) + : callback_(std::move(callback)) {} + void OnSuccess() override { callback_(); } + void OnFailure(RTCError error) override { + RTC_NOTREACHED() << error.message(); + } + void OnSetRemoteDescriptionComplete(RTCError error) override { + RTC_CHECK(error.ok()) << error.message(); + callback_(); + } + std::function callback_; + }; + return new rtc::RefCountedObject(std::move(callback)); +} + +CreateSessionDescriptionObserver* SdpCreateObserver( + std::function callback) { + class SdpCreateObserver : public CreateSessionDescriptionObserver { + public: + explicit SdpCreateObserver(decltype(callback) callback) + : callback_(std::move(callback)) {} + void OnSuccess(SessionDescriptionInterface* desc) override { + callback_(desc); + } + void OnFailure(RTCError error) override { + RTC_NOTREACHED() << error.message(); + } + decltype(callback) callback_; + }; + return new rtc::RefCountedObject(std::move(callback)); +} + +} // namespace test +} // namespace webrtc diff --git a/test/peer_scenario/sdp_callbacks.h b/test/peer_scenario/sdp_callbacks.h new file mode 100644 index 0000000000..413a467f96 --- /dev/null +++ b/test/peer_scenario/sdp_callbacks.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef TEST_PEER_SCENARIO_SDP_CALLBACKS_H_ +#define TEST_PEER_SCENARIO_SDP_CALLBACKS_H_ + +#include "api/peer_connection_interface.h" + +// Helpers to allow usage of std::function/lambdas to observe SDP operation in +// the peer conenction API. As they only have handlers for sucess, failures will +// cause a crash. + +namespace webrtc { +namespace test { +namespace webrtc_sdp_obs_impl { +class SdpSetObserversInterface : public SetSessionDescriptionObserver, + public SetRemoteDescriptionObserverInterface { +}; +} // namespace webrtc_sdp_obs_impl + +// Implementation of both SetSessionDescriptionObserver and +// SetRemoteDescriptionObserverInterface for use with SDP set operations. This +// return a raw owning pointer as it's only intended to be used as input to +// PeerConnection API which will take ownership. +webrtc_sdp_obs_impl::SdpSetObserversInterface* SdpSetObserver( + std::function callback); + +// Implementation of CreateSessionDescriptionObserver for use with SDP create +// operations. This return a raw owning pointer as it's only intended to be used +// as input to PeerConnection API which will take ownership. +CreateSessionDescriptionObserver* SdpCreateObserver( + std::function callback); + +} // namespace test +} // namespace webrtc + +#endif // TEST_PEER_SCENARIO_SDP_CALLBACKS_H_ diff --git a/test/peer_scenario/signaling_route.cc b/test/peer_scenario/signaling_route.cc new file mode 100644 index 0000000000..1e5b9aad9a --- /dev/null +++ b/test/peer_scenario/signaling_route.cc @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/peer_scenario/signaling_route.h" + +#include + +#include "test/network/network_emulation_manager.h" + +namespace webrtc { +namespace test { +namespace { +constexpr size_t kIcePacketSize = 400; +constexpr size_t kSdpPacketSize = 1200; + +struct IceMessage { + IceMessage() = default; + explicit IceMessage(const IceCandidateInterface* candidate) + : sdp_mid(candidate->sdp_mid()), + sdp_mline_index(candidate->sdp_mline_index()) { + RTC_CHECK(candidate->ToString(&sdp_line)); + } + std::unique_ptr AsCandidate() const { + SdpParseError err; + std::unique_ptr candidate( + CreateIceCandidate(sdp_mid, sdp_mline_index, sdp_line, &err)); + RTC_CHECK(candidate) << "Failed to parse: \"" << err.line + << "\". Reason: " << err.description; + return candidate; + } + std::string sdp_mid; + int sdp_mline_index; + std::string sdp_line; +}; + +void StartIceSignalingForRoute(PeerScenarioClient* caller, + PeerScenarioClient* callee, + TrafficRoute* send_route) { + caller->handlers()->on_ice_candidate.push_back( + [=](const IceCandidateInterface* candidate) { + IceMessage msg(candidate); + send_route->NetworkDelayedAction(kIcePacketSize, [callee, msg]() { + callee->thread()->PostTask(RTC_FROM_HERE, [callee, msg]() { + callee->AddIceCandidate(msg.AsCandidate()); + }); + }); + }); +} + +void StartSdpNegotiation( + PeerScenarioClient* caller, + PeerScenarioClient* callee, + TrafficRoute* send_route, + TrafficRoute* ret_route, + std::function modify_offer, + std::function exchange_finished) { + caller->CreateAndSetSdp([=](std::string sdp_offer) { + if (modify_offer) { + auto offer = CreateSessionDescription(SdpType::kOffer, sdp_offer); + modify_offer(offer.get()); + RTC_CHECK(offer->ToString(&sdp_offer)); + } + send_route->NetworkDelayedAction(kSdpPacketSize, [=] { + callee->SetSdpOfferAndGetAnswer(sdp_offer, [=](std::string answer) { + ret_route->NetworkDelayedAction(kSdpPacketSize, [=] { + caller->SetSdpAnswer(std::move(answer), std::move(exchange_finished)); + }); + }); + }); + }); +} +} // namespace + +SignalingRoute::SignalingRoute(PeerScenarioClient* caller, + PeerScenarioClient* callee, + TrafficRoute* send_route, + TrafficRoute* ret_route) + : caller_(caller), + callee_(callee), + send_route_(send_route), + ret_route_(ret_route) {} + +void SignalingRoute::StartIceSignaling() { + StartIceSignalingForRoute(caller_, callee_, send_route_); + StartIceSignalingForRoute(callee_, caller_, ret_route_); +} + +void SignalingRoute::NegotiateSdp( + std::function modify_offer, + std::function exchange_finished) { + StartSdpNegotiation(caller_, callee_, send_route_, ret_route_, modify_offer, + exchange_finished); +} + +void SignalingRoute::NegotiateSdp( + std::function exchange_finished) { + NegotiateSdp({}, exchange_finished); +} + +} // namespace test +} // namespace webrtc diff --git a/test/peer_scenario/signaling_route.h b/test/peer_scenario/signaling_route.h new file mode 100644 index 0000000000..189c4b6f3f --- /dev/null +++ b/test/peer_scenario/signaling_route.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef TEST_PEER_SCENARIO_SIGNALING_ROUTE_H_ +#define TEST_PEER_SCENARIO_SIGNALING_ROUTE_H_ + +#include +#include + +#include "test/network/network_emulation_manager.h" +#include "test/peer_scenario/peer_scenario_client.h" + +namespace webrtc { +namespace test { + +// Helper class to reduce the amount of boilerplate required for ICE signalling +// ad SDP negotiation. +class SignalingRoute { + public: + SignalingRoute(PeerScenarioClient* caller, + PeerScenarioClient* callee, + TrafficRoute* send_route, + TrafficRoute* ret_route); + + void StartIceSignaling(); + + // TODO(srte): Handle lossy links. + void NegotiateSdp( + std::function modify_offer, + std::function + exchange_finished); + void NegotiateSdp( + std::function + exchange_finished); + SignalingRoute reverse() { + return SignalingRoute(callee_, caller_, ret_route_, send_route_); + } + + private: + PeerScenarioClient* const caller_; + PeerScenarioClient* const callee_; + TrafficRoute* const send_route_; + TrafficRoute* const ret_route_; +}; + +} // namespace test +} // namespace webrtc + +#endif // TEST_PEER_SCENARIO_SIGNALING_ROUTE_H_ diff --git a/test/peer_scenario/tests/BUILD.gn b/test/peer_scenario/tests/BUILD.gn new file mode 100644 index 0000000000..6c1c75b79d --- /dev/null +++ b/test/peer_scenario/tests/BUILD.gn @@ -0,0 +1,24 @@ +# Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../webrtc.gni") + +if (rtc_include_tests) { + rtc_source_set("tests") { + testonly = true + sources = [ + "peer_scenario_quality_test.cc", + "remote_estimate_test.cc", + ] + deps = [ + "..:peer_scenario", + "../../:test_support", + "../../../pc:rtc_pc_base", + ] + } +} diff --git a/test/peer_scenario/tests/peer_scenario_quality_test.cc b/test/peer_scenario/tests/peer_scenario_quality_test.cc new file mode 100644 index 0000000000..17e5952d06 --- /dev/null +++ b/test/peer_scenario/tests/peer_scenario_quality_test.cc @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/gtest.h" +#include "test/peer_scenario/peer_scenario.h" + +namespace webrtc { +namespace test { + +TEST(PeerScenarioQualityTest, PsnrIsCollected) { + VideoQualityAnalyzerConfig analyzer_config; + analyzer_config.thread = rtc::Thread::Current(); + VideoQualityAnalyzer analyzer(analyzer_config); + PeerScenario s; + auto caller = s.CreateClient(PeerScenarioClient::Config()); + auto callee = s.CreateClient(PeerScenarioClient::Config()); + PeerScenarioClient::VideoSendTrackConfig video_conf; + video_conf.generator.squares_video->framerate = 20; + auto video = caller->CreateVideo("VIDEO", video_conf); + auto link_builder = s.net()->NodeBuilder().delay_ms(100).capacity_kbps(600); + s.AttachVideoQualityAnalyzer(&analyzer, video.track, callee); + s.SimpleConnection(caller, callee, {link_builder.Build().node}, + {link_builder.Build().node}); + s.ProcessMessages(TimeDelta::seconds(2)); + // We expect ca 40 frames to be produced, but to avoid flakiness on slow + // machines we only test for 10. + EXPECT_GT(analyzer.stats().render.count, 10); + EXPECT_GT(analyzer.stats().psnr_with_freeze.Mean(), 20); +} + +} // namespace test +} // namespace webrtc diff --git a/test/peer_scenario/tests/remote_estimate_test.cc b/test/peer_scenario/tests/remote_estimate_test.cc new file mode 100644 index 0000000000..05addc26ee --- /dev/null +++ b/test/peer_scenario/tests/remote_estimate_test.cc @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "pc/session_description.h" +#include "test/gtest.h" +#include "test/peer_scenario/peer_scenario.h" + +namespace webrtc { +namespace test { + +TEST(RemoteEstimateEndToEnd, OfferedCapabilityIsInAnswer) { + PeerScenario s; + + auto* caller = s.CreateClient(PeerScenarioClient::Config()); + auto* callee = s.CreateClient(PeerScenarioClient::Config()); + + auto send_link = {s.net()->NodeBuilder().Build().node}; + auto ret_link = {s.net()->NodeBuilder().Build().node}; + + s.net()->CreateRoute(caller->endpoint(), send_link, callee->endpoint()); + s.net()->CreateRoute(callee->endpoint(), ret_link, caller->endpoint()); + + auto signaling = s.ConnectSignaling(caller, callee, send_link, ret_link); + caller->CreateVideo("VIDEO", PeerScenarioClient::VideoSendTrackConfig()); + rtc::Event offer_exchange_done; + signaling.NegotiateSdp( + [](SessionDescriptionInterface* offer) { + for (auto& cont : offer->description()->contents()) { + cont.media_description()->set_remote_estimate(true); + } + }, + [&](const SessionDescriptionInterface& answer) { + for (auto& cont : answer.description()->contents()) { + EXPECT_TRUE(cont.media_description()->remote_estimate()); + } + offer_exchange_done.Set(); + }); + EXPECT_TRUE(s.WaitAndProcess(&offer_exchange_done)); +} + +} // namespace test +} // namespace webrtc diff --git a/test/scenario/stats_collection.cc b/test/scenario/stats_collection.cc index 964d62ac54..a78fb7eb3e 100644 --- a/test/scenario/stats_collection.cc +++ b/test/scenario/stats_collection.cc @@ -37,11 +37,26 @@ std::function VideoQualityAnalyzer::Handler() { return [this](VideoFramePair pair) { HandleFramePair(pair); }; } -void VideoQualityAnalyzer::HandleFramePair(VideoFramePair sample) { - layer_analyzers_[sample.layer_id].HandleFramePair(sample, writer_.get()); +void VideoQualityAnalyzer::HandleFramePair(VideoFramePair sample, double psnr) { + layer_analyzers_[sample.layer_id].HandleFramePair(sample, psnr, + writer_.get()); cached_.reset(); } +void VideoQualityAnalyzer::HandleFramePair(VideoFramePair sample) { + double psnr = NAN; + if (sample.decoded) + psnr = I420PSNR(*sample.captured->ToI420(), *sample.decoded->ToI420()); + + if (config_.thread) { + config_.thread->PostTask(RTC_FROM_HERE, [this, sample, psnr] { + HandleFramePair(std::move(sample), psnr); + }); + } else { + HandleFramePair(std::move(sample), psnr); + } +} + std::vector VideoQualityAnalyzer::layer_stats() const { std::vector res; for (auto& layer : layer_analyzers_) @@ -59,8 +74,8 @@ VideoQualityStats& VideoQualityAnalyzer::stats() { } void VideoLayerAnalyzer::HandleFramePair(VideoFramePair sample, + double psnr, RtcEventLogOutput* writer) { - double psnr = NAN; RTC_CHECK(sample.captured); HandleCapturedFrame(sample); if (!sample.decoded) { @@ -69,7 +84,6 @@ void VideoLayerAnalyzer::HandleFramePair(VideoFramePair sample, ++stats_.lost_count; ++skip_count_; } else { - psnr = I420PSNR(*sample.captured->ToI420(), *sample.decoded->ToI420()); stats_.psnr_with_freeze.AddSample(psnr); if (sample.repeated) { ++stats_.freeze_count; diff --git a/test/scenario/stats_collection.h b/test/scenario/stats_collection.h index 0b8b4a327f..64cb58cbe9 100644 --- a/test/scenario/stats_collection.h +++ b/test/scenario/stats_collection.h @@ -23,13 +23,16 @@ namespace test { struct VideoQualityAnalyzerConfig { double psnr_coverage = 1; + rtc::Thread* thread = nullptr; }; class VideoLayerAnalyzer { public: void HandleCapturedFrame(const VideoFramePair& sample); void HandleRenderedFrame(const VideoFramePair& sample); - void HandleFramePair(VideoFramePair sample, RtcEventLogOutput* writer); + void HandleFramePair(VideoFramePair sample, + double psnr, + RtcEventLogOutput* writer); VideoQualityStats stats_; Timestamp last_capture_time_ = Timestamp::MinusInfinity(); Timestamp last_render_time_ = Timestamp::MinusInfinity(); @@ -51,6 +54,7 @@ class VideoQualityAnalyzer { std::function Handler(); private: + void HandleFramePair(VideoFramePair sample, double psnr); const VideoQualityAnalyzerConfig config_; std::map layer_analyzers_; const std::unique_ptr writer_;