From db38972edaf0c2c748343e72bf64682a1351844c Mon Sep 17 00:00:00 2001 From: Sam Zackrisson Date: Thu, 21 Jun 2018 10:12:24 +0200 Subject: [PATCH] Remove nonlinear beamformer API from APM This CL removes the remaining beamformer parts from the APM. Bug: webrtc:9402 Change-Id: I9ab2795bd2813d17166ed0925125257b82d98a74 Reviewed-on: https://webrtc-review.googlesource.com/83340 Reviewed-by: Henrik Lundin Reviewed-by: Minyue Li Commit-Queue: Sam Zackrisson Cr-Commit-Position: refs/heads/master@{#23694} --- modules/audio_processing/BUILD.gn | 31 - .../audio_processing/audio_processing_impl.cc | 25 +- .../audio_processing/audio_processing_impl.h | 7 +- .../audio_processing/beamformer/array_util.cc | 119 ---- .../audio_processing/beamformer/array_util.h | 117 ---- .../beamformer/array_util_unittest.cc | 185 ------ .../beamformer/complex_matrix.h | 96 --- .../beamformer/complex_matrix_unittest.cc | 102 --- .../beamformer/covariance_matrix_generator.cc | 109 ---- .../beamformer/covariance_matrix_generator.h | 54 -- .../covariance_matrix_generator_unittest.cc | 217 ------- modules/audio_processing/beamformer/matrix.h | 369 ----------- .../beamformer/matrix_test_helpers.h | 98 --- .../beamformer/matrix_unittest.cc | 326 ---------- .../beamformer/mock_nonlinear_beamformer.h | 39 -- .../beamformer/nonlinear_beamformer.cc | 587 ------------------ .../beamformer/nonlinear_beamformer.h | 230 ------- .../beamformer/nonlinear_beamformer_test.cc | 120 ---- .../nonlinear_beamformer_unittest.cc | 365 ----------- .../include/audio_processing.cc | 19 - .../include/audio_processing.h | 24 - .../test/audio_processing_simulator.h | 1 - .../audio_processing_configs_fuzzer.cc | 4 - 23 files changed, 8 insertions(+), 3236 deletions(-) delete mode 100644 modules/audio_processing/beamformer/array_util.cc delete mode 100644 modules/audio_processing/beamformer/array_util.h delete mode 100644 modules/audio_processing/beamformer/array_util_unittest.cc delete mode 100644 modules/audio_processing/beamformer/complex_matrix.h delete mode 100644 modules/audio_processing/beamformer/complex_matrix_unittest.cc delete mode 100644 modules/audio_processing/beamformer/covariance_matrix_generator.cc delete mode 100644 modules/audio_processing/beamformer/covariance_matrix_generator.h delete mode 100644 modules/audio_processing/beamformer/covariance_matrix_generator_unittest.cc delete mode 100644 modules/audio_processing/beamformer/matrix.h delete mode 100644 modules/audio_processing/beamformer/matrix_test_helpers.h delete mode 100644 modules/audio_processing/beamformer/matrix_unittest.cc delete mode 100644 modules/audio_processing/beamformer/mock_nonlinear_beamformer.h delete mode 100644 modules/audio_processing/beamformer/nonlinear_beamformer.cc delete mode 100644 modules/audio_processing/beamformer/nonlinear_beamformer.h delete mode 100644 modules/audio_processing/beamformer/nonlinear_beamformer_test.cc delete mode 100644 modules/audio_processing/beamformer/nonlinear_beamformer_unittest.cc diff --git a/modules/audio_processing/BUILD.gn b/modules/audio_processing/BUILD.gn index ae6118c6bf..b5565fcfcc 100644 --- a/modules/audio_processing/BUILD.gn +++ b/modules/audio_processing/BUILD.gn @@ -48,14 +48,6 @@ rtc_static_library("audio_processing") { "audio_buffer.h", "audio_processing_impl.cc", "audio_processing_impl.h", - "beamformer/array_util.cc", - "beamformer/array_util.h", - "beamformer/complex_matrix.h", - "beamformer/covariance_matrix_generator.cc", - "beamformer/covariance_matrix_generator.h", - "beamformer/matrix.h", - "beamformer/nonlinear_beamformer.cc", - "beamformer/nonlinear_beamformer.h", "common.h", "echo_cancellation_impl.cc", "echo_cancellation_impl.h", @@ -437,7 +429,6 @@ if (rtc_include_tests) { deps = [ ":audioproc_test_utils", ":click_annotate", - ":nonlinear_beamformer_test", ":transient_suppression_test", ] @@ -468,12 +459,6 @@ if (rtc_include_tests) { "agc/mock_agc.h", "audio_buffer_unittest.cc", "audio_frame_view_unittest.cc", - "beamformer/array_util_unittest.cc", - "beamformer/complex_matrix_unittest.cc", - "beamformer/covariance_matrix_generator_unittest.cc", - "beamformer/matrix_test_helpers.h", - "beamformer/matrix_unittest.cc", - "beamformer/mock_nonlinear_beamformer.h", "config_unittest.cc", "echo_cancellation_impl_unittest.cc", "gain_controller2_unittest.cc", @@ -566,7 +551,6 @@ if (rtc_include_tests) { "audio_processing_impl_locking_unittest.cc", "audio_processing_impl_unittest.cc", "audio_processing_unittest.cc", - "beamformer/nonlinear_beamformer_unittest.cc", "echo_cancellation_bit_exact_unittest.cc", "echo_control_mobile_unittest.cc", "echo_detector/circular_buffer_unittest.cc", @@ -773,21 +757,6 @@ if (rtc_include_tests) { ] } - rtc_executable("nonlinear_beamformer_test") { - testonly = true - sources = [ - "beamformer/nonlinear_beamformer_test.cc", - ] - deps = [ - ":audio_processing", - ":audioproc_test_utils", - "../../common_audio:common_audio", - "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", - "../../system_wrappers:metrics_default", - ] - } - if (rtc_enable_intelligibility_enhancer) { rtc_executable("intelligibility_proc") { testonly = true diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc index f34e9c5bb3..38f67302db 100644 --- a/modules/audio_processing/audio_processing_impl.cc +++ b/modules/audio_processing/audio_processing_impl.cc @@ -22,7 +22,6 @@ #include "modules/audio_processing/agc/agc_manager_direct.h" #include "modules/audio_processing/agc2/gain_applier.h" #include "modules/audio_processing/audio_buffer.h" -#include "modules/audio_processing/beamformer/nonlinear_beamformer.h" #include "modules/audio_processing/common.h" #include "modules/audio_processing/echo_cancellation_impl.h" #include "modules/audio_processing/echo_control_mobile_impl.h" @@ -278,16 +277,13 @@ struct AudioProcessingImpl::ApmPublicSubmodules { }; struct AudioProcessingImpl::ApmPrivateSubmodules { - ApmPrivateSubmodules(NonlinearBeamformer* beamformer, - std::unique_ptr capture_post_processor, + ApmPrivateSubmodules(std::unique_ptr capture_post_processor, std::unique_ptr render_pre_processor, rtc::scoped_refptr echo_detector) - : beamformer(beamformer), - echo_detector(std::move(echo_detector)), + : echo_detector(std::move(echo_detector)), capture_post_processor(std::move(capture_post_processor)), render_pre_processor(std::move(render_pre_processor)) {} // Accessed internally from capture or during initialization - std::unique_ptr beamformer; std::unique_ptr agc_manager; std::unique_ptr gain_controller2; std::unique_ptr low_cut_filter; @@ -319,12 +315,6 @@ AudioProcessingBuilder& AudioProcessingBuilder::SetEchoControlFactory( return *this; } -AudioProcessingBuilder& AudioProcessingBuilder::SetNonlinearBeamformer( - std::unique_ptr nonlinear_beamformer) { - nonlinear_beamformer_ = std::move(nonlinear_beamformer); - return *this; -} - AudioProcessingBuilder& AudioProcessingBuilder::SetEchoDetector( rtc::scoped_refptr echo_detector) { echo_detector_ = std::move(echo_detector); @@ -340,7 +330,7 @@ AudioProcessing* AudioProcessingBuilder::Create(const webrtc::Config& config) { AudioProcessingImpl* apm = new rtc::RefCountedObject( config, std::move(capture_post_processing_), std::move(render_pre_processing_), std::move(echo_control_factory_), - std::move(echo_detector_), nonlinear_beamformer_.release()); + std::move(echo_detector_)); if (apm->Initialize() != AudioProcessing::kNoError) { delete apm; apm = nullptr; @@ -349,8 +339,7 @@ AudioProcessing* AudioProcessingBuilder::Create(const webrtc::Config& config) { } AudioProcessingImpl::AudioProcessingImpl(const webrtc::Config& config) - : AudioProcessingImpl(config, nullptr, nullptr, nullptr, nullptr, nullptr) { -} + : AudioProcessingImpl(config, nullptr, nullptr, nullptr, nullptr) {} int AudioProcessingImpl::instance_count_ = 0; @@ -359,8 +348,7 @@ AudioProcessingImpl::AudioProcessingImpl( std::unique_ptr capture_post_processor, std::unique_ptr render_pre_processor, std::unique_ptr echo_control_factory, - rtc::scoped_refptr echo_detector, - NonlinearBeamformer* beamformer) + rtc::scoped_refptr echo_detector) : data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), capture_runtime_settings_(kRuntimeSettingQueueSize), @@ -372,8 +360,7 @@ AudioProcessingImpl::AudioProcessingImpl( submodule_states_(!!capture_post_processor, !!render_pre_processor), public_submodules_(new ApmPublicSubmodules()), private_submodules_( - new ApmPrivateSubmodules(beamformer, - std::move(capture_post_processor), + new ApmPrivateSubmodules(std::move(capture_post_processor), std::move(render_pre_processor), std::move(echo_detector))), constants_(config.Get().startup_min_volume, diff --git a/modules/audio_processing/audio_processing_impl.h b/modules/audio_processing/audio_processing_impl.h index f0f823f489..d97e6763e5 100644 --- a/modules/audio_processing/audio_processing_impl.h +++ b/modules/audio_processing/audio_processing_impl.h @@ -31,21 +31,18 @@ namespace webrtc { class ApmDataDumper; class AudioConverter; -class NonlinearBeamformer; class AudioProcessingImpl : public AudioProcessing { public: // Methods forcing APM to run in a single-threaded manner. // Acquires both the render and capture locks. explicit AudioProcessingImpl(const webrtc::Config& config); - // AudioProcessingImpl takes ownership of capture post processor and - // beamformer. + // AudioProcessingImpl takes ownership of capture post processor. AudioProcessingImpl(const webrtc::Config& config, std::unique_ptr capture_post_processor, std::unique_ptr render_pre_processor, std::unique_ptr echo_control_factory, - rtc::scoped_refptr echo_detector, - NonlinearBeamformer* beamformer); + rtc::scoped_refptr echo_detector); ~AudioProcessingImpl() override; int Initialize() override; int Initialize(int capture_input_sample_rate_hz, diff --git a/modules/audio_processing/beamformer/array_util.cc b/modules/audio_processing/beamformer/array_util.cc deleted file mode 100644 index 0e461e9cc4..0000000000 --- a/modules/audio_processing/beamformer/array_util.cc +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/audio_processing/beamformer/array_util.h" - -#include -#include - -#include "rtc_base/checks.h" - -namespace webrtc { -namespace { - -const float kMaxDotProduct = 1e-6f; - -} // namespace - -float GetMinimumSpacing(const std::vector& array_geometry) { - RTC_CHECK_GT(array_geometry.size(), 1); - float mic_spacing = std::numeric_limits::max(); - for (size_t i = 0; i < (array_geometry.size() - 1); ++i) { - for (size_t j = i + 1; j < array_geometry.size(); ++j) { - mic_spacing = - std::min(mic_spacing, Distance(array_geometry[i], array_geometry[j])); - } - } - return mic_spacing; -} - -Point PairDirection(const Point& a, const Point& b) { - return {b.x() - a.x(), b.y() - a.y(), b.z() - a.z()}; -} - -float DotProduct(const Point& a, const Point& b) { - return a.x() * b.x() + a.y() * b.y() + a.z() * b.z(); -} - -Point CrossProduct(const Point& a, const Point& b) { - return {a.y() * b.z() - a.z() * b.y(), a.z() * b.x() - a.x() * b.z(), - a.x() * b.y() - a.y() * b.x()}; -} - -bool AreParallel(const Point& a, const Point& b) { - Point cross_product = CrossProduct(a, b); - return DotProduct(cross_product, cross_product) < kMaxDotProduct; -} - -bool ArePerpendicular(const Point& a, const Point& b) { - return std::abs(DotProduct(a, b)) < kMaxDotProduct; -} - -absl::optional GetDirectionIfLinear( - const std::vector& array_geometry) { - RTC_DCHECK_GT(array_geometry.size(), 1); - const Point first_pair_direction = - PairDirection(array_geometry[0], array_geometry[1]); - for (size_t i = 2u; i < array_geometry.size(); ++i) { - const Point pair_direction = - PairDirection(array_geometry[i - 1], array_geometry[i]); - if (!AreParallel(first_pair_direction, pair_direction)) { - return absl::nullopt; - } - } - return first_pair_direction; -} - -absl::optional GetNormalIfPlanar( - const std::vector& array_geometry) { - RTC_DCHECK_GT(array_geometry.size(), 1); - const Point first_pair_direction = - PairDirection(array_geometry[0], array_geometry[1]); - Point pair_direction(0.f, 0.f, 0.f); - size_t i = 2u; - bool is_linear = true; - for (; i < array_geometry.size() && is_linear; ++i) { - pair_direction = PairDirection(array_geometry[i - 1], array_geometry[i]); - if (!AreParallel(first_pair_direction, pair_direction)) { - is_linear = false; - } - } - if (is_linear) { - return absl::nullopt; - } - const Point normal_direction = - CrossProduct(first_pair_direction, pair_direction); - for (; i < array_geometry.size(); ++i) { - pair_direction = PairDirection(array_geometry[i - 1], array_geometry[i]); - if (!ArePerpendicular(normal_direction, pair_direction)) { - return absl::nullopt; - } - } - return normal_direction; -} - -absl::optional GetArrayNormalIfExists( - const std::vector& array_geometry) { - const absl::optional direction = GetDirectionIfLinear(array_geometry); - if (direction) { - return Point(direction->y(), -direction->x(), 0.f); - } - const absl::optional normal = GetNormalIfPlanar(array_geometry); - if (normal && normal->z() < kMaxDotProduct) { - return normal; - } - return absl::nullopt; -} - -Point AzimuthToPoint(float azimuth) { - return Point(std::cos(azimuth), std::sin(azimuth), 0.f); -} - -} // namespace webrtc diff --git a/modules/audio_processing/beamformer/array_util.h b/modules/audio_processing/beamformer/array_util.h deleted file mode 100644 index 9bdbecbcd8..0000000000 --- a/modules/audio_processing/beamformer/array_util.h +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_AUDIO_PROCESSING_BEAMFORMER_ARRAY_UTIL_H_ -#define MODULES_AUDIO_PROCESSING_BEAMFORMER_ARRAY_UTIL_H_ - -#include -#include - -#include "absl/types/optional.h" - -namespace webrtc { - -// Coordinates in meters. The convention used is: -// x: the horizontal dimension, with positive to the right from the camera's -// perspective. -// y: the depth dimension, with positive forward from the camera's -// perspective. -// z: the vertical dimension, with positive upwards. -template -struct CartesianPoint { - CartesianPoint() { - c[0] = 0; - c[1] = 0; - c[2] = 0; - } - CartesianPoint(T x, T y, T z) { - c[0] = x; - c[1] = y; - c[2] = z; - } - T x() const { return c[0]; } - T y() const { return c[1]; } - T z() const { return c[2]; } - T c[3]; -}; - -using Point = CartesianPoint; - -// Calculates the direction from a to b. -Point PairDirection(const Point& a, const Point& b); - -float DotProduct(const Point& a, const Point& b); -Point CrossProduct(const Point& a, const Point& b); - -bool AreParallel(const Point& a, const Point& b); -bool ArePerpendicular(const Point& a, const Point& b); - -// Returns the minimum distance between any two Points in the given -// |array_geometry|. -float GetMinimumSpacing(const std::vector& array_geometry); - -// If the given array geometry is linear it returns the direction without -// normalizing. -absl::optional GetDirectionIfLinear( - const std::vector& array_geometry); - -// If the given array geometry is planar it returns the normal without -// normalizing. -absl::optional GetNormalIfPlanar( - const std::vector& array_geometry); - -// Returns the normal of an array if it has one and it is in the xy-plane. -absl::optional GetArrayNormalIfExists( - const std::vector& array_geometry); - -// The resulting Point will be in the xy-plane. -Point AzimuthToPoint(float azimuth); - -template -float Distance(CartesianPoint a, CartesianPoint b) { - return std::sqrt((a.x() - b.x()) * (a.x() - b.x()) + - (a.y() - b.y()) * (a.y() - b.y()) + - (a.z() - b.z()) * (a.z() - b.z())); -} - -// The convention used: -// azimuth: zero is to the right from the camera's perspective, with positive -// angles in radians counter-clockwise. -// elevation: zero is horizontal, with positive angles in radians upwards. -// radius: distance from the camera in meters. -template -struct SphericalPoint { - SphericalPoint(T azimuth, T elevation, T radius) { - s[0] = azimuth; - s[1] = elevation; - s[2] = radius; - } - T azimuth() const { return s[0]; } - T elevation() const { return s[1]; } - T distance() const { return s[2]; } - T s[3]; -}; - -using SphericalPointf = SphericalPoint; - -// Helper functions to transform degrees to radians and the inverse. -template -T DegreesToRadians(T angle_degrees) { - return M_PI * angle_degrees / 180; -} - -template -T RadiansToDegrees(T angle_radians) { - return 180 * angle_radians / M_PI; -} - -} // namespace webrtc - -#endif // MODULES_AUDIO_PROCESSING_BEAMFORMER_ARRAY_UTIL_H_ diff --git a/modules/audio_processing/beamformer/array_util_unittest.cc b/modules/audio_processing/beamformer/array_util_unittest.cc deleted file mode 100644 index a5c075ab05..0000000000 --- a/modules/audio_processing/beamformer/array_util_unittest.cc +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -// MSVC++ requires this to be set before any other includes to get M_PI. -#define _USE_MATH_DEFINES - -#include "modules/audio_processing/beamformer/array_util.h" - -#include -#include - -#include "test/gtest.h" - -namespace webrtc { - -bool operator==(const Point& lhs, const Point& rhs) { - return lhs.x() == rhs.x() && lhs.y() == rhs.y() && lhs.z() == rhs.z(); -} - -TEST(ArrayUtilTest, PairDirection) { - EXPECT_EQ(Point(1.f, 2.f, 3.f), - PairDirection(Point(0.f, 0.f, 0.f), Point(1.f, 2.f, 3.f))); - EXPECT_EQ(Point(-1.f, -2.f, -3.f), - PairDirection(Point(1.f, 2.f, 3.f), Point(0.f, 0.f, 0.f))); - EXPECT_EQ(Point(0.f, 0.f, 0.f), - PairDirection(Point(1.f, 0.f, 0.f), Point(1.f, 0.f, 0.f))); - EXPECT_EQ(Point(-1.f, 2.f, 0.f), - PairDirection(Point(1.f, 0.f, 0.f), Point(0.f, 2.f, 0.f))); - EXPECT_EQ(Point(-4.f, 4.f, -4.f), - PairDirection(Point(1.f, -2.f, 3.f), Point(-3.f, 2.f, -1.f))); -} - -TEST(ArrayUtilTest, DotProduct) { - EXPECT_FLOAT_EQ(0.f, DotProduct(Point(0.f, 0.f, 0.f), Point(1.f, 2.f, 3.f))); - EXPECT_FLOAT_EQ(0.f, DotProduct(Point(1.f, 0.f, 2.f), Point(0.f, 3.f, 0.f))); - EXPECT_FLOAT_EQ(0.f, DotProduct(Point(1.f, 1.f, 0.f), Point(1.f, -1.f, 0.f))); - EXPECT_FLOAT_EQ(2.f, DotProduct(Point(1.f, 0.f, 0.f), Point(2.f, 0.f, 0.f))); - EXPECT_FLOAT_EQ(-6.f, - DotProduct(Point(-2.f, 0.f, 0.f), Point(3.f, 0.f, 0.f))); - EXPECT_FLOAT_EQ(-10.f, - DotProduct(Point(1.f, -2.f, 3.f), Point(-3.f, 2.f, -1.f))); -} - -TEST(ArrayUtilTest, CrossProduct) { - EXPECT_EQ(Point(0.f, 0.f, 0.f), - CrossProduct(Point(0.f, 0.f, 0.f), Point(1.f, 2.f, 3.f))); - EXPECT_EQ(Point(0.f, 0.f, 1.f), - CrossProduct(Point(1.f, 0.f, 0.f), Point(0.f, 1.f, 0.f))); - EXPECT_EQ(Point(1.f, 0.f, 0.f), - CrossProduct(Point(0.f, 1.f, 0.f), Point(0.f, 0.f, 1.f))); - EXPECT_EQ(Point(0.f, -1.f, 0.f), - CrossProduct(Point(1.f, 0.f, 0.f), Point(0.f, 0.f, 1.f))); - EXPECT_EQ(Point(-4.f, -8.f, -4.f), - CrossProduct(Point(1.f, -2.f, 3.f), Point(-3.f, 2.f, -1.f))); -} - -TEST(ArrayUtilTest, AreParallel) { - EXPECT_TRUE(AreParallel(Point(0.f, 0.f, 0.f), Point(1.f, 2.f, 3.f))); - EXPECT_FALSE(AreParallel(Point(1.f, 0.f, 2.f), Point(0.f, 3.f, 0.f))); - EXPECT_FALSE(AreParallel(Point(1.f, 2.f, 0.f), Point(1.f, -0.5f, 0.f))); - EXPECT_FALSE(AreParallel(Point(1.f, -2.f, 3.f), Point(-3.f, 2.f, -1.f))); - EXPECT_TRUE(AreParallel(Point(1.f, 0.f, 0.f), Point(2.f, 0.f, 0.f))); - EXPECT_TRUE(AreParallel(Point(1.f, 2.f, 3.f), Point(-2.f, -4.f, -6.f))); -} - -TEST(ArrayUtilTest, ArePerpendicular) { - EXPECT_TRUE(ArePerpendicular(Point(0.f, 0.f, 0.f), Point(1.f, 2.f, 3.f))); - EXPECT_TRUE(ArePerpendicular(Point(1.f, 0.f, 2.f), Point(0.f, 3.f, 0.f))); - EXPECT_TRUE(ArePerpendicular(Point(1.f, 2.f, 0.f), Point(1.f, -0.5f, 0.f))); - EXPECT_FALSE(ArePerpendicular(Point(1.f, -2.f, 3.f), Point(-3.f, 2.f, -1.f))); - EXPECT_FALSE(ArePerpendicular(Point(1.f, 0.f, 0.f), Point(2.f, 0.f, 0.f))); - EXPECT_FALSE(ArePerpendicular(Point(1.f, 2.f, 3.f), Point(-2.f, -4.f, -6.f))); -} - -TEST(ArrayUtilTest, GetMinimumSpacing) { - std::vector geometry; - geometry.push_back(Point(0.f, 0.f, 0.f)); - geometry.push_back(Point(0.1f, 0.f, 0.f)); - EXPECT_FLOAT_EQ(0.1f, GetMinimumSpacing(geometry)); - geometry.push_back(Point(0.f, 0.05f, 0.f)); - EXPECT_FLOAT_EQ(0.05f, GetMinimumSpacing(geometry)); - geometry.push_back(Point(0.f, 0.f, 0.02f)); - EXPECT_FLOAT_EQ(0.02f, GetMinimumSpacing(geometry)); - geometry.push_back(Point(-0.003f, -0.004f, 0.02f)); - EXPECT_FLOAT_EQ(0.005f, GetMinimumSpacing(geometry)); -} - -TEST(ArrayUtilTest, GetDirectionIfLinear) { - std::vector geometry; - geometry.push_back(Point(0.f, 0.f, 0.f)); - geometry.push_back(Point(0.1f, 0.f, 0.f)); - EXPECT_TRUE( - AreParallel(Point(1.f, 0.f, 0.f), *GetDirectionIfLinear(geometry))); - geometry.push_back(Point(0.15f, 0.f, 0.f)); - EXPECT_TRUE( - AreParallel(Point(1.f, 0.f, 0.f), *GetDirectionIfLinear(geometry))); - geometry.push_back(Point(-0.2f, 0.f, 0.f)); - EXPECT_TRUE( - AreParallel(Point(1.f, 0.f, 0.f), *GetDirectionIfLinear(geometry))); - geometry.push_back(Point(0.05f, 0.f, 0.f)); - EXPECT_TRUE( - AreParallel(Point(1.f, 0.f, 0.f), *GetDirectionIfLinear(geometry))); - geometry.push_back(Point(0.1f, 0.1f, 0.f)); - EXPECT_FALSE(GetDirectionIfLinear(geometry)); - geometry.push_back(Point(0.f, 0.f, -0.2f)); - EXPECT_FALSE(GetDirectionIfLinear(geometry)); -} - -TEST(ArrayUtilTest, GetNormalIfPlanar) { - std::vector geometry; - geometry.push_back(Point(0.f, 0.f, 0.f)); - geometry.push_back(Point(0.1f, 0.f, 0.f)); - EXPECT_FALSE(GetNormalIfPlanar(geometry)); - geometry.push_back(Point(0.15f, 0.f, 0.f)); - EXPECT_FALSE(GetNormalIfPlanar(geometry)); - geometry.push_back(Point(0.1f, 0.2f, 0.f)); - EXPECT_TRUE(AreParallel(Point(0.f, 0.f, 1.f), *GetNormalIfPlanar(geometry))); - geometry.push_back(Point(0.f, -0.15f, 0.f)); - EXPECT_TRUE(AreParallel(Point(0.f, 0.f, 1.f), *GetNormalIfPlanar(geometry))); - geometry.push_back(Point(0.f, 0.1f, 0.2f)); - EXPECT_FALSE(GetNormalIfPlanar(geometry)); - geometry.push_back(Point(0.f, 0.f, -0.15f)); - EXPECT_FALSE(GetNormalIfPlanar(geometry)); - geometry.push_back(Point(0.1f, 0.2f, 0.f)); - EXPECT_FALSE(GetNormalIfPlanar(geometry)); -} - -TEST(ArrayUtilTest, GetArrayNormalIfExists) { - std::vector geometry; - geometry.push_back(Point(0.f, 0.f, 0.f)); - geometry.push_back(Point(0.1f, 0.f, 0.f)); - EXPECT_TRUE( - AreParallel(Point(0.f, 1.f, 0.f), *GetArrayNormalIfExists(geometry))); - geometry.push_back(Point(0.15f, 0.f, 0.f)); - EXPECT_TRUE( - AreParallel(Point(0.f, 1.f, 0.f), *GetArrayNormalIfExists(geometry))); - geometry.push_back(Point(0.1f, 0.f, 0.2f)); - EXPECT_TRUE( - AreParallel(Point(0.f, 1.f, 0.f), *GetArrayNormalIfExists(geometry))); - geometry.push_back(Point(0.f, 0.f, -0.1f)); - EXPECT_TRUE( - AreParallel(Point(0.f, 1.f, 0.f), *GetArrayNormalIfExists(geometry))); - geometry.push_back(Point(0.1f, 0.2f, 0.3f)); - EXPECT_FALSE(GetArrayNormalIfExists(geometry)); - geometry.push_back(Point(0.f, -0.1f, 0.f)); - EXPECT_FALSE(GetArrayNormalIfExists(geometry)); - geometry.push_back(Point(1.f, 0.f, -0.2f)); - EXPECT_FALSE(GetArrayNormalIfExists(geometry)); -} - -TEST(ArrayUtilTest, DegreesToRadians) { - EXPECT_FLOAT_EQ(0.f, DegreesToRadians(0.f)); - EXPECT_FLOAT_EQ(static_cast(M_PI) / 6.f, DegreesToRadians(30.f)); - EXPECT_FLOAT_EQ(-static_cast(M_PI) / 4.f, DegreesToRadians(-45.f)); - EXPECT_FLOAT_EQ(static_cast(M_PI) / 3.f, DegreesToRadians(60.f)); - EXPECT_FLOAT_EQ(-static_cast(M_PI) / 2.f, DegreesToRadians(-90.f)); - EXPECT_FLOAT_EQ(2.f * static_cast(M_PI) / 3.f, - DegreesToRadians(120.f)); - EXPECT_FLOAT_EQ(-3.f * static_cast(M_PI) / 4.f, - DegreesToRadians(-135.f)); - EXPECT_FLOAT_EQ(5.f * static_cast(M_PI) / 6.f, - DegreesToRadians(150.f)); - EXPECT_FLOAT_EQ(-static_cast(M_PI), DegreesToRadians(-180.f)); -} - -TEST(ArrayUtilTest, RadiansToDegrees) { - EXPECT_FLOAT_EQ(0.f, RadiansToDegrees(0.f)); - EXPECT_FLOAT_EQ(30.f, RadiansToDegrees(M_PI / 6.f)); - EXPECT_FLOAT_EQ(-45.f, RadiansToDegrees(-M_PI / 4.f)); - EXPECT_FLOAT_EQ(60.f, RadiansToDegrees(M_PI / 3.f)); - EXPECT_FLOAT_EQ(-90.f, RadiansToDegrees(-M_PI / 2.f)); - EXPECT_FLOAT_EQ(120.f, RadiansToDegrees(2.f * M_PI / 3.f)); - EXPECT_FLOAT_EQ(-135.f, RadiansToDegrees(-3.f * M_PI / 4.f)); - EXPECT_FLOAT_EQ(150.f, RadiansToDegrees(5.f * M_PI / 6.f)); - EXPECT_FLOAT_EQ(-180.f, RadiansToDegrees(-M_PI)); -} - -} // namespace webrtc diff --git a/modules/audio_processing/beamformer/complex_matrix.h b/modules/audio_processing/beamformer/complex_matrix.h deleted file mode 100644 index 9960e1dd87..0000000000 --- a/modules/audio_processing/beamformer/complex_matrix.h +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_AUDIO_PROCESSING_BEAMFORMER_COMPLEX_MATRIX_H_ -#define MODULES_AUDIO_PROCESSING_BEAMFORMER_COMPLEX_MATRIX_H_ - -#include - -#include "modules/audio_processing/beamformer/matrix.h" -#include "rtc_base/checks.h" - -namespace webrtc { - -using std::complex; - -// An extension of Matrix for operations that only work on a complex type. -template -class ComplexMatrix : public Matrix > { - public: - ComplexMatrix() : Matrix >() {} - - ComplexMatrix(size_t num_rows, size_t num_columns) - : Matrix >(num_rows, num_columns) {} - - ComplexMatrix(const complex* data, size_t num_rows, size_t num_columns) - : Matrix >(data, num_rows, num_columns) {} - - // Complex Matrix operations. - ComplexMatrix& PointwiseConjugate() { - complex* const data = this->data(); - size_t size = this->num_rows() * this->num_columns(); - for (size_t i = 0; i < size; ++i) { - data[i] = conj(data[i]); - } - - return *this; - } - - ComplexMatrix& PointwiseConjugate(const ComplexMatrix& operand) { - this->CopyFrom(operand); - return PointwiseConjugate(); - } - - ComplexMatrix& ConjugateTranspose() { - this->CopyDataToScratch(); - size_t num_rows = this->num_rows(); - this->SetNumRows(this->num_columns()); - this->SetNumColumns(num_rows); - this->Resize(); - return ConjugateTranspose(this->scratch_elements()); - } - - ComplexMatrix& ConjugateTranspose(const ComplexMatrix& operand) { - RTC_CHECK_EQ(operand.num_rows(), this->num_columns()); - RTC_CHECK_EQ(operand.num_columns(), this->num_rows()); - return ConjugateTranspose(operand.elements()); - } - - ComplexMatrix& ZeroImag() { - complex* const data = this->data(); - size_t size = this->num_rows() * this->num_columns(); - for (size_t i = 0; i < size; ++i) { - data[i] = complex(data[i].real(), 0); - } - - return *this; - } - - ComplexMatrix& ZeroImag(const ComplexMatrix& operand) { - this->CopyFrom(operand); - return ZeroImag(); - } - - private: - ComplexMatrix& ConjugateTranspose(const complex* const* src) { - complex* const* elements = this->elements(); - for (size_t i = 0; i < this->num_rows(); ++i) { - for (size_t j = 0; j < this->num_columns(); ++j) { - elements[i][j] = conj(src[j][i]); - } - } - - return *this; - } -}; - -} // namespace webrtc - -#endif // MODULES_AUDIO_PROCESSING_BEAMFORMER_COMPLEX_MATRIX_H_ diff --git a/modules/audio_processing/beamformer/complex_matrix_unittest.cc b/modules/audio_processing/beamformer/complex_matrix_unittest.cc deleted file mode 100644 index f9c7ea5fe5..0000000000 --- a/modules/audio_processing/beamformer/complex_matrix_unittest.cc +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/audio_processing/beamformer/complex_matrix.h" -#include "modules/audio_processing/beamformer/matrix_test_helpers.h" -#include "test/gtest.h" - -namespace webrtc { - -TEST(ComplexMatrixTest, TestPointwiseConjugate) { - const int kNumRows = 2; - const int kNumCols = 4; - - const complex kValuesInitial[kNumRows][kNumCols] = { - {complex(1.1f, 1.1f), complex(2.2f, -2.2f), - complex(3.3f, 3.3f), complex(4.4f, -4.4f)}, - {complex(5.5f, 5.5f), complex(6.6f, -6.6f), - complex(7.7f, 7.7f), complex(8.8f, -8.8f)}}; - - const complex kValuesExpected[kNumRows][kNumCols] = { - {complex(1.1f, -1.1f), complex(2.2f, 2.2f), - complex(3.3f, -3.3f), complex(4.4f, 4.4f)}, - {complex(5.5f, -5.5f), complex(6.6f, 6.6f), - complex(7.7f, -7.7f), complex(8.8f, 8.8f)}}; - - ComplexMatrix initial_mat(*kValuesInitial, kNumRows, kNumCols); - ComplexMatrix expected_result(*kValuesExpected, kNumRows, kNumCols); - ComplexMatrix actual_result(kNumRows, kNumCols); - - actual_result.PointwiseConjugate(initial_mat); - MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(expected_result, - actual_result); - - initial_mat.PointwiseConjugate(); - MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(initial_mat, - actual_result); -} - -TEST(ComplexMatrixTest, TestConjugateTranspose) { - const int kNumInitialRows = 2; - const int kNumInitialCols = 4; - const int kNumResultRows = 4; - const int kNumResultCols = 2; - - const complex kValuesInitial[kNumInitialRows][kNumInitialCols] = { - {complex(1.1f, 1.1f), complex(2.2f, 2.2f), - complex(3.3f, 3.3f), complex(4.4f, 4.4f)}, - {complex(5.5f, 5.5f), complex(6.6f, 6.6f), - complex(7.7f, 7.7f), complex(8.8f, 8.8f)}}; - - const complex kValuesExpected[kNumResultRows][kNumResultCols] = { - {complex(1.1f, -1.1f), complex(5.5f, -5.5f)}, - {complex(2.2f, -2.2f), complex(6.6f, -6.6f)}, - {complex(3.3f, -3.3f), complex(7.7f, -7.7f)}, - {complex(4.4f, -4.4f), complex(8.8f, -8.8f)}}; - - ComplexMatrix initial_mat(*kValuesInitial, kNumInitialRows, - kNumInitialCols); - ComplexMatrix expected_result(*kValuesExpected, kNumResultRows, - kNumResultCols); - ComplexMatrix actual_result(kNumResultRows, kNumResultCols); - - actual_result.ConjugateTranspose(initial_mat); - MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(expected_result, - actual_result); - - initial_mat.ConjugateTranspose(); - MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(initial_mat, - actual_result); -} - -TEST(ComplexMatrixTest, TestZeroImag) { - const int kNumRows = 2; - const int kNumCols = 2; - const complex kValuesInitial[kNumRows][kNumCols] = { - {complex(1.1f, 1.1f), complex(2.2f, 2.2f)}, - {complex(3.3f, 3.3f), complex(4.4f, 4.4f)}}; - const complex kValuesExpected[kNumRows][kNumCols] = { - {complex(1.1f, 0.f), complex(2.2f, 0.f)}, - {complex(3.3f, 0.f), complex(4.4f, 0.f)}}; - - ComplexMatrix initial_mat(*kValuesInitial, kNumRows, kNumCols); - ComplexMatrix expected_result(*kValuesExpected, kNumRows, kNumCols); - ComplexMatrix actual_result; - - actual_result.ZeroImag(initial_mat); - MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(expected_result, - actual_result); - - initial_mat.ZeroImag(); - MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(initial_mat, - actual_result); -} - -} // namespace webrtc diff --git a/modules/audio_processing/beamformer/covariance_matrix_generator.cc b/modules/audio_processing/beamformer/covariance_matrix_generator.cc deleted file mode 100644 index afc6ddd3a1..0000000000 --- a/modules/audio_processing/beamformer/covariance_matrix_generator.cc +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#define _USE_MATH_DEFINES - -#include "modules/audio_processing/beamformer/covariance_matrix_generator.h" - -#include - -namespace webrtc { -namespace { - -float BesselJ0(float x) { -#ifdef WEBRTC_WIN - return _j0(x); -#else - return j0(x); -#endif -} - -// Calculates the Euclidean norm for a row vector. -float Norm(const ComplexMatrix& x) { - RTC_CHECK_EQ(1, x.num_rows()); - const size_t length = x.num_columns(); - const complex* elems = x.elements()[0]; - float result = 0.f; - for (size_t i = 0u; i < length; ++i) { - result += std::norm(elems[i]); - } - return std::sqrt(result); -} - -} // namespace - -void CovarianceMatrixGenerator::UniformCovarianceMatrix( - float wave_number, - const std::vector& geometry, - ComplexMatrix* mat) { - RTC_CHECK_EQ(geometry.size(), mat->num_rows()); - RTC_CHECK_EQ(geometry.size(), mat->num_columns()); - - complex* const* mat_els = mat->elements(); - for (size_t i = 0; i < geometry.size(); ++i) { - for (size_t j = 0; j < geometry.size(); ++j) { - if (wave_number > 0.f) { - mat_els[i][j] = - BesselJ0(wave_number * Distance(geometry[i], geometry[j])); - } else { - mat_els[i][j] = i == j ? 1.f : 0.f; - } - } - } -} - -void CovarianceMatrixGenerator::AngledCovarianceMatrix( - float sound_speed, - float angle, - size_t frequency_bin, - size_t fft_size, - size_t num_freq_bins, - int sample_rate, - const std::vector& geometry, - ComplexMatrix* mat) { - RTC_CHECK_EQ(geometry.size(), mat->num_rows()); - RTC_CHECK_EQ(geometry.size(), mat->num_columns()); - - ComplexMatrix interf_cov_vector(1, geometry.size()); - ComplexMatrix interf_cov_vector_transposed(geometry.size(), 1); - PhaseAlignmentMasks(frequency_bin, fft_size, sample_rate, sound_speed, - geometry, angle, &interf_cov_vector); - interf_cov_vector.Scale(1.f / Norm(interf_cov_vector)); - interf_cov_vector_transposed.Transpose(interf_cov_vector); - interf_cov_vector.PointwiseConjugate(); - mat->Multiply(interf_cov_vector_transposed, interf_cov_vector); -} - -void CovarianceMatrixGenerator::PhaseAlignmentMasks( - size_t frequency_bin, - size_t fft_size, - int sample_rate, - float sound_speed, - const std::vector& geometry, - float angle, - ComplexMatrix* mat) { - RTC_CHECK_EQ(1, mat->num_rows()); - RTC_CHECK_EQ(geometry.size(), mat->num_columns()); - - float freq_in_hertz = - (static_cast(frequency_bin) / fft_size) * sample_rate; - - complex* const* mat_els = mat->elements(); - for (size_t c_ix = 0; c_ix < geometry.size(); ++c_ix) { - float distance = std::cos(angle) * geometry[c_ix].x() + - std::sin(angle) * geometry[c_ix].y(); - float phase_shift = -2.f * M_PI * distance * freq_in_hertz / sound_speed; - - // Euler's formula for mat[0][c_ix] = e^(j * phase_shift). - mat_els[0][c_ix] = complex(cos(phase_shift), sin(phase_shift)); - } -} - -} // namespace webrtc diff --git a/modules/audio_processing/beamformer/covariance_matrix_generator.h b/modules/audio_processing/beamformer/covariance_matrix_generator.h deleted file mode 100644 index d26ed2fc0b..0000000000 --- a/modules/audio_processing/beamformer/covariance_matrix_generator.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_AUDIO_PROCESSING_BEAMFORMER_COVARIANCE_MATRIX_GENERATOR_H_ -#define MODULES_AUDIO_PROCESSING_BEAMFORMER_COVARIANCE_MATRIX_GENERATOR_H_ - -#include "modules/audio_processing/beamformer/array_util.h" -#include "modules/audio_processing/beamformer/complex_matrix.h" - -namespace webrtc { - -// Helper class for Beamformer in charge of generating covariance matrices. For -// each function, the passed-in ComplexMatrix is expected to be of size -// |num_input_channels| x |num_input_channels|. -class CovarianceMatrixGenerator { - public: - // A uniform covariance matrix with a gap at the target location. WARNING: - // The target angle is assumed to be 0. - static void UniformCovarianceMatrix(float wave_number, - const std::vector& geometry, - ComplexMatrix* mat); - - // The covariance matrix of a source at the given angle. - static void AngledCovarianceMatrix(float sound_speed, - float angle, - size_t frequency_bin, - size_t fft_size, - size_t num_freq_bins, - int sample_rate, - const std::vector& geometry, - ComplexMatrix* mat); - - // Calculates phase shifts that, when applied to a multichannel signal and - // added together, cause constructive interferernce for sources located at - // the given angle. - static void PhaseAlignmentMasks(size_t frequency_bin, - size_t fft_size, - int sample_rate, - float sound_speed, - const std::vector& geometry, - float angle, - ComplexMatrix* mat); -}; - -} // namespace webrtc - -#endif // MODULES_AUDIO_PROCESSING_BEAMFORMER_BF_HELPERS_H_ diff --git a/modules/audio_processing/beamformer/covariance_matrix_generator_unittest.cc b/modules/audio_processing/beamformer/covariance_matrix_generator_unittest.cc deleted file mode 100644 index 4f77744cc2..0000000000 --- a/modules/audio_processing/beamformer/covariance_matrix_generator_unittest.cc +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#define _USE_MATH_DEFINES - -#include "modules/audio_processing/beamformer/covariance_matrix_generator.h" - -#include - -#include "modules/audio_processing/beamformer/matrix_test_helpers.h" -#include "test/gtest.h" - -namespace webrtc { - -using std::complex; - -TEST(CovarianceMatrixGeneratorTest, TestUniformCovarianceMatrix2Mics) { - const float kWaveNumber = 0.5775f; - const int kNumberMics = 2; - const float kMicSpacing = 0.05f; - const float kTolerance = 0.0001f; - std::vector geometry; - float first_mic = (kNumberMics - 1) * kMicSpacing / 2.f; - for (int i = 0; i < kNumberMics; ++i) { - geometry.push_back(Point(i * kMicSpacing - first_mic, 0.f, 0.f)); - } - ComplexMatrix actual_covariance_matrix(kNumberMics, kNumberMics); - CovarianceMatrixGenerator::UniformCovarianceMatrix(kWaveNumber, geometry, - &actual_covariance_matrix); - - complex* const* actual_els = actual_covariance_matrix.elements(); - - EXPECT_NEAR(actual_els[0][0].real(), 1.f, kTolerance); - EXPECT_NEAR(actual_els[0][1].real(), 0.9998f, kTolerance); - EXPECT_NEAR(actual_els[1][0].real(), 0.9998f, kTolerance); - EXPECT_NEAR(actual_els[1][1].real(), 1.f, kTolerance); - - EXPECT_NEAR(actual_els[0][0].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[0][1].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[1][0].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[1][1].imag(), 0.f, kTolerance); -} - -TEST(CovarianceMatrixGeneratorTest, TestUniformCovarianceMatrix3Mics) { - const float kWaveNumber = 10.3861f; - const int kNumberMics = 3; - const float kMicSpacing = 0.04f; - const float kTolerance = 0.0001f; - std::vector geometry; - float first_mic = (kNumberMics - 1) * kMicSpacing / 2.f; - for (int i = 0; i < kNumberMics; ++i) { - geometry.push_back(Point(i * kMicSpacing - first_mic, 0.f, 0.f)); - } - ComplexMatrix actual_covariance_matrix(kNumberMics, kNumberMics); - CovarianceMatrixGenerator::UniformCovarianceMatrix(kWaveNumber, geometry, - &actual_covariance_matrix); - - complex* const* actual_els = actual_covariance_matrix.elements(); - - EXPECT_NEAR(actual_els[0][0].real(), 1.f, kTolerance); - EXPECT_NEAR(actual_els[0][1].real(), 0.9573f, kTolerance); - EXPECT_NEAR(actual_els[0][2].real(), 0.8347f, kTolerance); - EXPECT_NEAR(actual_els[1][0].real(), 0.9573f, kTolerance); - EXPECT_NEAR(actual_els[1][1].real(), 1.f, kTolerance); - EXPECT_NEAR(actual_els[1][2].real(), 0.9573f, kTolerance); - EXPECT_NEAR(actual_els[2][0].real(), 0.8347f, kTolerance); - EXPECT_NEAR(actual_els[2][1].real(), 0.9573f, kTolerance); - EXPECT_NEAR(actual_els[2][2].real(), 1.f, kTolerance); - - EXPECT_NEAR(actual_els[0][0].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[0][1].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[0][2].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[1][0].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[1][1].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[1][2].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[2][0].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[2][1].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[2][2].imag(), 0.f, kTolerance); -} - -TEST(CovarianceMatrixGeneratorTest, TestUniformCovarianceMatrix3DArray) { - const float kWaveNumber = 1.2345f; - const int kNumberMics = 4; - const float kTolerance = 0.0001f; - std::vector geometry; - geometry.push_back(Point(-0.025f, -0.05f, -0.075f)); - geometry.push_back(Point(0.075f, -0.05f, -0.075f)); - geometry.push_back(Point(-0.025f, 0.15f, -0.075f)); - geometry.push_back(Point(-0.025f, -0.05f, 0.225f)); - ComplexMatrix actual_covariance_matrix(kNumberMics, kNumberMics); - CovarianceMatrixGenerator::UniformCovarianceMatrix(kWaveNumber, geometry, - &actual_covariance_matrix); - - complex* const* actual_els = actual_covariance_matrix.elements(); - - EXPECT_NEAR(actual_els[0][0].real(), 1.f, kTolerance); - EXPECT_NEAR(actual_els[0][1].real(), 0.9962f, kTolerance); - EXPECT_NEAR(actual_els[0][2].real(), 0.9848f, kTolerance); - EXPECT_NEAR(actual_els[0][3].real(), 0.9660f, kTolerance); - EXPECT_NEAR(actual_els[1][0].real(), 0.9962f, kTolerance); - EXPECT_NEAR(actual_els[1][1].real(), 1.f, kTolerance); - EXPECT_NEAR(actual_els[1][2].real(), 0.9810f, kTolerance); - EXPECT_NEAR(actual_els[1][3].real(), 0.9623f, kTolerance); - EXPECT_NEAR(actual_els[2][0].real(), 0.9848f, kTolerance); - EXPECT_NEAR(actual_els[2][1].real(), 0.9810f, kTolerance); - EXPECT_NEAR(actual_els[2][2].real(), 1.f, kTolerance); - EXPECT_NEAR(actual_els[2][3].real(), 0.9511f, kTolerance); - EXPECT_NEAR(actual_els[3][0].real(), 0.9660f, kTolerance); - EXPECT_NEAR(actual_els[3][1].real(), 0.9623f, kTolerance); - EXPECT_NEAR(actual_els[3][2].real(), 0.9511f, kTolerance); - EXPECT_NEAR(actual_els[3][3].real(), 1.f, kTolerance); - - EXPECT_NEAR(actual_els[0][0].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[0][1].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[0][2].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[0][3].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[1][0].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[1][1].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[1][2].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[1][3].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[2][0].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[2][1].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[2][2].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[2][3].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[3][0].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[3][1].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[3][2].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[3][3].imag(), 0.f, kTolerance); -} - -TEST(CovarianceMatrixGeneratorTest, TestAngledCovarianceMatrix2Mics) { - const float kSpeedOfSound = 340; - const float kAngle = static_cast(M_PI) / 4.f; - const float kFrequencyBin = 6; - const float kFftSize = 512; - const int kNumberFrequencyBins = 257; - const int kSampleRate = 16000; - const int kNumberMics = 2; - const float kMicSpacing = 0.04f; - const float kTolerance = 0.0001f; - std::vector geometry; - float first_mic = (kNumberMics - 1) * kMicSpacing / 2.f; - for (int i = 0; i < kNumberMics; ++i) { - geometry.push_back(Point(i * kMicSpacing - first_mic, 0.f, 0.f)); - } - ComplexMatrix actual_covariance_matrix(kNumberMics, kNumberMics); - CovarianceMatrixGenerator::AngledCovarianceMatrix( - kSpeedOfSound, kAngle, kFrequencyBin, kFftSize, kNumberFrequencyBins, - kSampleRate, geometry, &actual_covariance_matrix); - - complex* const* actual_els = actual_covariance_matrix.elements(); - - EXPECT_NEAR(actual_els[0][0].real(), 0.5f, kTolerance); - EXPECT_NEAR(actual_els[0][1].real(), 0.4976f, kTolerance); - EXPECT_NEAR(actual_els[1][0].real(), 0.4976f, kTolerance); - EXPECT_NEAR(actual_els[1][1].real(), 0.5f, kTolerance); - - EXPECT_NEAR(actual_els[0][0].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[0][1].imag(), 0.0489f, kTolerance); - EXPECT_NEAR(actual_els[1][0].imag(), -0.0489f, kTolerance); - EXPECT_NEAR(actual_els[1][1].imag(), 0.f, kTolerance); -} - -TEST(CovarianceMatrixGeneratorTest, TestAngledCovarianceMatrix3Mics) { - const float kSpeedOfSound = 340; - const float kAngle = static_cast(M_PI) / 4.f; - const float kFrequencyBin = 9; - const float kFftSize = 512; - const int kNumberFrequencyBins = 257; - const int kSampleRate = 42000; - const int kNumberMics = 3; - const float kMicSpacing = 0.05f; - const float kTolerance = 0.0001f; - std::vector geometry; - float first_mic = (kNumberMics - 1) * kMicSpacing / 2.f; - for (int i = 0; i < kNumberMics; ++i) { - geometry.push_back(Point(i * kMicSpacing - first_mic, 0.f, 0.f)); - } - ComplexMatrix actual_covariance_matrix(kNumberMics, kNumberMics); - CovarianceMatrixGenerator::AngledCovarianceMatrix( - kSpeedOfSound, kAngle, kFrequencyBin, kFftSize, kNumberFrequencyBins, - kSampleRate, geometry, &actual_covariance_matrix); - - complex* const* actual_els = actual_covariance_matrix.elements(); - - EXPECT_NEAR(actual_els[0][0].real(), 0.3333f, kTolerance); - EXPECT_NEAR(actual_els[0][1].real(), 0.2953f, kTolerance); - EXPECT_NEAR(actual_els[0][2].real(), 0.1899f, kTolerance); - EXPECT_NEAR(actual_els[1][0].real(), 0.2953f, kTolerance); - EXPECT_NEAR(actual_els[1][1].real(), 0.3333f, kTolerance); - EXPECT_NEAR(actual_els[1][2].real(), 0.2953f, kTolerance); - EXPECT_NEAR(actual_els[2][0].real(), 0.1899f, kTolerance); - EXPECT_NEAR(actual_els[2][1].real(), 0.2953f, kTolerance); - EXPECT_NEAR(actual_els[2][2].real(), 0.3333f, kTolerance); - - EXPECT_NEAR(actual_els[0][0].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[0][1].imag(), 0.1546f, kTolerance); - EXPECT_NEAR(actual_els[0][2].imag(), 0.274f, kTolerance); - EXPECT_NEAR(actual_els[1][0].imag(), -0.1546f, kTolerance); - EXPECT_NEAR(actual_els[1][1].imag(), 0.f, kTolerance); - EXPECT_NEAR(actual_els[1][2].imag(), 0.1546f, kTolerance); - EXPECT_NEAR(actual_els[2][0].imag(), -0.274f, kTolerance); - EXPECT_NEAR(actual_els[2][1].imag(), -0.1546f, kTolerance); - EXPECT_NEAR(actual_els[2][2].imag(), 0.f, kTolerance); -} - -// PhaseAlignmentMasks is tested by AngledCovarianceMatrix and by -// InitBeamformerWeights in BeamformerUnittest. - -} // namespace webrtc diff --git a/modules/audio_processing/beamformer/matrix.h b/modules/audio_processing/beamformer/matrix.h deleted file mode 100644 index 3d1a709d52..0000000000 --- a/modules/audio_processing/beamformer/matrix.h +++ /dev/null @@ -1,369 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_AUDIO_PROCESSING_BEAMFORMER_MATRIX_H_ -#define MODULES_AUDIO_PROCESSING_BEAMFORMER_MATRIX_H_ - -#include -#include -#include -#include - -#include "rtc_base/checks.h" -#include "rtc_base/constructormagic.h" - -namespace { - -// Wrappers to get around the compiler warning resulting from the fact that -// there's no std::sqrt overload for ints. We cast all non-complex types to -// a double for the sqrt method. -template -T sqrt_wrapper(T x) { - return sqrt(static_cast(x)); -} - -template -std::complex sqrt_wrapper(std::complex x) { - return sqrt(x); -} -} // namespace - -namespace webrtc { - -// Matrix is a class for doing standard matrix operations on 2 dimensional -// matrices of any size. Results of matrix operations are stored in the -// calling object. Function overloads exist for both in-place (the calling -// object is used as both an operand and the result) and out-of-place (all -// operands are passed in as parameters) operations. If operand dimensions -// mismatch, the program crashes. Out-of-place operations change the size of -// the calling object, if necessary, before operating. -// -// 'In-place' operations that inherently change the size of the matrix (eg. -// Transpose, Multiply on different-sized matrices) must make temporary copies -// (|scratch_elements_| and |scratch_data_|) of existing data to complete the -// operations. -// -// The data is stored contiguously. Data can be accessed internally as a flat -// array, |data_|, or as an array of row pointers, |elements_|, but is -// available to users only as an array of row pointers through |elements()|. -// Memory for storage is allocated when a matrix is resized only if the new -// size overflows capacity. Memory needed temporarily for any operations is -// similarly resized only if the new size overflows capacity. -// -// If you pass in storage through the ctor, that storage is copied into the -// matrix. TODO(claguna): albeit tricky, allow for data to be referenced -// instead of copied, and owned by the user. -template -class Matrix { - public: - Matrix() : num_rows_(0), num_columns_(0) {} - - // Allocates space for the elements and initializes all values to zero. - Matrix(size_t num_rows, size_t num_columns) - : num_rows_(num_rows), num_columns_(num_columns) { - Resize(); - scratch_data_.resize(num_rows_ * num_columns_); - scratch_elements_.resize(num_rows_); - } - - // Copies |data| into the new Matrix. - Matrix(const T* data, size_t num_rows, size_t num_columns) - : num_rows_(0), num_columns_(0) { - CopyFrom(data, num_rows, num_columns); - scratch_data_.resize(num_rows_ * num_columns_); - scratch_elements_.resize(num_rows_); - } - - virtual ~Matrix() {} - - // Deep copy an existing matrix. - void CopyFrom(const Matrix& other) { - CopyFrom(&other.data_[0], other.num_rows_, other.num_columns_); - } - - // Copy |data| into the Matrix. The current data is lost. - void CopyFrom(const T* const data, size_t num_rows, size_t num_columns) { - Resize(num_rows, num_columns); - memcpy(&data_[0], data, num_rows_ * num_columns_ * sizeof(data_[0])); - } - - Matrix& CopyFromColumn(const T* const* src, - size_t column_index, - size_t num_rows) { - Resize(1, num_rows); - for (size_t i = 0; i < num_columns_; ++i) { - data_[i] = src[i][column_index]; - } - - return *this; - } - - void Resize(size_t num_rows, size_t num_columns) { - if (num_rows != num_rows_ || num_columns != num_columns_) { - num_rows_ = num_rows; - num_columns_ = num_columns; - Resize(); - } - } - - // Accessors and mutators. - size_t num_rows() const { return num_rows_; } - size_t num_columns() const { return num_columns_; } - T* const* elements() { return &elements_[0]; } - const T* const* elements() const { return &elements_[0]; } - - T Trace() { - RTC_CHECK_EQ(num_rows_, num_columns_); - - T trace = 0; - for (size_t i = 0; i < num_rows_; ++i) { - trace += elements_[i][i]; - } - return trace; - } - - // Matrix Operations. Returns *this to support method chaining. - Matrix& Transpose() { - CopyDataToScratch(); - Resize(num_columns_, num_rows_); - return Transpose(scratch_elements()); - } - - Matrix& Transpose(const Matrix& operand) { - RTC_CHECK_EQ(operand.num_rows_, num_columns_); - RTC_CHECK_EQ(operand.num_columns_, num_rows_); - - return Transpose(operand.elements()); - } - - template - Matrix& Scale(const S& scalar) { - for (size_t i = 0; i < data_.size(); ++i) { - data_[i] *= scalar; - } - - return *this; - } - - template - Matrix& Scale(const Matrix& operand, const S& scalar) { - CopyFrom(operand); - return Scale(scalar); - } - - Matrix& Add(const Matrix& operand) { - RTC_CHECK_EQ(num_rows_, operand.num_rows_); - RTC_CHECK_EQ(num_columns_, operand.num_columns_); - - for (size_t i = 0; i < data_.size(); ++i) { - data_[i] += operand.data_[i]; - } - - return *this; - } - - Matrix& Add(const Matrix& lhs, const Matrix& rhs) { - CopyFrom(lhs); - return Add(rhs); - } - - Matrix& Subtract(const Matrix& operand) { - RTC_CHECK_EQ(num_rows_, operand.num_rows_); - RTC_CHECK_EQ(num_columns_, operand.num_columns_); - - for (size_t i = 0; i < data_.size(); ++i) { - data_[i] -= operand.data_[i]; - } - - return *this; - } - - Matrix& Subtract(const Matrix& lhs, const Matrix& rhs) { - CopyFrom(lhs); - return Subtract(rhs); - } - - Matrix& PointwiseMultiply(const Matrix& operand) { - RTC_CHECK_EQ(num_rows_, operand.num_rows_); - RTC_CHECK_EQ(num_columns_, operand.num_columns_); - - for (size_t i = 0; i < data_.size(); ++i) { - data_[i] *= operand.data_[i]; - } - - return *this; - } - - Matrix& PointwiseMultiply(const Matrix& lhs, const Matrix& rhs) { - CopyFrom(lhs); - return PointwiseMultiply(rhs); - } - - Matrix& PointwiseDivide(const Matrix& operand) { - RTC_CHECK_EQ(num_rows_, operand.num_rows_); - RTC_CHECK_EQ(num_columns_, operand.num_columns_); - - for (size_t i = 0; i < data_.size(); ++i) { - data_[i] /= operand.data_[i]; - } - - return *this; - } - - Matrix& PointwiseDivide(const Matrix& lhs, const Matrix& rhs) { - CopyFrom(lhs); - return PointwiseDivide(rhs); - } - - Matrix& PointwiseSquareRoot() { - for (size_t i = 0; i < data_.size(); ++i) { - data_[i] = sqrt_wrapper(data_[i]); - } - - return *this; - } - - Matrix& PointwiseSquareRoot(const Matrix& operand) { - CopyFrom(operand); - return PointwiseSquareRoot(); - } - - Matrix& PointwiseAbsoluteValue() { - for (size_t i = 0; i < data_.size(); ++i) { - data_[i] = abs(data_[i]); - } - - return *this; - } - - Matrix& PointwiseAbsoluteValue(const Matrix& operand) { - CopyFrom(operand); - return PointwiseAbsoluteValue(); - } - - Matrix& PointwiseSquare() { - for (size_t i = 0; i < data_.size(); ++i) { - data_[i] *= data_[i]; - } - - return *this; - } - - Matrix& PointwiseSquare(const Matrix& operand) { - CopyFrom(operand); - return PointwiseSquare(); - } - - Matrix& Multiply(const Matrix& lhs, const Matrix& rhs) { - RTC_CHECK_EQ(lhs.num_columns_, rhs.num_rows_); - RTC_CHECK_EQ(num_rows_, lhs.num_rows_); - RTC_CHECK_EQ(num_columns_, rhs.num_columns_); - - return Multiply(lhs.elements(), rhs.num_rows_, rhs.elements()); - } - - Matrix& Multiply(const Matrix& rhs) { - RTC_CHECK_EQ(num_columns_, rhs.num_rows_); - - CopyDataToScratch(); - Resize(num_rows_, rhs.num_columns_); - return Multiply(scratch_elements(), rhs.num_rows_, rhs.elements()); - } - - std::string ToString() const { - std::ostringstream ss; - ss << std::endl << "Matrix" << std::endl; - - for (size_t i = 0; i < num_rows_; ++i) { - for (size_t j = 0; j < num_columns_; ++j) { - ss << elements_[i][j] << " "; - } - ss << std::endl; - } - ss << std::endl; - - return ss.str(); - } - - protected: - void SetNumRows(const size_t num_rows) { num_rows_ = num_rows; } - void SetNumColumns(const size_t num_columns) { num_columns_ = num_columns; } - T* data() { return &data_[0]; } - const T* data() const { return &data_[0]; } - const T* const* scratch_elements() const { return &scratch_elements_[0]; } - - // Resize the matrix. If an increase in capacity is required, the current - // data is lost. - void Resize() { - size_t size = num_rows_ * num_columns_; - data_.resize(size); - elements_.resize(num_rows_); - - for (size_t i = 0; i < num_rows_; ++i) { - elements_[i] = &data_[i * num_columns_]; - } - } - - // Copies data_ into scratch_data_ and updates scratch_elements_ accordingly. - void CopyDataToScratch() { - scratch_data_ = data_; - scratch_elements_.resize(num_rows_); - - for (size_t i = 0; i < num_rows_; ++i) { - scratch_elements_[i] = &scratch_data_[i * num_columns_]; - } - } - - private: - size_t num_rows_; - size_t num_columns_; - std::vector data_; - std::vector elements_; - - // Stores temporary copies of |data_| and |elements_| for in-place operations - // where referring to original data is necessary. - std::vector scratch_data_; - std::vector scratch_elements_; - - // Helpers for Transpose and Multiply operations that unify in-place and - // out-of-place solutions. - Matrix& Transpose(const T* const* src) { - for (size_t i = 0; i < num_rows_; ++i) { - for (size_t j = 0; j < num_columns_; ++j) { - elements_[i][j] = src[j][i]; - } - } - - return *this; - } - - Matrix& Multiply(const T* const* lhs, - size_t num_rows_rhs, - const T* const* rhs) { - for (size_t row = 0; row < num_rows_; ++row) { - for (size_t col = 0; col < num_columns_; ++col) { - T cur_element = 0; - for (size_t i = 0; i < num_rows_rhs; ++i) { - cur_element += lhs[row][i] * rhs[i][col]; - } - - elements_[row][col] = cur_element; - } - } - - return *this; - } - - RTC_DISALLOW_COPY_AND_ASSIGN(Matrix); -}; - -} // namespace webrtc - -#endif // MODULES_AUDIO_PROCESSING_BEAMFORMER_MATRIX_H_ diff --git a/modules/audio_processing/beamformer/matrix_test_helpers.h b/modules/audio_processing/beamformer/matrix_test_helpers.h deleted file mode 100644 index e7b2f09d58..0000000000 --- a/modules/audio_processing/beamformer/matrix_test_helpers.h +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_AUDIO_PROCESSING_BEAMFORMER_MATRIX_TEST_HELPERS_H_ -#define MODULES_AUDIO_PROCESSING_BEAMFORMER_MATRIX_TEST_HELPERS_H_ - -#include "modules/audio_processing/beamformer/complex_matrix.h" -#include "modules/audio_processing/beamformer/matrix.h" -#include "test/gtest.h" - -namespace { -const float kTolerance = 0.001f; -} - -namespace webrtc { - -using std::complex; - -// Functions used in both matrix_unittest and complex_matrix_unittest. -class MatrixTestHelpers { - public: - template - static void ValidateMatrixEquality(const Matrix& expected, - const Matrix& actual) { - EXPECT_EQ(expected.num_rows(), actual.num_rows()); - EXPECT_EQ(expected.num_columns(), actual.num_columns()); - - const T* const* expected_elements = expected.elements(); - const T* const* actual_elements = actual.elements(); - for (size_t i = 0; i < expected.num_rows(); ++i) { - for (size_t j = 0; j < expected.num_columns(); ++j) { - EXPECT_EQ(expected_elements[i][j], actual_elements[i][j]); - } - } - } - - static void ValidateMatrixEqualityFloat(const Matrix& expected, - const Matrix& actual) { - EXPECT_EQ(expected.num_rows(), actual.num_rows()); - EXPECT_EQ(expected.num_columns(), actual.num_columns()); - - const float* const* expected_elements = expected.elements(); - const float* const* actual_elements = actual.elements(); - for (size_t i = 0; i < expected.num_rows(); ++i) { - for (size_t j = 0; j < expected.num_columns(); ++j) { - EXPECT_NEAR(expected_elements[i][j], actual_elements[i][j], kTolerance); - } - } - } - - static void ValidateMatrixEqualityComplexFloat( - const Matrix >& expected, - const Matrix >& actual) { - EXPECT_EQ(expected.num_rows(), actual.num_rows()); - EXPECT_EQ(expected.num_columns(), actual.num_columns()); - - const complex* const* expected_elements = expected.elements(); - const complex* const* actual_elements = actual.elements(); - for (size_t i = 0; i < expected.num_rows(); ++i) { - for (size_t j = 0; j < expected.num_columns(); ++j) { - EXPECT_NEAR(expected_elements[i][j].real(), - actual_elements[i][j].real(), kTolerance); - EXPECT_NEAR(expected_elements[i][j].imag(), - actual_elements[i][j].imag(), kTolerance); - } - } - } - - static void ValidateMatrixNearEqualityComplexFloat( - const Matrix >& expected, - const Matrix >& actual, - float tolerance) { - EXPECT_EQ(expected.num_rows(), actual.num_rows()); - EXPECT_EQ(expected.num_columns(), actual.num_columns()); - - const complex* const* expected_elements = expected.elements(); - const complex* const* actual_elements = actual.elements(); - for (size_t i = 0; i < expected.num_rows(); ++i) { - for (size_t j = 0; j < expected.num_columns(); ++j) { - EXPECT_NEAR(expected_elements[i][j].real(), - actual_elements[i][j].real(), tolerance); - EXPECT_NEAR(expected_elements[i][j].imag(), - actual_elements[i][j].imag(), tolerance); - } - } - } -}; - -} // namespace webrtc - -#endif // MODULES_AUDIO_PROCESSING_BEAMFORMER_MATRIX_TEST_HELPERS_H_ diff --git a/modules/audio_processing/beamformer/matrix_unittest.cc b/modules/audio_processing/beamformer/matrix_unittest.cc deleted file mode 100644 index 4dcf91b4a0..0000000000 --- a/modules/audio_processing/beamformer/matrix_unittest.cc +++ /dev/null @@ -1,326 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include - -#include "modules/audio_processing/beamformer/matrix.h" -#include "modules/audio_processing/beamformer/matrix_test_helpers.h" -#include "test/gtest.h" - -namespace webrtc { - -using std::complex; - -TEST(MatrixTest, TestMultiplySameSize) { - const int kNumRows = 2; - const int kNumCols = 2; - const float kValuesLeft[kNumRows][kNumCols] = {{1.1f, 2.2f}, {3.3f, 4.4f}}; - const float kValuesRight[kNumRows][kNumCols] = {{5.4f, 127.f}, - {4600.f, -555.f}}; - const float kValuesExpected[kNumRows][kNumCols] = {{10125.94f, -1081.3f}, - {20257.82f, -2022.9f}}; - - Matrix lh_mat(*kValuesLeft, kNumRows, kNumCols); - Matrix rh_mat(*kValuesRight, kNumRows, kNumCols); - Matrix expected_result(*kValuesExpected, kNumRows, kNumCols); - Matrix actual_result(kNumRows, kNumCols); - - actual_result.Multiply(lh_mat, rh_mat); - MatrixTestHelpers::ValidateMatrixEquality(expected_result, actual_result); - - lh_mat.Multiply(rh_mat); - MatrixTestHelpers::ValidateMatrixEquality(lh_mat, actual_result); -} - -TEST(MatrixTest, TestMultiplyDifferentSize) { - const int kNumRowsLeft = 2; - const int kNumColsLeft = 3; - const int kNumRowsRight = 3; - const int kNumColsRight = 2; - const int kValuesLeft[kNumRowsLeft][kNumColsLeft] = {{35, 466, -15}, - {-3, 3422, 9}}; - const int kValuesRight[kNumRowsRight][kNumColsRight] = { - {765, -42}, {0, 194}, {625, 66321}}; - const int kValuesExpected[kNumRowsLeft][kNumColsRight] = {{17400, -905881}, - {3330, 1260883}}; - - Matrix lh_mat(*kValuesLeft, kNumRowsLeft, kNumColsLeft); - Matrix rh_mat(*kValuesRight, kNumRowsRight, kNumColsRight); - Matrix expected_result(*kValuesExpected, kNumRowsLeft, kNumColsRight); - Matrix actual_result(kNumRowsLeft, kNumColsRight); - - actual_result.Multiply(lh_mat, rh_mat); - MatrixTestHelpers::ValidateMatrixEquality(expected_result, actual_result); - - lh_mat.Multiply(rh_mat); - MatrixTestHelpers::ValidateMatrixEquality(lh_mat, actual_result); -} - -TEST(MatrixTest, TestTranspose) { - const int kNumInitialRows = 2; - const int kNumInitialCols = 4; - const int kNumResultRows = 4; - const int kNumResultCols = 2; - const float kValuesInitial[kNumInitialRows][kNumInitialCols] = { - {1.1f, 2.2f, 3.3f, 4.4f}, {5.5f, 6.6f, 7.7f, 8.8f}}; - const float kValuesExpected[kNumResultRows][kNumResultCols] = { - {1.1f, 5.5f}, {2.2f, 6.6f}, {3.3f, 7.7f}, {4.4f, 8.8f}}; - - Matrix initial_mat(*kValuesInitial, kNumInitialRows, kNumInitialCols); - Matrix expected_result(*kValuesExpected, kNumResultRows, - kNumResultCols); - Matrix actual_result(kNumResultRows, kNumResultCols); - - actual_result.Transpose(initial_mat); - MatrixTestHelpers::ValidateMatrixEqualityFloat(expected_result, - actual_result); - initial_mat.Transpose(); - MatrixTestHelpers::ValidateMatrixEqualityFloat(initial_mat, actual_result); -} - -TEST(MatrixTest, TestScale) { - const int kNumRows = 3; - const int kNumCols = 3; - const int kScaleFactor = -9; - const int kValuesInitial[kNumRows][kNumCols] = { - {1, 20, 5000}, {-3, -29, 66}, {7654, 0, -23455}}; - const int kValuesExpected[kNumRows][kNumCols] = { - {-9, -180, -45000}, {27, 261, -594}, {-68886, 0, 211095}}; - - Matrix initial_mat(*kValuesInitial, kNumRows, kNumCols); - Matrix expected_result(*kValuesExpected, kNumRows, kNumCols); - Matrix actual_result; - - actual_result.Scale(initial_mat, kScaleFactor); - MatrixTestHelpers::ValidateMatrixEquality(expected_result, actual_result); - - initial_mat.Scale(kScaleFactor); - MatrixTestHelpers::ValidateMatrixEquality(initial_mat, actual_result); -} - -TEST(MatrixTest, TestPointwiseAdd) { - const int kNumRows = 2; - const int kNumCols = 3; - const float kValuesLeft[kNumRows][kNumCols] = {{1.1f, 210.45f, -549.2f}, - {11.876f, 586.7f, -64.35f}}; - const float kValuesRight[kNumRows][kNumCols] = {{-50.4f, 1.f, 0.5f}, - {460.f, -554.2f, 4566.f}}; - const float kValuesExpected[kNumRows][kNumCols] = { - {-49.3f, 211.45f, -548.7f}, {471.876f, 32.5f, 4501.65f}}; - - Matrix lh_mat(*kValuesLeft, kNumRows, kNumCols); - Matrix rh_mat(*kValuesRight, kNumRows, kNumCols); - Matrix expected_result(*kValuesExpected, kNumRows, kNumCols); - Matrix actual_result; - - actual_result.Add(lh_mat, rh_mat); - MatrixTestHelpers::ValidateMatrixEqualityFloat(expected_result, - actual_result); - lh_mat.Add(rh_mat); - MatrixTestHelpers::ValidateMatrixEqualityFloat(lh_mat, actual_result); -} - -TEST(MatrixTest, TestPointwiseSubtract) { - const int kNumRows = 3; - const int kNumCols = 2; - const float kValuesLeft[kNumRows][kNumCols] = { - {1.1f, 210.45f}, {-549.2f, 11.876f}, {586.7f, -64.35f}}; - const float kValuesRight[kNumRows][kNumCols] = { - {-50.4f, 1.f}, {0.5f, 460.f}, {-554.2f, 4566.f}}; - const float kValuesExpected[kNumRows][kNumCols] = { - {51.5f, 209.45f}, {-549.7f, -448.124f}, {1140.9f, -4630.35f}}; - - Matrix lh_mat(*kValuesLeft, kNumRows, kNumCols); - Matrix rh_mat(*kValuesRight, kNumRows, kNumCols); - Matrix expected_result(*kValuesExpected, kNumRows, kNumCols); - Matrix actual_result; - - actual_result.Subtract(lh_mat, rh_mat); - MatrixTestHelpers::ValidateMatrixEqualityFloat(expected_result, - actual_result); - - lh_mat.Subtract(rh_mat); - MatrixTestHelpers::ValidateMatrixEqualityFloat(lh_mat, actual_result); -} - -TEST(MatrixTest, TestPointwiseMultiply) { - const int kNumRows = 1; - const int kNumCols = 5; - const float kValuesLeft[kNumRows][kNumCols] = { - {1.1f, 6.4f, 0.f, -1.f, -88.3f}}; - const float kValuesRight[kNumRows][kNumCols] = { - {53.2f, -210.45f, -549.2f, 99.99f, -45.2f}}; - const float kValuesExpected[kNumRows][kNumCols] = { - {58.52f, -1346.88f, 0.f, -99.99f, 3991.16f}}; - - Matrix lh_mat(*kValuesLeft, kNumRows, kNumCols); - Matrix rh_mat(*kValuesRight, kNumRows, kNumCols); - Matrix expected_result(*kValuesExpected, kNumRows, kNumCols); - Matrix actual_result; - - actual_result.PointwiseMultiply(lh_mat, rh_mat); - MatrixTestHelpers::ValidateMatrixEqualityFloat(expected_result, - actual_result); - - lh_mat.PointwiseMultiply(rh_mat); - MatrixTestHelpers::ValidateMatrixEqualityFloat(lh_mat, actual_result); -} - -TEST(MatrixTest, TestPointwiseDivide) { - const int kNumRows = 5; - const int kNumCols = 1; - const float kValuesLeft[kNumRows][kNumCols] = { - {1.1f}, {6.4f}, {0.f}, {-1.f}, {-88.3f}}; - const float kValuesRight[kNumRows][kNumCols] = { - {53.2f}, {-210.45f}, {-549.2f}, {99.99f}, {-45.2f}}; - const float kValuesExpected[kNumRows][kNumCols] = { - {0.020676691f}, {-0.03041102399f}, {0.f}, {-0.010001f}, {1.9535398f}}; - - Matrix lh_mat(*kValuesLeft, kNumRows, kNumCols); - Matrix rh_mat(*kValuesRight, kNumRows, kNumCols); - Matrix expected_result(*kValuesExpected, kNumRows, kNumCols); - Matrix actual_result; - - actual_result.PointwiseDivide(lh_mat, rh_mat); - MatrixTestHelpers::ValidateMatrixEqualityFloat(expected_result, - actual_result); - - lh_mat.PointwiseDivide(rh_mat); - MatrixTestHelpers::ValidateMatrixEqualityFloat(lh_mat, actual_result); -} - -TEST(MatrixTest, TestPointwiseSquareRoot) { - const int kNumRows = 2; - const int kNumCols = 2; - const int kValues[kNumRows][kNumCols] = {{4, 9}, {16, 0}}; - const int kValuesExpected[kNumRows][kNumCols] = {{2, 3}, {4, 0}}; - - Matrix operand_mat(*kValues, kNumRows, kNumCols); - Matrix expected_result(*kValuesExpected, kNumRows, kNumCols); - Matrix actual_result; - - actual_result.PointwiseSquareRoot(operand_mat); - MatrixTestHelpers::ValidateMatrixEquality(expected_result, actual_result); - - operand_mat.PointwiseSquareRoot(); - MatrixTestHelpers::ValidateMatrixEquality(operand_mat, actual_result); -} - -TEST(MatrixTest, TestPointwiseSquareRootComplex) { - const int kNumRows = 1; - const int kNumCols = 3; - const complex kValues[kNumRows][kNumCols] = { - {complex(-4.f, 0), complex(0, 9), complex(3, -4)}}; - const complex kValuesExpected[kNumRows][kNumCols] = { - {complex(0.f, 2.f), complex(2.1213202f, 2.1213202f), - complex(2.f, -1.f)}}; - - Matrix > operand_mat(*kValues, kNumRows, kNumCols); - Matrix > expected_result(*kValuesExpected, kNumRows, kNumCols); - Matrix > actual_result; - - actual_result.PointwiseSquareRoot(operand_mat); - MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(expected_result, - actual_result); - - operand_mat.PointwiseSquareRoot(); - MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(operand_mat, - actual_result); -} - -TEST(MatrixTest, TestPointwiseAbsoluteValue) { - const int kNumRows = 1; - const int kNumCols = 3; - const complex kValues[kNumRows][kNumCols] = { - {complex(-4.f, 0), complex(0, 9), complex(3, -4)}}; - const complex kValuesExpected[kNumRows][kNumCols] = { - {complex(4.f, 0), complex(9.f, 0), complex(5.f, 0)}}; - - Matrix > operand_mat(*kValues, kNumRows, kNumCols); - Matrix > expected_result(*kValuesExpected, kNumRows, kNumCols); - Matrix > actual_result; - - actual_result.PointwiseAbsoluteValue(operand_mat); - MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(expected_result, - actual_result); - - operand_mat.PointwiseAbsoluteValue(); - MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(operand_mat, - actual_result); -} - -TEST(MatrixTest, TestPointwiseSquare) { - const int kNumRows = 1; - const int kNumCols = 3; - const float kValues[kNumRows][kNumCols] = {{2.4f, -4.f, 3.3f}}; - const float kValuesExpected[kNumRows][kNumCols] = {{5.76f, 16.f, 10.89f}}; - - Matrix operand_mat(*kValues, kNumRows, kNumCols); - Matrix expected_result(*kValuesExpected, kNumRows, kNumCols); - Matrix actual_result; - - actual_result.PointwiseSquare(operand_mat); - MatrixTestHelpers::ValidateMatrixEqualityFloat(expected_result, - actual_result); - - operand_mat.PointwiseSquare(); - MatrixTestHelpers::ValidateMatrixEqualityFloat(operand_mat, actual_result); -} - -TEST(MatrixTest, TestComplexOperations) { - const int kNumRows = 2; - const int kNumCols = 2; - - const complex kValuesLeft[kNumRows][kNumCols] = { - {complex(1.f, 1.f), complex(2.f, 2.f)}, - {complex(3.f, 3.f), complex(4.f, 4.f)}}; - - const complex kValuesRight[kNumRows][kNumCols] = { - {complex(5.f, 5.f), complex(6.f, 6.f)}, - {complex(7.f, 7.f), complex(8.f, 8.f)}}; - - const complex kValuesExpectedAdd[kNumRows][kNumCols] = { - {complex(6.f, 6.f), complex(8.f, 8.f)}, - {complex(10.f, 10.f), complex(12.f, 12.f)}}; - - const complex kValuesExpectedMultiply[kNumRows][kNumCols] = { - {complex(0.f, 38.f), complex(0.f, 44.f)}, - {complex(0.f, 86.f), complex(0.f, 100.f)}}; - - const complex kValuesExpectedPointwiseDivide[kNumRows][kNumCols] = { - {complex(0.2f, 0.f), complex(0.33333333f, 0.f)}, - {complex(0.42857143f, 0.f), complex(0.5f, 0.f)}}; - - Matrix > lh_mat(*kValuesLeft, kNumRows, kNumCols); - Matrix > rh_mat(*kValuesRight, kNumRows, kNumCols); - Matrix > expected_result_add(*kValuesExpectedAdd, kNumRows, - kNumCols); - Matrix > expected_result_multiply(*kValuesExpectedMultiply, - kNumRows, kNumCols); - Matrix > expected_result_pointwise_divide( - *kValuesExpectedPointwiseDivide, kNumRows, kNumCols); - Matrix > actual_result_add; - Matrix > actual_result_multiply(kNumRows, kNumCols); - Matrix > actual_result_pointwise_divide; - - actual_result_add.Add(lh_mat, rh_mat); - MatrixTestHelpers::ValidateMatrixEqualityComplexFloat(expected_result_add, - actual_result_add); - - actual_result_multiply.Multiply(lh_mat, rh_mat); - MatrixTestHelpers::ValidateMatrixEqualityComplexFloat( - expected_result_multiply, actual_result_multiply); - - actual_result_pointwise_divide.PointwiseDivide(lh_mat, rh_mat); - MatrixTestHelpers::ValidateMatrixEqualityComplexFloat( - expected_result_pointwise_divide, actual_result_pointwise_divide); -} - -} // namespace webrtc diff --git a/modules/audio_processing/beamformer/mock_nonlinear_beamformer.h b/modules/audio_processing/beamformer/mock_nonlinear_beamformer.h deleted file mode 100644 index c4c7358b63..0000000000 --- a/modules/audio_processing/beamformer/mock_nonlinear_beamformer.h +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_AUDIO_PROCESSING_BEAMFORMER_MOCK_BEAMFORMER_H_ -#define MODULES_AUDIO_PROCESSING_BEAMFORMER_MOCK_BEAMFORMER_H_ - -#include - -#include "modules/audio_processing/beamformer/nonlinear_beamformer.h" -#include "test/gmock.h" - -namespace webrtc { - -class MockNonlinearBeamformer : public NonlinearBeamformer { - public: - MockNonlinearBeamformer(const std::vector& array_geometry, - size_t num_postfilter_channels) - : NonlinearBeamformer(array_geometry, num_postfilter_channels) {} - - MockNonlinearBeamformer(const std::vector& array_geometry) - : NonlinearBeamformer(array_geometry, 1u) {} - - MOCK_METHOD2(Initialize, void(int chunk_size_ms, int sample_rate_hz)); - MOCK_METHOD1(AnalyzeChunk, void(const ChannelBuffer& data)); - MOCK_METHOD1(PostFilter, void(ChannelBuffer* data)); - MOCK_METHOD1(IsInBeam, bool(const SphericalPointf& spherical_point)); - MOCK_METHOD0(is_target_present, bool()); -}; - -} // namespace webrtc - -#endif // MODULES_AUDIO_PROCESSING_BEAMFORMER_MOCK_BEAMFORMER_H_ diff --git a/modules/audio_processing/beamformer/nonlinear_beamformer.cc b/modules/audio_processing/beamformer/nonlinear_beamformer.cc deleted file mode 100644 index 89dc44c8b8..0000000000 --- a/modules/audio_processing/beamformer/nonlinear_beamformer.cc +++ /dev/null @@ -1,587 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#define _USE_MATH_DEFINES - -#include "modules/audio_processing/beamformer/nonlinear_beamformer.h" - -#include -#include -#include -#include - -#include "common_audio/window_generator.h" -#include "modules/audio_processing/beamformer/covariance_matrix_generator.h" -#include "rtc_base/arraysize.h" - -namespace webrtc { -namespace { - -// Alpha for the Kaiser Bessel Derived window. -const float kKbdAlpha = 1.5f; - -const float kSpeedOfSoundMeterSeconds = 343; - -// The minimum separation in radians between the target direction and an -// interferer scenario. -const float kMinAwayRadians = 0.2f; - -// The separation between the target direction and the closest interferer -// scenario is proportional to this constant. -const float kAwaySlope = 0.008f; - -// When calculating the interference covariance matrix, this is the weight for -// the weighted average between the uniform covariance matrix and the angled -// covariance matrix. -// Rpsi = Rpsi_angled * kBalance + Rpsi_uniform * (1 - kBalance) -const float kBalance = 0.95f; - -// Alpha coefficients for mask smoothing. -const float kMaskTimeSmoothAlpha = 0.2f; -const float kMaskFrequencySmoothAlpha = 0.6f; - -// The average mask is computed from masks in this mid-frequency range. If these -// ranges are changed |kMaskQuantile| might need to be adjusted. -const int kLowMeanStartHz = 200; -const int kLowMeanEndHz = 400; - -// Range limiter for subtractive terms in the nominator and denominator of the -// postfilter expression. It handles the scenario mismatch between the true and -// model sources (target and interference). -const float kCutOffConstant = 0.9999f; - -// Quantile of mask values which is used to estimate target presence. -const float kMaskQuantile = 0.7f; -// Mask threshold over which the data is considered signal and not interference. -// It has to be updated every time the postfilter calculation is changed -// significantly. -// TODO(aluebs): Write a tool to tune the target threshold automatically based -// on files annotated with target and interference ground truth. -const float kMaskTargetThreshold = 0.01f; -// Time in seconds after which the data is considered interference if the mask -// does not pass |kMaskTargetThreshold|. -const float kHoldTargetSeconds = 0.25f; - -// To compensate for the attenuation this algorithm introduces to the target -// signal. It was estimated empirically from a low-noise low-reverberation -// recording from broadside. -const float kCompensationGain = 2.f; - -// Does conjugate(|norm_mat|) * |mat| * transpose(|norm_mat|). No extra space is -// used; to accomplish this, we compute both multiplications in the same loop. -// The returned norm is clamped to be non-negative. -float Norm(const ComplexMatrix& mat, - const ComplexMatrix& norm_mat) { - RTC_CHECK_EQ(1, norm_mat.num_rows()); - RTC_CHECK_EQ(norm_mat.num_columns(), mat.num_rows()); - RTC_CHECK_EQ(norm_mat.num_columns(), mat.num_columns()); - - complex first_product = complex(0.f, 0.f); - complex second_product = complex(0.f, 0.f); - - const complex* const* mat_els = mat.elements(); - const complex* const* norm_mat_els = norm_mat.elements(); - - for (size_t i = 0; i < norm_mat.num_columns(); ++i) { - for (size_t j = 0; j < norm_mat.num_columns(); ++j) { - first_product += conj(norm_mat_els[0][j]) * mat_els[j][i]; - } - second_product += first_product * norm_mat_els[0][i]; - first_product = 0.f; - } - return std::max(second_product.real(), 0.f); -} - -// Does conjugate(|lhs|) * |rhs| for row vectors |lhs| and |rhs|. -complex ConjugateDotProduct(const ComplexMatrix& lhs, - const ComplexMatrix& rhs) { - RTC_CHECK_EQ(1, lhs.num_rows()); - RTC_CHECK_EQ(1, rhs.num_rows()); - RTC_CHECK_EQ(lhs.num_columns(), rhs.num_columns()); - - const complex* const* lhs_elements = lhs.elements(); - const complex* const* rhs_elements = rhs.elements(); - - complex result = complex(0.f, 0.f); - for (size_t i = 0; i < lhs.num_columns(); ++i) { - result += conj(lhs_elements[0][i]) * rhs_elements[0][i]; - } - - return result; -} - -// Works for positive numbers only. -size_t Round(float x) { - return static_cast(std::floor(x + 0.5f)); -} - -// Calculates the sum of squares of a complex matrix. -float SumSquares(const ComplexMatrix& mat) { - float sum_squares = 0.f; - const complex* const* mat_els = mat.elements(); - for (size_t i = 0; i < mat.num_rows(); ++i) { - for (size_t j = 0; j < mat.num_columns(); ++j) { - float abs_value = std::abs(mat_els[i][j]); - sum_squares += abs_value * abs_value; - } - } - return sum_squares; -} - -// Does |out| = |in|.' * conj(|in|) for row vector |in|. -void TransposedConjugatedProduct(const ComplexMatrix& in, - ComplexMatrix* out) { - RTC_CHECK_EQ(1, in.num_rows()); - RTC_CHECK_EQ(out->num_rows(), in.num_columns()); - RTC_CHECK_EQ(out->num_columns(), in.num_columns()); - const complex* in_elements = in.elements()[0]; - complex* const* out_elements = out->elements(); - for (size_t i = 0; i < out->num_rows(); ++i) { - for (size_t j = 0; j < out->num_columns(); ++j) { - out_elements[i][j] = in_elements[i] * conj(in_elements[j]); - } - } -} - -std::vector GetCenteredArray(std::vector array_geometry) { - for (size_t dim = 0; dim < 3; ++dim) { - float center = 0.f; - for (size_t i = 0; i < array_geometry.size(); ++i) { - center += array_geometry[i].c[dim]; - } - center /= array_geometry.size(); - for (size_t i = 0; i < array_geometry.size(); ++i) { - array_geometry[i].c[dim] -= center; - } - } - return array_geometry; -} - -} // namespace - -const float NonlinearBeamformer::kHalfBeamWidthRadians = DegreesToRadians(20.f); - -// static -const size_t NonlinearBeamformer::kNumFreqBins; - -PostFilterTransform::PostFilterTransform(size_t num_channels, - size_t chunk_length, - float* window, - size_t fft_size) - : transform_(num_channels, - num_channels, - chunk_length, - window, - fft_size, - fft_size / 2, - this), - num_freq_bins_(fft_size / 2 + 1) {} - -void PostFilterTransform::ProcessChunk(float* const* data, float* final_mask) { - final_mask_ = final_mask; - transform_.ProcessChunk(data, data); -} - -void PostFilterTransform::ProcessAudioBlock(const complex* const* input, - size_t num_input_channels, - size_t num_freq_bins, - size_t num_output_channels, - complex* const* output) { - RTC_DCHECK_EQ(num_freq_bins_, num_freq_bins); - RTC_DCHECK_EQ(num_input_channels, num_output_channels); - - for (size_t ch = 0; ch < num_input_channels; ++ch) { - for (size_t f_ix = 0; f_ix < num_freq_bins_; ++f_ix) { - output[ch][f_ix] = - kCompensationGain * final_mask_[f_ix] * input[ch][f_ix]; - } - } -} - -NonlinearBeamformer::NonlinearBeamformer( - const std::vector& array_geometry, - size_t num_postfilter_channels, - SphericalPointf target_direction) - : num_input_channels_(array_geometry.size()), - num_postfilter_channels_(num_postfilter_channels), - array_geometry_(GetCenteredArray(array_geometry)), - array_normal_(GetArrayNormalIfExists(array_geometry)), - min_mic_spacing_(GetMinimumSpacing(array_geometry)), - target_angle_radians_(target_direction.azimuth()), - away_radians_(std::min( - static_cast(M_PI), - std::max(kMinAwayRadians, - kAwaySlope * static_cast(M_PI) / min_mic_spacing_))) { - WindowGenerator::KaiserBesselDerived(kKbdAlpha, kFftSize, window_); -} - -NonlinearBeamformer::~NonlinearBeamformer() = default; - -void NonlinearBeamformer::Initialize(int chunk_size_ms, int sample_rate_hz) { - chunk_length_ = - static_cast(sample_rate_hz / (1000.f / chunk_size_ms)); - sample_rate_hz_ = sample_rate_hz; - - high_pass_postfilter_mask_ = 1.f; - is_target_present_ = false; - hold_target_blocks_ = kHoldTargetSeconds * 2 * sample_rate_hz / kFftSize; - interference_blocks_count_ = hold_target_blocks_; - - process_transform_.reset(new LappedTransform(num_input_channels_, 0u, - chunk_length_, window_, kFftSize, - kFftSize / 2, this)); - postfilter_transform_.reset(new PostFilterTransform( - num_postfilter_channels_, chunk_length_, window_, kFftSize)); - const float wave_number_step = - (2.f * M_PI * sample_rate_hz_) / (kFftSize * kSpeedOfSoundMeterSeconds); - for (size_t i = 0; i < kNumFreqBins; ++i) { - time_smooth_mask_[i] = 1.f; - final_mask_[i] = 1.f; - wave_numbers_[i] = i * wave_number_step; - } - - InitLowFrequencyCorrectionRanges(); - InitDiffuseCovMats(); - AimAt(SphericalPointf(target_angle_radians_, 0.f, 1.f)); -} - -// These bin indexes determine the regions over which a mean is taken. This is -// applied as a constant value over the adjacent end "frequency correction" -// regions. -// -// low_mean_start_bin_ high_mean_start_bin_ -// v v constant -// |----------------|--------|----------------|-------|----------------| -// constant ^ ^ -// low_mean_end_bin_ high_mean_end_bin_ -// -void NonlinearBeamformer::InitLowFrequencyCorrectionRanges() { - low_mean_start_bin_ = - Round(static_cast(kLowMeanStartHz) * kFftSize / sample_rate_hz_); - low_mean_end_bin_ = - Round(static_cast(kLowMeanEndHz) * kFftSize / sample_rate_hz_); - - RTC_DCHECK_GT(low_mean_start_bin_, 0U); - RTC_DCHECK_LT(low_mean_start_bin_, low_mean_end_bin_); -} - -void NonlinearBeamformer::InitHighFrequencyCorrectionRanges() { - const float kAliasingFreqHz = - kSpeedOfSoundMeterSeconds / - (min_mic_spacing_ * (1.f + std::abs(std::cos(target_angle_radians_)))); - const float kHighMeanStartHz = - std::min(0.5f * kAliasingFreqHz, sample_rate_hz_ / 2.f); - const float kHighMeanEndHz = - std::min(0.75f * kAliasingFreqHz, sample_rate_hz_ / 2.f); - high_mean_start_bin_ = Round(kHighMeanStartHz * kFftSize / sample_rate_hz_); - high_mean_end_bin_ = Round(kHighMeanEndHz * kFftSize / sample_rate_hz_); - - RTC_DCHECK_LT(low_mean_end_bin_, high_mean_end_bin_); - RTC_DCHECK_LT(high_mean_start_bin_, high_mean_end_bin_); - RTC_DCHECK_LT(high_mean_end_bin_, kNumFreqBins - 1); -} - -void NonlinearBeamformer::InitInterfAngles() { - interf_angles_radians_.clear(); - const Point target_direction = AzimuthToPoint(target_angle_radians_); - const Point clockwise_interf_direction = - AzimuthToPoint(target_angle_radians_ - away_radians_); - if (!array_normal_ || - DotProduct(*array_normal_, target_direction) * - DotProduct(*array_normal_, clockwise_interf_direction) >= - 0.f) { - // The target and clockwise interferer are in the same half-plane defined - // by the array. - interf_angles_radians_.push_back(target_angle_radians_ - away_radians_); - } else { - // Otherwise, the interferer will begin reflecting back at the target. - // Instead rotate it away 180 degrees. - interf_angles_radians_.push_back(target_angle_radians_ - away_radians_ + - M_PI); - } - const Point counterclock_interf_direction = - AzimuthToPoint(target_angle_radians_ + away_radians_); - if (!array_normal_ || - DotProduct(*array_normal_, target_direction) * - DotProduct(*array_normal_, counterclock_interf_direction) >= - 0.f) { - // The target and counter-clockwise interferer are in the same half-plane - // defined by the array. - interf_angles_radians_.push_back(target_angle_radians_ + away_radians_); - } else { - // Otherwise, the interferer will begin reflecting back at the target. - // Instead rotate it away 180 degrees. - interf_angles_radians_.push_back(target_angle_radians_ + away_radians_ - - M_PI); - } -} - -void NonlinearBeamformer::InitDelaySumMasks() { - for (size_t f_ix = 0; f_ix < kNumFreqBins; ++f_ix) { - delay_sum_masks_[f_ix].Resize(1, num_input_channels_); - CovarianceMatrixGenerator::PhaseAlignmentMasks( - f_ix, kFftSize, sample_rate_hz_, kSpeedOfSoundMeterSeconds, - array_geometry_, target_angle_radians_, &delay_sum_masks_[f_ix]); - - complex_f norm_factor = sqrt( - ConjugateDotProduct(delay_sum_masks_[f_ix], delay_sum_masks_[f_ix])); - delay_sum_masks_[f_ix].Scale(1.f / norm_factor); - } -} - -void NonlinearBeamformer::InitTargetCovMats() { - for (size_t i = 0; i < kNumFreqBins; ++i) { - target_cov_mats_[i].Resize(num_input_channels_, num_input_channels_); - TransposedConjugatedProduct(delay_sum_masks_[i], &target_cov_mats_[i]); - } -} - -void NonlinearBeamformer::InitDiffuseCovMats() { - for (size_t i = 0; i < kNumFreqBins; ++i) { - uniform_cov_mat_[i].Resize(num_input_channels_, num_input_channels_); - CovarianceMatrixGenerator::UniformCovarianceMatrix( - wave_numbers_[i], array_geometry_, &uniform_cov_mat_[i]); - complex_f normalization_factor = uniform_cov_mat_[i].elements()[0][0]; - uniform_cov_mat_[i].Scale(1.f / normalization_factor); - uniform_cov_mat_[i].Scale(1 - kBalance); - } -} - -void NonlinearBeamformer::InitInterfCovMats() { - for (size_t i = 0; i < kNumFreqBins; ++i) { - interf_cov_mats_[i].clear(); - for (size_t j = 0; j < interf_angles_radians_.size(); ++j) { - interf_cov_mats_[i].push_back(std::unique_ptr( - new ComplexMatrixF(num_input_channels_, num_input_channels_))); - ComplexMatrixF angled_cov_mat(num_input_channels_, num_input_channels_); - CovarianceMatrixGenerator::AngledCovarianceMatrix( - kSpeedOfSoundMeterSeconds, interf_angles_radians_[j], i, kFftSize, - kNumFreqBins, sample_rate_hz_, array_geometry_, &angled_cov_mat); - // Normalize matrices before averaging them. - complex_f normalization_factor = angled_cov_mat.elements()[0][0]; - angled_cov_mat.Scale(1.f / normalization_factor); - // Weighted average of matrices. - angled_cov_mat.Scale(kBalance); - interf_cov_mats_[i][j]->Add(uniform_cov_mat_[i], angled_cov_mat); - } - } -} - -void NonlinearBeamformer::NormalizeCovMats() { - for (size_t i = 0; i < kNumFreqBins; ++i) { - rxiws_[i] = Norm(target_cov_mats_[i], delay_sum_masks_[i]); - rpsiws_[i].clear(); - for (size_t j = 0; j < interf_angles_radians_.size(); ++j) { - rpsiws_[i].push_back(Norm(*interf_cov_mats_[i][j], delay_sum_masks_[i])); - } - } -} - -void NonlinearBeamformer::AnalyzeChunk(const ChannelBuffer& data) { - RTC_DCHECK_EQ(data.num_channels(), num_input_channels_); - RTC_DCHECK_EQ(data.num_frames_per_band(), chunk_length_); - - old_high_pass_mask_ = high_pass_postfilter_mask_; - process_transform_->ProcessChunk(data.channels(0), nullptr); -} - -void NonlinearBeamformer::PostFilter(ChannelBuffer* data) { - RTC_DCHECK_EQ(data->num_frames_per_band(), chunk_length_); - // TODO(aluebs): Change to RTC_CHECK_EQ once the ChannelBuffer is updated. - RTC_DCHECK_GE(data->num_channels(), num_postfilter_channels_); - - postfilter_transform_->ProcessChunk(data->channels(0), final_mask_); - - // Ramp up/down for smoothing is needed in order to avoid discontinuities in - // the transitions between 10 ms frames. - const float ramp_increment = - (high_pass_postfilter_mask_ - old_high_pass_mask_) / - data->num_frames_per_band(); - for (size_t i = 1; i < data->num_bands(); ++i) { - float smoothed_mask = old_high_pass_mask_; - for (size_t j = 0; j < data->num_frames_per_band(); ++j) { - smoothed_mask += ramp_increment; - for (size_t k = 0; k < num_postfilter_channels_; ++k) { - data->channels(i)[k][j] *= smoothed_mask; - } - } - } -} - -void NonlinearBeamformer::AimAt(const SphericalPointf& target_direction) { - target_angle_radians_ = target_direction.azimuth(); - InitHighFrequencyCorrectionRanges(); - InitInterfAngles(); - InitDelaySumMasks(); - InitTargetCovMats(); - InitInterfCovMats(); - NormalizeCovMats(); -} - -bool NonlinearBeamformer::IsInBeam(const SphericalPointf& spherical_point) { - // If more than half-beamwidth degrees away from the beam's center, - // you are out of the beam. - return fabs(spherical_point.azimuth() - target_angle_radians_) < - kHalfBeamWidthRadians; -} - -bool NonlinearBeamformer::is_target_present() { - return is_target_present_; -} - -void NonlinearBeamformer::ProcessAudioBlock(const complex_f* const* input, - size_t num_input_channels, - size_t num_freq_bins, - size_t num_output_channels, - complex_f* const* output) { - RTC_CHECK_EQ(kNumFreqBins, num_freq_bins); - RTC_CHECK_EQ(num_input_channels_, num_input_channels); - RTC_CHECK_EQ(0, num_output_channels); - - // Calculating the post-filter masks. Note that we need two for each - // frequency bin to account for the positive and negative interferer - // angle. - for (size_t i = low_mean_start_bin_; i <= high_mean_end_bin_; ++i) { - eig_m_.CopyFromColumn(input, i, num_input_channels_); - float eig_m_norm_factor = std::sqrt(SumSquares(eig_m_)); - if (eig_m_norm_factor != 0.f) { - eig_m_.Scale(1.f / eig_m_norm_factor); - } - - float rxim = Norm(target_cov_mats_[i], eig_m_); - float ratio_rxiw_rxim = 0.f; - if (rxim > 0.f) { - ratio_rxiw_rxim = rxiws_[i] / rxim; - } - - complex_f rmw = abs(ConjugateDotProduct(delay_sum_masks_[i], eig_m_)); - rmw *= rmw; - float rmw_r = rmw.real(); - - new_mask_[i] = CalculatePostfilterMask( - *interf_cov_mats_[i][0], rpsiws_[i][0], ratio_rxiw_rxim, rmw_r); - for (size_t j = 1; j < interf_angles_radians_.size(); ++j) { - float tmp_mask = CalculatePostfilterMask( - *interf_cov_mats_[i][j], rpsiws_[i][j], ratio_rxiw_rxim, rmw_r); - if (tmp_mask < new_mask_[i]) { - new_mask_[i] = tmp_mask; - } - } - } - - ApplyMaskTimeSmoothing(); - EstimateTargetPresence(); - ApplyLowFrequencyCorrection(); - ApplyHighFrequencyCorrection(); - ApplyMaskFrequencySmoothing(); -} - -float NonlinearBeamformer::CalculatePostfilterMask( - const ComplexMatrixF& interf_cov_mat, - float rpsiw, - float ratio_rxiw_rxim, - float rmw_r) { - float rpsim = Norm(interf_cov_mat, eig_m_); - - float ratio = 0.f; - if (rpsim > 0.f) { - ratio = rpsiw / rpsim; - } - - float numerator = 1.f - kCutOffConstant; - if (rmw_r > 0.f) { - numerator = 1.f - std::min(kCutOffConstant, ratio / rmw_r); - } - - float denominator = 1.f - kCutOffConstant; - if (ratio_rxiw_rxim > 0.f) { - denominator = 1.f - std::min(kCutOffConstant, ratio / ratio_rxiw_rxim); - } - - return numerator / denominator; -} - -// Smooth new_mask_ into time_smooth_mask_. -void NonlinearBeamformer::ApplyMaskTimeSmoothing() { - for (size_t i = low_mean_start_bin_; i <= high_mean_end_bin_; ++i) { - time_smooth_mask_[i] = kMaskTimeSmoothAlpha * new_mask_[i] + - (1 - kMaskTimeSmoothAlpha) * time_smooth_mask_[i]; - } -} - -// Copy time_smooth_mask_ to final_mask_ and smooth over frequency. -void NonlinearBeamformer::ApplyMaskFrequencySmoothing() { - // Smooth over frequency in both directions. The "frequency correction" - // regions have constant value, but we enter them to smooth over the jump - // that exists at the boundary. However, this does mean when smoothing "away" - // from the region that we only need to use the last element. - // - // Upward smoothing: - // low_mean_start_bin_ - // v - // |------|------------|------| - // ^------------------>^ - // - // Downward smoothing: - // high_mean_end_bin_ - // v - // |------|------------|------| - // ^<------------------^ - std::copy(time_smooth_mask_, time_smooth_mask_ + kNumFreqBins, final_mask_); - for (size_t i = low_mean_start_bin_; i < kNumFreqBins; ++i) { - final_mask_[i] = kMaskFrequencySmoothAlpha * final_mask_[i] + - (1 - kMaskFrequencySmoothAlpha) * final_mask_[i - 1]; - } - for (size_t i = high_mean_end_bin_ + 1; i > 0; --i) { - final_mask_[i - 1] = kMaskFrequencySmoothAlpha * final_mask_[i - 1] + - (1 - kMaskFrequencySmoothAlpha) * final_mask_[i]; - } -} - -// Apply low frequency correction to time_smooth_mask_. -void NonlinearBeamformer::ApplyLowFrequencyCorrection() { - const float low_frequency_mask = - MaskRangeMean(low_mean_start_bin_, low_mean_end_bin_ + 1); - std::fill(time_smooth_mask_, time_smooth_mask_ + low_mean_start_bin_, - low_frequency_mask); -} - -// Apply high frequency correction to time_smooth_mask_. Update -// high_pass_postfilter_mask_ to use for the high frequency time-domain bands. -void NonlinearBeamformer::ApplyHighFrequencyCorrection() { - high_pass_postfilter_mask_ = - MaskRangeMean(high_mean_start_bin_, high_mean_end_bin_ + 1); - std::fill(time_smooth_mask_ + high_mean_end_bin_ + 1, - time_smooth_mask_ + kNumFreqBins, high_pass_postfilter_mask_); -} - -// Compute mean over the given range of time_smooth_mask_, [first, last). -float NonlinearBeamformer::MaskRangeMean(size_t first, size_t last) { - RTC_DCHECK_GT(last, first); - const float sum = - std::accumulate(time_smooth_mask_ + first, time_smooth_mask_ + last, 0.f); - return sum / (last - first); -} - -void NonlinearBeamformer::EstimateTargetPresence() { - const size_t quantile = static_cast( - (high_mean_end_bin_ - low_mean_start_bin_) * kMaskQuantile + - low_mean_start_bin_); - std::nth_element(new_mask_ + low_mean_start_bin_, new_mask_ + quantile, - new_mask_ + high_mean_end_bin_ + 1); - if (new_mask_[quantile] > kMaskTargetThreshold) { - is_target_present_ = true; - interference_blocks_count_ = 0; - } else { - is_target_present_ = interference_blocks_count_++ < hold_target_blocks_; - } -} - -} // namespace webrtc diff --git a/modules/audio_processing/beamformer/nonlinear_beamformer.h b/modules/audio_processing/beamformer/nonlinear_beamformer.h deleted file mode 100644 index b58165f0ad..0000000000 --- a/modules/audio_processing/beamformer/nonlinear_beamformer.h +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_AUDIO_PROCESSING_BEAMFORMER_NONLINEAR_BEAMFORMER_H_ -#define MODULES_AUDIO_PROCESSING_BEAMFORMER_NONLINEAR_BEAMFORMER_H_ - -// MSVC++ requires this to be set before any other includes to get M_PI. -#ifndef _USE_MATH_DEFINES -#define _USE_MATH_DEFINES -#endif - -#include - -#include -#include - -#include "common_audio/channel_buffer.h" -#include "common_audio/lapped_transform.h" -#include "modules/audio_processing/beamformer/array_util.h" -#include "modules/audio_processing/beamformer/complex_matrix.h" - -namespace webrtc { - -class PostFilterTransform : public LappedTransform::Callback { - public: - PostFilterTransform(size_t num_channels, - size_t chunk_length, - float* window, - size_t fft_size); - - void ProcessChunk(float* const* data, float* final_mask); - - protected: - void ProcessAudioBlock(const complex* const* input, - size_t num_input_channels, - size_t num_freq_bins, - size_t num_output_channels, - complex* const* output) override; - - private: - LappedTransform transform_; - const size_t num_freq_bins_; - float* final_mask_; -}; - -// Enhances sound sources coming directly in front of a uniform linear array -// and suppresses sound sources coming from all other directions. Operates on -// multichannel signals and produces single-channel output. -// -// The implemented nonlinear postfilter algorithm taken from "A Robust Nonlinear -// Beamforming Postprocessor" by Bastiaan Kleijn. -class NonlinearBeamformer : public LappedTransform::Callback { - public: - static const float kHalfBeamWidthRadians; - - explicit NonlinearBeamformer( - const std::vector& array_geometry, - size_t num_postfilter_channels = 1u, - SphericalPointf target_direction = - SphericalPointf(static_cast(M_PI) / 2.f, 0.f, 1.f)); - ~NonlinearBeamformer() override; - - // Sample rate corresponds to the lower band. - // Needs to be called before the NonlinearBeamformer can be used. - virtual void Initialize(int chunk_size_ms, int sample_rate_hz); - - // Analyzes one time-domain chunk of audio. The audio is expected to be split - // into frequency bands inside the ChannelBuffer. The number of frames and - // channels must correspond to the constructor parameters. - virtual void AnalyzeChunk(const ChannelBuffer& data); - - // Applies the postfilter mask to one chunk of audio. The audio is expected to - // be split into frequency bands inside the ChannelBuffer. The number of - // frames and channels must correspond to the constructor parameters. - virtual void PostFilter(ChannelBuffer* data); - - virtual void AimAt(const SphericalPointf& target_direction); - - virtual bool IsInBeam(const SphericalPointf& spherical_point); - - // After processing each block |is_target_present_| is set to true if the - // target signal es present and to false otherwise. This methods can be called - // to know if the data is target signal or interference and process it - // accordingly. - virtual bool is_target_present(); - - protected: - // Process one frequency-domain block of audio. This is where the fun - // happens. Implements LappedTransform::Callback. - void ProcessAudioBlock(const complex* const* input, - size_t num_input_channels, - size_t num_freq_bins, - size_t num_output_channels, - complex* const* output) override; - - private: - FRIEND_TEST_ALL_PREFIXES(NonlinearBeamformerTest, - InterfAnglesTakeAmbiguityIntoAccount); - - typedef Matrix MatrixF; - typedef ComplexMatrix ComplexMatrixF; - typedef complex complex_f; - - void InitLowFrequencyCorrectionRanges(); - void InitHighFrequencyCorrectionRanges(); - void InitInterfAngles(); - void InitDelaySumMasks(); - void InitTargetCovMats(); - void InitDiffuseCovMats(); - void InitInterfCovMats(); - void NormalizeCovMats(); - - // Calculates postfilter masks that minimize the mean squared error of our - // estimation of the desired signal. - float CalculatePostfilterMask(const ComplexMatrixF& interf_cov_mat, - float rpsiw, - float ratio_rxiw_rxim, - float rmxi_r); - - // Prevents the postfilter masks from degenerating too quickly (a cause of - // musical noise). - void ApplyMaskTimeSmoothing(); - void ApplyMaskFrequencySmoothing(); - - // The postfilter masks are unreliable at low frequencies. Calculates a better - // mask by averaging mid-low frequency values. - void ApplyLowFrequencyCorrection(); - - // Postfilter masks are also unreliable at high frequencies. Average mid-high - // frequency masks to calculate a single mask per block which can be applied - // in the time-domain. Further, we average these block-masks over a chunk, - // resulting in one postfilter mask per audio chunk. This allows us to skip - // both transforming and blocking the high-frequency signal. - void ApplyHighFrequencyCorrection(); - - // Compute the means needed for the above frequency correction. - float MaskRangeMean(size_t start_bin, size_t end_bin); - - // Applies post-filter mask to |input| and store in |output|. - void ApplyPostFilter(const complex_f* input, complex_f* output); - - void EstimateTargetPresence(); - - static const size_t kFftSize = 256; - static const size_t kNumFreqBins = kFftSize / 2 + 1; - - // Deals with the fft transform and blocking. - size_t chunk_length_; - std::unique_ptr process_transform_; - std::unique_ptr postfilter_transform_; - float window_[kFftSize]; - - // Parameters exposed to the user. - const size_t num_input_channels_; - const size_t num_postfilter_channels_; - int sample_rate_hz_; - - const std::vector array_geometry_; - // The normal direction of the array if it has one and it is in the xy-plane. - const absl::optional array_normal_; - - // Minimum spacing between microphone pairs. - const float min_mic_spacing_; - - // Calculated based on user-input and constants in the .cc file. - size_t low_mean_start_bin_; - size_t low_mean_end_bin_; - size_t high_mean_start_bin_; - size_t high_mean_end_bin_; - - // Quickly varying mask updated every block. - float new_mask_[kNumFreqBins]; - // Time smoothed mask. - float time_smooth_mask_[kNumFreqBins]; - // Time and frequency smoothed mask. - float final_mask_[kNumFreqBins]; - - float target_angle_radians_; - // Angles of the interferer scenarios. - std::vector interf_angles_radians_; - // The angle between the target and the interferer scenarios. - const float away_radians_; - - // Array of length |kNumFreqBins|, Matrix of size |1| x |num_channels_|. - ComplexMatrixF delay_sum_masks_[kNumFreqBins]; - - // Arrays of length |kNumFreqBins|, Matrix of size |num_input_channels_| x - // |num_input_channels_|. - ComplexMatrixF target_cov_mats_[kNumFreqBins]; - ComplexMatrixF uniform_cov_mat_[kNumFreqBins]; - // Array of length |kNumFreqBins|, Matrix of size |num_input_channels_| x - // |num_input_channels_|. The vector has a size equal to the number of - // interferer scenarios. - std::vector> interf_cov_mats_[kNumFreqBins]; - - // Of length |kNumFreqBins|. - float wave_numbers_[kNumFreqBins]; - - // Preallocated for ProcessAudioBlock() - // Of length |kNumFreqBins|. - float rxiws_[kNumFreqBins]; - // The vector has a size equal to the number of interferer scenarios. - std::vector rpsiws_[kNumFreqBins]; - - // The microphone normalization factor. - ComplexMatrixF eig_m_; - - // For processing the high-frequency input signal. - float high_pass_postfilter_mask_; - float old_high_pass_mask_; - - // True when the target signal is present. - bool is_target_present_; - // Number of blocks after which the data is considered interference if the - // mask does not pass |kMaskSignalThreshold|. - size_t hold_target_blocks_; - // Number of blocks since the last mask that passed |kMaskSignalThreshold|. - size_t interference_blocks_count_; -}; - -} // namespace webrtc - -#endif // MODULES_AUDIO_PROCESSING_BEAMFORMER_NONLINEAR_BEAMFORMER_H_ diff --git a/modules/audio_processing/beamformer/nonlinear_beamformer_test.cc b/modules/audio_processing/beamformer/nonlinear_beamformer_test.cc deleted file mode 100644 index 10f5cc022e..0000000000 --- a/modules/audio_processing/beamformer/nonlinear_beamformer_test.cc +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include - -#include "common_audio/channel_buffer.h" -#include "common_audio/wav_file.h" -#include "modules/audio_processing/beamformer/nonlinear_beamformer.h" -#include "modules/audio_processing/test/test_utils.h" -#include "rtc_base/checks.h" -#include "rtc_base/flags.h" -#include "rtc_base/format_macros.h" - -DEFINE_string(i, "", "The name of the input file to read from."); -DEFINE_string(o, "out.wav", "Name of the output file to write to."); -DEFINE_string(mic_positions, - "", - "Space delimited cartesian coordinates of microphones in meters. " - "The coordinates of each point are contiguous. " - "For a two element array: \"x1 y1 z1 x2 y2 z2\""); -DEFINE_bool(help, false, "Prints this message."); - -namespace webrtc { -namespace { - -const int kChunksPerSecond = 100; -const int kChunkSizeMs = 1000 / kChunksPerSecond; - -const char kUsage[] = - "Command-line tool to run beamforming on WAV files. The signal is passed\n" - "in as a single band, unlike the audio processing interface which splits\n" - "signals into multiple bands.\n"; - -std::vector ParseArrayGeometry(const std::string& mic_positions) { - const std::vector values = ParseList(mic_positions); - const size_t num_mics = - rtc::CheckedDivExact(values.size(), static_cast(3)); - RTC_CHECK_GT(num_mics, 0) << "mic_positions is not large enough."; - - std::vector result; - result.reserve(num_mics); - for (size_t i = 0; i < values.size(); i += 3) { - result.push_back(Point(values[i + 0], values[i + 1], values[i + 2])); - } - - return result; -} - -std::vector ParseArrayGeometry(const std::string& mic_positions, - size_t num_mics) { - std::vector result = ParseArrayGeometry(mic_positions); - RTC_CHECK_EQ(result.size(), num_mics) - << "Could not parse mic_positions or incorrect number of points."; - return result; -} - -} // namespace - -int main(int argc, char* argv[]) { - if (rtc::FlagList::SetFlagsFromCommandLine(&argc, argv, true) || FLAG_help || - argc != 1) { - printf("%s", kUsage); - if (FLAG_help) { - rtc::FlagList::Print(nullptr, false); - return 0; - } - return 1; - } - - WavReader in_file(FLAG_i); - WavWriter out_file(FLAG_o, in_file.sample_rate(), in_file.num_channels()); - - const size_t num_mics = in_file.num_channels(); - const std::vector array_geometry = - ParseArrayGeometry(FLAG_mic_positions, num_mics); - RTC_CHECK_EQ(array_geometry.size(), num_mics); - - NonlinearBeamformer bf(array_geometry, array_geometry.size()); - bf.Initialize(kChunkSizeMs, in_file.sample_rate()); - - printf("Input file: %s\nChannels: %" PRIuS ", Sample rate: %d Hz\n\n", FLAG_i, - in_file.num_channels(), in_file.sample_rate()); - printf("Output file: %s\nChannels: %" PRIuS ", Sample rate: %d Hz\n\n", - FLAG_o, out_file.num_channels(), out_file.sample_rate()); - - ChannelBuffer buf( - rtc::CheckedDivExact(in_file.sample_rate(), kChunksPerSecond), - in_file.num_channels()); - - std::vector interleaved(buf.size()); - while (in_file.ReadSamples(interleaved.size(), &interleaved[0]) == - interleaved.size()) { - FloatS16ToFloat(&interleaved[0], interleaved.size(), &interleaved[0]); - Deinterleave(&interleaved[0], buf.num_frames(), buf.num_channels(), - buf.channels()); - - bf.AnalyzeChunk(buf); - bf.PostFilter(&buf); - - Interleave(buf.channels(), buf.num_frames(), buf.num_channels(), - &interleaved[0]); - FloatToFloatS16(&interleaved[0], interleaved.size(), &interleaved[0]); - out_file.WriteSamples(&interleaved[0], interleaved.size()); - } - - return 0; -} - -} // namespace webrtc - -int main(int argc, char* argv[]) { - return webrtc::main(argc, argv); -} diff --git a/modules/audio_processing/beamformer/nonlinear_beamformer_unittest.cc b/modules/audio_processing/beamformer/nonlinear_beamformer_unittest.cc deleted file mode 100644 index 7d81b88af7..0000000000 --- a/modules/audio_processing/beamformer/nonlinear_beamformer_unittest.cc +++ /dev/null @@ -1,365 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -// MSVC++ requires this to be set before any other includes to get M_PI. -#define _USE_MATH_DEFINES - -#include "modules/audio_processing/beamformer/nonlinear_beamformer.h" - -#include - -#include "api/array_view.h" -#include "modules/audio_processing/audio_buffer.h" -#include "modules/audio_processing/test/audio_buffer_tools.h" -#include "modules/audio_processing/test/bitexactness_tools.h" -#include "test/gtest.h" - -namespace webrtc { -namespace { - -const int kChunkSizeMs = 10; -const int kSampleRateHz = 16000; - -SphericalPointf AzimuthToSphericalPoint(float azimuth_radians) { - return SphericalPointf(azimuth_radians, 0.f, 1.f); -} - -void Verify(NonlinearBeamformer* bf, float target_azimuth_radians) { - EXPECT_TRUE(bf->IsInBeam(AzimuthToSphericalPoint(target_azimuth_radians))); - EXPECT_TRUE(bf->IsInBeam(AzimuthToSphericalPoint( - target_azimuth_radians - NonlinearBeamformer::kHalfBeamWidthRadians + - 0.001f))); - EXPECT_TRUE(bf->IsInBeam(AzimuthToSphericalPoint( - target_azimuth_radians + NonlinearBeamformer::kHalfBeamWidthRadians - - 0.001f))); - EXPECT_FALSE(bf->IsInBeam(AzimuthToSphericalPoint( - target_azimuth_radians - NonlinearBeamformer::kHalfBeamWidthRadians - - 0.001f))); - EXPECT_FALSE(bf->IsInBeam(AzimuthToSphericalPoint( - target_azimuth_radians + NonlinearBeamformer::kHalfBeamWidthRadians + - 0.001f))); -} - -void AimAndVerify(NonlinearBeamformer* bf, float target_azimuth_radians) { - bf->AimAt(AzimuthToSphericalPoint(target_azimuth_radians)); - Verify(bf, target_azimuth_radians); -} - -// Bitexactness test code. -const size_t kNumFramesToProcess = 1000; - -void ProcessOneFrame(int sample_rate_hz, - AudioBuffer* capture_audio_buffer, - NonlinearBeamformer* beamformer) { - if (sample_rate_hz > AudioProcessing::kSampleRate16kHz) { - capture_audio_buffer->SplitIntoFrequencyBands(); - } - - beamformer->AnalyzeChunk(*capture_audio_buffer->split_data_f()); - capture_audio_buffer->set_num_channels(1); - beamformer->PostFilter(capture_audio_buffer->split_data_f()); - - if (sample_rate_hz > AudioProcessing::kSampleRate16kHz) { - capture_audio_buffer->MergeFrequencyBands(); - } -} - -int BeamformerSampleRate(int sample_rate_hz) { - return (sample_rate_hz > AudioProcessing::kSampleRate16kHz - ? AudioProcessing::kSampleRate16kHz - : sample_rate_hz); -} - -void RunBitExactnessTest(int sample_rate_hz, - const std::vector& array_geometry, - const SphericalPointf& target_direction, - rtc::ArrayView output_reference) { - NonlinearBeamformer beamformer(array_geometry, 1u, target_direction); - beamformer.Initialize(AudioProcessing::kChunkSizeMs, - BeamformerSampleRate(sample_rate_hz)); - - const StreamConfig capture_config(sample_rate_hz, array_geometry.size(), - false); - AudioBuffer capture_buffer( - capture_config.num_frames(), capture_config.num_channels(), - capture_config.num_frames(), capture_config.num_channels(), - capture_config.num_frames()); - test::InputAudioFile capture_file( - test::GetApmCaptureTestVectorFileName(sample_rate_hz)); - std::vector capture_input(capture_config.num_frames() * - capture_config.num_channels()); - for (size_t frame_no = 0u; frame_no < kNumFramesToProcess; ++frame_no) { - ReadFloatSamplesFromStereoFile(capture_config.num_frames(), - capture_config.num_channels(), &capture_file, - capture_input); - - test::CopyVectorToAudioBuffer(capture_config, capture_input, - &capture_buffer); - - ProcessOneFrame(sample_rate_hz, &capture_buffer, &beamformer); - } - - // Extract and verify the test results. - std::vector capture_output; - test::ExtractVectorFromAudioBuffer(capture_config, &capture_buffer, - &capture_output); - - const float kElementErrorBound = 1.f / static_cast(1 << 15); - - // Compare the output with the reference. Only the first values of the output - // from last frame processed are compared in order not having to specify all - // preceeding frames as testvectors. As the algorithm being tested has a - // memory, testing only the last frame implicitly also tests the preceeding - // frames. - EXPECT_TRUE(test::VerifyDeinterleavedArray( - capture_config.num_frames(), capture_config.num_channels(), - output_reference, capture_output, kElementErrorBound)); -} - -// TODO(peah): Add bitexactness tests for scenarios with more than 2 input -// channels. -std::vector CreateArrayGeometry(int variant) { - std::vector array_geometry; - switch (variant) { - case 1: - array_geometry.push_back(Point(-0.025f, 0.f, 0.f)); - array_geometry.push_back(Point(0.025f, 0.f, 0.f)); - break; - case 2: - array_geometry.push_back(Point(-0.035f, 0.f, 0.f)); - array_geometry.push_back(Point(0.035f, 0.f, 0.f)); - break; - case 3: - array_geometry.push_back(Point(-0.5f, 0.f, 0.f)); - array_geometry.push_back(Point(0.5f, 0.f, 0.f)); - break; - default: - RTC_CHECK(false); - } - return array_geometry; -} - -const SphericalPointf TargetDirection1(0.4f * static_cast(M_PI) / 2.f, - 0.f, - 1.f); -const SphericalPointf TargetDirection2(static_cast(M_PI) / 2.f, - 1.f, - 2.f); - -} // namespace - -TEST(NonlinearBeamformerTest, AimingModifiesBeam) { - std::vector array_geometry; - array_geometry.push_back(Point(-0.025f, 0.f, 0.f)); - array_geometry.push_back(Point(0.025f, 0.f, 0.f)); - NonlinearBeamformer bf(array_geometry, 1u); - bf.Initialize(kChunkSizeMs, kSampleRateHz); - // The default constructor parameter sets the target angle to PI / 2. - Verify(&bf, static_cast(M_PI) / 2.f); - AimAndVerify(&bf, static_cast(M_PI) / 3.f); - AimAndVerify(&bf, 3.f * static_cast(M_PI) / 4.f); - AimAndVerify(&bf, static_cast(M_PI) / 6.f); - AimAndVerify(&bf, static_cast(M_PI)); -} - -TEST(NonlinearBeamformerTest, InterfAnglesTakeAmbiguityIntoAccount) { - { - // For linear arrays there is ambiguity. - std::vector array_geometry; - array_geometry.push_back(Point(-0.1f, 0.f, 0.f)); - array_geometry.push_back(Point(0.f, 0.f, 0.f)); - array_geometry.push_back(Point(0.2f, 0.f, 0.f)); - NonlinearBeamformer bf(array_geometry, 1u); - bf.Initialize(kChunkSizeMs, kSampleRateHz); - EXPECT_EQ(2u, bf.interf_angles_radians_.size()); - EXPECT_FLOAT_EQ(M_PI / 2.f - bf.away_radians_, - bf.interf_angles_radians_[0]); - EXPECT_FLOAT_EQ(M_PI / 2.f + bf.away_radians_, - bf.interf_angles_radians_[1]); - bf.AimAt(AzimuthToSphericalPoint(bf.away_radians_ / 2.f)); - EXPECT_EQ(2u, bf.interf_angles_radians_.size()); - EXPECT_FLOAT_EQ(M_PI - bf.away_radians_ / 2.f, - bf.interf_angles_radians_[0]); - EXPECT_FLOAT_EQ(3.f * bf.away_radians_ / 2.f, bf.interf_angles_radians_[1]); - } - { - // For planar arrays with normal in the xy-plane there is ambiguity. - std::vector array_geometry; - array_geometry.push_back(Point(-0.1f, 0.f, 0.f)); - array_geometry.push_back(Point(0.f, 0.f, 0.f)); - array_geometry.push_back(Point(0.2f, 0.f, 0.f)); - array_geometry.push_back(Point(0.1f, 0.f, 0.2f)); - array_geometry.push_back(Point(0.f, 0.f, -0.1f)); - NonlinearBeamformer bf(array_geometry, 1u); - bf.Initialize(kChunkSizeMs, kSampleRateHz); - EXPECT_EQ(2u, bf.interf_angles_radians_.size()); - EXPECT_FLOAT_EQ(M_PI / 2.f - bf.away_radians_, - bf.interf_angles_radians_[0]); - EXPECT_FLOAT_EQ(M_PI / 2.f + bf.away_radians_, - bf.interf_angles_radians_[1]); - bf.AimAt(AzimuthToSphericalPoint(bf.away_radians_ / 2.f)); - EXPECT_EQ(2u, bf.interf_angles_radians_.size()); - EXPECT_FLOAT_EQ(M_PI - bf.away_radians_ / 2.f, - bf.interf_angles_radians_[0]); - EXPECT_FLOAT_EQ(3.f * bf.away_radians_ / 2.f, bf.interf_angles_radians_[1]); - } - { - // For planar arrays with normal not in the xy-plane there is no ambiguity. - std::vector array_geometry; - array_geometry.push_back(Point(0.f, 0.f, 0.f)); - array_geometry.push_back(Point(0.2f, 0.f, 0.f)); - array_geometry.push_back(Point(0.f, 0.1f, -0.2f)); - NonlinearBeamformer bf(array_geometry, 1u); - bf.Initialize(kChunkSizeMs, kSampleRateHz); - EXPECT_EQ(2u, bf.interf_angles_radians_.size()); - EXPECT_FLOAT_EQ(M_PI / 2.f - bf.away_radians_, - bf.interf_angles_radians_[0]); - EXPECT_FLOAT_EQ(M_PI / 2.f + bf.away_radians_, - bf.interf_angles_radians_[1]); - bf.AimAt(AzimuthToSphericalPoint(bf.away_radians_ / 2.f)); - EXPECT_EQ(2u, bf.interf_angles_radians_.size()); - EXPECT_FLOAT_EQ(-bf.away_radians_ / 2.f, bf.interf_angles_radians_[0]); - EXPECT_FLOAT_EQ(3.f * bf.away_radians_ / 2.f, bf.interf_angles_radians_[1]); - } - { - // For arrays which are not linear or planar there is no ambiguity. - std::vector array_geometry; - array_geometry.push_back(Point(0.f, 0.f, 0.f)); - array_geometry.push_back(Point(0.1f, 0.f, 0.f)); - array_geometry.push_back(Point(0.f, 0.2f, 0.f)); - array_geometry.push_back(Point(0.f, 0.f, 0.3f)); - NonlinearBeamformer bf(array_geometry, 1u); - bf.Initialize(kChunkSizeMs, kSampleRateHz); - EXPECT_EQ(2u, bf.interf_angles_radians_.size()); - EXPECT_FLOAT_EQ(M_PI / 2.f - bf.away_radians_, - bf.interf_angles_radians_[0]); - EXPECT_FLOAT_EQ(M_PI / 2.f + bf.away_radians_, - bf.interf_angles_radians_[1]); - bf.AimAt(AzimuthToSphericalPoint(bf.away_radians_ / 2.f)); - EXPECT_EQ(2u, bf.interf_angles_radians_.size()); - EXPECT_FLOAT_EQ(-bf.away_radians_ / 2.f, bf.interf_angles_radians_[0]); - EXPECT_FLOAT_EQ(3.f * bf.away_radians_ / 2.f, bf.interf_angles_radians_[1]); - } -} - -// TODO(peah): Investigate why the nonlinear_beamformer.cc causes a DCHECK in -// this setup. -TEST(BeamformerBitExactnessTest, - DISABLED_Stereo8kHz_ArrayGeometry1_TargetDirection1) { - const float kOutputReference[] = {0.001318f, -0.001091f, 0.000990f, - 0.001318f, -0.001091f, 0.000990f}; - - RunBitExactnessTest(AudioProcessing::kSampleRate8kHz, CreateArrayGeometry(1), - TargetDirection1, kOutputReference); -} - -TEST(BeamformerBitExactnessTest, Stereo16kHz_ArrayGeometry1_TargetDirection1) { - const float kOutputReference[] = {-0.000077f, -0.000147f, -0.000138f, - -0.000077f, -0.000147f, -0.000138f}; - - RunBitExactnessTest(AudioProcessing::kSampleRate16kHz, CreateArrayGeometry(1), - TargetDirection1, kOutputReference); -} - -TEST(BeamformerBitExactnessTest, Stereo32kHz_ArrayGeometry1_TargetDirection1) { - const float kOutputReference[] = {-0.000061f, -0.000061f, -0.000061f, - -0.000061f, -0.000061f, -0.000061f}; - - RunBitExactnessTest(AudioProcessing::kSampleRate32kHz, CreateArrayGeometry(1), - TargetDirection1, kOutputReference); -} - -TEST(BeamformerBitExactnessTest, Stereo48kHz_ArrayGeometry1_TargetDirection1) { - const float kOutputReference[] = {0.000450f, 0.000436f, 0.000433f, - 0.000450f, 0.000436f, 0.000433f}; - - RunBitExactnessTest(AudioProcessing::kSampleRate48kHz, CreateArrayGeometry(1), - TargetDirection1, kOutputReference); -} - -// TODO(peah): Investigate why the nonlinear_beamformer.cc causes a DCHECK in -// this setup. -TEST(BeamformerBitExactnessTest, - DISABLED_Stereo8kHz_ArrayGeometry1_TargetDirection2) { - const float kOutputReference[] = {0.001144f, -0.001026f, 0.001074f, - -0.016205f, -0.007324f, -0.015656f}; - - RunBitExactnessTest(AudioProcessing::kSampleRate8kHz, CreateArrayGeometry(1), - TargetDirection2, kOutputReference); -} - -TEST(BeamformerBitExactnessTest, Stereo16kHz_ArrayGeometry1_TargetDirection2) { - const float kOutputReference[] = {0.000221f, -0.000249f, 0.000140f, - 0.000221f, -0.000249f, 0.000140f}; - - RunBitExactnessTest(AudioProcessing::kSampleRate16kHz, CreateArrayGeometry(1), - TargetDirection2, kOutputReference); -} - -TEST(BeamformerBitExactnessTest, Stereo32kHz_ArrayGeometry1_TargetDirection2) { - const float kOutputReference[] = {0.000763f, -0.000336f, 0.000549f, - 0.000763f, -0.000336f, 0.000549f}; - - RunBitExactnessTest(AudioProcessing::kSampleRate32kHz, CreateArrayGeometry(1), - TargetDirection2, kOutputReference); -} - -TEST(BeamformerBitExactnessTest, Stereo48kHz_ArrayGeometry1_TargetDirection2) { - const float kOutputReference[] = {-0.000004f, -0.000494f, 0.000255f, - -0.000004f, -0.000494f, 0.000255f}; - - RunBitExactnessTest(AudioProcessing::kSampleRate48kHz, CreateArrayGeometry(1), - TargetDirection2, kOutputReference); -} - -TEST(BeamformerBitExactnessTest, Stereo8kHz_ArrayGeometry2_TargetDirection2) { - const float kOutputReference[] = {-0.000914f, 0.002170f, -0.002382f, - -0.000914f, 0.002170f, -0.002382f}; - - RunBitExactnessTest(AudioProcessing::kSampleRate8kHz, CreateArrayGeometry(2), - TargetDirection2, kOutputReference); -} - -TEST(BeamformerBitExactnessTest, Stereo16kHz_ArrayGeometry2_TargetDirection2) { - const float kOutputReference[] = {0.000179f, -0.000179f, 0.000081f, - 0.000179f, -0.000179f, 0.000081f}; - - RunBitExactnessTest(AudioProcessing::kSampleRate16kHz, CreateArrayGeometry(2), - TargetDirection2, kOutputReference); -} - -TEST(BeamformerBitExactnessTest, Stereo32kHz_ArrayGeometry2_TargetDirection2) { - const float kOutputReference[] = {0.000549f, -0.000214f, 0.000366f, - 0.000549f, -0.000214f, 0.000366f}; - - RunBitExactnessTest(AudioProcessing::kSampleRate32kHz, CreateArrayGeometry(2), - TargetDirection2, kOutputReference); -} - -TEST(BeamformerBitExactnessTest, Stereo48kHz_ArrayGeometry2_TargetDirection2) { - const float kOutputReference[] = {0.000019f, -0.000310f, 0.000182f, - 0.000019f, -0.000310f, 0.000182f}; - - RunBitExactnessTest(AudioProcessing::kSampleRate48kHz, CreateArrayGeometry(2), - TargetDirection2, kOutputReference); -} - -// TODO(peah): Investigate why the nonlinear_beamformer.cc causes a DCHECK in -// this setup. -TEST(BeamformerBitExactnessTest, - DISABLED_Stereo16kHz_ArrayGeometry3_TargetDirection1) { - const float kOutputReference[] = {-0.000161f, 0.000171f, -0.000096f, - 0.001007f, 0.000427f, 0.000977f}; - - RunBitExactnessTest(AudioProcessing::kSampleRate16kHz, CreateArrayGeometry(3), - TargetDirection1, kOutputReference); -} - -} // namespace webrtc diff --git a/modules/audio_processing/include/audio_processing.cc b/modules/audio_processing/include/audio_processing.cc index fa025b9280..75eedafc29 100644 --- a/modules/audio_processing/include/audio_processing.cc +++ b/modules/audio_processing/include/audio_processing.cc @@ -14,25 +14,6 @@ namespace webrtc { -Beamforming::Beamforming() - : enabled(false), - array_geometry(), - target_direction( - SphericalPointf(static_cast(M_PI) / 2.f, 0.f, 1.f)) {} -Beamforming::Beamforming(bool enabled, const std::vector& array_geometry) - : Beamforming(enabled, - array_geometry, - SphericalPointf(static_cast(M_PI) / 2.f, 0.f, 1.f)) {} - -Beamforming::Beamforming(bool enabled, - const std::vector& array_geometry, - SphericalPointf target_direction) - : enabled(enabled), - array_geometry(array_geometry), - target_direction(target_direction) {} - -Beamforming::~Beamforming() {} - void CustomProcessing::SetRuntimeSetting( AudioProcessing::RuntimeSetting setting) {} diff --git a/modules/audio_processing/include/audio_processing.h b/modules/audio_processing/include/audio_processing.h index 38a97ca3dc..d24502e873 100644 --- a/modules/audio_processing/include/audio_processing.h +++ b/modules/audio_processing/include/audio_processing.h @@ -25,7 +25,6 @@ #include "absl/types/optional.h" #include "api/audio/echo_canceller3_config.h" #include "api/audio/echo_control.h" -#include "modules/audio_processing/beamformer/array_util.h" #include "modules/audio_processing/include/audio_generator.h" #include "modules/audio_processing/include/audio_processing_statistics.h" #include "modules/audio_processing/include/config.h" @@ -44,8 +43,6 @@ class AecDump; class AudioBuffer; class AudioFrame; -class NonlinearBeamformer; - class StreamConfig; class ProcessingConfig; @@ -147,22 +144,6 @@ struct ExperimentalNs { bool enabled; }; -// Use to enable beamforming. Must be provided through the constructor. It will -// have no impact if used with AudioProcessing::SetExtraOptions(). -struct Beamforming { - Beamforming(); - Beamforming(bool enabled, const std::vector& array_geometry); - Beamforming(bool enabled, - const std::vector& array_geometry, - SphericalPointf target_direction); - ~Beamforming(); - - static const ConfigOptionID identifier = ConfigOptionID::kBeamforming; - const bool enabled; - const std::vector array_geometry; - const SphericalPointf target_direction; -}; - // Use to enable intelligibility enhancer in audio processing. // // Note: If enabled and the reverse stream has more than one output channel, @@ -673,10 +654,6 @@ class AudioProcessingBuilder { // The AudioProcessingBuilder takes ownership of the render_pre_processing. AudioProcessingBuilder& SetRenderPreProcessing( std::unique_ptr render_pre_processing); - // The AudioProcessingBuilder takes ownership of the nonlinear beamformer. - RTC_DEPRECATED - AudioProcessingBuilder& SetNonlinearBeamformer( - std::unique_ptr nonlinear_beamformer); // The AudioProcessingBuilder takes ownership of the echo_detector. AudioProcessingBuilder& SetEchoDetector( rtc::scoped_refptr echo_detector); @@ -689,7 +666,6 @@ class AudioProcessingBuilder { std::unique_ptr echo_control_factory_; std::unique_ptr capture_post_processing_; std::unique_ptr render_pre_processing_; - std::unique_ptr nonlinear_beamformer_; rtc::scoped_refptr echo_detector_; RTC_DISALLOW_COPY_AND_ASSIGN(AudioProcessingBuilder); }; diff --git a/modules/audio_processing/test/audio_processing_simulator.h b/modules/audio_processing/test/audio_processing_simulator.h index 522f778939..e82e7d5d69 100644 --- a/modules/audio_processing/test/audio_processing_simulator.h +++ b/modules/audio_processing/test/audio_processing_simulator.h @@ -56,7 +56,6 @@ struct SimulationSettings { absl::optional use_hpf; absl::optional use_ns; absl::optional use_ts; - absl::optional use_bf; absl::optional use_ie; absl::optional use_vad; absl::optional use_le; diff --git a/test/fuzzers/audio_processing_configs_fuzzer.cc b/test/fuzzers/audio_processing_configs_fuzzer.cc index d02ac16758..03437f7fa5 100644 --- a/test/fuzzers/audio_processing_configs_fuzzer.cc +++ b/test/fuzzers/audio_processing_configs_fuzzer.cc @@ -46,7 +46,6 @@ std::unique_ptr CreateApm(test::FuzzDataHelper* fuzz_data, // configurable public components of APM. bool exp_agc = fuzz_data->ReadOrDefaultValue(true); bool exp_ns = fuzz_data->ReadOrDefaultValue(true); - bool bf = fuzz_data->ReadOrDefaultValue(true); bool ef = fuzz_data->ReadOrDefaultValue(true); bool raf = fuzz_data->ReadOrDefaultValue(true); static_cast(fuzz_data->ReadOrDefaultValue(true)); @@ -105,9 +104,6 @@ std::unique_ptr CreateApm(test::FuzzDataHelper* fuzz_data, config.Set(new ExperimentalAgc(exp_agc)); config.Set(new ExperimentalNs(exp_ns)); - if (bf) { - config.Set(new Beamforming()); - } config.Set(new ExtendedFilter(ef)); config.Set(new RefinedAdaptiveFilter(raf)); config.Set(new DelayAgnostic(true));