diff --git a/modules/audio_processing/BUILD.gn b/modules/audio_processing/BUILD.gn index a733612ccc..506c8212fd 100644 --- a/modules/audio_processing/BUILD.gn +++ b/modules/audio_processing/BUILD.gn @@ -184,6 +184,7 @@ rtc_library("audio_processing") { "../../rtc_base/synchronization:mutex", "../../rtc_base/system:rtc_export", "../../system_wrappers", + "../../system_wrappers:denormal_disabler", "../../system_wrappers:field_trial", "../../system_wrappers:metrics", "aec3", @@ -373,6 +374,7 @@ if (rtc_include_tests) { "../../rtc_base/system:arch", "../../rtc_base/system:file_wrapper", "../../system_wrappers", + "../../system_wrappers:denormal_disabler", "../../test:fileutils", "../../test:rtc_expect_death", "../../test:test_support", diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc index 5acf6932c0..7facd25c12 100644 --- a/modules/audio_processing/audio_processing_impl.cc +++ b/modules/audio_processing/audio_processing_impl.cc @@ -35,6 +35,7 @@ #include "rtc_base/ref_counted_object.h" #include "rtc_base/time_utils.h" #include "rtc_base/trace_event.h" +#include "system_wrappers/include/denormal_disabler.h" #include "system_wrappers/include/field_trial.h" #include "system_wrappers/include/metrics.h" @@ -254,6 +255,8 @@ AudioProcessingImpl::AudioProcessingImpl( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), use_setup_specific_default_aec3_config_( UseSetupSpecificDefaultAec3Congfig()), + use_denormal_disabler_( + !field_trial::IsEnabled("WebRTC-ApmDenormalDisablerKillSwitch")), capture_runtime_settings_(RuntimeSettingQueueSize()), render_runtime_settings_(RuntimeSettingQueueSize()), capture_runtime_settings_enqueuer_(&capture_runtime_settings_), @@ -284,6 +287,9 @@ AudioProcessingImpl::AudioProcessingImpl( << !!submodules_.capture_post_processor << "\nRender pre processor: " << !!submodules_.render_pre_processor; + RTC_LOG(LS_INFO) << "Denormal disabler: " + << (DenormalDisabler::IsSupported() ? "supported" + : "unsupported"); // Mark Echo Controller enabled if a factory is injected. capture_nonlocked_.echo_controller_enabled = @@ -791,6 +797,7 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, RETURN_ON_ERR(MaybeInitializeCapture(input_config, output_config)); MutexLock lock_capture(&mutex_capture_); + DenormalDisabler denormal_disabler(use_denormal_disabler_); if (aec_dump_) { RecordUnprocessedCaptureStream(src); @@ -1080,6 +1087,7 @@ int AudioProcessingImpl::ProcessStream(const int16_t* const src, RETURN_ON_ERR(MaybeInitializeCapture(input_config, output_config)); MutexLock lock_capture(&mutex_capture_); + DenormalDisabler denormal_disabler(use_denormal_disabler_); if (aec_dump_) { RecordUnprocessedCaptureStream(src, input_config); @@ -1109,6 +1117,7 @@ int AudioProcessingImpl::ProcessStream(const int16_t* const src, int AudioProcessingImpl::ProcessCaptureStreamLocked() { EmptyQueuedRenderAudioLocked(); HandleCaptureRuntimeSettings(); + DenormalDisabler denormal_disabler(use_denormal_disabler_); // Ensure that not both the AEC and AECM are active at the same time. // TODO(peah): Simplify once the public API Enable functions for these @@ -1436,6 +1445,8 @@ int AudioProcessingImpl::ProcessReverseStream(const float* const* src, float* const* dest) { TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_StreamConfig"); MutexLock lock(&mutex_render_); + DenormalDisabler denormal_disabler(use_denormal_disabler_); + RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, input_config, output_config)); if (submodule_states_.RenderMultiBandProcessingActive() || submodule_states_.RenderFullBandProcessingActive()) { @@ -1473,6 +1484,8 @@ int AudioProcessingImpl::AnalyzeReverseStreamLocked( RTC_DCHECK_EQ(input_config.num_frames(), formats_.api_format.reverse_input_stream().num_frames()); + DenormalDisabler denormal_disabler(use_denormal_disabler_); + if (aec_dump_) { const size_t channel_size = formats_.api_format.reverse_input_stream().num_frames(); @@ -1497,6 +1510,8 @@ int AudioProcessingImpl::ProcessReverseStream(const int16_t* const src, } MutexLock lock(&mutex_render_); + DenormalDisabler denormal_disabler(use_denormal_disabler_); + ProcessingConfig processing_config = formats_.api_format; processing_config.reverse_input_stream().set_sample_rate_hz( input_config.sample_rate_hz()); @@ -1531,6 +1546,7 @@ int AudioProcessingImpl::ProcessRenderStreamLocked() { AudioBuffer* render_buffer = render_.render_audio.get(); // For brevity. HandleRenderRuntimeSettings(); + DenormalDisabler denormal_disabler(use_denormal_disabler_); if (submodules_.render_pre_processor) { submodules_.render_pre_processor->Process(render_buffer); diff --git a/modules/audio_processing/audio_processing_impl.h b/modules/audio_processing/audio_processing_impl.h index 2c2253676c..27abbd4c20 100644 --- a/modules/audio_processing/audio_processing_impl.h +++ b/modules/audio_processing/audio_processing_impl.h @@ -187,6 +187,8 @@ class AudioProcessingImpl : public AudioProcessing { static int instance_count_; const bool use_setup_specific_default_aec3_config_; + const bool use_denormal_disabler_; + SwapQueue capture_runtime_settings_; SwapQueue render_runtime_settings_; diff --git a/system_wrappers/BUILD.gn b/system_wrappers/BUILD.gn index 80088e0d01..fc956a5a84 100644 --- a/system_wrappers/BUILD.gn +++ b/system_wrappers/BUILD.gn @@ -108,11 +108,25 @@ rtc_library("metrics") { ] } +rtc_library("denormal_disabler") { + visibility = [ "*" ] + public = [ "include/denormal_disabler.h" ] + sources = [ "source/denormal_disabler.cc" ] + deps = [ + "../rtc_base:checks", + "../rtc_base/system:arch", + ] + if (is_clang) { + cflags_cc = [ "-Wno-unused-private-field" ] + } +} + if (rtc_include_tests && !build_with_chromium) { rtc_test("system_wrappers_unittests") { testonly = true sources = [ "source/clock_unittest.cc", + "source/denormal_disabler_unittest.cc", "source/field_trial_unittest.cc", "source/metrics_default_unittest.cc", "source/metrics_unittest.cc", @@ -121,6 +135,7 @@ if (rtc_include_tests && !build_with_chromium) { ] deps = [ + ":denormal_disabler", ":field_trial", ":metrics", ":system_wrappers", diff --git a/system_wrappers/include/denormal_disabler.h b/system_wrappers/include/denormal_disabler.h new file mode 100644 index 0000000000..ca1e19ea68 --- /dev/null +++ b/system_wrappers/include/denormal_disabler.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SYSTEM_WRAPPERS_INCLUDE_DENORMAL_DISABLER_H_ +#define SYSTEM_WRAPPERS_INCLUDE_DENORMAL_DISABLER_H_ + +#include "rtc_base/system/arch.h" + +namespace webrtc { + +// Activates the hardware (HW) way to flush denormals (see [1]) to zero as they +// can very seriously impact performance. At destruction time restores the +// denormals handling state read by the ctor; hence, supports nested calls. +// Equals a no-op if the architecture is not x86 or ARM or if the compiler is +// not CLANG. +// [1] https://en.wikipedia.org/wiki/Denormal_number +// +// Example usage: +// +// void Foo() { +// DenormalDisabler d; +// ... +// } +class DenormalDisabler { + public: + // Ctor. If `enabled` is true and architecture and compiler are supported, + // stores the HW settings for denormals, disables denormals and sets + // `disabling_activated_` to true. Otherwise, only sets `disabling_activated_` + // to false. + explicit DenormalDisabler(bool enabled); + DenormalDisabler(const DenormalDisabler&) = delete; + DenormalDisabler& operator=(const DenormalDisabler&) = delete; + // Dtor. If `disabling_activated_` is true, restores the denormals HW settings + // read by the ctor before denormals were disabled. Otherwise it's a no-op. + ~DenormalDisabler(); + + // Returns true if architecture and compiler are supported. + static bool IsSupported(); + + private: + const int status_word_; + const bool disabling_activated_; +}; + +} // namespace webrtc + +#endif // SYSTEM_WRAPPERS_INCLUDE_DENORMAL_DISABLER_H_ diff --git a/system_wrappers/source/denormal_disabler.cc b/system_wrappers/source/denormal_disabler.cc new file mode 100644 index 0000000000..fe8ec1afdc --- /dev/null +++ b/system_wrappers/source/denormal_disabler.cc @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "system_wrappers/include/denormal_disabler.h" + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +#if defined(WEBRTC_ARCH_X86_FAMILY) && defined(__clang__) +#define WEBRTC_DENORMAL_DISABLER_X86_SUPPORTED +#endif + +#if defined(WEBRTC_DENORMAL_DISABLER_X86_SUPPORTED) || \ + defined(WEBRTC_ARCH_ARM_FAMILY) +#define WEBRTC_DENORMAL_DISABLER_SUPPORTED +#endif + +constexpr int kUnspecifiedStatusWord = -1; + +#if defined(WEBRTC_DENORMAL_DISABLER_SUPPORTED) + +// Control register bit mask to disable denormals on the hardware. +#if defined(WEBRTC_DENORMAL_DISABLER_X86_SUPPORTED) +// On x86 two bits are used: flush-to-zero (FTZ) and denormals-are-zero (DAZ). +constexpr int kDenormalBitMask = 0x8040; +#elif defined(WEBRTC_ARCH_ARM_FAMILY) +// On ARM one bit is used: flush-to-zero (FTZ). +constexpr int kDenormalBitMask = 1 << 24; +#endif + +// Reads the relevant CPU control register and returns its value for supported +// architectures and compilers. Otherwise returns `kUnspecifiedStatusWord`. +int ReadStatusWord() { + int result = kUnspecifiedStatusWord; +#if defined(WEBRTC_DENORMAL_DISABLER_X86_SUPPORTED) + asm volatile("stmxcsr %0" : "=m"(result)); +#elif defined(WEBRTC_ARCH_ARM_FAMILY) && defined(WEBRTC_ARCH_32_BITS) + asm volatile("vmrs %[result], FPSCR" : [result] "=r"(result)); +#elif defined(WEBRTC_ARCH_ARM_FAMILY) && defined(WEBRTC_ARCH_64_BITS) + asm volatile("mrs %x[result], FPCR" : [result] "=r"(result)); +#endif + return result; +} + +// Writes `status_word` in the relevant CPU control register if the architecture +// and the compiler are supported. +void SetStatusWord(int status_word) { +#if defined(WEBRTC_DENORMAL_DISABLER_X86_SUPPORTED) + asm volatile("ldmxcsr %0" : : "m"(status_word)); +#elif defined(WEBRTC_ARCH_ARM_FAMILY) && defined(WEBRTC_ARCH_32_BITS) + asm volatile("vmsr FPSCR, %[src]" : : [src] "r"(status_word)); +#elif defined(WEBRTC_ARCH_ARM_FAMILY) && defined(WEBRTC_ARCH_64_BITS) + asm volatile("msr FPCR, %x[src]" : : [src] "r"(status_word)); +#endif +} + +// Returns true if the status word indicates that denormals are enabled. +constexpr bool DenormalsEnabled(int status_word) { + return (status_word & kDenormalBitMask) != kDenormalBitMask; +} + +#endif // defined(WEBRTC_DENORMAL_DISABLER_SUPPORTED) + +} // namespace + +#if defined(WEBRTC_DENORMAL_DISABLER_SUPPORTED) +DenormalDisabler::DenormalDisabler(bool enabled) + : status_word_(enabled ? ReadStatusWord() : kUnspecifiedStatusWord), + disabling_activated_(enabled && DenormalsEnabled(status_word_)) { + if (disabling_activated_) { + RTC_DCHECK_NE(status_word_, kUnspecifiedStatusWord); + SetStatusWord(status_word_ | kDenormalBitMask); + RTC_DCHECK(!DenormalsEnabled(ReadStatusWord())); + } +} + +bool DenormalDisabler::IsSupported() { + return true; +} + +DenormalDisabler::~DenormalDisabler() { + if (disabling_activated_) { + RTC_DCHECK_NE(status_word_, kUnspecifiedStatusWord); + SetStatusWord(status_word_); + } +} +#else +DenormalDisabler::DenormalDisabler(bool enabled) + : status_word_(kUnspecifiedStatusWord), disabling_activated_(false) {} + +bool DenormalDisabler::IsSupported() { + return false; +} + +DenormalDisabler::~DenormalDisabler() = default; +#endif + +} // namespace webrtc diff --git a/system_wrappers/source/denormal_disabler_unittest.cc b/system_wrappers/source/denormal_disabler_unittest.cc new file mode 100644 index 0000000000..32516db045 --- /dev/null +++ b/system_wrappers/source/denormal_disabler_unittest.cc @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "system_wrappers/include/denormal_disabler.h" + +#include +#include +#include + +#include "rtc_base/checks.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +constexpr float kSmallest = std::numeric_limits::min(); + +// Float values such that, if used as divisors of `kSmallest`, the division +// produces a denormal or zero depending on whether denormals are enabled. +constexpr float kDenormalDivisors[] = {123.125f, 97.0f, 32.0f, 5.0f, 1.5f}; + +// Returns true if the result of `dividend` / `divisor` is a denormal. +// `dividend` and `divisor` must not be denormals. +bool DivisionIsDenormal(float dividend, float divisor) { + RTC_DCHECK_GE(std::fabsf(dividend), kSmallest); + RTC_DCHECK_GE(std::fabsf(divisor), kSmallest); + volatile float division = dividend / divisor; + return division != 0.0f && std::fabsf(division) < kSmallest; +} + +} // namespace + +class DenormalDisablerParametrization : public ::testing::TestWithParam { +}; + +// Checks that +inf and -inf are not zeroed regardless of whether +// architecture and compiler are supported. +TEST_P(DenormalDisablerParametrization, InfNotZeroed) { + DenormalDisabler denormal_disabler(/*enabled=*/GetParam()); + constexpr float kMax = std::numeric_limits::max(); + for (float x : {-2.0f, 2.0f}) { + SCOPED_TRACE(x); + volatile float multiplication = kMax * x; + EXPECT_TRUE(std::isinf(multiplication)); + } +} + +// Checks that a NaN is not zeroed regardless of whether architecture and +// compiler are supported. +TEST_P(DenormalDisablerParametrization, NanNotZeroed) { + DenormalDisabler denormal_disabler(/*enabled=*/GetParam()); + volatile float kNan = std::sqrt(-1.0f); + EXPECT_TRUE(std::isnan(kNan)); +} + +INSTANTIATE_TEST_SUITE_P(DenormalDisabler, + DenormalDisablerParametrization, + ::testing::Values(false, true), + [](const ::testing::TestParamInfo& info) { + return info.param ? "enabled" : "disabled"; + }); + +// Checks that denormals are not zeroed if `DenormalDisabler` is disabled and +// architecture and compiler are supported. +TEST(DenormalDisabler, DoNotZeroDenormalsIfDisabled) { + if (!DenormalDisabler::IsSupported()) { + GTEST_SKIP() << "Unsupported platform."; + } + ASSERT_TRUE(DivisionIsDenormal(kSmallest, kDenormalDivisors[0])) + << "Precondition not met: denormals must be enabled."; + DenormalDisabler denormal_disabler(/*enabled=*/false); + for (float x : kDenormalDivisors) { + SCOPED_TRACE(x); + EXPECT_TRUE(DivisionIsDenormal(-kSmallest, x)); + EXPECT_TRUE(DivisionIsDenormal(kSmallest, x)); + } +} + +// Checks that denormals are zeroed if `DenormalDisabler` is enabled if +// architecture and compiler are supported. +TEST(DenormalDisabler, ZeroDenormals) { + if (!DenormalDisabler::IsSupported()) { + GTEST_SKIP() << "Unsupported platform."; + } + DenormalDisabler denormal_disabler(/*enabled=*/true); + for (float x : kDenormalDivisors) { + SCOPED_TRACE(x); + EXPECT_FALSE(DivisionIsDenormal(-kSmallest, x)); + EXPECT_FALSE(DivisionIsDenormal(kSmallest, x)); + } +} + +// Checks that the `DenormalDisabler` dtor re-enables denormals if previously +// enabled and architecture and compiler are supported. +TEST(DenormalDisabler, RestoreDenormalsEnabled) { + if (!DenormalDisabler::IsSupported()) { + GTEST_SKIP() << "Unsupported platform."; + } + ASSERT_TRUE(DivisionIsDenormal(kSmallest, kDenormalDivisors[0])) + << "Precondition not met: denormals must be enabled."; + { + DenormalDisabler denormal_disabler(/*enabled=*/true); + ASSERT_FALSE(DivisionIsDenormal(kSmallest, kDenormalDivisors[0])); + } + EXPECT_TRUE(DivisionIsDenormal(kSmallest, kDenormalDivisors[0])); +} + +// Checks that the `DenormalDisabler` dtor keeps denormals disabled if +// architecture and compiler are supported and if previously disabled - i.e., +// nested usage is supported. +TEST(DenormalDisabler, ZeroDenormalsNested) { + if (!DenormalDisabler::IsSupported()) { + GTEST_SKIP() << "Unsupported platform."; + } + DenormalDisabler d1(/*enabled=*/true); + ASSERT_FALSE(DivisionIsDenormal(kSmallest, kDenormalDivisors[0])); + { + DenormalDisabler d2(/*enabled=*/true); + ASSERT_FALSE(DivisionIsDenormal(kSmallest, kDenormalDivisors[0])); + } + EXPECT_FALSE(DivisionIsDenormal(kSmallest, kDenormalDivisors[0])); +} + +// Checks that `DenormalDisabler` does not zero denormals if architecture and +// compiler are not supported. +TEST(DenormalDisabler, DoNotZeroDenormalsIfUnsupported) { + if (DenormalDisabler::IsSupported()) { + // TODO(bugs.webrtc.org/13057): Use GTEST_SKIP() once fixed. + return; + } + DenormalDisabler denormal_disabler(/*enabled=*/true); + for (float x : kDenormalDivisors) { + SCOPED_TRACE(x); + EXPECT_TRUE(DivisionIsDenormal(-kSmallest, x)); + EXPECT_TRUE(DivisionIsDenormal(kSmallest, x)); + } +} + +} // namespace webrtc