diff --git a/BUILD.gn b/BUILD.gn index fd5b799c7a..3caa4c50cc 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -52,7 +52,6 @@ if (!build_with_chromium) { ":voip_unittests", ":webrtc_nonparallel_tests", ":webrtc_perf_tests", - "api/test/metrics:metrics_unittests", "common_audio:common_audio_unittests", "common_video:common_video_unittests", "examples:examples_unittests", @@ -565,6 +564,7 @@ if (rtc_include_tests && !build_with_chromium) { "api/audio_codecs/test:audio_codecs_api_unittests", "api/numerics:numerics_unittests", "api/task_queue:pending_task_safety_flag_unittests", + "api/test/metrics:metrics_unittests", "api/transport:stun_unittest", "api/video/test:rtc_api_video_unittests", "api/video_codecs/test:video_codecs_api_unittests", diff --git a/api/test/metrics/BUILD.gn b/api/test/metrics/BUILD.gn index 0d1ffffc7c..099ee8a283 100644 --- a/api/test/metrics/BUILD.gn +++ b/api/test/metrics/BUILD.gn @@ -9,14 +9,18 @@ import("../../../webrtc.gni") group("metrics") { - deps = [ ":metric" ] + deps = [ + ":metric", + ":metrics_exporter", + ":stdout_metrics_exporter", + ] } if (rtc_include_tests) { group("metrics_unittests") { testonly = true - deps = [] + deps = [ ":stdout_metrics_exporter_test" ] } } @@ -29,3 +33,40 @@ rtc_library("metric") { absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] deps = [ "../../../api/units:timestamp" ] } + +rtc_library("metrics_exporter") { + visibility = [ "*" ] + sources = [ "metrics_exporter.h" ] + deps = [ + ":metric", + "../../../api:array_view", + ] +} + +rtc_library("stdout_metrics_exporter") { + visibility = [ "*" ] + sources = [ + "stdout_metrics_exporter.cc", + "stdout_metrics_exporter.h", + ] + deps = [ + ":metric", + ":metrics_exporter", + "../../../api:array_view", + "../../../rtc_base:stringutils", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +if (rtc_include_tests) { + rtc_library("stdout_metrics_exporter_test") { + testonly = true + sources = [ "stdout_metrics_exporter_test.cc" ] + deps = [ + ":metric", + ":stdout_metrics_exporter", + "../../../api/units:timestamp", + "../../../test:test_support", + ] + } +} diff --git a/api/test/metrics/metrics_exporter.h b/api/test/metrics/metrics_exporter.h new file mode 100644 index 0000000000..23954b6b1f --- /dev/null +++ b/api/test/metrics/metrics_exporter.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_METRICS_METRICS_EXPORTER_H_ +#define API_TEST_METRICS_METRICS_EXPORTER_H_ + +#include "api/array_view.h" +#include "api/test/metrics/metric.h" + +namespace webrtc { +namespace test { + +// Exports metrics in the requested format. +class MetricsExporter { + public: + virtual ~MetricsExporter() = default; + + // Exports specified metrics in a format that depends on the implementation. + // Returns true if export succeeded, false otherwise. + virtual bool Export(rtc::ArrayView metrics) = 0; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_METRICS_METRICS_EXPORTER_H_ diff --git a/api/test/metrics/stdout_metrics_exporter.cc b/api/test/metrics/stdout_metrics_exporter.cc new file mode 100644 index 0000000000..f509591ac1 --- /dev/null +++ b/api/test/metrics/stdout_metrics_exporter.cc @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/stdout_metrics_exporter.h" + +#include + +#include +#include + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/test/metrics/metric.h" +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { +namespace test { +namespace { + +// Returns positive integral part of the number. +int64_t IntegralPart(double value) { + return std::lround(std::floor(std::abs(value))); +} + +void AppendWithPrecision(double value, + int digits_after_comma, + rtc::StringBuilder& out) { + int64_t multiplier = std::lround(std::pow(10, digits_after_comma)); + int64_t integral_part = IntegralPart(value); + double decimal_part = std::abs(value) - integral_part; + + // If decimal part has leading zeros then when it will be multiplied on + // `multiplier`, leading zeros will be lost. To preserve them we add "1" + // so then leading digit will be greater than 0 and won't be removed. + // + // During conversion to the string leading digit has to be stripped. + // + // Also due to rounding it may happen that leading digit may be incremented, + // like with `digits_after_comma` 3 number 1.9995 will be rounded to 2. In + // such case this increment has to be propagated to the `integral_part`. + int64_t decimal_holder = std::lround((1 + decimal_part) * multiplier); + if (decimal_holder >= 2 * multiplier) { + // Rounding incremented added leading digit, so we need to transfer 1 to + // integral part. + integral_part++; + decimal_holder -= multiplier; + } + // Remove trailing zeros. + while (decimal_holder % 10 == 0) { + decimal_holder /= 10; + } + + // Print serialized number to output. + if (value < 0) { + out << "-"; + } + out << integral_part; + if (decimal_holder != 1) { + out << "." << std::to_string(decimal_holder).substr(1, digits_after_comma); + } +} + +} // namespace + +StdoutMetricsExporter::StdoutMetricsExporter() : output_(stdout) {} + +bool StdoutMetricsExporter::Export(rtc::ArrayView metrics) { + for (const Metric& metric : metrics) { + PrintMetric(metric); + } + return true; +} + +void StdoutMetricsExporter::PrintMetric(const Metric& metric) { + rtc::StringBuilder value_stream; + value_stream << metric.test_case << "/" << metric.name << "= {mean="; + if (metric.stats.mean.has_value()) { + AppendWithPrecision(*metric.stats.mean, 8, value_stream); + } else { + value_stream << "-"; + } + value_stream << ", stddev="; + if (metric.stats.stddev.has_value()) { + AppendWithPrecision(*metric.stats.stddev, 8, value_stream); + } else { + value_stream << "-"; + } + value_stream << "} " << ToString(metric.unit) << " (" + << ToString(metric.improvement_direction) << ")"; + + fprintf(output_, "RESULT: %s\n", value_stream.str().c_str()); +} + +} // namespace test +} // namespace webrtc diff --git a/api/test/metrics/stdout_metrics_exporter.h b/api/test/metrics/stdout_metrics_exporter.h new file mode 100644 index 0000000000..2c572cb2ea --- /dev/null +++ b/api/test/metrics/stdout_metrics_exporter.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_METRICS_STDOUT_METRICS_EXPORTER_H_ +#define API_TEST_METRICS_STDOUT_METRICS_EXPORTER_H_ + +#include "api/array_view.h" +#include "api/test/metrics/metric.h" +#include "api/test/metrics/metrics_exporter.h" + +namespace webrtc { +namespace test { + +// Exports all collected metrics to stdout. +class StdoutMetricsExporter : public MetricsExporter { + public: + StdoutMetricsExporter(); + ~StdoutMetricsExporter() override = default; + + StdoutMetricsExporter(const StdoutMetricsExporter&) = delete; + StdoutMetricsExporter& operator=(const StdoutMetricsExporter&) = delete; + + bool Export(rtc::ArrayView metrics) override; + + private: + void PrintMetric(const Metric& metric); + + FILE* const output_; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_METRICS_STDOUT_METRICS_EXPORTER_H_ diff --git a/api/test/metrics/stdout_metrics_exporter_test.cc b/api/test/metrics/stdout_metrics_exporter_test.cc new file mode 100644 index 0000000000..b69addabfb --- /dev/null +++ b/api/test/metrics/stdout_metrics_exporter_test.cc @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/stdout_metrics_exporter.h" + +#include +#include +#include + +#include "api/test/metrics/metric.h" +#include "api/units/timestamp.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { +namespace { + +using ::testing::TestWithParam; + +std::map DefaultMetadata() { + return std::map{{"key", "value"}}; +} + +Metric::TimeSeries::Sample Sample(double value) { + return Metric::TimeSeries::Sample{.timestamp = Timestamp::Seconds(1), + .value = value, + .sample_metadata = DefaultMetadata()}; +} + +Metric PsnrForTestFoo(double mean, double stddev) { + return Metric{.name = "psnr", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .test_case = "foo", + .time_series = Metric::TimeSeries{}, + .stats = Metric::Stats{.mean = mean, .stddev = stddev}}; +} + +TEST(StdoutMetricsExporterTest, MAYBE_ExportMetricFormatCorrect) { + Metric metric1{ + .name = "test_metric1", + .unit = Unit::kTimeMs, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .test_case = "test_case_name1", + .metric_metadata = DefaultMetadata(), + .time_series = + Metric::TimeSeries{.samples = std::vector{Sample(10), Sample(20)}}, + .stats = + Metric::Stats{.mean = 15.0, .stddev = 5.0, .min = 10.0, .max = 20.0}}; + Metric metric2{ + .name = "test_metric2", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .test_case = "test_case_name2", + .metric_metadata = DefaultMetadata(), + .time_series = + Metric::TimeSeries{.samples = std::vector{Sample(20), Sample(40)}}, + .stats = Metric::Stats{ + .mean = 30.0, .stddev = 10.0, .min = 20.0, .max = 40.0}}; + + testing::internal::CaptureStdout(); + StdoutMetricsExporter exporter; + + std::string expected = + "RESULT: test_case_name1/test_metric1= " + "{mean=15, stddev=5} TimeMs (BiggerIsBetter)\n" + "RESULT: test_case_name2/test_metric2= " + "{mean=30, stddev=10} KilobitsPerSecond (SmallerIsBetter)\n"; + + EXPECT_TRUE(exporter.Export(std::vector{metric1, metric2})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(StdoutMetricsExporterNumberFormatTest, PositiveNumberMaxPrecision) { + testing::internal::CaptureStdout(); + StdoutMetricsExporter exporter; + + Metric metric = PsnrForTestFoo(15.00000001, 0.00000001); + std::string expected = + "RESULT: foo/psnr= " + "{mean=15.00000001, stddev=0.00000001} Unitless (BiggerIsBetter)\n"; + EXPECT_TRUE(exporter.Export(std::vector{metric})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(StdoutMetricsExporterNumberFormatTest, + PositiveNumberTrailingZeroNotAdded) { + testing::internal::CaptureStdout(); + StdoutMetricsExporter exporter; + + Metric metric = PsnrForTestFoo(15.12345, 0.12); + std::string expected = + "RESULT: foo/psnr= " + "{mean=15.12345, stddev=0.12} Unitless (BiggerIsBetter)\n"; + EXPECT_TRUE(exporter.Export(std::vector{metric})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(StdoutMetricsExporterNumberFormatTest, + PositiveNumberTrailingZeroAreRemoved) { + testing::internal::CaptureStdout(); + StdoutMetricsExporter exporter; + + Metric metric = PsnrForTestFoo(15.123450000, 0.120000000); + std::string expected = + "RESULT: foo/psnr= " + "{mean=15.12345, stddev=0.12} Unitless (BiggerIsBetter)\n"; + EXPECT_TRUE(exporter.Export(std::vector{metric})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(StdoutMetricsExporterNumberFormatTest, + PositiveNumberRoundsUpOnPrecisionCorrectly) { + testing::internal::CaptureStdout(); + StdoutMetricsExporter exporter; + + Metric metric = PsnrForTestFoo(15.000000009, 0.999999999); + std::string expected = + "RESULT: foo/psnr= " + "{mean=15.00000001, stddev=1} Unitless (BiggerIsBetter)\n"; + EXPECT_TRUE(exporter.Export(std::vector{metric})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(StdoutMetricsExporterNumberFormatTest, + PositiveNumberRoundsDownOnPrecisionCorrectly) { + testing::internal::CaptureStdout(); + StdoutMetricsExporter exporter; + + Metric metric = PsnrForTestFoo(15.0000000049, 0.9999999949); + std::string expected = + "RESULT: foo/psnr= " + "{mean=15, stddev=0.99999999} Unitless (BiggerIsBetter)\n"; + EXPECT_TRUE(exporter.Export(std::vector{metric})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(StdoutMetricsExporterNumberFormatTest, NegativeNumberMaxPrecision) { + testing::internal::CaptureStdout(); + StdoutMetricsExporter exporter; + + Metric metric = PsnrForTestFoo(-15.00000001, -0.00000001); + std::string expected = + "RESULT: foo/psnr= " + "{mean=-15.00000001, stddev=-0.00000001} Unitless (BiggerIsBetter)\n"; + EXPECT_TRUE(exporter.Export(std::vector{metric})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(StdoutMetricsExporterNumberFormatTest, + NegativeNumberTrailingZeroNotAdded) { + testing::internal::CaptureStdout(); + StdoutMetricsExporter exporter; + + Metric metric = PsnrForTestFoo(-15.12345, -0.12); + std::string expected = + "RESULT: foo/psnr= " + "{mean=-15.12345, stddev=-0.12} Unitless (BiggerIsBetter)\n"; + EXPECT_TRUE(exporter.Export(std::vector{metric})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(StdoutMetricsExporterNumberFormatTest, + NegativeNumberTrailingZeroAreRemoved) { + testing::internal::CaptureStdout(); + StdoutMetricsExporter exporter; + + Metric metric = PsnrForTestFoo(-15.123450000, -0.120000000); + std::string expected = + "RESULT: foo/psnr= " + "{mean=-15.12345, stddev=-0.12} Unitless (BiggerIsBetter)\n"; + EXPECT_TRUE(exporter.Export(std::vector{metric})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(StdoutMetricsExporterNumberFormatTest, + NegativeNumberRoundsUpOnPrecisionCorrectly) { + testing::internal::CaptureStdout(); + StdoutMetricsExporter exporter; + + Metric metric = PsnrForTestFoo(-15.000000009, -0.999999999); + std::string expected = + "RESULT: foo/psnr= " + "{mean=-15.00000001, stddev=-1} Unitless (BiggerIsBetter)\n"; + EXPECT_TRUE(exporter.Export(std::vector{metric})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(StdoutMetricsExporterNumberFormatTest, + NegativeNumberRoundsDownOnPrecisionCorrectly) { + testing::internal::CaptureStdout(); + StdoutMetricsExporter exporter; + + Metric metric = PsnrForTestFoo(-15.0000000049, -0.9999999949); + std::string expected = + "RESULT: foo/psnr= " + "{mean=-15, stddev=-0.99999999} Unitless (BiggerIsBetter)\n"; + EXPECT_TRUE(exporter.Export(std::vector{metric})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +} // namespace +} // namespace test +} // namespace webrtc