Make WebRTC perf tests report to histograms.

This converts the old RESULT lines to the new perf test reporter.
It would be hard to port the result reporter to WebRTC, so let's keep
the WebRTC code ignorant of that.

Bug: chromium:1874811
Change-Id: I524fd3f24fa2739043e4facfa93cb1300a22a5c8
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1899865Reviewed-by: Brian Sheedy <bsheedy@chromium.org>
Commit-Queue: Patrik Höglund <phoglund@chromium.org>
Cr-Commit-Position: refs/heads/master@{#715228}

chrome/browser/media/webrtc/webrtc_browsertest_perf.cc

@@ -6,9 +6,12 @@
 
 #include <stddef.h>
 
+#include "base/strings/string_number_conversions.h"
+#include "base/strings/string_split.h"
 #include "base/strings/stringprintf.h"
 #include "base/values.h"
 #include "chrome/test/base/in_process_browser_test.h"
+#include "testing/perf/perf_result_reporter.h"
 #include "testing/perf/perf_test.h"
 
 static std::string Statistic(const std::string& statistic,
@@ -262,4 +265,113 @@ void PrintMetricsForRecvStreams(const base::DictionaryValue& pc_dict,
   }
 }
 
+constexpr char kMetricPrefixVideoQualityTest[] =
+    "WebRtcVideoQualityBrowserTest.";
+constexpr char kMetricUniqueFramesCount[] = "Unique_frames_count";
+constexpr char kMetricPsnrUnitless[] = "PSNR";
+constexpr char kMetricSsimUnitless[] = "SSIM";
+constexpr char kMetricMaxRepeatedCount[] = "Max_repeated";
+constexpr char kMetricMaxSkippedCount[] = "Max_skipped";
+constexpr char kMetricTotalSkippedCount[] = "Total_skipped";
+constexpr char kMetricDecodeErrorsReferenceCount[] = "Decode_errors_reference";
+constexpr char kMetricDecodeErrorsTestCount[] = "Decode_errors_test";
+
+bool ParseDoubleFromOutput(const std::string& line,
+                           const char* expected_label,
+                           double* value) {
+  auto fields = base::SplitString(line, " ", base::TRIM_WHITESPACE,
+                                  base::SPLIT_WANT_NONEMPTY);
+  auto actual_label = std::string(expected_label) + ":";
+  if (fields.size() < 4 || fields[1] != actual_label) {
+    LOG(ERROR) << "Expected line with " << actual_label
+               << " and four or five space-separated fields, got " << line;
+    return false;
+  }
+  if (!base::StringToDouble(fields[3], value)) {
+    LOG(ERROR) << "Expected " << fields[3] << " to be an int";
+    return false;
+  }
+  return true;
+}
+
+bool ParseListFromOutput(const std::string& line,
+                         const char* expected_label,
+                         std::string* value) {
+  auto fields = base::SplitString(line, " ", base::TRIM_WHITESPACE,
+                                  base::SPLIT_WANT_NONEMPTY);
+  auto actual_label = std::string(expected_label) + ":";
+  if (fields.size() < 4 || fields[1] != actual_label) {
+    LOG(ERROR) << "Expected line with " << actual_label
+               << " and four or five space-separated fields, got " << line;
+    return false;
+  }
+
+  // Strip out [].
+  std::string values = fields[3];
+  if (values.length() < 2) {
+    LOG(ERROR) << "Malformed values, expected [val1, val2], got " << values;
+    return false;
+  }
+  *value = values.substr(1, values.length() - 2);
+  return true;
+}
+
+bool WriteCompareVideosOutputAsHistogram(const std::string& test_label,
+                                         const std::string& output) {
+  perf_test::PerfResultReporter reporter(kMetricPrefixVideoQualityTest,
+                                         test_label);
+
+  reporter.RegisterFyiMetric(kMetricUniqueFramesCount, "count");
+  reporter.RegisterFyiMetric(kMetricPsnrUnitless, "unitless");
+  reporter.RegisterFyiMetric(kMetricSsimUnitless, "unitless");
+  reporter.RegisterFyiMetric(kMetricMaxRepeatedCount, "count");
+  reporter.RegisterFyiMetric(kMetricMaxSkippedCount, "count");
+  reporter.RegisterFyiMetric(kMetricTotalSkippedCount, "count");
+  reporter.RegisterFyiMetric(kMetricDecodeErrorsReferenceCount, "count");
+  reporter.RegisterFyiMetric(kMetricDecodeErrorsTestCount, "count");
+  auto lines = base::SplitString(output, "\n", base::TRIM_WHITESPACE,
+                                 base::SPLIT_WANT_NONEMPTY);
+
+  if (lines.size() == 12) {
+    // Remove warning about colorspace conversion.
+    lines.erase(lines.begin(), lines.begin() + 4);
+  }
+
+  if (lines.size() != 8) {
+    LOG(ERROR) << "Expected 8 lines, got " << lines.size()
+               << " lines. Output:\n\n"
+               << output;
+    return false;
+  }
+
+  double value;
+  if (!ParseDoubleFromOutput(lines[0], kMetricUniqueFramesCount, &value))
+    return false;
+  reporter.AddResult(kMetricUniqueFramesCount, value);
+  std::string value_list;
+  if (!ParseListFromOutput(lines[1], kMetricPsnrUnitless, &value_list))
+    return false;
+  reporter.AddResultList(kMetricPsnrUnitless, value_list);
+  if (!ParseListFromOutput(lines[2], kMetricSsimUnitless, &value_list))
+    return false;
+  reporter.AddResultList(kMetricSsimUnitless, value_list);
+  if (!ParseDoubleFromOutput(lines[3], kMetricMaxRepeatedCount, &value))
+    return false;
+  reporter.AddResult(kMetricMaxRepeatedCount, value);
+  if (!ParseDoubleFromOutput(lines[4], kMetricMaxSkippedCount, &value))
+    return false;
+  reporter.AddResult(kMetricMaxSkippedCount, value);
+  if (!ParseDoubleFromOutput(lines[5], kMetricTotalSkippedCount, &value))
+    return false;
+  reporter.AddResult(kMetricTotalSkippedCount, value);
+  if (!ParseDoubleFromOutput(lines[6], kMetricDecodeErrorsReferenceCount,
+                             &value))
+    return false;
+  reporter.AddResult(kMetricDecodeErrorsReferenceCount, value);
+  if (!ParseDoubleFromOutput(lines[7], kMetricDecodeErrorsTestCount, &value))
+    return false;
+  reporter.AddResult(kMetricDecodeErrorsTestCount, value);
+  return true;
+}
+
 }  // namespace test

chrome/browser/media/webrtc/webrtc_browsertest_perf.h

@@ -39,6 +39,9 @@ void PrintMetricsForRecvStreams(const base::DictionaryValue& pc_dict,
                                 const std::string& modifier,
                                 const std::string& video_codec);
 
+bool WriteCompareVideosOutputAsHistogram(const std::string& test_label,
+                                         const std::string& output);
+
 }  // namespace test
 
 #endif  // CHROME_BROWSER_MEDIA_WEBRTC_WEBRTC_BROWSERTEST_PERF_H_

chrome/browser/media/webrtc/webrtc_browsertest_perf_unittest.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/media/webrtc/webrtc_browsertest_perf.h"
+
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace test {
+
+constexpr char kTypicalCompareVideosOutput[] =
+    "Adjusting test video with color transformation: \n"
+    " 1.00   0.01   0.01  -2.91 \n"
+    "-0.01   0.99  -0.04   7.21 \n"
+    " 0.00  -0.06   0.99   7.47 \n"
+    "RESULT Unique_frames_count: 360p_VP8= 80 score\n"
+    "RESULT PSNR: 360p_VP8= "
+    "[38.453419,38.453419,38.453419,38.453419,38.453419,38.453419,38.453419,38."
+    "453419,38.453419,38.453419,39.102536,39.102536,39.102536,39.767288,39."
+    "767288,39.767288,40.023144,40.023144,40.023144,40.562812,40.562812,40."
+    "562812,40.72701,40.72701,40.72701,40.72701,40.926442,40.926442,40.926442,"
+    "41.198192,41.198192,41.198192,41.397378,41.397378,41.397378,41.435832,41."
+    "435832,41.435832,41.456998,41.456998,41.456998,41.66108,41.66108,41.66108,"
+    "41.722977,41.722977,41.722977,41.471985,41.471985,41.471985,41.471985,41."
+    "471985,41.263275,41.263275,41.263275,40.953795,40.953795,40.890606,40."
+    "890606,40.890606,41.055124,41.055124,41.055124,41.371183,41.371183,41."
+    "371183,41.64044,41.64044,41.64044,41.64044,41.64044,41.725886,41.725886,"
+    "41.725886,41.578544,41.578544,41.646766,41.646766,41.490909,41.490909] "
+    "dB\n"
+    "RESULT SSIM: 360p_VP8= "
+    "[0.96503067,0.96503067,0.96503067,0.96503067,0.96503067,0.96503067,0."
+    "96503067,0.96503067,0.96503067,0.96503067,0.96694655,0.96694655,0."
+    "96694655,0.97058175,0.97058175,0.97058175,0.97440174,0.97440174,0."
+    "97440174,0.97723814,0.97723814,0.97723814,0.97804682,0.97804682,0."
+    "97804682,0.97804682,0.98044036,0.98044036,0.98044036,0.98102023,0."
+    "98102023,0.98102023,0.98076329,0.98076329,0.98076329,0.98025288,0."
+    "98025288,0.98025288,0.98084894,0.98084894,0.98084894,0.98137786,0."
+    "98137786,0.98137786,0.9812953,0.9812953,0.9812953,0.97990543,0.97990543,0."
+    "97990543,0.97990543,0.97990543,0.97811092,0.97811092,0.97811092,0."
+    "97576317,0.97576317,0.97655883,0.97655883,0.97655883,0.97669573,0."
+    "97669573,0.97669573,0.9795819,0.9795819,0.9795819,0.98144956,0.98144956,0."
+    "98144956,0.98144956,0.98144956,0.98165894,0.98165894,0.98165894,0."
+    "98185588,0.98185588,0.98135814,0.98135814,0.98102463,0.98102463] score\n"
+    "RESULT Max_repeated: 360p_VP8= 10\n"
+    "RESULT Max_skipped: 360p_VP8= 1\n"
+    "RESULT Total_skipped: 360p_VP8= 23\n"
+    "RESULT Decode_errors_reference: 360p_VP8= 0\n"
+    "RESULT Decode_errors_test: 360p_VP8= 0\n";
+
+TEST(WebrtcBrowserTestPerfTest, ParsesTypicalCompareVideosOutput) {
+  EXPECT_TRUE(WriteCompareVideosOutputAsHistogram("someLabel",
+                                                  kTypicalCompareVideosOutput));
+}
+
+TEST(WebrtcBrowserTestPerfTest, FailsOnWrongNumberOfLines) {
+  EXPECT_FALSE(WriteCompareVideosOutputAsHistogram(
+      "whatever", "RESULT bad_label: 360p_VP8= 80 score\n"));
+}
+
+TEST(WebrtcBrowserTestPerfTest, FailsOnBadLabels) {
+  EXPECT_FALSE(WriteCompareVideosOutputAsHistogram(
+      "whatever", "RESULT bad_label: 360p_VP8= 80 score\na\nb\nc\nd\ne\nf\ng"));
+}
+
+TEST(WebrtcBrowserTestPerfTest, FailsOnBadValues) {
+  EXPECT_FALSE(WriteCompareVideosOutputAsHistogram(
+      "whatever",
+      "RESULT bad_label: 360p_VP8= meh score\na\nb\nc\nd\ne\nf\ng"));
+}
+
+TEST(WebrtcBrowserTestPerfTest, FailsIfLabelsInWrongOrder) {
+  EXPECT_FALSE(WriteCompareVideosOutputAsHistogram(
+      "whatever", "RESULT PSNR: 360p_VP8= 80 score\na\nb\nc\nd\ne\nf\ng"));
+}
+
+}  // namespace test

chrome/browser/media/webrtc/webrtc_video_quality_browsertest.cc

@@ -23,6 +23,7 @@
 #include "chrome/browser/infobars/infobar_service.h"
 #include "chrome/browser/media/webrtc/webrtc_browsertest_base.h"
 #include "chrome/browser/media/webrtc/webrtc_browsertest_common.h"
+#include "chrome/browser/media/webrtc/webrtc_browsertest_perf.h"
 #include "chrome/browser/profiles/profile.h"
 #include "chrome/browser/ui/browser.h"
 #include "chrome/browser/ui/browser_tabstrip.h"
@@ -224,10 +225,16 @@ class WebRtcVideoQualityBrowserTest : public WebRtcTestBase,
     printf("Output was:\n\n%s\n", output.c_str());
     bool has_result_lines = output.find("RESULT") != std::string::npos;
     if (!ok || !has_result_lines) {
-      LOG(ERROR) << "Failed to compare videos; see output above to see what "
-                 << "the error was.";
+      LOG(ERROR) << "Failed to compare videos; see output to see what "
+                 << "the error was:\n\n"
+                 << output;
       return false;
     }
+    // TODO(http://crbug.com/1874811): Enable this and drop the printf above
+    // when ready to switch to histogram sets.
+    // if (!test::WriteCompareVideosOutputAsHistogram(test_label, output))
+    //  return false;
+
     return true;
   }
 

chrome/test/BUILD.gn

@@ -960,6 +960,7 @@ if (!is_android) {
       "../browser/media/webrtc/webrtc_browsertest_common.h",
       "../browser/media/webrtc/webrtc_browsertest_perf.cc",
       "../browser/media/webrtc/webrtc_browsertest_perf.h",
+      "../browser/media/webrtc/webrtc_browsertest_perf_unittest.cc",
       "../browser/media/webrtc/webrtc_desktop_capture_browsertest.cc",
       "../browser/media/webrtc/webrtc_disable_encryption_flag_browsertest.cc",
       "../browser/media/webrtc/webrtc_getdisplaymedia_browsertest.cc",