[tools/perf] Allow all available formats in standalone mode

This CL allows to run a standalone Results Processor with any output format
that has a formatter defined in processor.py. This is useful for external
clients which do not need to wait until Telemetry is fully switched over to
results_processor formatters.

Bug: 981349
Change-Id: I4c7850a7c044d3926d9359769ab23e780ab31910
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1823864
Commit-Queue: Mikhail Khokhlov <khokhlov@google.com>
Reviewed-by: Juan Antonio Navarro Pérez <perezju@chromium.org>
Cr-Commit-Position: refs/heads/master@{#700694}

tools/perf/core/results_processor/command_line.py

@@ -15,14 +15,20 @@ import sys
 
 from py_utils import cloud_storage
 
+from core.results_processor import formatters
 
-SUPPORTED_FORMATS = ['none', 'json-test-results']
+
+# These formats are always handled natively, and never handed over to Telemetry.
+HANDLED_NATIVELY = ['none', 'json-test-results']
 
 
 def ArgumentParser(standalone=False, legacy_formats=None):
   """Create an ArgumentParser defining options required by the processor."""
-  all_output_formats = sorted(
-      set(SUPPORTED_FORMATS).union(legacy_formats or ()))
+  if standalone:
+    all_output_formats = formatters.FORMATTERS.keys()
+  else:
+    all_output_formats = sorted(
+        set(HANDLED_NATIVELY).union(legacy_formats or ()))
   parser, group = _CreateTopLevelParser(standalone)
   group.add_argument(
       '--output-format', action='append', dest='output_formats',
@@ -65,7 +71,7 @@ def ArgumentParser(standalone=False, legacy_formats=None):
   return parser
 
 
-def ProcessOptions(options):
+def ProcessOptions(options, standalone=False):
   """Adjust result processing options as needed before running benchmarks.
 
   Note: The intended scope of this function is limited to only adjust options
@@ -78,6 +84,8 @@ def ProcessOptions(options):
 
   Args:
     options: An options object with values parsed from the command line.
+    standalone: Whether this is a standalone Results Processor run (as
+      opposed to the run with Telemetry).
   """
   # The output_dir option is None or missing if the selected Telemetry command
   # does not involve output generation, e.g. "run_benchmark list", and the
@@ -116,7 +124,7 @@ def ProcessOptions(options):
   for output_format in chosen_formats:
     if output_format == 'none':
       continue
-    elif output_format in SUPPORTED_FORMATS:
+    elif standalone or output_format in HANDLED_NATIVELY:
       options.output_formats.append(output_format)
     else:
       options.legacy_output_formats.append(output_format)

tools/perf/core/results_processor/command_line_unittest.py

@@ -17,7 +17,7 @@ import unittest
 import mock
 
 from core.results_processor import command_line
-from core.results_processor import processor
+from core.results_processor import formatters
 
 
 # To easily mock module level symbols within the command_line module.
@@ -133,7 +133,7 @@ class TestProcessOptions(ProcessOptionsTestCase):
     with self.assertRaises(SystemExit):
       self.ParseArgs(['--output-format', 'unknown'])
 
-  @mock.patch(module('SUPPORTED_FORMATS'), ['new-format'])
+  @mock.patch(module('HANDLED_NATIVELY'), ['new-format'])
   def testOutputFormatsSplit(self):
     self.legacy_formats = ['old-format']
     options = self.ParseArgs(
@@ -141,7 +141,7 @@ class TestProcessOptions(ProcessOptionsTestCase):
     self.assertEqual(options.output_formats, ['new-format'])
     self.assertEqual(options.legacy_output_formats, ['old-format'])
 
-  @mock.patch(module('SUPPORTED_FORMATS'), ['new-format'])
+  @mock.patch(module('HANDLED_NATIVELY'), ['new-format'])
   def testNoDuplicateOutputFormats(self):
     self.legacy_formats = ['old-format']
     options = self.ParseArgs(
@@ -160,23 +160,22 @@ class StandaloneTestProcessOptions(ProcessOptionsTestCase):
     with self.assertRaises(SystemExit):
       self.ParseArgs([])
 
-  @mock.patch(module('SUPPORTED_FORMATS'), ['new-format'])
   def testIntermediateDirRequired(self):
     with self.assertRaises(SystemExit):
-      self.ParseArgs(['--output-format', 'new-format'])
+      self.ParseArgs(['--output-format', 'json-test-results'])
 
-  @mock.patch(module('SUPPORTED_FORMATS'), ['new-format'])
   def testSuccessful(self):
     options = self.ParseArgs(
-        ['--output-format', 'new-format', '--intermediate-dir', 'some_dir'])
-    self.assertEqual(options.output_formats, ['new-format'])
+        ['--output-format', 'json-test-results',
+         '--intermediate-dir', 'some_dir'])
+    self.assertEqual(options.output_formats, ['json-test-results'])
     self.assertEqual(options.intermediate_dir, '/path/to/curdir/some_dir')
     self.assertEqual(options.output_dir, '/path/to/output_dir')
 
 
-class TestSupportedFormats(unittest.TestCase):
-  def testAllSupportedFormatsHaveFormatters(self):
-    for output_format in command_line.SUPPORTED_FORMATS:
+class TestNativelyHandledFormats(unittest.TestCase):
+  def testNativelyHandledFormatsHaveFormatters(self):
+    for output_format in command_line.HANDLED_NATIVELY:
       if output_format == 'none':
         continue
-      self.assertIn(output_format, processor.FORMATTERS)
+      self.assertIn(output_format, formatters.FORMATTERS)

tools/perf/core/results_processor/formatters/__init__.py

+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from core.results_processor.formatters import histograms_output
+from core.results_processor.formatters import html_output
+from core.results_processor.formatters import json3_output
+
+
+FORMATTERS = {
+    'json-test-results': json3_output,
+    'histograms': histograms_output,
+    'html': html_output,
+}

tools/perf/core/results_processor/histograms_output_unittest.py → tools/perf/core/results_processor/formatters/histograms_output_unittest.py

@@ -9,7 +9,7 @@ import os
 import shutil
 import tempfile
 
-from core.results_processor import histograms_output
+from core.results_processor.formatters import histograms_output
 from core.results_processor import command_line
 from core.results_processor import testing
 

tools/perf/core/results_processor/html_output.py → tools/perf/core/results_processor/formatters/html_output.py

@@ -7,7 +7,7 @@
 import codecs
 import os
 
-from core.results_processor import histograms_output
+from core.results_processor.formatters import histograms_output
 from tracing_build import vulcanize_histograms_viewer
 
 

tools/perf/core/results_processor/json3_output_unittest.py → tools/perf/core/results_processor/formatters/json3_output_unittest.py

@@ -5,7 +5,7 @@
 import copy
 import unittest
 
-from core.results_processor import json3_output
+from core.results_processor.formatters import json3_output
 from core.results_processor import testing
 
 

tools/perf/core/results_processor/processor.py

@@ -12,18 +12,11 @@ import json
 import os
 
 from core.results_processor import command_line
-from core.results_processor import json3_output
-from core.results_processor import histograms_output
-from core.results_processor import html_output
+from core.results_processor import formatters
 
 
 HTML_TRACE_NAME = 'trace.html'
 TELEMETRY_RESULTS = '_telemetry_results.jsonl'
-FORMATTERS = {
-    'json-test-results': json3_output,
-    'histograms': histograms_output,
-    'html': html_output,
-}
 
 
 def ProcessResults(options):
@@ -48,10 +41,10 @@ def ProcessResults(options):
   _UploadArtifacts(intermediate_results, options.upload_bucket)
 
   for output_format in options.output_formats:
-    if output_format not in FORMATTERS:
+    if output_format not in formatters.FORMATTERS:
       raise NotImplementedError(output_format)
 
-    formatter = FORMATTERS[output_format]
+    formatter = formatters.FORMATTERS[output_format]
     formatter.Process(intermediate_results, options)
 
 
@@ -106,5 +99,5 @@ def main(args=None):
   """Entry point for the standalone version of the results_processor script."""
   parser = command_line.ArgumentParser(standalone=True)
   options = parser.parse_args(args)
-  command_line.ProcessOptions(options)
+  command_line.ProcessOptions(options, standalone=True)
   return ProcessResults(options)

tools/perf/core/results_processor/processor_test.py

@@ -15,11 +15,9 @@ import shutil
 import tempfile
 import unittest
 
-import mock
-
-from core.results_processor import json3_output
-from core.results_processor import histograms_output
-from core.results_processor import html_output
+from core.results_processor.formatters import json3_output
+from core.results_processor.formatters import histograms_output
+from core.results_processor.formatters import html_output
 from core.results_processor import processor
 from core.results_processor import testing
 
@@ -105,10 +103,6 @@ class ResultsProcessorIntegrationTests(unittest.TestCase):
     self.assertEqual(artifacts['logs'], ['gs://logs.txt'])
     self.assertEqual(artifacts['trace.html'], ['gs://trace.html'])
 
-  # TODO(crbug.com/981349): Remove this mock when histograms format
-  # is enabled in results_processor.
-  @mock.patch('core.results_processor.command_line.SUPPORTED_FORMATS',
-              ['histograms'])
   def testHistogramsOutput(self):
     hist_file = os.path.join(self.output_dir,
                              histograms_output.HISTOGRAM_DICTS_NAME)
@@ -154,10 +148,6 @@ class ResultsProcessorIntegrationTests(unittest.TestCase):
     self.assertIn([['documentation', 'url']], diag_values)
     self.assertIn(['label'], diag_values)
 
-  # TODO(crbug.com/981349): Remove this mock when histograms format
-  # is enabled in results_processor.
-  @mock.patch('core.results_processor.command_line.SUPPORTED_FORMATS',
-              ['histograms'])
   def testHistogramsOutputResetResults(self):
     hist_file = os.path.join(self.output_dir,
                              histograms_output.HISTOGRAM_DICTS_NAME)
@@ -199,10 +189,6 @@ class ResultsProcessorIntegrationTests(unittest.TestCase):
     self.assertNotIn(['label1'], diag_values)
     self.assertIn(['label2'], diag_values)
 
-  # TODO(crbug.com/981349): Remove this mock when histograms format
-  # is enabled in results_processor.
-  @mock.patch('core.results_processor.command_line.SUPPORTED_FORMATS',
-              ['histograms'])
   def testHistogramsOutputAppendResults(self):
     hist_file = os.path.join(self.output_dir,
                              histograms_output.HISTOGRAM_DICTS_NAME)
@@ -243,10 +229,6 @@ class ResultsProcessorIntegrationTests(unittest.TestCase):
     self.assertIn(['label1'], diag_values)
     self.assertIn(['label2'], diag_values)
 
-  # TODO(crbug.com/981349): Remove this mock when html format
-  # is enabled in results_processor.
-  @mock.patch('core.results_processor.command_line.SUPPORTED_FORMATS',
-              ['html'])
   def testHtmlOutput(self):
     hist_file = os.path.join(self.output_dir,
                              histograms_output.HISTOGRAM_DICTS_NAME)
@@ -292,10 +274,6 @@ class ResultsProcessorIntegrationTests(unittest.TestCase):
     self.assertIn([['documentation', 'url']], diag_values)
     self.assertIn(['label'], diag_values)
 
-  # TODO(crbug.com/981349): Remove this mock when html format
-  # is enabled in results_processor.
-  @mock.patch('core.results_processor.command_line.SUPPORTED_FORMATS',
-              ['html'])
   def testHtmlOutputResetResults(self):
     self.SerializeIntermediateResults([])
 
@@ -324,10 +302,6 @@ class ResultsProcessorIntegrationTests(unittest.TestCase):
     self.assertNotIn(['label1'], diag_values)
     self.assertIn(['label2'], diag_values)
 
-  # TODO(crbug.com/981349): Remove this mock when html format
-  # is enabled in results_processor.
-  @mock.patch('core.results_processor.command_line.SUPPORTED_FORMATS',
-              ['html'])
   def testHtmlOutputAppendResults(self):
     self.SerializeIntermediateResults([])