Add trace rerun options to benchmark and pass them through chartjson.

After discussion with Annie/Dave/Fadi/Kari and myself, seems like we settled on not exposing an arbitrary # of new command lines. Rather, there'd be an optional param to rerun with extra tracing categories enabled. The benchmark would specify that there's a debug option available, and has a way of customizing the exact categories it needs.

Example of v8 benchmark modified:

class V8GarbageCollectionCases(benchmark.Benchmark):
  """Measure V8 metrics on the garbage collection cases."""
  def CreateTimelineBasedMeasurementOptions(self):
    # TODO(ernstm): Remove v8-overhead when benchmark relevant v8 events become
    # available in the 'benchmark' category.
    # This can theoretically be handled now by the "default" override
    return timeline_based_measurement.Options()

  @classmethod
  def HasBenchmarkTraceRerunDebugOption(cls):
    return True

  def SetupBenchmarkDefaultTraceRerunOptions(self, tbm_options):
    tbm_options.ExtendTraceCategoryFilters(['v8'])

  def SetupBenchmarkDebugTraceRerunOptions(self, tbm_options):
    tbm_options.ExtendTraceCategoryFilters(['*'])

BUG=

Review URL: https://codereview.chromium.org/840043004

Cr-Commit-Position: refs/heads/master@{#313080}

tools/telemetry/telemetry/benchmark.py

@@ -33,9 +33,10 @@ class InvalidOptionsError(Exception):
 
 
 class BenchmarkMetadata(object):
-  def __init__(self, name, description=''):
+  def __init__(self, name, description='', rerun_options=None):
     self._name = name
     self._description = description
+    self._rerun_options = rerun_options
 
   @property
   def name(self):
@@ -45,6 +46,10 @@ class BenchmarkMetadata(object):
   def description(self):
       return self._description
 
+  @property
+  def rerun_options(self):
+      return self._rerun_options
+
 
 class Benchmark(command_line.Command):
   """Base class for a Telemetry benchmark.
@@ -81,11 +86,44 @@ class Benchmark(command_line.Command):
 
   @classmethod
   def AddCommandLineArgs(cls, parser):
+    group = optparse.OptionGroup(parser, '%s test options' % cls.Name())
     if hasattr(cls, 'AddBenchmarkCommandLineArgs'):
-      group = optparse.OptionGroup(parser, '%s test options' % cls.Name())
       cls.AddBenchmarkCommandLineArgs(group)
+
+    if cls.HasTraceRerunDebugOption():
+      group.add_option(
+          '--rerun-with-debug-trace',
+          action='store_true',
+          help='Rerun option that enables more extensive tracing.')
+
+    if group.option_list:
       parser.add_option_group(group)
 
+  @classmethod
+  def HasTraceRerunDebugOption(cls):
+    if hasattr(cls, 'HasBenchmarkTraceRerunDebugOption'):
+      if cls.HasBenchmarkTraceRerunDebugOption():
+        return True
+    return False
+
+  def GetTraceRerunCommands(self):
+    if self.HasTraceRerunDebugOption():
+      return [['Debug Trace', '--rerun-with-debug-trace']]
+    return []
+
+  def SetupTraceRerunOptions(self, browser_options, tbm_options):
+    if self.HasTraceRerunDebugOption():
+      if browser_options.rerun_with_debug_trace:
+        self.SetupBenchmarkDebugTraceRerunOptions(tbm_options)
+      else:
+        self.SetupBenchmarkDefaultTraceRerunOptions(tbm_options)
+
+  def SetupBenchmarkDefaultTraceRerunOptions(self, tbm_options):
+    """Setup tracing categories associated with default trace option."""
+
+  def SetupBenchmarkDebugTraceRerunOptions(self, tbm_options):
+    """Setup tracing categories associated with debug trace option."""
+
   @classmethod
   def SetArgumentDefaults(cls, parser):
     default_values = parser.get_default_values()
@@ -104,7 +142,8 @@ class Benchmark(command_line.Command):
     """Add browser options that are required by this benchmark."""
 
   def GetMetadata(self):
-    return BenchmarkMetadata(self.Name(), self.__doc__)
+    return BenchmarkMetadata(
+        self.Name(), self.__doc__, self.GetTraceRerunCommands())
 
   def Run(self, finder_options):
     """Run this test with the given options.
@@ -248,8 +287,10 @@ class Benchmark(command_line.Command):
           'Cannot override CreateTimelineBasedMeasurementOptions '
           'with a PageTest.')
       return self.test()  # pylint: disable=no-value-for-parameter
-    return timeline_based_measurement.TimelineBasedMeasurement(
-        self.CreateTimelineBasedMeasurementOptions())
+
+    opts = self.CreateTimelineBasedMeasurementOptions()
+    self.SetupTraceRerunOptions(options, opts)
+    return timeline_based_measurement.TimelineBasedMeasurement(opts)
 
   def CreatePageSet(self, options):  # pylint: disable=unused-argument
     """Get the page set this test will run on.

tools/telemetry/telemetry/results/chart_json_output_formatter.py

@@ -54,6 +54,7 @@ def _ResultsAsChartDict(benchmark_metadata, page_specific_values,
     'format_version': '0.1',
     'benchmark_name': benchmark_metadata.name,
     'benchmark_description': benchmark_metadata.description,
+    'benchmark_rerun_options': benchmark_metadata.rerun_options,
     'charts': charts,
   }
 

tools/telemetry/telemetry/web_perf/timeline_based_measurement.py

@@ -139,6 +139,10 @@ class Options(object):
         The v8 overhead level is a temporary solution that may be removed.
     """
     self._overhead_level = overhead_level
+    self._category_filters = []
+
+  def ExtendTraceCategoryFilters(self, filters):
+    self._category_filters.extend(filters)
 
 
 class TimelineBasedMeasurement(object):
@@ -165,6 +169,7 @@ class TimelineBasedMeasurement(object):
   """
   def __init__(self, options):
     self._overhead_level = options._overhead_level
+    self._category_filters = options._category_filters
 
   def WillRunUserStory(self, tracing_controller,
                        synthetic_delay_categories=None):
@@ -192,6 +197,9 @@ class TimelineBasedMeasurement(object):
     else:
       category_filter = tracing_category_filter.CreateDebugOverheadFilter()
 
+    for new_category_filter in self._category_filters:
+      category_filter.AddIncludedCategory(new_category_filter)
+
     # TODO(slamm): Move synthetic_delay_categories to the TBM options.
     for delay in synthetic_delay_categories or []:
       category_filter.AddSyntheticDelay(delay)