Telemetry: finally move rendering to TBMv2

tasks_per_frame metrics are added in
https://chromium-review.googlesource.com/c/catapult/+/1318249.

mean_frame_time metrics are deleted since we already frame_times for
display compositor frames and we decided to not use renderer
compositor frames in crbug.com/888551.

Bug: 890757
Change-Id: Ib4d0b9cedc4f1012873f82da01839b9173d4e6dd
Reviewed-on: https://chromium-review.googlesource.com/c/1318395Reviewed-by: Ned Nguyen <nednguyen@google.com>
Commit-Queue: Ehsan Chiniforooshan <chiniforooshan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#605718}

tools/perf/benchmarks/rendering.py

@@ -4,17 +4,16 @@
 from core import perf_benchmark
 
 import page_sets
-from measurements import rendering
 from telemetry import benchmark
 from telemetry import story as story_module
-
+from telemetry.timeline import chrome_trace_category_filter
+from telemetry.web_perf import timeline_based_measurement
 
 @benchmark.Info(emails=['sadrul@chromium.org', 'vmiura@chromium.org'],
                 documentation_url='https://bit.ly/rendering-benchmarks',
                 component='Internals>GPU>Metrics')
 class RenderingDesktop(perf_benchmark.PerfBenchmark):
 
-  test = rendering.Rendering
   SUPPORTED_PLATFORMS = [story_module.expectations.ALL_DESKTOP]
 
   @classmethod
@@ -31,13 +30,18 @@ class RenderingDesktop(perf_benchmark.PerfBenchmark):
   def CreateStorySet(self, options):
     return page_sets.RenderingStorySet(platform='desktop')
 
+  def CreateCoreTimelineBasedMeasurementOptions(self):
+    category_filter = chrome_trace_category_filter.CreateLowOverheadFilter()
+    options = timeline_based_measurement.Options(category_filter)
+    options.SetTimelineBasedMetrics(['renderingMetric'])
+    return options
+
 
 @benchmark.Info(emails=['sadrul@chromium.org', 'vmiura@chromium.org'],
                 documentation_url='https://bit.ly/rendering-benchmarks',
                 component='Internals>GPU>Metrics')
 class RenderingMobile(perf_benchmark.PerfBenchmark):
 
-  test = rendering.Rendering
   SUPPORTED_PLATFORMS = [story_module.expectations.ALL_MOBILE]
 
   @classmethod
@@ -53,3 +57,9 @@ class RenderingMobile(perf_benchmark.PerfBenchmark):
 
   def CreateStorySet(self, options):
     return page_sets.RenderingStorySet(platform='mobile')
+
+  def CreateCoreTimelineBasedMeasurementOptions(self):
+    category_filter = chrome_trace_category_filter.CreateLowOverheadFilter()
+    options = timeline_based_measurement.Options(category_filter)
+    options.SetTimelineBasedMetrics(['renderingMetric'])
+    return options

tools/perf/contrib/cluster_telemetry/rendering_ct.py

@@ -4,9 +4,10 @@
 
 from contrib.cluster_telemetry import ct_benchmarks_util
 from contrib.cluster_telemetry import page_set
+from telemetry.timeline import chrome_trace_category_filter
+from telemetry.web_perf import timeline_based_measurement
 
 from core import perf_benchmark
-from measurements import rendering
 
 def ScrollToEndOfPage(action_runner):
   action_runner.Wait(1)
@@ -19,8 +20,6 @@ class RenderingCT(perf_benchmark.PerfBenchmark):
 
   options = {'upload_results': True}
 
-  test = rendering.Rendering
-
   @classmethod
   def Name(cls):
     return 'rendering.cluster_telemetry'
@@ -37,3 +36,9 @@ class RenderingCT(perf_benchmark.PerfBenchmark):
     return page_set.CTPageSet(
         options.urls_list, options.user_agent, options.archive_data_file,
         run_page_interaction_callback=ScrollToEndOfPage)
+
+  def CreateCoreTimelineBasedMeasurementOptions(self):
+    category_filter = chrome_trace_category_filter.CreateLowOverheadFilter()
+    options = timeline_based_measurement.Options(category_filter)
+    options.SetTimelineBasedMetrics(['renderingMetric'])
+    return options

tools/perf/contrib/cluster_telemetry/repaint.py

@@ -5,16 +5,15 @@
 from contrib.cluster_telemetry import ct_benchmarks_util
 from contrib.cluster_telemetry import page_set
 from contrib.cluster_telemetry import repaint_helpers
+from telemetry.timeline import chrome_trace_category_filter
+from telemetry.web_perf import timeline_based_measurement
 
 from core import perf_benchmark
-from measurements import rendering
 
 
 class RepaintCT(perf_benchmark.PerfBenchmark):
   """Measures repaint performance for Cluster Telemetry."""
 
-  test = rendering.Rendering
-
   @classmethod
   def Name(cls):
     return 'repaint_ct'
@@ -41,3 +40,9 @@ class RepaintCT(perf_benchmark.PerfBenchmark):
     return page_set.CTPageSet(
         options.urls_list, options.user_agent, options.archive_data_file,
         run_page_interaction_callback=repaint_helpers.WaitThenRepaint)
+
+  def CreateCoreTimelineBasedMeasurementOptions(self):
+    category_filter = chrome_trace_category_filter.CreateLowOverheadFilter()
+    options = timeline_based_measurement.Options(category_filter)
+    options.SetTimelineBasedMetrics(['renderingMetric'])
+    return options

tools/perf/measurements/rendering.py

-# Copyright 2017 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-from telemetry.page import legacy_page_test
-from telemetry.timeline import model as model_module
-from telemetry.value import trace
-from telemetry.web_perf import smooth_gesture_util
-from telemetry.web_perf import timeline_interaction_record as tir_module
-from telemetry.timeline import tracing_config
-
-from metrics import timeline
-from measurements import rendering_util
-
-
-def _CollectRecordsFromRendererThreads(model, renderer_thread):
-  records = []
-  for event in renderer_thread.async_slices:
-    if tir_module.IsTimelineInteractionRecord(event.name):
-      interaction = tir_module.TimelineInteractionRecord.FromAsyncEvent(event)
-      # Adjust the interaction record to match the synthetic gesture
-      # controller if needed.
-      interaction = (
-          smooth_gesture_util.GetAdjustedInteractionIfContainGesture(
-              model, interaction))
-      records.append(interaction)
-  return records
-
-
-class Rendering(legacy_page_test.LegacyPageTest):
-
-  def __init__(self):
-    super(Rendering, self).__init__()
-    self._results = None
-
-  @classmethod
-  def CustomizeBrowserOptions(cls, options):
-    options.AppendExtraBrowserArgs('--enable-gpu-benchmarking')
-    options.AppendExtraBrowserArgs('--touch-events=enabled')
-
-  def WillNavigateToPage(self, page, tab):
-    config = tracing_config.TracingConfig()
-    config.enable_chrome_trace = True
-    config.enable_platform_display_trace = True
-
-    # Basic categories for smoothness.
-    config.chrome_trace_config.SetLowOverheadFilter()
-
-    # Extra categories from commandline flag.
-    if self.options and self.options.extra_chrome_categories:
-      config.chrome_trace_config.category_filter.AddFilterString(
-          self.options.extra_chrome_categories)
-
-    tab.browser.platform.tracing_controller.StartTracing(config)
-
-  def ValidateAndMeasurePage(self, _, tab, results):
-    self._results = results
-    tab.browser.platform.tracing_controller.telemetry_info = (
-        results.telemetry_info)
-    trace_result = tab.browser.platform.tracing_controller.StopTracing()
-
-    # TODO(charliea): This is part of a three-sided Chromium/Telemetry patch
-    # where we're changing the return type of StopTracing from a TraceValue to a
-    # (TraceValue, nonfatal_exception_list) tuple. Once the tuple return value
-    # lands in Chromium, the non-tuple logic should be deleted.
-    if isinstance(trace_result, tuple):
-      trace_result = trace_result[0]
-
-    trace_value = trace.TraceValue(
-        results.current_page, trace_result,
-        file_path=results.telemetry_info.trace_local_path,
-        remote_path=results.telemetry_info.trace_remote_path,
-        upload_bucket=results.telemetry_info.upload_bucket,
-        cloud_url=results.telemetry_info.trace_remote_url)
-    results.AddValue(trace_value)
-
-    model = model_module.TimelineModel(trace_result)
-    renderer_thread = model.GetFirstRendererThread(tab.id)
-    records = _CollectRecordsFromRendererThreads(model, renderer_thread)
-
-    thread_times_metric = timeline.ThreadTimesTimelineMetric()
-    thread_times_metric.AddResults(model, renderer_thread, records, results)
-
-    rendering_util.AddTBMv2RenderingMetrics(
-        trace_value, results, import_experimental_metrics=True)
-
-  def DidRunPage(self, platform):
-    if platform.tracing_controller.is_tracing_running:
-      trace_result = platform.tracing_controller.StopTracing()
-      if self._results:
-
-        # TODO(charliea): This is part of a three-sided Chromium/Telemetry patch
-        # where we're changing the return type of StopTracing from a TraceValue
-        # to a (TraceValue, nonfatal_exception_list) tuple. Once the tuple
-        # return value lands in Chromium, the non-tuple logic should be deleted.
-        if isinstance(trace_result, tuple):
-          trace_result = trace_result[0]
-
-        trace_value = trace.TraceValue(
-            self._results.current_page, trace_result,
-            file_path=self._results.telemetry_info.trace_local_path,
-            remote_path=self._results.telemetry_info.trace_remote_path,
-            upload_bucket=self._results.telemetry_info.upload_bucket,
-            cloud_url=self._results.telemetry_info.trace_remote_url)
-
-        self._results.AddValue(trace_value)

tools/perf/measurements/rendering_unittest.py

-# Copyright 2018 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-from telemetry.testing import options_for_unittests
-from telemetry.testing import page_test_test_case
-from telemetry.util import wpr_modes
-
-from measurements import rendering
-from measurements import rendering_util
-
-RENDERING_THREAD_GROUPS = ['GPU', 'IO', 'browser', 'display_compositor',
-                           'other', 'raster', 'renderer_compositor',
-                           'renderer_main', 'total_all', 'total_fast_path']
-
-class RenderingUnitTest(page_test_test_case.PageTestTestCase):
-  """Test for rendering measurement
-
-     Runs rendering measurement on a simple page and verifies the existence of
-     all metrics in the results. The correctness of metrics is not tested here.
-  """
-
-  def setUp(self):
-    self._options = options_for_unittests.GetCopy()
-    self._options.browser_options.wpr_mode = wpr_modes.WPR_OFF
-    self._options.pageset_repeat = 2
-
-  def testRendering(self):
-    ps = self.CreateStorySetFromFileInUnittestDataDir('scrollable_page.html')
-    results = self.RunMeasurement(
-        rendering.Rendering(), ps, options=self._options)
-    self.assertFalse(results.had_failures)
-    stat = rendering_util.ExtractStat(results)
-
-    self.assertGreater(stat['frame_times'].count, 1)
-    self.assertGreater(stat['percentage_smooth'].count, 1)
-    for thread_group in RENDERING_THREAD_GROUPS:
-      # We should have at least two sample values for each metric, since
-      # pageset_repeat is 2.
-      histogram_name = 'thread_%s_cpu_time_per_frame' % thread_group
-      self.assertGreater(stat[histogram_name].count, 1)

tools/perf/measurements/rendering_util.py

-# Copyright 2018 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-from tracing.metrics import metric_runner
-from tracing.value import histogram as histogram_module
-
-def AddTBMv2RenderingMetrics(trace_value, results, import_experimental_metrics):
-  mre_result = metric_runner.RunMetric(
-      trace_value.filename, metrics=['renderingMetric'],
-      extra_import_options={'trackDetailedModelStats': True},
-      report_progress=False, canonical_url=results.telemetry_info.trace_url)
-
-  for f in mre_result.failures:
-    results.Fail(f.stack)
-
-  existing_values = {v.name for v in results.all_page_specific_values}
-  histograms = []
-  for histogram in mre_result.pairs.get('histograms', []):
-    name = histogram.get('name', '')
-    if ((import_experimental_metrics or name.find('_tbmv2') < 0) and
-        not name in existing_values):
-      histograms.append(histogram)
-  results.ImportHistogramDicts(histograms, import_immediately=False)
-
-def ExtractStat(results):
-  stat = {}
-  for histogram_dict in results.AsHistogramDicts():
-    # It would be nicer if instead of converting results._histograms to dicts
-    # and then parsing them back in the following line, results had a getter
-    # returning results._histograms. But, since this is a temporary code that
-    # will be deleted after transitioning Smoothness to TBMv2, we don't change
-    # page_test_results.py for a temporary usecase.
-    if 'name' in histogram_dict:
-      histogram = histogram_module.Histogram.FromDict(histogram_dict)
-      if histogram.running is None:
-        continue
-      if histogram.name in stat:
-        stat[histogram.name] = stat[histogram.name].Merge(histogram.running)
-      else:
-        stat[histogram.name] = histogram.running
-  return stat

tools/perf/metrics/timeline.py

-# Copyright 2014 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-import collections
-
-from telemetry.util.statistics import DivideIfPossibleOrZero
-from telemetry.value import scalar
-from telemetry.web_perf.metrics import timeline_based_metric
-
-
-# We want to generate a consistent picture of our thread usage, despite
-# having several process configurations (in-proc-gpu/single-proc).
-# Since we can't isolate renderer threads in single-process mode, we
-# always sum renderer-process threads' times. We also sum all io-threads
-# for simplicity.
-TimelineThreadCategories = {
-    "Chrome_InProcGpuThread": "GPU",
-    "CrGpuMain": "GPU",
-    "VizCompositorThread": "display_compositor",
-    "CrBrowserMain": "browser",
-    "Browser Compositor": "browser",
-    "CrRendererMain": "renderer_main",
-    "Compositor": "renderer_compositor",
-    "IOThread": "IO",
-    "CompositorTileWorker": "raster",
-    "DummyThreadName1": "other",
-    "DummyThreadName2": "total_fast_path",
-    "DummyThreadName3": "total_all"
-}
-
-_MatchBySubString = ["IOThread", "CompositorTileWorker"]
-
-AllThreads = TimelineThreadCategories.values()
-NoThreads = []
-FastPathThreads = [
-  "GPU", "display_compositor", "renderer_compositor", "browser", "IO"]
-
-ReportMainThreadOnly = ["renderer_main"]
-ReportSilkDetails = ["renderer_main"]
-
-# TODO(epenner): Thread names above are likely fairly stable but trace names
-# could change. We should formalize these traces to keep this robust.
-OverheadTraceCategory = "trace_event_overhead"
-OverheadTraceName = "overhead"
-FrameTraceName = "Graphics.Pipeline"
-FrameTraceStepName = "GenerateCompositorFrame"
-FrameTraceThreadName = "renderer_compositor"
-
-
-def Rate(numerator, denominator):
-  return DivideIfPossibleOrZero(numerator, denominator)
-
-
-def ThreadCategoryName(thread_name):
-  thread_category = "other"
-  for substring, category in TimelineThreadCategories.iteritems():
-    if substring in _MatchBySubString and substring in thread_name:
-      thread_category = category
-  if thread_name in TimelineThreadCategories:
-    thread_category = TimelineThreadCategories[thread_name]
-  return thread_category
-
-
-def ThreadTasksResultName(thread_category):
-  return "tasks_per_frame_" + thread_category
-
-
-def ThreadMeanFrameTimeResultName(thread_category):
-  return "mean_frame_time_" + thread_category
-
-
-def ThreadDetailResultName(thread_category, detail):
-  detail_sanitized = detail.replace(".", "_")
-  return "thread_" + thread_category + "|" + detail_sanitized
-
-
-class ResultsForThread(object):
-
-  def __init__(self, model, record_ranges, name):
-    self.model = model
-    self.toplevel_slices = []
-    self.all_slices = []
-    self.name = name
-    self.record_ranges = record_ranges
-    self.all_action_time = \
-        sum([record_range.bounds for record_range in self.record_ranges])
-
-  def SlicesInActions(self, slices):
-    slices_in_actions = []
-    for event in slices:
-      for record_range in self.record_ranges:
-        if record_range.ContainsInterval(event.start, event.end):
-          slices_in_actions.append(event)
-          break
-    return slices_in_actions
-
-  def AppendThreadSlices(self, thread):
-    self.all_slices.extend(self.SlicesInActions(thread.all_slices))
-    self.toplevel_slices.extend(self.SlicesInActions(thread.toplevel_slices))
-
-  # Reports cpu-time per interval and tasks per interval.
-  def AddResults(self, num_intervals, results):
-    tasks_per_interval = Rate(len(self.toplevel_slices), num_intervals)
-    results.AddValue(scalar.ScalarValue(
-        results.current_page,
-        ThreadTasksResultName(self.name),
-        "tasks", tasks_per_interval))
-
-  def AddDetailedResults(self, num_intervals, results):
-    slices_by_category = collections.defaultdict(list)
-    for s in self.all_slices:
-      slices_by_category[s.category].append(s)
-    all_self_times = []
-    for category, slices_in_category in slices_by_category.iteritems():
-      self_time = sum([x.self_time for x in slices_in_category])
-      all_self_times.append(self_time)
-      self_time_per_interval = Rate(self_time, num_intervals)
-      results.AddValue(scalar.ScalarValue(
-          results.current_page,
-          ThreadDetailResultName(self.name, category),
-          "ms",
-          self_time_per_interval))
-    all_measured_time = sum(all_self_times)
-    idle_time = max(0, self.all_action_time - all_measured_time)
-    idle_time_per_interval = Rate(idle_time, num_intervals)
-    results.AddValue(scalar.ScalarValue(
-        results.current_page,
-        ThreadDetailResultName(self.name, "idle"),
-        "ms",
-        idle_time_per_interval))
-
-  def CountTracesWithNameAndArg(self, trace_name, step):
-    count = 0
-    for event in self.all_slices:
-      if trace_name in event.name and event.args and event.args["step"] == step:
-        count += 1
-    return count
-
-
-class ThreadTimesTimelineMetric(timeline_based_metric.TimelineBasedMetric):
-
-  def __init__(self):
-    super(ThreadTimesTimelineMetric, self).__init__()
-    # Minimal traces, for minimum noise in CPU-time measurements.
-    self.results_to_report = AllThreads
-    self.details_to_report = NoThreads
-
-  def AddResults(self, model, _, interaction_records, results):
-    # Set up each thread category for consistent results.
-    thread_category_results = {}
-    for name in TimelineThreadCategories.values():
-      thread_category_results[name] = ResultsForThread(
-          model, [r.GetBounds() for r in interaction_records], name)
-
-    # Group the slices by their thread category.
-    for thread in model.GetAllThreads():
-      thread_category = ThreadCategoryName(thread.name)
-      thread_category_results[thread_category].AppendThreadSlices(thread)
-
-    # Group all threads.
-    for thread in model.GetAllThreads():
-      thread_category_results["total_all"].AppendThreadSlices(thread)
-
-    # Also group fast-path threads.
-    for thread in model.GetAllThreads():
-      if ThreadCategoryName(thread.name) in FastPathThreads:
-        thread_category_results["total_fast_path"].AppendThreadSlices(thread)
-
-    # Calculate the interaction's number of frames.
-    frame_rate_thread = thread_category_results[FrameTraceThreadName]
-    num_frames = frame_rate_thread.CountTracesWithNameAndArg(FrameTraceName,
-        FrameTraceStepName)
-
-    # Report the desired results and details for each interval type.
-    for thread_results in thread_category_results.values():
-      if thread_results.name in self.results_to_report:
-        thread_results.AddResults(num_frames, results)
-      # TODO(nduca): When generic results objects are done, this special case
-      # can be replaced with a generic UI feature.
-      if thread_results.name in self.details_to_report:
-        thread_results.AddDetailedResults(num_frames, results)
-
-    # Report mean frame time for the frame rate thread. We could report other
-    # frame rates (e.g. renderer_main) but this might get confusing.
-    mean_frame_time = Rate(frame_rate_thread.all_action_time, num_frames)
-    results.AddValue(scalar.ScalarValue(
-        results.current_page,
-        ThreadMeanFrameTimeResultName(FrameTraceThreadName),
-        "ms", mean_frame_time))

tools/perf/metrics/timeline_unittest.py

-# Copyright 2013 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import unittest
-
-from telemetry.testing import test_page_test_results
-from telemetry.timeline import model as model_module
-from telemetry.web_perf import timeline_interaction_record as tir_module
-
-from metrics import timeline
-
-
-def _GetInteractionRecord(start, end):
-  return tir_module.TimelineInteractionRecord('test-record', start, end)
-
-
-class ThreadTimesTimelineMetricUnittest(unittest.TestCase):
-
-  def GetResults(self, metric, model, renderer_thread, interaction_record):
-    results = test_page_test_results.TestPageTestResults(self)
-    metric.AddResults(model, renderer_thread, interaction_record,
-                      results)
-    return results
-
-  def testResults(self):
-    model = model_module.TimelineModel()
-    renderer_main = model.GetOrCreateProcess(1).GetOrCreateThread(2)
-    renderer_main.name = 'CrRendererMain'
-
-    metric = timeline.ThreadTimesTimelineMetric()
-    metric.details_to_report = timeline.ReportMainThreadOnly
-    results = self.GetResults(metric, model, renderer_main.parent,
-                              [_GetInteractionRecord(1, 2)])
-
-    # Test that all result thread categories exist
-    for name in timeline.TimelineThreadCategories.values():
-      results.GetPageSpecificValueNamed(timeline.ThreadTasksResultName(name))
-
-  def testBasic(self):
-    model = model_module.TimelineModel()
-    renderer_main = model.GetOrCreateProcess(1).GetOrCreateThread(2)
-    renderer_main.name = 'CrRendererMain'
-
-    # Create two frame swaps (Results times should be divided by two) for
-    # an interaction that lasts 20 milliseconds.
-    cc_main = model.GetOrCreateProcess(1).GetOrCreateThread(3)
-    cc_main.name = 'Compositor'
-    cc_main.BeginSlice('cc_cat', timeline.FrameTraceName, 10, 10,
-        args={'step': 'GenerateCompositorFrame'})
-    cc_main.EndSlice(11, 11)
-    cc_main.BeginSlice('cc_cat', timeline.FrameTraceName, 12, 12,
-        args={'step': 'GenerateCompositorFrame'})
-    cc_main.EndSlice(13, 13)
-
-    # [      X       ]   [ Z ]
-    #      [  Y  ]
-    renderer_main.BeginSlice('cat1', 'X', 10, 0)
-    renderer_main.BeginSlice('cat2', 'Y', 15, 5)
-    renderer_main.EndSlice(16, 5.5)
-    renderer_main.EndSlice(30, 19.5)
-    renderer_main.BeginSlice('cat1', 'Z', 31, 20)
-    renderer_main.BeginSlice('cat1', 'Z', 33, 21)
-    model.FinalizeImport()
-
-    # Exclude 'Z' using an action-range.
-    metric = timeline.ThreadTimesTimelineMetric()
-    metric.details_to_report = timeline.ReportMainThreadOnly
-    results = self.GetResults(metric, model, renderer_main.parent,
-                              [_GetInteractionRecord(10, 30)])
-
-    # Test for the results we expect.
-    main_thread = 'renderer_main'
-    cc_thread = 'renderer_compositor'
-    assert_results = [
-        (timeline.ThreadMeanFrameTimeResultName(cc_thread), 'ms', 10.0),
-        (timeline.ThreadTasksResultName(main_thread), 'tasks', 0.5),
-        (timeline.ThreadTasksResultName(cc_thread), 'tasks', 1.0),
-        (timeline.ThreadDetailResultName(main_thread, 'cat1'), 'ms', 9.5),
-        (timeline.ThreadDetailResultName(main_thread, 'cat2'), 'ms', 0.5),
-        (timeline.ThreadDetailResultName(main_thread, 'idle'), 'ms', 0),
-    ]
-    for name, unit, value in assert_results:
-      results.AssertHasPageSpecificScalarValue(name, unit, value)