perf: Rendering benchmark that combines smoothness and thread_times

This CL introduces rendering benchmarks that incorporate both smoothness
and thread_times measurements. The benchmarks are split into two page
sets, desktop and mobile, in which all smoothness page sets will be
consolidated.

To begin, rendering.desktop contains top_25_smooth pages, and
rendering.mobile contains top_25_smooth (default and gpu_raster) and
key_mobile_sites_smooth (default and sync_scroll). More page sets will
be merged into these over time.

Bug: 760553
Change-Id: I8179a14d9b9af96be509a63112fd9bf879e0bcf6
Reviewed-on: https://chromium-review.googlesource.com/821473
Commit-Queue: Ned Nguyen <nednguyen@google.com>
Reviewed-by: Ned Nguyen <nednguyen@google.com>
Cr-Commit-Position: refs/heads/master@{#546642}

tools/perf/benchmarks/rendering.py

+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+from core import perf_benchmark
+
+import page_sets
+from measurements import rendering
+from telemetry import benchmark
+from telemetry import story as story_module
+
+
+@benchmark.Owner(emails=['sadrul@chromium.org', 'vmiura@chromium.org'])
+class RenderingDesktop(perf_benchmark.PerfBenchmark):
+
+  test = rendering.Rendering
+  page_set = page_sets.RenderingDesktopPageSet
+  SUPPORTED_PLATFORMS = [story_module.expectations.ALL_DESKTOP]
+
+  @classmethod
+  def Name(cls):
+    return 'rendering.desktop'
+
+  @classmethod
+  def AddBenchmarkCommandLineArgs(cls, parser):
+    parser.add_option('--scroll-forever', action='store_true',
+                      help='If set, continuously scroll up and down forever. '
+                           'This is useful for analysing scrolling behaviour '
+                           'with tools such as perf.')
+
+
+@benchmark.Owner(emails=['sadrul@chromium.org', 'vmiura@chromium.org'])
+class RenderingMobile(perf_benchmark.PerfBenchmark):
+
+  test = rendering.Rendering
+  page_set = page_sets.RenderingMobilePageSet
+  SUPPORTED_PLATFORMS = [story_module.expectations.ALL_MOBILE]
+
+  @classmethod
+  def Name(cls):
+    return 'rendering.mobile'
+
+  @classmethod
+  def AddBenchmarkCommandLineArgs(cls, parser):
+    parser.add_option('--scroll-forever', action='store_true',
+                      help='If set, continuously scroll up and down forever. '
+                           'This is useful for analysing scrolling behaviour '
+                           'with tools such as perf.')

tools/perf/core/perf_data_generator.py

@@ -35,7 +35,9 @@ from core.sharding_map_generator import load_benchmark_sharding_map
 
 
 _UNSCHEDULED_TELEMETRY_BENCHMARKS = set([
-  'experimental.startup.android.coldish'
+  'experimental.startup.android.coldish',
+  'rendering.desktop',
+  'rendering.mobile'
   ])
 
 

tools/perf/expectations.config

@@ -113,6 +113,33 @@ crbug.com/773949 [ Nexus_6 ] power.idle_platform/* [ Skip ]
 crbug.com/773949 [ Nexus_7 ] power.idle_platform/* [ Skip ]
 crbug.com/773949 [ Android_One ] power.idle_platform/* [ Skip ]
 
+# Benchmark: rendering.desktop
+crbug.com/762165 [ Win ] rendering.desktop/google_calendar [ Skip ]
+crbug.com/762165 [ Win ] rendering.desktop/youtube [ Skip ]
+crbug.com/815205 [ Mac ] rendering.desktop/youtube [ Skip ]
+crbug.com/667432 [ All ] rendering.desktop/amazon [ Skip ]
+crbug.com/528474 [ All ] rendering.desktop/cnn [ Skip ]
+crbug.com/762165 [ Win ] rendering.desktop/google_plus [ Skip ]
+crbug.com/762165 [ Win ] rendering.desktop/google_image_search [ Skip ]
+crbug.com/762165 [ Win ] rendering.desktop/google_docs [ Skip ]
+
+# Benchmark: rendering.mobile
+crbug.com/667432 [ All ] rendering.mobile/amazon_desktop [ Skip ]
+crbug.com/667432 [ All ] rendering.mobile/amazon_desktop_gpu_raster [ Skip ]
+crbug.com/528474 [ All ] rendering.mobile/cnn_desktop [ Skip ]
+crbug.com/528474 [ All ] rendering.mobile/cnn_desktop_gpu_raster [ Skip ]
+crbug.com/667432 [ All ] rendering.mobile/pinterest_desktop_gpu_raster [ Skip ]
+crbug.com/803869 [ Nexus_5X ] rendering.mobile/youtube_desktop_gpu_raster [ Skip ]
+crbug.com/756119 [ All ] rendering.mobile/digg [ Skip ]
+crbug.com/756119 [ All ] rendering.mobile/digg_sync_scroll [ Skip ]
+crbug.com/667432 [ All ] rendering.mobile/amazon [ Skip ]
+crbug.com/667432 [ All ] rendering.mobile/amazon_sync_scroll [ Skip ]
+crbug.com/169980 [ All ] rendering.mobile/twitter [ Skip ]
+crbug.com/169980 [ All ] rendering.mobile/twitter_sync_scroll [ Skip ]
+crbug.com/249722 [ All ] rendering.mobile/espn [ Skip ]
+crbug.com/249722 [ All ] rendering.mobile/espn_sync_scroll [ Skip ]
+crbug.com/249736 [ All ] rendering.mobile/forecast.io [ Skip ]
+crbug.com/249736 [ All ] rendering.mobile/forecast.io_sync_scroll [ Skip ]
 
 # Benchmark: smoothness.gpu_rasterization.polymer
 [ All ] smoothness.gpu_rasterization.polymer/* [ Skip ] # Test needs to be modernized.

tools/perf/measurements/rendering.py

+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from telemetry.page import legacy_page_test
+from telemetry.timeline import model as model_module
+from telemetry.value import trace
+from telemetry.web_perf.metrics import smoothness
+from telemetry.web_perf import smooth_gesture_util
+from telemetry.web_perf import timeline_interaction_record as tir_module
+from telemetry.timeline import tracing_config
+
+from metrics import timeline
+
+
+def _CollectRecordsFromRendererThreads(model, renderer_thread):
+  records = []
+  for event in renderer_thread.async_slices:
+    if tir_module.IsTimelineInteractionRecord(event.name):
+      interaction = tir_module.TimelineInteractionRecord.FromAsyncEvent(event)
+      # Adjust the interaction record to match the synthetic gesture
+      # controller if needed.
+      interaction = (
+          smooth_gesture_util.GetAdjustedInteractionIfContainGesture(
+              model, interaction))
+      records.append(interaction)
+  return records
+
+
+class Rendering(legacy_page_test.LegacyPageTest):
+
+  def __init__(self):
+    super(Rendering, self).__init__()
+    self._results = None
+
+  @classmethod
+  def CustomizeBrowserOptions(cls, options):
+    options.AppendExtraBrowserArgs('--enable-gpu-benchmarking')
+    options.AppendExtraBrowserArgs('--touch-events=enabled')
+
+  def WillNavigateToPage(self, page, tab):
+    config = tracing_config.TracingConfig()
+    config.enable_chrome_trace = True
+    config.enable_platform_display_trace = True
+
+    # Basic categories for smoothness.
+    config.chrome_trace_config.SetLowOverheadFilter()
+
+    # Extra categories from commandline flag.
+    if self.options and self.options.extra_chrome_categories:
+      config.chrome_trace_config.category_filter.AddFilterString(
+          self.options.extra_chrome_categories)
+
+    tab.browser.platform.tracing_controller.StartTracing(config)
+
+  def ValidateAndMeasurePage(self, _, tab, results):
+    self._results = results
+    trace_result = tab.browser.platform.tracing_controller.StopTracing()
+
+    # TODO(charliea): This is part of a three-sided Chromium/Telemetry patch
+    # where we're changing the return type of StopTracing from a TraceValue to a
+    # (TraceValue, nonfatal_exception_list) tuple. Once the tuple return value
+    # lands in Chromium, the non-tuple logic should be deleted.
+    if isinstance(trace_result, tuple):
+      trace_result = trace_result[0]
+
+    trace_value = trace.TraceValue(
+        results.current_page, trace_result,
+        file_path=results.telemetry_info.trace_local_path,
+        remote_path=results.telemetry_info.trace_remote_path,
+        upload_bucket=results.telemetry_info.upload_bucket,
+        cloud_url=results.telemetry_info.trace_remote_url)
+    results.AddValue(trace_value)
+
+    model = model_module.TimelineModel(trace_result)
+    renderer_thread = model.GetRendererThreadFromTabId(tab.id)
+    records = _CollectRecordsFromRendererThreads(model, renderer_thread)
+
+    smoothness_metric = smoothness.SmoothnessMetric()
+    smoothness_metric.AddResults(model, renderer_thread, records, results)
+
+    thread_times_metric = timeline.ThreadTimesTimelineMetric()
+    thread_times_metric.AddResults(model, renderer_thread, records, results)
+
+  def DidRunPage(self, platform):
+    if platform.tracing_controller.is_tracing_running:
+      trace_result = platform.tracing_controller.StopTracing()
+      if self._results:
+
+        # TODO(charliea): This is part of a three-sided Chromium/Telemetry patch
+        # where we're changing the return type of StopTracing from a TraceValue
+        # to a (TraceValue, nonfatal_exception_list) tuple. Once the tuple
+        # return value lands in Chromium, the non-tuple logic should be deleted.
+        if isinstance(trace_result, tuple):
+          trace_result = trace_result[0]
+
+        trace_value = trace.TraceValue(
+            self._results.current_page, trace_result,
+            file_path=self._results.telemetry_info.trace_local_path,
+            remote_path=self._results.telemetry_info.trace_remote_path,
+            upload_bucket=self._results.telemetry_info.upload_bucket,
+            cloud_url=self._results.telemetry_info.trace_remote_url)
+
+        self._results.AddValue(trace_value)

tools/perf/page_sets/data/rendering_desktop.json

+{
+    "archives": {
+        "google_docs": {
+            "DEFAULT": "top_25_009.wprgo"
+        },
+        "facebook": {
+            "DEFAULT": "top_25_003.wprgo"
+        },
+        "wikipedia": {
+            "DEFAULT": "top_25_004.wprgo"
+        },
+        "yahoo_answers": {
+            "DEFAULT": "top_25_000.wprgo"
+        },
+        "booking.com": {
+            "DEFAULT": "top_25_000.wprgo"
+        },
+        "wordpress": {
+            "DEFAULT": "top_25_000.wprgo"
+        },
+        "espn": {
+            "DEFAULT": "top_25_000.wprgo"
+        },
+        "yahoo_games": {
+            "DEFAULT": "top_25_000.wprgo"
+        },
+        "blogspot": {
+            "DEFAULT": "top_25_000.wprgo"
+        },
+        "yahoo_news": {
+            "DEFAULT": "top_25_000.wprgo"
+        },
+        "pinterest": {
+            "DEFAULT": "top_25_000.wprgo"
+        },
+        "yahoo_sports": {
+            "DEFAULT": "top_25_000.wprgo"
+        },
+        "techcrunch": {
+            "DEFAULT": "top_25_000.wprgo"
+        },
+        "amazon": {
+            "DEFAULT": "top_25_000.wprgo"
+        },
+        "cnn": {
+            "DEFAULT": "top_25_000.wprgo"
+        },
+        "ebay": {
+            "DEFAULT": "top_25_000.wprgo"
+        },
+        "linkedin": {
+            "DEFAULT": "top_25_000.wprgo"
+        },
+        "weather.com": {
+            "DEFAULT": "top_25_000.wprgo"
+        },
+        "youtube": {
+            "DEFAULT": "top_25_011.wprgo"
+        },
+        "google_drive": {
+            "DEFAULT": "top_25_000.wprgo"
+        },
+        "gmail": {
+            "DEFAULT": "top_25_006.wprgo"
+        },
+        "google_plus": {
+            "DEFAULT": "top_25_010.wprgo"
+        },
+        "twitter": {
+            "DEFAULT": "top_25_000.wprgo"
+        },
+        "google_web_search": {
+            "DEFAULT": "top_25_000.wprgo"
+        },
+        "google_calendar": {
+            "DEFAULT": "top_25_008.wprgo"
+        },
+        "google_image_search": {
+            "DEFAULT": "top_25_007.wprgo"
+        }
+    },
+    "description": "Describes the Web Page Replay archives for a story set. Don't edit by hand! Use record_wpr for updating.",
+    "platform_specific": true
+}

tools/perf/page_sets/rendering_desktop.py

+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+from telemetry.page import shared_page_state
+from telemetry import story
+
+from page_sets import top_25_smooth
+
+
+class _SharedPageState(shared_page_state.SharedDesktopPageState):
+  pass
+
+
+class RenderingDesktopPageSet(story.StorySet):
+  """ Page set for measuring rendering performance on desktop. """
+
+  def __init__(self, scroll_forever=False):
+    super(RenderingDesktopPageSet, self).__init__(
+        archive_data_file='data/rendering_desktop.json',
+        cloud_storage_bucket=story.PARTNER_BUCKET)
+
+    self.scroll_forever = scroll_forever
+
+    top_25_smooth.AddPagesToPageSet(
+        page_set=self,
+        shared_page_state_class=_SharedPageState)

tools/perf/page_sets/rendering_mobile.py

+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+from telemetry.page import shared_page_state
+from telemetry import story
+
+from page_sets import key_mobile_sites_smooth
+from page_sets import top_25_smooth
+
+
+class _SharedPageState(shared_page_state.SharedMobilePageState):
+  pass
+
+
+class RenderingMobilePageSet(story.StorySet):
+  """ Page set for measuring rendering performance on desktop. """
+
+  def __init__(self, scroll_forever=False):
+    super(RenderingMobilePageSet, self).__init__(
+        archive_data_file='data/rendering_mobile.json',
+        cloud_storage_bucket=story.PARTNER_BUCKET)
+
+    self.scroll_forever = scroll_forever
+
+    top_25_smooth.AddPagesToPageSet(
+        self,
+        shared_page_state_class=_SharedPageState,
+        name_func=lambda name: name + '_desktop')
+    top_25_smooth.AddPagesToPageSet(
+        self,
+        shared_page_state_class=_SharedPageState,
+        name_func=lambda name: name + '_desktop_gpu_raster',
+        extra_browser_args=['--force-gpu-rasterization'])
+
+    key_mobile_sites_smooth.AddPagesToPageSet(
+        self,
+        shared_page_state_class=_SharedPageState)
+    key_mobile_sites_smooth.AddPagesToPageSet(
+        self,
+        shared_page_state_class=_SharedPageState,
+        name_func=lambda name: name + '_sync_scroll',
+        extra_browser_args=['--disable-threaded-scrolling'])