Tab Switching Benchmark for ChromeOS

Copy the tab switching benchmark to cros_benchmarks. Using key events for
simulate tab switching behavior to handle the case that some tab contexts
may be discarded.

Example usage to open 120 tabs:
$ ./run_benchmark --browser=cros-chrome --remote=DUT_IP
cros_tab_switching.typical_24 --tabset-repeat=5

BUG=724381

Review-Url: https://codereview.chromium.org/2890333002
Cr-Commit-Position: refs/heads/master@{#474975}

tools/perf/contrib/cros_benchmarks/.gitignore

+*.wpr
+data/send_key_tab_switch

tools/perf/contrib/cros_benchmarks/cros_utils.py

+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import logging
+import os
+import subprocess
+
+import py_utils
+
+from telemetry.core import exceptions
+from telemetry.value import histogram_util
+
+
+def _RunRemoteCommand(dut_ip, cmd):
+  return os.system('ssh root@%s %s' % (dut_ip, cmd))
+
+
+def _GetTabSwitchHistogram(browser):
+  """Gets MPArch.RWH_TabSwitchPaintDuration histogram.
+
+  Catches exceptions to handle devtools context lost cases.
+
+  Any tab context can be used to get the TabSwitchPaintDuration histogram.
+  browser.tabs[-1] is the last valid context. Ex: If the browser opens
+  A, B, ..., I, J tabs, E, F, G tabs have valid contexts (not discarded),
+  then browser.tab[0] is tab E, browser.tab[-1] is tab G.
+
+  In the tab switching benchmark, the tabs are opened and switched to in
+  1, 2, ..., n order. The tabs are discarded in roughly 1, 2, ..., n order
+  (LRU order). The chance of discarding the last valid context is lower
+  than discarding the first valid context.
+
+  Args:
+    browser: Gets histogram from this browser.
+
+  Returns:
+    A json serialization of a histogram or None if get histogram failed.
+  """
+  histogram_name = 'MPArch.RWH_TabSwitchPaintDuration'
+  histogram_type = histogram_util.BROWSER_HISTOGRAM
+  try:
+    return histogram_util.GetHistogram(
+        histogram_type, histogram_name, browser.tabs[-1])
+  except (exceptions.DevtoolsTargetCrashException, KeyError):
+    logging.warning('GetHistogram: Devtools context lost.')
+  except exceptions.TimeoutException:
+    logging.warning('GetHistogram: Timed out getting histogram.')
+  return None
+
+
+def GetTabSwitchHistogramRetry(browser):
+  """Retries getting histogram as it may fail when a context was discarded.
+
+  Args:
+    browser: Gets histogram from this browser.
+
+  Returns:
+    A json serialization of a histogram.
+
+  Raises:
+    py_utils.TimeoutException: There is no valid histogram in 10 seconds.
+  """
+  return py_utils.WaitFor(lambda: _GetTabSwitchHistogram(browser), 10)
+
+
+def WaitTabSwitching(browser, prev_histogram):
+  """Waits for tab switching completion.
+
+  It's done by checking browser histogram to see if
+  RWH_TabSwitchPaintDuration count increases.
+
+  Args:
+    browser: Gets histogram from this browser.
+    prev_histogram: Checks histogram change against this histogram.
+  """
+  def _IsDone():
+    cur_histogram = _GetTabSwitchHistogram(browser)
+    if not cur_histogram:
+      return False
+
+    diff_histogram = histogram_util.SubtractHistogram(
+        cur_histogram, prev_histogram)
+    diff_histogram_count = json.loads(diff_histogram).get('count', 0)
+    return diff_histogram_count > 0
+
+  try:
+    py_utils.WaitFor(_IsDone, 10)
+  except py_utils.TimeoutException:
+    logging.warning('Timed out waiting for histogram count increasing.')
+
+
+class KeyboardEmulator(object):
+  """Sets up a remote emulated keyboard and sends key events to switch tab.
+
+  Example usage:
+    with KeyboardEmulator(DUT_IP) as kbd:
+      for i in range(5):
+        kbd.SwitchTab()
+  """
+  REMOTE_LOG_KEY_FILENAME = '/usr/local/tmp/log_key_tab_switch'
+
+  def __init__(self, dut_ip):
+    """Inits KeyboardEmulator.
+
+    Args:
+      dut_ip: DUT IP or hostname.
+    """
+    self._dut_ip = dut_ip
+    self._root_dut_ip = 'root@' + dut_ip
+    self._key_device_name = None
+
+  def _StartRemoteKeyboardEmulator(self):
+    """Starts keyboard emulator on the DUT.
+
+    Returns:
+      The device name of the emulated keyboard.
+
+    Raises:
+      RuntimeError: Keyboard emulation failed.
+    """
+    kbd_prop_filename = '/usr/local/autotest/cros/input_playback/keyboard.prop'
+
+    ret = _RunRemoteCommand(self._dut_ip, 'test -e %s' % kbd_prop_filename)
+    if ret != 0:
+      raise RuntimeError('Keyboard property file does not exist.')
+
+    cmd = ['ssh', self._root_dut_ip, 'evemu-device', kbd_prop_filename]
+    ssh_process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+
+    # The evemu-device output format:
+    # Emulated Keyboard: /dev/input/event10
+    output = ssh_process.stdout.readline()
+    # The remote process would live when the ssh process was terminated.
+    ssh_process.kill()
+    if not output.startswith('Emulated Keyboard:'):
+      raise RuntimeError('Keyboard emulation failed.')
+    key_device_name = output.split()[2]
+    return key_device_name
+
+  def _SetupKeyDispatch(self):
+    """Uploads the script to send key to switch tabs."""
+    cur_dir = os.path.dirname(os.path.abspath(__file__))
+    log_key_filename = os.path.join(cur_dir, 'data', 'log_key_tab_switch')
+    os.system('scp -q %s %s:%s' %
+              (log_key_filename, self._root_dut_ip,
+               KeyboardEmulator.REMOTE_LOG_KEY_FILENAME))
+
+  def __enter__(self):
+    self._key_device_name = self._StartRemoteKeyboardEmulator()
+    self._SetupKeyDispatch()
+    return self
+
+  def SwitchTab(self):
+    """Sending Ctrl-tab key to trigger tab switching."""
+    cmd = ('"evemu-play --insert-slot0 %s < %s"' %
+           (self._key_device_name,
+            KeyboardEmulator.REMOTE_LOG_KEY_FILENAME))
+    _RunRemoteCommand(self._dut_ip, cmd)
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    # Kills the remote emulator process explicitly.
+    _RunRemoteCommand(self._dut_ip, 'pkill evemu-device')
+
+
+def NoScreenOff(dut_ip):
+  """Sets screen always on for 1 hour.
+
+  Args:
+    dut_ip: DUT IP or hostname.
+  """
+  _RunRemoteCommand(dut_ip, 'set_power_policy --ac_screen_off_delay=3600')
+  _RunRemoteCommand(dut_ip, 'set_power_policy --ac_screen_dim_delay=3600')

tools/perf/contrib/cros_benchmarks/data/log_key_tab_switch

+E: 1494933813.802943 0004 0004 29
+E: 1494933813.802943 0001 001d 1
+E: 1494933813.802943 0000 0000 0
+E: 1494933813.986151 0004 0004 15
+E: 1494933813.986151 0001 000f 1
+E: 1494933813.986151 0000 0000 0
+E: 1494933814.128580 0004 0004 29
+E: 1494933814.128580 0001 001d 0
+E: 1494933814.128580 0000 0000 0
+E: 1494933814.131678 0004 0004 15
+E: 1494933814.131678 0001 000f 0
+E: 1494933814.131678 0000 0000 0

tools/perf/contrib/cros_benchmarks/data/tab_switching.json

+{
+    "archives": {
+        "cros_tab_switching_typical24": {
+            "DEFAULT": "tab_switching.wpr"
+        }
+    },
+    "description": "Describes the Web Page Replay archives for a story set. Don't edit by hand! Use record_wpr for updating.",
+    "platform_specific": true
+}

tools/perf/contrib/cros_benchmarks/data/tab_switching.wpr.sha1

+fdf3c3844b38e9ff07fd3031b7c7f893867460a7
\ No newline at end of file

tools/perf/contrib/cros_benchmarks/tab_switching_bench.py

+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import logging
+
+from core import perf_benchmark
+
+from contrib.cros_benchmarks import tab_switching_measure
+from contrib.cros_benchmarks import tab_switching_stories
+from telemetry import benchmark
+from telemetry import story
+
+
+@benchmark.Owner(emails=['vovoy@chromium.org'],
+                 component='OS>Performance')
+@benchmark.Enabled('chromeos')
+class CrosTabSwitchingTypical24(perf_benchmark.PerfBenchmark):
+  """Measures tab switching performance with 24 tabs.
+
+  The script opens 24 pages in 24 different tabs, waits for them to load,
+  and then switches to each tabs and measures the tabs paint time. The 24
+  pages were chosen from Alexa top ranking sites. Tab paint time is the
+  time between tab being requested and the first paint event.
+
+  Benchmark specific option:
+    --tabset-repeat=N: Duplicate tab set for N times.
+  The following usage example opens 120 tabs.
+  $ ./run_benchmark --browser=cros-chrome --remote=DUT_IP
+  cros_tab_switching.typical_24 --tabset-repeat=5
+  """
+  test = tab_switching_measure.CrosTabSwitchingMeasurement
+
+  @classmethod
+  def AddBenchmarkCommandLineArgs(cls, parser):
+    parser.add_option('--tabset-repeat', type='int', default=1,
+                      help='repeat tab page set')
+
+  def CreateStorySet(self, options):
+    if not options.cros_remote:
+      # Raise exception here would fail the presubmit check.
+      logging.error('Must specify --remote=DUT_IP to run this test.')
+    story_set = story.StorySet(
+        archive_data_file='data/tab_switching.json',
+        base_dir=os.path.dirname(os.path.abspath(__file__)),
+        cloud_storage_bucket=story.PARTNER_BUCKET)
+    story_set.AddStory(tab_switching_stories.CrosMultiTabTypical24Story(
+        story_set, options.cros_remote, options.tabset_repeat))
+    return story_set
+
+  @classmethod
+  def Name(cls):
+    return 'cros_tab_switching.typical_24'

tools/perf/contrib/cros_benchmarks/tab_switching_measure.py

+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""The tab switching measurement.
+
+This measurement record the MPArch.RWH_TabSwitchPaintDuration histogram
+of each tab swithcing.
+"""
+
+from telemetry.page import legacy_page_test
+from telemetry.value import histogram
+from telemetry.value import histogram_util
+
+from contrib.cros_benchmarks import cros_utils
+
+
+class CrosTabSwitchingMeasurement(legacy_page_test.LegacyPageTest):
+  """Measures tab switching performance."""
+
+  def __init__(self):
+    super(CrosTabSwitchingMeasurement, self).__init__()
+    self._first_histogram = None
+
+  def CustomizeBrowserOptions(self, options):
+    """Adding necessary browser flag to collect histogram."""
+    options.AppendExtraBrowserArgs(['--enable-stats-collection-bindings'])
+
+  def DidNavigateToPage(self, page, tab):
+    """Record the starting histogram."""
+    self._first_histogram = cros_utils.GetTabSwitchHistogramRetry(tab.browser)
+
+  def ValidateAndMeasurePage(self, page, tab, results):
+    """Record the ending histogram for the tab switching metric."""
+    last_histogram = cros_utils.GetTabSwitchHistogramRetry(tab.browser)
+    total_diff_histogram = histogram_util.SubtractHistogram(
+        last_histogram, self._first_histogram)
+
+    display_name = 'MPArch_RWH_TabSwitchPaintDuration'
+    results.AddSummaryValue(
+        histogram.HistogramValue(
+            None, display_name, 'ms',
+            raw_value_json=total_diff_histogram,
+            important=False))

tools/perf/contrib/cros_benchmarks/tab_switching_stories.py

+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import py_utils
+import logging
+
+from telemetry.core import exceptions
+from telemetry.page import page as page_module
+from telemetry.page import shared_page_state
+from contrib.cros_benchmarks import cros_utils
+
+
+# NOTE: When tab count is high, some tabs may be discarded, and the tab
+# context would be invalidated. Avoid storing tab object for later use.
+class CrosMultiTabStory(page_module.Page):
+  """Base class for multi-tab stories."""
+
+  def __init__(self, story_set, cros_remote, tabset_repeat=1):
+    super(CrosMultiTabStory, self).__init__(
+        shared_page_state_class=shared_page_state.SharedPageState,
+        page_set=story_set, name=self.NAME, url=self.URL)
+    self._cros_remote = cros_remote
+    self._tabset_repeat = tabset_repeat
+
+  def RunNavigateSteps(self, action_runner):
+    """Opening tabs and waiting for them to load."""
+    if not self._cros_remote:
+      raise ValueError('Must specify --remote=DUT_IP to run this test.')
+
+    # As this story may run for a long time, adjusting screen off time to
+    # avoid screen off.
+    cros_utils.NoScreenOff(self._cros_remote)
+
+    tabs = action_runner.tab.browser.tabs
+
+    # No need to create the first tab as there is already one
+    # when the browser is ready.
+    url_list = self.URL_LIST * self._tabset_repeat
+    if url_list:
+      action_runner.Navigate(url_list[0])
+    for i, url in enumerate(url_list[1:]):
+      new_tab = tabs.New()
+      new_tab.action_runner.Navigate(url)
+      if i % 10 == 0:
+        print 'opening tab:', i
+
+    # Waiting for every tabs to be stable.
+    for i, url in enumerate(url_list):
+      try:
+        tabs[i].action_runner.WaitForNetworkQuiescence()
+      except py_utils.TimeoutException:
+        logging.info('WaitForNetworkQuiescence() timeout, url[%d]: %s',
+                     i, url)
+      except exceptions.DevtoolsTargetCrashException:
+        logging.info('RunNavigateSteps: devtools context lost')
+
+  def RunPageInteractions(self, action_runner):
+    """Tab switching to each tabs."""
+    url_list = self.URL_LIST * self._tabset_repeat
+    browser = action_runner.tab.browser
+
+    total_tab_count = len(url_list)
+    live_tab_count = len(browser.tabs)
+    if live_tab_count != total_tab_count:
+      logging.warning('live tab: %d, tab discarded: %d',
+                      live_tab_count, total_tab_count - live_tab_count)
+
+    with cros_utils.KeyboardEmulator(self._cros_remote) as keyboard:
+      for i in range(total_tab_count):
+        prev_histogram = cros_utils.GetTabSwitchHistogramRetry(browser)
+        keyboard.SwitchTab()
+        cros_utils.WaitTabSwitching(browser, prev_histogram)
+
+        if i % 10 == 0:
+          print 'switching tab:', i
+
+
+class CrosMultiTabTypical24Story(CrosMultiTabStory):
+  """Multi-tab stories to test 24 typical webpages."""
+  NAME = 'cros_tab_switching_typical24'
+  URL_LIST = [
+      # Why: Alexa games #48
+      'http://www.nick.com/games',
+      # Why: Alexa sports #45
+      'http://www.rei.com/',
+      # Why: Alexa sports #50
+      'http://www.fifa.com/',
+      # Why: Alexa shopping #41
+      'http://www.gamestop.com/ps3',
+      # Why: Alexa news #55
+      ('http://www.economist.com/news/science-and-technology/21573529-small-'
+       'models-cosmic-phenomena-are-shedding-light-real-thing-how-build'),
+      # Why: Alexa news #67
+      'http://www.theonion.com',
+      'http://arstechnica.com/',
+      # Why: Alexa home #10
+      'http://allrecipes.com/Recipe/Pull-Apart-Hot-Cross-Buns/Detail.aspx',
+      'http://www.html5rocks.com/en/',
+      'http://www.mlb.com/',
+      ('http://gawker.com/5939683/based-on-a-true-story-is-a-rotten-lie-i-'
+       'hope-you-never-believe'),
+      'http://www.imdb.com/title/tt0910970/',
+      'http://www.flickr.com/search/?q=monkeys&f=hp',
+      'http://money.cnn.com/',
+      'http://www.nationalgeographic.com/',
+      'http://premierleague.com',
+      'http://www.osubeavers.com/',
+      'http://walgreens.com',
+      'http://colorado.edu',
+      ('http://www.ticketmaster.com/JAY-Z-and-Justin-Timberlake-tickets/artist/'
+       '1837448?brand=none&tm_link=tm_homeA_rc_name2'),
+      # pylint: disable=line-too-long
+      'http://www.theverge.com/2013/3/5/4061684/inside-ted-the-smartest-bubble-in-the-world',
+      'http://www.airbnb.com/',
+      'http://www.ign.com/',
+      # Why: Alexa health #25
+      'http://www.fda.gov',
+  ]
+  URL = URL_LIST[0]