Commit 2675d3c7 authored by vovoy's avatar vovoy Committed by Commit bot

Tab Switching Benchmark for ChromeOS

Copy the tab switching benchmark to cros_benchmarks. Using key events for
simulate tab switching behavior to handle the case that some tab contexts
may be discarded.

Example usage to open 120 tabs:
$ ./run_benchmark --browser=cros-chrome --remote=DUT_IP
cros_tab_switching.typical_24 --tabset-repeat=5

BUG=724381

Review-Url: https://codereview.chromium.org/2890333002
Cr-Commit-Position: refs/heads/master@{#474975}
parent b0a2e707
# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import json
import logging
import os
import subprocess
import py_utils
from telemetry.core import exceptions
from telemetry.value import histogram_util
def _RunRemoteCommand(dut_ip, cmd):
return os.system('ssh root@%s %s' % (dut_ip, cmd))
def _GetTabSwitchHistogram(browser):
"""Gets MPArch.RWH_TabSwitchPaintDuration histogram.
Catches exceptions to handle devtools context lost cases.
Any tab context can be used to get the TabSwitchPaintDuration histogram.
browser.tabs[-1] is the last valid context. Ex: If the browser opens
A, B, ..., I, J tabs, E, F, G tabs have valid contexts (not discarded),
then browser.tab[0] is tab E, browser.tab[-1] is tab G.
In the tab switching benchmark, the tabs are opened and switched to in
1, 2, ..., n order. The tabs are discarded in roughly 1, 2, ..., n order
(LRU order). The chance of discarding the last valid context is lower
than discarding the first valid context.
Args:
browser: Gets histogram from this browser.
Returns:
A json serialization of a histogram or None if get histogram failed.
"""
histogram_name = 'MPArch.RWH_TabSwitchPaintDuration'
histogram_type = histogram_util.BROWSER_HISTOGRAM
try:
return histogram_util.GetHistogram(
histogram_type, histogram_name, browser.tabs[-1])
except (exceptions.DevtoolsTargetCrashException, KeyError):
logging.warning('GetHistogram: Devtools context lost.')
except exceptions.TimeoutException:
logging.warning('GetHistogram: Timed out getting histogram.')
return None
def GetTabSwitchHistogramRetry(browser):
"""Retries getting histogram as it may fail when a context was discarded.
Args:
browser: Gets histogram from this browser.
Returns:
A json serialization of a histogram.
Raises:
py_utils.TimeoutException: There is no valid histogram in 10 seconds.
"""
return py_utils.WaitFor(lambda: _GetTabSwitchHistogram(browser), 10)
def WaitTabSwitching(browser, prev_histogram):
"""Waits for tab switching completion.
It's done by checking browser histogram to see if
RWH_TabSwitchPaintDuration count increases.
Args:
browser: Gets histogram from this browser.
prev_histogram: Checks histogram change against this histogram.
"""
def _IsDone():
cur_histogram = _GetTabSwitchHistogram(browser)
if not cur_histogram:
return False
diff_histogram = histogram_util.SubtractHistogram(
cur_histogram, prev_histogram)
diff_histogram_count = json.loads(diff_histogram).get('count', 0)
return diff_histogram_count > 0
try:
py_utils.WaitFor(_IsDone, 10)
except py_utils.TimeoutException:
logging.warning('Timed out waiting for histogram count increasing.')
class KeyboardEmulator(object):
"""Sets up a remote emulated keyboard and sends key events to switch tab.
Example usage:
with KeyboardEmulator(DUT_IP) as kbd:
for i in range(5):
kbd.SwitchTab()
"""
REMOTE_LOG_KEY_FILENAME = '/usr/local/tmp/log_key_tab_switch'
def __init__(self, dut_ip):
"""Inits KeyboardEmulator.
Args:
dut_ip: DUT IP or hostname.
"""
self._dut_ip = dut_ip
self._root_dut_ip = 'root@' + dut_ip
self._key_device_name = None
def _StartRemoteKeyboardEmulator(self):
"""Starts keyboard emulator on the DUT.
Returns:
The device name of the emulated keyboard.
Raises:
RuntimeError: Keyboard emulation failed.
"""
kbd_prop_filename = '/usr/local/autotest/cros/input_playback/keyboard.prop'
ret = _RunRemoteCommand(self._dut_ip, 'test -e %s' % kbd_prop_filename)
if ret != 0:
raise RuntimeError('Keyboard property file does not exist.')
cmd = ['ssh', self._root_dut_ip, 'evemu-device', kbd_prop_filename]
ssh_process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
# The evemu-device output format:
# Emulated Keyboard: /dev/input/event10
output = ssh_process.stdout.readline()
# The remote process would live when the ssh process was terminated.
ssh_process.kill()
if not output.startswith('Emulated Keyboard:'):
raise RuntimeError('Keyboard emulation failed.')
key_device_name = output.split()[2]
return key_device_name
def _SetupKeyDispatch(self):
"""Uploads the script to send key to switch tabs."""
cur_dir = os.path.dirname(os.path.abspath(__file__))
log_key_filename = os.path.join(cur_dir, 'data', 'log_key_tab_switch')
os.system('scp -q %s %s:%s' %
(log_key_filename, self._root_dut_ip,
KeyboardEmulator.REMOTE_LOG_KEY_FILENAME))
def __enter__(self):
self._key_device_name = self._StartRemoteKeyboardEmulator()
self._SetupKeyDispatch()
return self
def SwitchTab(self):
"""Sending Ctrl-tab key to trigger tab switching."""
cmd = ('"evemu-play --insert-slot0 %s < %s"' %
(self._key_device_name,
KeyboardEmulator.REMOTE_LOG_KEY_FILENAME))
_RunRemoteCommand(self._dut_ip, cmd)
def __exit__(self, exc_type, exc_value, traceback):
# Kills the remote emulator process explicitly.
_RunRemoteCommand(self._dut_ip, 'pkill evemu-device')
def NoScreenOff(dut_ip):
"""Sets screen always on for 1 hour.
Args:
dut_ip: DUT IP or hostname.
"""
_RunRemoteCommand(dut_ip, 'set_power_policy --ac_screen_off_delay=3600')
_RunRemoteCommand(dut_ip, 'set_power_policy --ac_screen_dim_delay=3600')
E: 1494933813.802943 0004 0004 29
E: 1494933813.802943 0001 001d 1
E: 1494933813.802943 0000 0000 0
E: 1494933813.986151 0004 0004 15
E: 1494933813.986151 0001 000f 1
E: 1494933813.986151 0000 0000 0
E: 1494933814.128580 0004 0004 29
E: 1494933814.128580 0001 001d 0
E: 1494933814.128580 0000 0000 0
E: 1494933814.131678 0004 0004 15
E: 1494933814.131678 0001 000f 0
E: 1494933814.131678 0000 0000 0
{
"archives": {
"cros_tab_switching_typical24": {
"DEFAULT": "tab_switching.wpr"
}
},
"description": "Describes the Web Page Replay archives for a story set. Don't edit by hand! Use record_wpr for updating.",
"platform_specific": true
}
fdf3c3844b38e9ff07fd3031b7c7f893867460a7
\ No newline at end of file
# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import os
import logging
from core import perf_benchmark
from contrib.cros_benchmarks import tab_switching_measure
from contrib.cros_benchmarks import tab_switching_stories
from telemetry import benchmark
from telemetry import story
@benchmark.Owner(emails=['vovoy@chromium.org'],
component='OS>Performance')
@benchmark.Enabled('chromeos')
class CrosTabSwitchingTypical24(perf_benchmark.PerfBenchmark):
"""Measures tab switching performance with 24 tabs.
The script opens 24 pages in 24 different tabs, waits for them to load,
and then switches to each tabs and measures the tabs paint time. The 24
pages were chosen from Alexa top ranking sites. Tab paint time is the
time between tab being requested and the first paint event.
Benchmark specific option:
--tabset-repeat=N: Duplicate tab set for N times.
The following usage example opens 120 tabs.
$ ./run_benchmark --browser=cros-chrome --remote=DUT_IP
cros_tab_switching.typical_24 --tabset-repeat=5
"""
test = tab_switching_measure.CrosTabSwitchingMeasurement
@classmethod
def AddBenchmarkCommandLineArgs(cls, parser):
parser.add_option('--tabset-repeat', type='int', default=1,
help='repeat tab page set')
def CreateStorySet(self, options):
if not options.cros_remote:
# Raise exception here would fail the presubmit check.
logging.error('Must specify --remote=DUT_IP to run this test.')
story_set = story.StorySet(
archive_data_file='data/tab_switching.json',
base_dir=os.path.dirname(os.path.abspath(__file__)),
cloud_storage_bucket=story.PARTNER_BUCKET)
story_set.AddStory(tab_switching_stories.CrosMultiTabTypical24Story(
story_set, options.cros_remote, options.tabset_repeat))
return story_set
@classmethod
def Name(cls):
return 'cros_tab_switching.typical_24'
# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""The tab switching measurement.
This measurement record the MPArch.RWH_TabSwitchPaintDuration histogram
of each tab swithcing.
"""
from telemetry.page import legacy_page_test
from telemetry.value import histogram
from telemetry.value import histogram_util
from contrib.cros_benchmarks import cros_utils
class CrosTabSwitchingMeasurement(legacy_page_test.LegacyPageTest):
"""Measures tab switching performance."""
def __init__(self):
super(CrosTabSwitchingMeasurement, self).__init__()
self._first_histogram = None
def CustomizeBrowserOptions(self, options):
"""Adding necessary browser flag to collect histogram."""
options.AppendExtraBrowserArgs(['--enable-stats-collection-bindings'])
def DidNavigateToPage(self, page, tab):
"""Record the starting histogram."""
self._first_histogram = cros_utils.GetTabSwitchHistogramRetry(tab.browser)
def ValidateAndMeasurePage(self, page, tab, results):
"""Record the ending histogram for the tab switching metric."""
last_histogram = cros_utils.GetTabSwitchHistogramRetry(tab.browser)
total_diff_histogram = histogram_util.SubtractHistogram(
last_histogram, self._first_histogram)
display_name = 'MPArch_RWH_TabSwitchPaintDuration'
results.AddSummaryValue(
histogram.HistogramValue(
None, display_name, 'ms',
raw_value_json=total_diff_histogram,
important=False))
# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import py_utils
import logging
from telemetry.core import exceptions
from telemetry.page import page as page_module
from telemetry.page import shared_page_state
from contrib.cros_benchmarks import cros_utils
# NOTE: When tab count is high, some tabs may be discarded, and the tab
# context would be invalidated. Avoid storing tab object for later use.
class CrosMultiTabStory(page_module.Page):
"""Base class for multi-tab stories."""
def __init__(self, story_set, cros_remote, tabset_repeat=1):
super(CrosMultiTabStory, self).__init__(
shared_page_state_class=shared_page_state.SharedPageState,
page_set=story_set, name=self.NAME, url=self.URL)
self._cros_remote = cros_remote
self._tabset_repeat = tabset_repeat
def RunNavigateSteps(self, action_runner):
"""Opening tabs and waiting for them to load."""
if not self._cros_remote:
raise ValueError('Must specify --remote=DUT_IP to run this test.')
# As this story may run for a long time, adjusting screen off time to
# avoid screen off.
cros_utils.NoScreenOff(self._cros_remote)
tabs = action_runner.tab.browser.tabs
# No need to create the first tab as there is already one
# when the browser is ready.
url_list = self.URL_LIST * self._tabset_repeat
if url_list:
action_runner.Navigate(url_list[0])
for i, url in enumerate(url_list[1:]):
new_tab = tabs.New()
new_tab.action_runner.Navigate(url)
if i % 10 == 0:
print 'opening tab:', i
# Waiting for every tabs to be stable.
for i, url in enumerate(url_list):
try:
tabs[i].action_runner.WaitForNetworkQuiescence()
except py_utils.TimeoutException:
logging.info('WaitForNetworkQuiescence() timeout, url[%d]: %s',
i, url)
except exceptions.DevtoolsTargetCrashException:
logging.info('RunNavigateSteps: devtools context lost')
def RunPageInteractions(self, action_runner):
"""Tab switching to each tabs."""
url_list = self.URL_LIST * self._tabset_repeat
browser = action_runner.tab.browser
total_tab_count = len(url_list)
live_tab_count = len(browser.tabs)
if live_tab_count != total_tab_count:
logging.warning('live tab: %d, tab discarded: %d',
live_tab_count, total_tab_count - live_tab_count)
with cros_utils.KeyboardEmulator(self._cros_remote) as keyboard:
for i in range(total_tab_count):
prev_histogram = cros_utils.GetTabSwitchHistogramRetry(browser)
keyboard.SwitchTab()
cros_utils.WaitTabSwitching(browser, prev_histogram)
if i % 10 == 0:
print 'switching tab:', i
class CrosMultiTabTypical24Story(CrosMultiTabStory):
"""Multi-tab stories to test 24 typical webpages."""
NAME = 'cros_tab_switching_typical24'
URL_LIST = [
# Why: Alexa games #48
'http://www.nick.com/games',
# Why: Alexa sports #45
'http://www.rei.com/',
# Why: Alexa sports #50
'http://www.fifa.com/',
# Why: Alexa shopping #41
'http://www.gamestop.com/ps3',
# Why: Alexa news #55
('http://www.economist.com/news/science-and-technology/21573529-small-'
'models-cosmic-phenomena-are-shedding-light-real-thing-how-build'),
# Why: Alexa news #67
'http://www.theonion.com',
'http://arstechnica.com/',
# Why: Alexa home #10
'http://allrecipes.com/Recipe/Pull-Apart-Hot-Cross-Buns/Detail.aspx',
'http://www.html5rocks.com/en/',
'http://www.mlb.com/',
('http://gawker.com/5939683/based-on-a-true-story-is-a-rotten-lie-i-'
'hope-you-never-believe'),
'http://www.imdb.com/title/tt0910970/',
'http://www.flickr.com/search/?q=monkeys&f=hp',
'http://money.cnn.com/',
'http://www.nationalgeographic.com/',
'http://premierleague.com',
'http://www.osubeavers.com/',
'http://walgreens.com',
'http://colorado.edu',
('http://www.ticketmaster.com/JAY-Z-and-Justin-Timberlake-tickets/artist/'
'1837448?brand=none&tm_link=tm_homeA_rc_name2'),
# pylint: disable=line-too-long
'http://www.theverge.com/2013/3/5/4061684/inside-ted-the-smartest-bubble-in-the-world',
'http://www.airbnb.com/',
'http://www.ign.com/',
# Why: Alexa health #25
'http://www.fda.gov',
]
URL = URL_LIST[0]
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment