Skip to content

GitLab

T
tangled
Commit b8951864 authored by prasadv's avatar prasadv Committed by Commit bot
Browse files
Options

Add feature to run perf try jobs across all platforms on tryserver.chromium.perf. 

This provides a list of new browser to run perf try jobs on tryserver.chromium.perf:
try-all: For tryjobs on all bisect bots (android, mac, win, winx64, linux).
trybot-all-android: tryjobs on all android bisect bots.
trybot-all-win: tryjobs on all windows bisect bots (win, win x64).
trybot-all-mac: tryjobs on all mac bisect bots.
trybot-all-linux: tryjobs on all linux bisect bots.

BUG=467124

Review URL: https://codereview.chromium.org/1020103006

Cr-Commit-Position: refs/heads/master@{#322064}
parent 5d6b995d
Show whitespace changes
Inline Side-by-side
Showing with 487 additions and 151 deletions
tools/telemetry/telemetry/core/backends/remote/trybot_browser_finder.py
View file @ b8951864
...@@ -20,7 +20,30 @@ from telemetry.core.platform import trybot_device ...@@ -20,7 +20,30 @@ from telemetry.core.platform import trybot_device
CHROMIUM_CONFIG_FILENAME = 'tools/run-perf-test.cfg' CHROMIUM_CONFIG_FILENAME = 'tools/run-perf-test.cfg'
BLINK_CONFIG_FILENAME = 'Tools/run-perf-test.cfg' BLINK_CONFIG_FILENAME = 'Tools/run-perf-test.cfg'
SUCCESS, NO_CHANGES, ERROR = range(3) SUCCESS, NO_CHANGES, ERROR = range(3)
# Unsupported Perf bisect bots.
EXCLUDED_BOTS = {
'win_xp_perf_bisect',
'linux_perf_tester',
'linux_perf_bisector',
'win_perf_bisect_builder',
'win_x64_perf_bisect_builder',
'linux_perf_bisect_builder',
'mac_perf_bisect_builder',
'android_perf_bisect_builder'
}
INCLUDE_BOTS = [
'trybot-all',
'trybot-all-win',
'trybot-all-mac',
'trybot-all-linux',
'trybot-all-android'
]
class TrybotError(Exception):
def __str__(self):
return '%s\nError running tryjob.' % self.args[0]
class PossibleTrybotBrowser(possible_browser.PossibleBrowser): class PossibleTrybotBrowser(possible_browser.PossibleBrowser):
...@@ -28,8 +51,7 @@ class PossibleTrybotBrowser(possible_browser.PossibleBrowser): ...@@ -28,8 +51,7 @@ class PossibleTrybotBrowser(possible_browser.PossibleBrowser):
def __init__(self, browser_type, _): def __init__(self, browser_type, _):
target_os = browser_type.split('-')[1] target_os = browser_type.split('-')[1]
self._buildername = '%s_perf_bisect' % browser_type.replace( self._builder_names = _GetBuilderNames(browser_type)
'trybot-', '').replace('-', '_')
super(PossibleTrybotBrowser, self).__init__(browser_type, target_os, True) super(PossibleTrybotBrowser, self).__init__(browser_type, target_os, True)
def Create(self, finder_options): def Create(self, finder_options):
...@@ -55,13 +77,12 @@ class PossibleTrybotBrowser(possible_browser.PossibleBrowser): ...@@ -55,13 +77,12 @@ class PossibleTrybotBrowser(possible_browser.PossibleBrowser):
returncode = proc.poll() returncode = proc.poll()
return (returncode, out, err) return (returncode, out, err)
def _AttemptTryjob(self, cfg_file_path): def _UpdateConfigAndRunTryjob(self, bot_platform, cfg_file_path):
"""Attempts to run a tryjob from the current directory. """Updates perf config file, uploads changes and excutes perf try job.
This is run once for chromium, and if it returns NO_CHANGES, once for blink.
Args: Args:
cfg_file_path: Path to the config file for the try job. bot_platform: Name of the platform to be generated.
cfg_file_path: Perf config file path.
Returns: Returns:
(result, msg) where result is one of: (result, msg) where result is one of:
...@@ -69,121 +90,173 @@ class PossibleTrybotBrowser(possible_browser.PossibleBrowser): ...@@ -69,121 +90,173 @@ class PossibleTrybotBrowser(possible_browser.PossibleBrowser):
NO_CHANGES if there was nothing to try, NO_CHANGES if there was nothing to try,
ERROR if a tryjob was attempted but an error encountered ERROR if a tryjob was attempted but an error encountered
and msg is an error message if an error was encountered, or rietveld and msg is an error message if an error was encountered, or rietveld
url if success. url if success, otherwise throws TrybotError exception.
""" """
returncode, original_branchname, err = self._RunProcess( config = self._GetPerfConfig(bot_platform)
['git', 'rev-parse', '--abbrev-ref', 'HEAD']) try:
if returncode: config_file = open(cfg_file_path, 'w')
msg = 'Must be in a git repository to send changes to trybots.' except IOError:
if err: msg = 'Cannot find %s. Please run from src dir.' % cfg_file_path
msg += '\nGit error: %s' % err
return (ERROR, msg)
original_branchname = original_branchname.strip()
# Check if the tree is dirty: make sure the index is up to date and then
# run diff-index
self._RunProcess(['git', 'update-index', '--refresh', '-q'])
returncode, out, err = self._RunProcess(['git', 'diff-index', 'HEAD'])
if out:
msg = 'Cannot send a try job with a dirty tree. Commit locally first.'
return (ERROR, msg) return (ERROR, msg)
config_file.write('config = %s' % json.dumps(
# Make sure the tree does have local commits. config, sort_keys=True, indent=2, separators=(',', ': ')))
returncode, out, err = self._RunProcess( config_file.close()
['git', 'log', 'origin/master..HEAD']) # Commit the config changes locally.
if not out:
return (NO_CHANGES, None)
# Create/check out the telemetry-tryjob branch, and edit the configs
# for the tryjob there.
returncode, out, err = self._RunProcess( returncode, out, err = self._RunProcess(
['git', 'checkout', '-b', 'telemetry-tryjob']) ['git', 'commit', '-a', '-m', 'bisect config: %s' % bot_platform])
if returncode: if returncode:
msg = ('Error creating branch telemetry-tryjob. ' raise TrybotError('Could not commit bisect config change for %s,'
'Please delete it if it exists.\n%s' % err) ' error %s' % (bot_platform, err))
return (ERROR, msg) # Upload the CL to rietveld and run a try job.
returncode, out, err = self._RunProcess( returncode, out, err = self._RunProcess([
['git', 'branch', '--set-upstream-to', 'origin/master']) 'git', 'cl', 'upload', '-f', '--bypass-hooks', '-m',
'CL for perf tryjob on %s' % bot_platform
])
if returncode: if returncode:
return (ERROR, 'Error in git branch --set-upstream-to: %s' % err) raise TrybotError('Could upload to rietveld for %s, error %s' %
(bot_platform, err))
match = re.search(r'https://codereview.chromium.org/[\d]+', out)
if not match:
raise TrybotError('Could not upload CL to rietveld for %s! Output %s' %
(bot_platform, out))
rietveld_url = match.group(0)
# Generate git try command for available bots.
git_try_command = ['git', 'cl', 'try', '-m', 'tryserver.chromium.perf']
for bot in self._builder_names[bot_platform]:
git_try_command.extend(['-b', bot])
returncode, out, err = self._RunProcess(git_try_command)
if returncode:
raise TrybotError('Could not try CL for %s, error %s' %
(bot_platform, err))
return (SUCCESS, rietveld_url)
def _GetPerfConfig(self, bot_platform):
"""Generates the perf config for try job.
Args:
bot_platform: Name of the platform to be generated.
Returns:
A dictionary with perf config parameters.
"""
# Generate the command line for the perf trybots # Generate the command line for the perf trybots
target_arch = 'ia32'
arguments = sys.argv arguments = sys.argv
if self._target_os == 'win': if bot_platform in ['win', 'win-x64']:
arguments[0] = 'python tools\\perf\\run_benchmark' arguments[0] = 'python tools\\perf\\run_benchmark'
else: else:
arguments[0] = './tools/perf/run_benchmark' arguments[0] = './tools/perf/run_benchmark'
for index, arg in enumerate(arguments): for index, arg in enumerate(arguments):
if arg.startswith('--browser='): if arg.startswith('--browser='):
if self._target_os == 'android': if bot_platform == 'android':
arguments[index] = '--browser=android-chrome-shell' arguments[index] = '--browser=android-chrome-shell'
elif 'x64' in self._buildername: elif any('x64' in bot for bot in self._builder_names[bot_platform]):
arguments[index] = '--browser=release_x64' arguments[index] = '--browser=release_x64'
target_arch = 'x64'
else: else:
arguments[index] = '--browser=release' arguments[index] = '--browser=release'
command = ' '.join(arguments) command = ' '.join(arguments)
# Set target architecture to build 32 or 64 bit binaries. return {
target_arch = 'x64' if 'x64' in self._buildername else 'ia32'
# Add the correct command to the config file and commit it.
config = {
'command': command, 'command': command,
'repeat_count': '1', 'repeat_count': '1',
'max_time_minutes': '120', 'max_time_minutes': '120',
'truncate_percent': '0', 'truncate_percent': '0',
'target_arch': target_arch, 'target_arch': target_arch,
} }
try:
config_file = open(cfg_file_path, 'w') def _AttemptTryjob(self, cfg_file_path):
except IOError: """Attempts to run a tryjob from the current directory.
msg = 'Cannot find %s. Please run from src dir.' % cfg_file_path
return (ERROR, msg) This is run once for chromium, and if it returns NO_CHANGES, once for blink.
config_file.write('config = %s' % json.dumps(
config, sort_keys=True, indent=2, separators=(',', ': '))) Args:
config_file.close() cfg_file_path: Path to the config file for the try job.
returncode, out, err = self._RunProcess(
['git', 'commit', '-a', '-m', 'bisect config']) Returns:
Returns SUCCESS if a tryjob was sent, NO_CHANGES if there was nothing to
try, ERROR if a tryjob was attempted but an error encountered.
"""
source_repo = 'chromium'
if cfg_file_path == BLINK_CONFIG_FILENAME:
source_repo = 'blink'
# TODO(prasadv): This method is quite long, we should consider refactor
# this by extracting to helper methods.
returncode, original_branchname, err = self._RunProcess(
['git', 'rev-parse', '--abbrev-ref', 'HEAD'])
if returncode: if returncode:
msg = 'Could not commit bisect config change, error %s' % err msg = 'Must be in a git repository to send changes to trybots.'
return (ERROR, msg) if err:
msg += '\nGit error: %s' % err
logging.error(msg)
return ERROR
# Upload the CL to rietveld and run a try job. original_branchname = original_branchname.strip()
returncode, out, err = self._RunProcess([
'git', 'cl', 'upload', '-f', '--bypass-hooks', '-m', # Check if the tree is dirty: make sure the index is up to date and then
'CL for perf tryjob' # run diff-index
]) self._RunProcess(['git', 'update-index', '--refresh', '-q'])
returncode, out, err = self._RunProcess(['git', 'diff-index', 'HEAD'])
if out:
logging.error(
'Cannot send a try job with a dirty tree. Commit locally first.')
return ERROR
# Make sure the tree does have local commits.
returncode, out, err = self._RunProcess(
['git', 'log', 'origin/master..HEAD'])
if not out:
return NO_CHANGES
# Create/check out the telemetry-tryjob branch, and edit the configs
# for the tryjob there.
returncode, out, err = self._RunProcess(
['git', 'checkout', '-b', 'telemetry-tryjob'])
if returncode: if returncode:
msg = 'Could upload to rietveld, error %s' % err logging.error('Error creating branch telemetry-tryjob. '
return (ERROR, msg) 'Please delete it if it exists.\n%s', err)
match = re.search(r'https://codereview.chromium.org/[\d]+', out) return ERROR
if not match: try:
msg = 'Could not upload CL to rietveld! Output %s' % out returncode, out, err = self._RunProcess(
return (ERROR, msg) ['git', 'branch', '--set-upstream-to', 'origin/master'])
rietveld_url = match.group(0)
returncode, out, err = self._RunProcess([
'git', 'cl', 'try', '-m', 'tryserver.chromium.perf', '-b',
self._buildername])
if returncode: if returncode:
msg = 'Could not try CL, error %s' % err logging.error('Error in git branch --set-upstream-to: %s', err)
return (ERROR, msg) return ERROR
for bot_platform in self._builder_names:
try:
results, output = self._UpdateConfigAndRunTryjob(
bot_platform, cfg_file_path)
if results == ERROR:
logging.error(output)
return ERROR
print ('Uploaded %s try job to rietveld for %s platform. '
'View progress at %s' % (source_repo, bot_platform, output))
except TrybotError, err:
print err
logging.error(err)
finally:
# Checkout original branch and delete telemetry-tryjob branch. # Checkout original branch and delete telemetry-tryjob branch.
# TODO(prasadv): This finally block could be extracted out to be a
# separate function called _CleanupBranch.
returncode, out, err = self._RunProcess( returncode, out, err = self._RunProcess(
['git', 'checkout', original_branchname]) ['git', 'checkout', original_branchname])
if returncode: if returncode:
msg = ( logging.error('Could not check out %s. Please check it out and '
('Could not check out %s. Please check it out and manually ' 'manually delete the telemetry-tryjob branch. '
'delete the telemetry-tryjob branch. Error message: %s') % ': %s', original_branchname, err)
(original_branchname, err)) return ERROR # pylint: disable=lost-exception
return (ERROR, msg) logging.info('Checked out original branch: %s', original_branchname)
returncode, out, err = self._RunProcess( returncode, out, err = self._RunProcess(
['git', 'branch', '-D', 'telemetry-tryjob']) ['git', 'branch', '-D', 'telemetry-tryjob'])
if returncode: if returncode:
msg = (('Could not delete telemetry-tryjob branch. ' logging.error('Could not delete telemetry-tryjob branch. '
'Please delete it manually. Error %s') % err) 'Please delete it manually: %s', err)
return (ERROR, msg) return ERROR # pylint: disable=lost-exception
return (SUCCESS, rietveld_url) logging.info('Deleted temp branch: telemetry-tryjob')
return SUCCESS
def RunRemote(self): def RunRemote(self):
"""Sends a tryjob to a perf trybot. """Sends a tryjob to a perf trybot.
...@@ -193,29 +266,17 @@ class PossibleTrybotBrowser(possible_browser.PossibleBrowser): ...@@ -193,29 +266,17 @@ class PossibleTrybotBrowser(possible_browser.PossibleBrowser):
tryjob on the given bot. tryjob on the given bot.
""" """
# First check if there are chromium changes to upload. # First check if there are chromium changes to upload.
status, msg = self._AttemptTryjob(CHROMIUM_CONFIG_FILENAME) status = self._AttemptTryjob(CHROMIUM_CONFIG_FILENAME)
if status == SUCCESS: if status not in [SUCCESS, ERROR]:
print 'Uploaded chromium try job to rietveld. View progress at %s' % msg
return
elif status == ERROR:
logging.error(msg)
return
# If we got here, there are no chromium changes to upload. Try blink. # If we got here, there are no chromium changes to upload. Try blink.
os.chdir('third_party/WebKit/') os.chdir('third_party/WebKit/')
status, msg = self._AttemptTryjob(BLINK_CONFIG_FILENAME) status = self._AttemptTryjob(BLINK_CONFIG_FILENAME)
os.chdir('../..') os.chdir('../..')
if status == SUCCESS: if status not in [SUCCESS, ERROR]:
print 'Uploaded blink try job to rietveld. View progress at %s' % msg
return
elif status == ERROR:
logging.error(msg)
return
else:
logging.error('No local changes found in chromium or blink trees. ' logging.error('No local changes found in chromium or blink trees. '
'browser=%s argument sends local changes to the %s ' 'browser=%s argument sends local changes to the '
'perf trybot.', self.browser_type, self._buildername) 'perf trybot(s): %s.', self.browser_type,
return self._builder_names.values())
def _InitPlatformIfNeeded(self): def _InitPlatformIfNeeded(self):
if self._platform: if self._platform:
...@@ -240,12 +301,49 @@ def _GetTrybotList(): ...@@ -240,12 +301,49 @@ def _GetTrybotList():
f = urllib2.urlopen( f = urllib2.urlopen(
'http://build.chromium.org/p/tryserver.chromium.perf/json') 'http://build.chromium.org/p/tryserver.chromium.perf/json')
builders = json.loads(f.read()).get('builders', {}).keys() builders = json.loads(f.read()).get('builders', {}).keys()
builders = ['trybot-%s' % b.replace('_perf_bisect', '').replace('_', '-') builders = ['trybot-%s' % bot.replace('_perf_bisect', '').replace('_', '-')
for b in builders if not b.endswith('_perf_bisect_builder')] for bot in builders if bot not in EXCLUDED_BOTS]
# Perf try jobs do not work on Windows XP builders.extend(INCLUDE_BOTS)
if 'trybot-win-xp' in builders: return sorted(builders)
builders.remove('trybot-win-xp')
return builders
def _GetBuilderNames(browser_type):
""" Return platform and its available bot name as dictionary."""
if 'all' not in browser_type:
bot = ['%s_perf_bisect' % browser_type.replace(
'trybot-', '').replace('-', '_')]
bot_platform = browser_type.split('-')[1]
if 'x64' in browser_type:
bot_platform += '-x64'
return {bot_platform: bot}
f = urllib2.urlopen(
'http://build.chromium.org/p/tryserver.chromium.perf/json')
builders = json.loads(f.read()).get('builders', {}).keys()
# Exclude unsupported bots like win xp and some dummy bots.
builders = [bot for bot in builders if bot not in EXCLUDED_BOTS]
platform_and_bots = {}
for os_name in ['linux', 'android', 'mac', 'win']:
platform_and_bots[os_name] = [bot for bot in builders if os_name in bot]
# Special case for Windows x64, consider it as separate platform
# config config should contain target_arch=x64 and --browser=release_x64.
win_x64_bots = [platform_and_bots['win'].pop(i)
for i, win_bot in enumerate(platform_and_bots['win']) if 'x64' in win_bot]
platform_and_bots['win-x64'] = win_x64_bots
if 'all-win' in browser_type:
return {'win': platform_and_bots['win'],
'win-x64': platform_and_bots['win-x64']}
if 'all-mac' in browser_type:
return {'mac': platform_and_bots['mac']}
if 'all-android' in browser_type:
return {'android': platform_and_bots['android']}
if 'all-linux' in browser_type:
return {'linux': platform_and_bots['linux']}
return platform_and_bots
def FindAllBrowserTypes(finder_options): def FindAllBrowserTypes(finder_options):
......
tools/telemetry/telemetry/core/backends/remote/trybot_browser_finder_unittest.py
View file @ b8951864
...@@ -42,35 +42,52 @@ class TrybotBrowserFinderTest(unittest.TestCase): ...@@ -42,35 +42,52 @@ class TrybotBrowserFinderTest(unittest.TestCase):
'Popen').WithArgs(arg[0]).WillReturn(mock_popen) 'Popen').WithArgs(arg[0]).WillReturn(mock_popen)
trybot_browser_finder.subprocess = mock_subprocess trybot_browser_finder.subprocess = mock_subprocess
def test_find_all_browser_types_list(self): def _MockTryserverJson(self, bots_dict):
finder_options = browser_options.BrowserFinderOptions(browser_type='list')
trybot_browser_finder.urllib2 = simple_mock.MockObject() trybot_browser_finder.urllib2 = simple_mock.MockObject()
trybot_browser_finder.urllib2.ExpectCall('urlopen').WithArgs( trybot_browser_finder.urllib2.ExpectCall('urlopen').WithArgs(
'http://build.chromium.org/p/tryserver.chromium.perf/json').WillReturn( 'http://build.chromium.org/p/tryserver.chromium.perf/json').WillReturn(
StringIO.StringIO(json.dumps({'builders': { StringIO.StringIO(json.dumps({'builders': bots_dict})))
def test_find_all_browser_types_list(self):
finder_options = browser_options.BrowserFinderOptions(browser_type='list')
self._MockTryserverJson({
'android_nexus4_perf_bisect': 'stuff', 'android_nexus4_perf_bisect': 'stuff',
'mac_10_9_perf_bisect': 'otherstuff', 'mac_10_9_perf_bisect': 'otherstuff',
'win_perf_bisect_builder': 'not a trybot', 'win_perf_bisect_builder': 'not a trybot',
}}))) })
expected_trybots_list = [
'trybot-all',
'trybot-all-android',
'trybot-all-linux',
'trybot-all-mac',
'trybot-all-win',
'trybot-android-nexus4',
'trybot-mac-10-9'
]
self.assertEquals( self.assertEquals(
['trybot-android-nexus4', 'trybot-mac-10-9'], expected_trybots_list,
# pylint: disable=W0212
sorted(trybot_browser_finder.FindAllBrowserTypes(finder_options))) sorted(trybot_browser_finder.FindAllBrowserTypes(finder_options)))
def test_find_all_browser_types_trybot(self): def test_find_all_browser_types_trybot(self):
finder_options = browser_options.BrowserFinderOptions( finder_options = browser_options.BrowserFinderOptions(
browser_type='trybot-win') browser_type='trybot-win')
trybot_browser_finder.urllib2 = simple_mock.MockObject() self._MockTryserverJson({
trybot_browser_finder.urllib2.ExpectCall('urlopen').WithArgs(
'http://build.chromium.org/p/tryserver.chromium.perf/json').WillReturn(
StringIO.StringIO(json.dumps({'builders': {
'android_nexus4_perf_bisect': 'stuff', 'android_nexus4_perf_bisect': 'stuff',
'mac_10_9_perf_bisect': 'otherstuff', 'mac_10_9_perf_bisect': 'otherstuff',
'win_perf_bisect_builder': 'not a trybot', 'win_perf_bisect_builder': 'not a trybot',
}}))) })
expected_trybots_list = [
'trybot-all',
'trybot-all-android',
'trybot-all-linux',
'trybot-all-mac',
'trybot-all-win',
'trybot-android-nexus4',
'trybot-mac-10-9'
]
self.assertEquals( self.assertEquals(
['trybot-android-nexus4', 'trybot-mac-10-9'], expected_trybots_list,
# pylint: disable=W0212
sorted(trybot_browser_finder.FindAllBrowserTypes(finder_options))) sorted(trybot_browser_finder.FindAllBrowserTypes(finder_options)))
def test_find_all_browser_types_non_trybot_browser(self): def test_find_all_browser_types_non_trybot_browser(self):
...@@ -84,15 +101,134 @@ class TrybotBrowserFinderTest(unittest.TestCase): ...@@ -84,15 +101,134 @@ class TrybotBrowserFinderTest(unittest.TestCase):
def test_constructor(self): def test_constructor(self):
finder_options = browser_options.BrowserFinderOptions() finder_options = browser_options.BrowserFinderOptions()
self._MockTryserverJson({
'android_nexus4_perf_bisect': 'stuff',
'mac_10_9_perf_bisect': 'otherstuff',
'win_perf_bisect_builder': 'not a trybot',
})
browser = trybot_browser_finder.PossibleTrybotBrowser( browser = trybot_browser_finder.PossibleTrybotBrowser(
'trybot-android-nexus4', finder_options) 'trybot-android-nexus4', finder_options)
self.assertEquals('android', browser.target_os)
# pylint: disable=W0212
self.assertTrue('android' in browser._builder_names)
self.assertEquals(['android_nexus4_perf_bisect'],
browser._builder_names.get('android'))
def test_constructor_trybot_all(self):
finder_options = browser_options.BrowserFinderOptions()
self._MockTryserverJson({
'android_nexus4_perf_bisect': 'stuff',
'android_nexus5_perf_bisect': 'stuff2',
'mac_10_9_perf_bisect': 'otherstuff',
'mac_perf_bisect': 'otherstuff1',
'win_perf_bisect': 'otherstuff2',
'linux_perf_bisect': 'otherstuff3',
'win_x64_perf_bisect': 'otherstuff4',
'win_perf_bisect_builder': 'not a trybot',
})
browser = trybot_browser_finder.PossibleTrybotBrowser(
'trybot-all', finder_options)
self.assertEquals('all', browser.target_os)
# pylint: disable=W0212
self.assertEquals(
['android', 'linux', 'mac', 'win', 'win-x64'],
sorted(browser._builder_names))
self.assertEquals(
['android_nexus4_perf_bisect', 'android_nexus5_perf_bisect'],
sorted(browser._builder_names.get('android')))
self.assertEquals(
['mac_10_9_perf_bisect', 'mac_perf_bisect'],
sorted(browser._builder_names.get('mac')))
self.assertEquals(
['linux_perf_bisect'], sorted(browser._builder_names.get('linux')))
self.assertEquals(
['win_perf_bisect'], sorted(browser._builder_names.get('win')))
self.assertEquals(
['win_x64_perf_bisect'], sorted(browser._builder_names.get('win-x64')))
def test_constructor_trybot_all_win(self):
finder_options = browser_options.BrowserFinderOptions()
self._MockTryserverJson({
'android_nexus4_perf_bisect': 'stuff',
'android_nexus5_perf_bisect': 'stuff2',
'win_8_perf_bisect': 'otherstuff',
'win_perf_bisect': 'otherstuff2',
'linux_perf_bisect': 'otherstuff3',
'win_x64_perf_bisect': 'otherstuff4',
'win_perf_bisect_builder': 'not a trybot',
})
browser = trybot_browser_finder.PossibleTrybotBrowser(
'trybot-all-win', finder_options)
self.assertEquals('all', browser.target_os)
# pylint: disable=W0212
self.assertEquals(
['win', 'win-x64'],
sorted(browser._builder_names))
self.assertEquals(
['win_8_perf_bisect', 'win_perf_bisect'],
sorted(browser._builder_names.get('win')))
self.assertEquals(
['win_x64_perf_bisect'], sorted(browser._builder_names.get('win-x64')))
def test_constructor_trybot_all_android(self):
finder_options = browser_options.BrowserFinderOptions()
self._MockTryserverJson({
'android_nexus4_perf_bisect': 'stuff',
'android_nexus5_perf_bisect': 'stuff2',
'win_8_perf_bisect': 'otherstuff',
'win_perf_bisect': 'otherstuff2',
'linux_perf_bisect': 'otherstuff3',
'win_x64_perf_bisect': 'otherstuff4',
'win_perf_bisect_builder': 'not a trybot',
})
browser = trybot_browser_finder.PossibleTrybotBrowser(
'trybot-all-android', finder_options)
self.assertEquals(
['android_nexus4_perf_bisect', 'android_nexus5_perf_bisect'],
sorted(browser._builder_names.get('android')))
def test_constructor_trybot_all_mac(self):
finder_options = browser_options.BrowserFinderOptions()
self._MockTryserverJson({
'android_nexus4_perf_bisect': 'stuff',
'win_8_perf_bisect': 'otherstuff',
'mac_perf_bisect': 'otherstuff2',
'win_perf_bisect_builder': 'not a trybot',
})
browser = trybot_browser_finder.PossibleTrybotBrowser(
'trybot-all-mac', finder_options)
self.assertEquals('all', browser.target_os)
# pylint: disable=W0212 # pylint: disable=W0212
self.assertEquals('android', browser._target_os) self.assertEquals(
['mac'],
sorted(browser._builder_names))
self.assertEquals(
['mac_perf_bisect'],
sorted(browser._builder_names.get('mac')))
def test_constructor_trybot_all_linux(self):
finder_options = browser_options.BrowserFinderOptions()
self._MockTryserverJson({
'android_nexus4_perf_bisect': 'stuff',
'linux_perf_bisect': 'stuff1',
'win_8_perf_bisect': 'otherstuff',
'mac_perf_bisect': 'otherstuff2',
'win_perf_bisect_builder': 'not a trybot',
})
browser = trybot_browser_finder.PossibleTrybotBrowser(
'trybot-all-linux', finder_options)
self.assertEquals('all', browser.target_os)
# pylint: disable=W0212 # pylint: disable=W0212
self.assertEquals('android_nexus4_perf_bisect', browser._buildername) self.assertEquals(
['linux'],
sorted(browser._builder_names))
self.assertEquals(
['linux_perf_bisect'],
sorted(browser._builder_names.get('linux')))
def test_no_git(self): def test_no_git(self):
finder_options = browser_options.BrowserFinderOptions() finder_options = browser_options.BrowserFinderOptions()
self._MockTryserverJson({'android_nexus4_perf_bisect': 'stuff'})
browser = trybot_browser_finder.PossibleTrybotBrowser( browser = trybot_browser_finder.PossibleTrybotBrowser(
'trybot-android-nexus4', finder_options) 'trybot-android-nexus4', finder_options)
self._ExpectProcesses(( self._ExpectProcesses((
...@@ -105,6 +241,7 @@ class TrybotBrowserFinderTest(unittest.TestCase): ...@@ -105,6 +241,7 @@ class TrybotBrowserFinderTest(unittest.TestCase):
def test_dirty_tree(self): def test_dirty_tree(self):
finder_options = browser_options.BrowserFinderOptions() finder_options = browser_options.BrowserFinderOptions()
self._MockTryserverJson({'android_nexus4_perf_bisect': 'stuff'})
browser = trybot_browser_finder.PossibleTrybotBrowser( browser = trybot_browser_finder.PossibleTrybotBrowser(
'trybot-android-nexus4', finder_options) 'trybot-android-nexus4', finder_options)
self._ExpectProcesses(( self._ExpectProcesses((
...@@ -120,6 +257,7 @@ class TrybotBrowserFinderTest(unittest.TestCase): ...@@ -120,6 +257,7 @@ class TrybotBrowserFinderTest(unittest.TestCase):
def test_no_local_commits(self): def test_no_local_commits(self):
finder_options = browser_options.BrowserFinderOptions() finder_options = browser_options.BrowserFinderOptions()
self._MockTryserverJson({'android_nexus4_perf_bisect': 'stuff'})
browser = trybot_browser_finder.PossibleTrybotBrowser( browser = trybot_browser_finder.PossibleTrybotBrowser(
'trybot-android-nexus4', finder_options) 'trybot-android-nexus4', finder_options)
self._ExpectProcesses(( self._ExpectProcesses((
...@@ -137,11 +275,13 @@ class TrybotBrowserFinderTest(unittest.TestCase): ...@@ -137,11 +275,13 @@ class TrybotBrowserFinderTest(unittest.TestCase):
self.assertEquals( self.assertEquals(
('No local changes found in chromium or blink trees. ' ('No local changes found in chromium or blink trees. '
'browser=trybot-android-nexus4 argument sends local changes to the ' 'browser=trybot-android-nexus4 argument sends local changes to the '
'android_nexus4_perf_bisect perf trybot.\n'), 'perf trybot(s): '
'[[\'android_nexus4_perf_bisect\']].\n'),
self.log_output.getvalue()) self.log_output.getvalue())
def test_branch_checkout_fails(self): def test_branch_checkout_fails(self):
finder_options = browser_options.BrowserFinderOptions() finder_options = browser_options.BrowserFinderOptions()
self._MockTryserverJson({'android_nexus4_perf_bisect': 'stuff'})
browser = trybot_browser_finder.PossibleTrybotBrowser( browser = trybot_browser_finder.PossibleTrybotBrowser(
'trybot-android-nexus4', finder_options) 'trybot-android-nexus4', finder_options)
self._ExpectProcesses(( self._ExpectProcesses((
...@@ -160,10 +300,12 @@ class TrybotBrowserFinderTest(unittest.TestCase): ...@@ -160,10 +300,12 @@ class TrybotBrowserFinderTest(unittest.TestCase):
'fatal: A branch named \'telemetry-try\' already exists.\n'), 'fatal: A branch named \'telemetry-try\' already exists.\n'),
self.log_output.getvalue()) self.log_output.getvalue())
def _GetConfigForBrowser(self, name, branch, cfg_filename, is_blink=False): def _GetConfigForBrowser(self, name, platform, branch, cfg_filename,
is_blink=False):
finder_options = browser_options.BrowserFinderOptions() finder_options = browser_options.BrowserFinderOptions()
browser = trybot_browser_finder.PossibleTrybotBrowser(name, finder_options)
bot = '%s_perf_bisect' % name.replace('trybot-', '').replace('-', '_') bot = '%s_perf_bisect' % name.replace('trybot-', '').replace('-', '_')
self._MockTryserverJson({bot: 'stuff'})
browser = trybot_browser_finder.PossibleTrybotBrowser(name, finder_options)
first_processes = () first_processes = ()
if is_blink: if is_blink:
first_processes = ( first_processes = (
...@@ -180,9 +322,10 @@ class TrybotBrowserFinderTest(unittest.TestCase): ...@@ -180,9 +322,10 @@ class TrybotBrowserFinderTest(unittest.TestCase):
(['git', 'checkout', '-b', 'telemetry-tryjob'], (0, None, None)), (['git', 'checkout', '-b', 'telemetry-tryjob'], (0, None, None)),
(['git', 'branch', '--set-upstream-to', 'origin/master'], (['git', 'branch', '--set-upstream-to', 'origin/master'],
(0, None, None)), (0, None, None)),
(['git', 'commit', '-a', '-m', 'bisect config'], (0, None, None)), (['git', 'commit', '-a', '-m', 'bisect config: %s' % platform],
(0, None, None)),
(['git', 'cl', 'upload', '-f', '--bypass-hooks', '-m', (['git', 'cl', 'upload', '-f', '--bypass-hooks', '-m',
'CL for perf tryjob'], 'CL for perf tryjob on %s' % platform],
(0, 'stuff https://codereview.chromium.org/12345 stuff', None)), (0, 'stuff https://codereview.chromium.org/12345 stuff', None)),
(['git', 'cl', 'try', '-m', 'tryserver.chromium.perf', '-b', bot], (['git', 'cl', 'try', '-m', 'tryserver.chromium.perf', '-b', bot],
(0, None, None)), (0, None, None)),
...@@ -201,7 +344,8 @@ class TrybotBrowserFinderTest(unittest.TestCase): ...@@ -201,7 +344,8 @@ class TrybotBrowserFinderTest(unittest.TestCase):
def test_config_android(self): def test_config_android(self):
config = self._GetConfigForBrowser( config = self._GetConfigForBrowser(
'trybot-android-nexus4', 'somebranch', 'tools/run-perf-test.cfg') 'trybot-android-nexus4', 'android','somebranch',
'tools/run-perf-test.cfg')
self.assertEquals( self.assertEquals(
('config = {\n' ('config = {\n'
' "command": "./tools/perf/run_benchmark ' ' "command": "./tools/perf/run_benchmark '
...@@ -214,7 +358,7 @@ class TrybotBrowserFinderTest(unittest.TestCase): ...@@ -214,7 +358,7 @@ class TrybotBrowserFinderTest(unittest.TestCase):
def test_config_mac(self): def test_config_mac(self):
config = self._GetConfigForBrowser( config = self._GetConfigForBrowser(
'trybot-mac-10-9', 'currentwork', 'tools/run-perf-test.cfg') 'trybot-mac-10-9', 'mac', 'currentwork', 'tools/run-perf-test.cfg')
self.assertEquals( self.assertEquals(
('config = {\n' ('config = {\n'
' "command": "./tools/perf/run_benchmark ' ' "command": "./tools/perf/run_benchmark '
...@@ -227,7 +371,7 @@ class TrybotBrowserFinderTest(unittest.TestCase): ...@@ -227,7 +371,7 @@ class TrybotBrowserFinderTest(unittest.TestCase):
def test_config_win_x64(self): def test_config_win_x64(self):
config = self._GetConfigForBrowser( config = self._GetConfigForBrowser(
'trybot-win-x64', 'currentwork', 'tools/run-perf-test.cfg') 'trybot-win-x64', 'win-x64', 'currentwork', 'tools/run-perf-test.cfg')
self.assertEquals( self.assertEquals(
('config = {\n' ('config = {\n'
' "command": "python tools\\\\perf\\\\run_benchmark ' ' "command": "python tools\\\\perf\\\\run_benchmark '
...@@ -240,7 +384,8 @@ class TrybotBrowserFinderTest(unittest.TestCase): ...@@ -240,7 +384,8 @@ class TrybotBrowserFinderTest(unittest.TestCase):
def test_config_blink(self): def test_config_blink(self):
config = self._GetConfigForBrowser( config = self._GetConfigForBrowser(
'trybot-mac-10-9', 'blinkbranch', 'Tools/run-perf-test.cfg', True) 'trybot-mac-10-9', 'mac', 'blinkbranch',
'Tools/run-perf-test.cfg', True)
self.assertEquals( self.assertEquals(
('config = {\n' ('config = {\n'
' "command": "./tools/perf/run_benchmark ' ' "command": "./tools/perf/run_benchmark '
...@@ -250,3 +395,96 @@ class TrybotBrowserFinderTest(unittest.TestCase): ...@@ -250,3 +395,96 @@ class TrybotBrowserFinderTest(unittest.TestCase):
' "target_arch": "ia32",\n' ' "target_arch": "ia32",\n'
' "truncate_percent": "0"\n' ' "truncate_percent": "0"\n'
'}'), config) '}'), config)
def test_update_config_git_commit_tryboterror(self):
finder_options = browser_options.BrowserFinderOptions()
self._MockTryserverJson({'android_nexus4_perf_bisect': 'stuff'})
browser = trybot_browser_finder.PossibleTrybotBrowser(
'trybot-android-nexus4', finder_options)
self._ExpectProcesses((
(['git', 'commit', '-a', '-m', 'bisect config: android'],
(128, 'None', 'commit failed')),
(['git', 'cl', 'upload', '-f', '--bypass-hooks', '-m',
'CL for perf tryjob on android'],
(0, 'stuff https://codereview.chromium.org/12345 stuff', None)),
(['git', 'cl', 'try', '-m', 'tryserver.chromium.perf', '-b',
'android_nexus4_perf_bisect'], (0, None, None))))
self._stubs.sys.argv = [
'tools/perf/run_benchmark',
'--browser=%s' % browser,
'sunspider']
cfg_filename = 'tools/run-perf-test.cfg'
cfg = StringIO.StringIO()
self._stubs.open.files = {cfg_filename: cfg}
self.assertRaises(trybot_browser_finder.TrybotError,
browser._UpdateConfigAndRunTryjob, 'android', cfg_filename)
def test_update_config_git_upload_tryboterror(self):
finder_options = browser_options.BrowserFinderOptions()
self._MockTryserverJson({'android_nexus4_perf_bisect': 'stuff'})
browser = trybot_browser_finder.PossibleTrybotBrowser(
'trybot-android-nexus4', finder_options)
self._ExpectProcesses((
(['git', 'commit', '-a', '-m', 'bisect config: android'],
(0, 'None', None)),
(['git', 'cl', 'upload', '-f', '--bypass-hooks', '-m',
'CL for perf tryjob on android'],
(128, None, 'error')),
(['git', 'cl', 'try', '-m', 'tryserver.chromium.perf', '-b',
'android_nexus4_perf_bisect'], (0, None, None))))
self._stubs.sys.argv = [
'tools/perf/run_benchmark',
'--browser=%s' % browser,
'sunspider']
cfg_filename = 'tools/run-perf-test.cfg'
cfg = StringIO.StringIO()
self._stubs.open.files = {cfg_filename: cfg}
self.assertRaises(trybot_browser_finder.TrybotError,
browser._UpdateConfigAndRunTryjob, 'android', cfg_filename)
def test_update_config_git_try_tryboterror(self):
finder_options = browser_options.BrowserFinderOptions()
self._MockTryserverJson({'android_nexus4_perf_bisect': 'stuff'})
browser = trybot_browser_finder.PossibleTrybotBrowser(
'trybot-android-nexus4', finder_options)
self._ExpectProcesses((
(['git', 'commit', '-a', '-m', 'bisect config: android'],
(0, 'None', None)),
(['git', 'cl', 'upload', '-f', '--bypass-hooks', '-m',
'CL for perf tryjob on android'],
(0, 'stuff https://codereview.chromium.org/12345 stuff', None)),
(['git', 'cl', 'try', '-m', 'tryserver.chromium.perf', '-b',
'android_nexus4_perf_bisect'], (128, None, None))))
self._stubs.sys.argv = [
'tools/perf/run_benchmark',
'--browser=%s' % browser,
'sunspider']
cfg_filename = 'tools/run-perf-test.cfg'
cfg = StringIO.StringIO()
self._stubs.open.files = {cfg_filename: cfg}
self.assertRaises(trybot_browser_finder.TrybotError,
browser._UpdateConfigAndRunTryjob, 'android', cfg_filename)
def test_update_config_git_try(self):
finder_options = browser_options.BrowserFinderOptions()
self._MockTryserverJson({'android_nexus4_perf_bisect': 'stuff'})
browser = trybot_browser_finder.PossibleTrybotBrowser(
'trybot-android-nexus4', finder_options)
self._ExpectProcesses((
(['git', 'commit', '-a', '-m', 'bisect config: android'],
(0, 'None', None)),
(['git', 'cl', 'upload', '-f', '--bypass-hooks', '-m',
'CL for perf tryjob on android'],
(0, 'stuff https://codereview.chromium.org/12345 stuff', None)),
(['git', 'cl', 'try', '-m', 'tryserver.chromium.perf', '-b',
'android_nexus4_perf_bisect'], (0, None, None))))
self._stubs.sys.argv = [
'tools/perf/run_benchmark',
'--browser=%s' % browser,
'sunspider']
cfg_filename = 'tools/run-perf-test.cfg'
cfg = StringIO.StringIO()
self._stubs.open.files = {cfg_filename: cfg}
self.assertEquals((0, 'https://codereview.chromium.org/12345'),
browser._UpdateConfigAndRunTryjob('android', cfg_filename))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment