Skip to content

GitLab

T
tangled
Commit b9b19c3b authored by Caleb Rouleau's avatar Caleb Rouleau
Browse files
Options

[Benchmarking] Remove duplication in waterfall configs. 

We now only define shard_num and shard map location
in bot_platforms.py.

Bug: 902089
Change-Id: I101743f3a4aaa2291d2dba03d7338d9dbe03798c
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1941273
Commit-Queue: Caleb Rouleau <crouleau@chromium.org>
Reviewed-by: default avatarJohn Chen <johnchen@chromium.org>
Cr-Commit-Position: refs/heads/master@{#720609}
parent ea6a45cd
Show whitespace changes
Inline Side-by-side
Showing with 49 additions and 88 deletions
testing/buildbot/chromium.perf.fyi.json
View file @ b9b19c3b
......@@ -8,8 +8,8 @@
"-v",
"--browser=android-chrome",
"--upload-results",
"--output-format=histograms",
"--test-shard-map-filename=android-nexus5x-perf-fyi_map.json"
"--test-shard-map-filename=android-nexus5x-perf-fyi_map.json",
"--output-format=histograms"
],
"isolate_name": "performance_test_suite",
"merge": {
......@@ -54,9 +54,9 @@
"-v",
"--browser=exact",
"--upload-results",
"--test-shard-map-filename=android-pixel2-perf-fyi_map.json",
"--browser-executable=../../out/Release/bin/monochrome_64_32_bundle",
"--device=android",
"--test-shard-map-filename=android-pixel2-perf-fyi_map.json"
"--device=android"
],
"isolate_name": "performance_test_suite",
"merge": {
......@@ -146,9 +146,9 @@
"-v",
"--browser=release",
"--upload-results",
"--test-shard-map-filename=linux-perf-fyi_map.json",
"--output-format=histograms",
"--experimental-proto-trace-format",
"--test-shard-map-filename=linux-perf-fyi_map.json"
"--experimental-proto-trace-format"
],
"isolate_name": "performance_test_suite",
"merge": {
......@@ -192,8 +192,8 @@
"-v",
"--browser=release_x64",
"--upload-results",
"--run-ref-build",
"--test-shard-map-filename=win-10_laptop_low_end-perf_HP-Candidate_map.json"
"--test-shard-map-filename=win-10_laptop_low_end-perf_hp-candidate_map.json",
"--run-ref-build"
],
"isolate_name": "performance_test_suite",
"merge": {
......
testing/buildbot/chromium.perf.json
View file @ b9b19c3b
......@@ -125,8 +125,8 @@
"-v",
"--browser=android-chromium",
"--upload-results",
"--run-ref-build",
"--test-shard-map-filename=android_nexus5_perf_map.json",
"--run-ref-build",
"--assert-gpu-compositing"
],
"isolate_name": "performance_test_suite",
......@@ -172,8 +172,8 @@
"-v",
"--browser=android-webview",
"--upload-results",
"--webview-embedder-apk=../../out/Release/apks/SystemWebViewShell.apk",
"--test-shard-map-filename=android_nexus5x_webview_perf_map.json",
"--webview-embedder-apk=../../out/Release/apks/SystemWebViewShell.apk",
"--assert-gpu-compositing"
],
"isolate_name": "performance_webview_test_suite",
......@@ -219,8 +219,8 @@
"-v",
"--browser=android-webview",
"--upload-results",
"--webview-embedder-apk=../../out/Release/apks/SystemWebViewShell.apk",
"--test-shard-map-filename=android_nexus6_webview_perf_map.json",
"--webview-embedder-apk=../../out/Release/apks/SystemWebViewShell.apk",
"--assert-gpu-compositing"
],
"isolate_name": "performance_webview_test_suite",
......@@ -460,8 +460,8 @@
"-v",
"--browser=release_x64",
"--upload-results",
"--run-ref-build",
"--test-shard-map-filename=win_7_nvidia_gpu_perf_map.json",
"--run-ref-build",
"--assert-gpu-compositing"
],
"isolate_name": "performance_test_suite",
......@@ -620,8 +620,8 @@
"-v",
"--browser=release",
"--upload-results",
"--run-ref-build",
"--test-shard-map-filename=win_7_perf_map.json"
"--test-shard-map-filename=win_7_perf_map.json",
"--run-ref-build"
],
"isolate_name": "performance_test_suite",
"merge": {
......@@ -866,8 +866,8 @@
"-v",
"--browser=android-chrome",
"--upload-results",
"--run-ref-build",
"--test-shard-map-filename=android-go-perf_map.json"
"--test-shard-map-filename=android-go-perf_map.json",
"--run-ref-build"
],
"isolate_name": "performance_test_suite",
"merge": {
......@@ -912,8 +912,8 @@
"-v",
"--browser=android-webview-google",
"--upload-results",
"--webview-embedder-apk=../../out/Release/apks/SystemWebViewShell.apk",
"--test-shard-map-filename=android-go_webview-perf_map.json"
"--test-shard-map-filename=android-go_webview-perf_map.json",
"--webview-embedder-apk=../../out/Release/apks/SystemWebViewShell.apk"
],
"isolate_name": "performance_webview_test_suite",
"merge": {
......@@ -1193,8 +1193,8 @@
"-v",
"--browser=android-chromium",
"--upload-results",
"--run-ref-build",
"--test-shard-map-filename=android-nexus5x-perf_map.json",
"--run-ref-build",
"--assert-gpu-compositing"
],
"isolate_name": "performance_test_suite",
......@@ -1240,8 +1240,8 @@
"-v",
"--browser=android-chrome",
"--upload-results",
"--run-ref-build",
"--test-shard-map-filename=android-pixel2-perf_map.json"
"--test-shard-map-filename=android-pixel2-perf_map.json",
"--run-ref-build"
],
"isolate_name": "performance_test_suite",
"merge": {
......@@ -1286,9 +1286,9 @@
"-v",
"--browser=android-weblayer",
"--upload-results",
"--test-shard-map-filename=android-pixel2_weblayer-perf_map.json",
"--webview-embedder-apk=../../out/Release/apks/WebLayerShell.apk",
"--webview-embedder-apk=../../out/Release/apks/WebLayerSupport.apk",
"--test-shard-map-filename=android-pixel2_weblayer-perf_map.json"
"--webview-embedder-apk=../../out/Release/apks/WebLayerSupport.apk"
],
"isolate_name": "performance_weblayer_test_suite",
"merge": {
......@@ -1333,8 +1333,8 @@
"-v",
"--browser=android-webview-google",
"--upload-results",
"--webview-embedder-apk=../../out/Release/apks/SystemWebViewShell.apk",
"--test-shard-map-filename=android-pixel2_webview-perf_map.json"
"--test-shard-map-filename=android-pixel2_webview-perf_map.json",
"--webview-embedder-apk=../../out/Release/apks/SystemWebViewShell.apk"
],
"isolate_name": "performance_webview_test_suite",
"merge": {
......@@ -1789,8 +1789,8 @@
"-v",
"--browser=release",
"--upload-results",
"--run-ref-build",
"--test-shard-map-filename=linux-perf_map.json",
"--run-ref-build",
"--assert-gpu-compositing"
],
"isolate_name": "performance_test_suite",
......@@ -1909,8 +1909,8 @@
"-v",
"--browser=release",
"--upload-results",
"--run-ref-build",
"--test-shard-map-filename=mac-10_12_laptop_low_end-perf_map.json",
"--run-ref-build",
"--assert-gpu-compositing"
],
"isolate_name": "performance_test_suite",
......@@ -2182,8 +2182,8 @@
"-v",
"--browser=release",
"--upload-results",
"--run-ref-build",
"--test-shard-map-filename=mac-10_13_laptop_high_end-perf_map.json",
"--run-ref-build",
"--assert-gpu-compositing"
],
"isolate_name": "performance_test_suite",
......@@ -2463,8 +2463,8 @@
"-v",
"--browser=release_x64",
"--upload-results",
"--run-ref-build",
"--test-shard-map-filename=win-10-perf_map.json",
"--run-ref-build",
"--assert-gpu-compositing"
],
"isolate_name": "performance_test_suite",
......
tools/perf/core/bot_platforms.py
View file @ b9b19c3b
......@@ -47,8 +47,9 @@ class PerfPlatform(object):
base_file_name = name.replace(' ', '_').lower()
self._timing_file_path = os.path.join(
_SHARD_MAP_DIR, 'timing_data', base_file_name + '_timing.json')
self.shards_map_file_name = base_file_name + '_map.json'
self._shards_map_file_path = os.path.join(
_SHARD_MAP_DIR, base_file_name + '_map.json')
_SHARD_MAP_DIR, self.shards_map_file_name)
def __lt__(self, other):
if not isinstance(other, type(self)):
......@@ -295,6 +296,7 @@ LINUX_PERF_FYI = PerfPlatform(
ALL_PLATFORMS = {
p for p in locals().values() if isinstance(p, PerfPlatform)
}
PLATFORMS_BY_NAME = {p.name: p for p in ALL_PLATFORMS}
FYI_PLATFORMS = {
p for p in ALL_PLATFORMS if p.is_fyi
}
......
tools/perf/core/perf_data_generator.py
View file @ b9b19c3b
......@@ -58,6 +58,8 @@ from telemetry import decorators
# 'name': 'presentation_name',
#
# # The number of shards for this test as an int.
# # This is only required for GTEST tests since this is defined
# # in bot_platforms.py for Telemetry tests.
# 'num_shards': 2,
#
# # What kind of test this is; for options, see TEST_TYPES
......@@ -88,8 +90,6 @@ class TEST_TYPES(object):
ALL = (GENERIC, GTEST, TELEMETRY)
# TODO(crbug.com/902089): automatically generate --test-shard-map-filename
# arguments once we track all the perf FYI builders to core/bot_platforms.py
FYI_BUILDERS = {
'android-nexus5x-perf-fyi': {
'tests': [
......@@ -97,9 +97,7 @@ FYI_BUILDERS = {
'isolate': 'performance_test_suite',
'extra_args': [
'--output-format=histograms',
'--test-shard-map-filename=android-nexus5x-perf-fyi_map.json',
],
'num_shards': 3
}
],
'platform': 'android-chrome',
......@@ -120,9 +118,7 @@ FYI_BUILDERS = {
# --browser=exact (used by this bot to have it run Monochrome6432)
# and --browser=reference together.
#'--run-ref-build',
'--test-shard-map-filename=android-pixel2-perf-fyi_map.json',
],
'num_shards': 4
}
],
'platform': 'android-chrome',
......@@ -139,11 +135,9 @@ FYI_BUILDERS = {
'tests': [
{
'isolate': 'performance_test_suite',
'num_shards': 1,
'extra_args': [
'--output-format=histograms',
'--experimental-proto-trace-format',
'--test-shard-map-filename=linux-perf-fyi_map.json',
],
}
],
......@@ -159,11 +153,8 @@ FYI_BUILDERS = {
'tests': [
{
'isolate': 'performance_test_suite',
'num_shards': 1,
'extra_args': [
'--run-ref-build',
'--test-shard-map-filename='
'win-10_laptop_low_end-perf_HP-Candidate_map.json',
],
},
],
......@@ -182,9 +173,7 @@ FYI_BUILDERS = {
'tests': [
{
'isolate': 'performance_test_suite',
'num_shards': 4,
'extra_args': [
'--test-shard-map-filename=chromeos-kevin-perf-fyi_map.json',
# The magic hostname that resolves to a CrOS device in the test lab
'--remote=variable_chromeos_device_hostname',
],
......@@ -338,9 +327,7 @@ BUILDERS = {
'isolate': 'performance_test_suite',
'extra_args': [
'--run-ref-build',
'--test-shard-map-filename=android-go-perf_map.json',
],
'num_shards': 19
}
],
'platform': 'android-chrome',
......@@ -356,10 +343,6 @@ BUILDERS = {
'tests': [
{
'isolate': 'performance_webview_test_suite',
'extra_args': [
'--test-shard-map-filename=android-go_webview-perf_map.json',
],
'num_shards': 13
}
],
'platform': 'android-webview-google',
......@@ -375,10 +358,8 @@ BUILDERS = {
'tests': [
{
'isolate': 'performance_test_suite',
'num_shards': 10,
'extra_args': [
'--run-ref-build',
'--test-shard-map-filename=android-nexus5x-perf_map.json',
'--assert-gpu-compositing',
],
},
......@@ -429,10 +410,8 @@ BUILDERS = {
'tests': [
{
'isolate': 'performance_test_suite',
'num_shards': 16,
'extra_args': [
'--run-ref-build',
'--test-shard-map-filename=android_nexus5_perf_map.json',
'--assert-gpu-compositing',
],
},
......@@ -465,9 +444,7 @@ BUILDERS = {
'tests': [
{
'isolate': 'performance_webview_test_suite',
'num_shards': 16,
'extra_args': [
'--test-shard-map-filename=android_nexus5x_webview_perf_map.json',
'--assert-gpu-compositing',
],
}
......@@ -485,9 +462,7 @@ BUILDERS = {
'tests': [
{
'isolate': 'performance_webview_test_suite',
'num_shards': 12,
'extra_args': [
'--test-shard-map-filename=android_nexus6_webview_perf_map.json',
'--assert-gpu-compositing',
],
}
......@@ -506,9 +481,7 @@ BUILDERS = {
{
'isolate': 'performance_webview_test_suite',
'extra_args': [
'--test-shard-map-filename=android-pixel2_webview-perf_map.json',
],
'num_shards': 28,
}
],
'platform': 'android-webview-google',
......@@ -525,9 +498,7 @@ BUILDERS = {
{
'isolate': 'performance_weblayer_test_suite',
'extra_args': [
'--test-shard-map-filename=android-pixel2_weblayer-perf_map.json',
],
'num_shards': 4,
}
],
'platform': 'android-weblayer',
......@@ -545,9 +516,7 @@ BUILDERS = {
'isolate': 'performance_test_suite',
'extra_args': [
'--run-ref-build',
'--test-shard-map-filename=android-pixel2-perf_map.json',
],
'num_shards': 35
}
],
'platform': 'android-chrome',
......@@ -563,10 +532,8 @@ BUILDERS = {
'tests': [
{
'isolate': 'performance_test_suite',
'num_shards': 26,
'extra_args': [
'--run-ref-build',
'--test-shard-map-filename=win-10-perf_map.json',
'--assert-gpu-compositing',
],
},
......@@ -624,10 +591,8 @@ BUILDERS = {
'tests': [
{
'isolate': 'performance_test_suite',
'num_shards': 5,
'extra_args': [
'--run-ref-build',
'--test-shard-map-filename=win_7_perf_map.json',
],
},
{
......@@ -659,10 +624,8 @@ BUILDERS = {
'tests': [
{
'isolate': 'performance_test_suite',
'num_shards': 5,
'extra_args': [
'--run-ref-build',
'--test-shard-map-filename=win_7_nvidia_gpu_perf_map.json',
'--assert-gpu-compositing',
],
},
......@@ -715,11 +678,8 @@ BUILDERS = {
'tests': [
{
'isolate': 'performance_test_suite',
'num_shards': 26,
'extra_args': [
'--run-ref-build',
('--test-shard-map-filename='
'mac-10_12_laptop_low_end-perf_map.json'),
'--assert-gpu-compositing',
],
},
......@@ -746,10 +706,8 @@ BUILDERS = {
# Add views_perftests, crbug.com/811766
{
'isolate': 'performance_test_suite',
'num_shards': 26,
'extra_args': [
'--run-ref-build',
'--test-shard-map-filename=linux-perf_map.json',
'--assert-gpu-compositing',
],
},
......@@ -798,10 +756,8 @@ BUILDERS = {
'isolate': 'performance_test_suite',
'extra_args': [
'--run-ref-build',
'--test-shard-map-filename=mac-10_13_laptop_high_end-perf_map.json',
'--assert-gpu-compositing',
],
'num_shards': 26
},
{
'isolate': 'performance_browser_tests',
......@@ -852,7 +808,7 @@ def update_all_tests(builders_dict, file_path):
tests['AAAAA2 See //tools/perf/generate_perf_data to make changes'] = {}
for name, config in builders_dict.iteritems():
tests[name] = generate_builder_config(config)
tests[name] = generate_builder_config(config, name)
with open(file_path, 'w') as fp:
json.dump(tests, fp, indent=2, separators=(',', ': '), sort_keys=True)
......@@ -1140,7 +1096,7 @@ def validate_docs(labs_docs_file):
os.remove(labs_docs_tempfile)
def generate_telemetry_args(tester_config):
def generate_telemetry_args(tester_config, platform):
# First determine the browser that you need based on the tester
browser_name = ''
# For trybot testing we always use the reference build
......@@ -1159,13 +1115,12 @@ def generate_telemetry_args(tester_config):
browser_name = 'release_x64'
else:
browser_name ='release'
test_args = [
'-v',
'--browser=%s' % browser_name,
'--upload-results'
'--upload-results',
'--test-shard-map-filename=%s' % platform.shards_map_file_name,
]
if 'browser' in tester_config:
test_args.append('--browser-executable=../../out/Release/%s' %
tester_config['browser'])
......@@ -1180,7 +1135,6 @@ def generate_telemetry_args(tester_config):
'--webview-embedder-apk=../../out/Release/apks/WebLayerShell.apk')
test_args.append(
'--webview-embedder-apk=../../out/Release/apks/WebLayerSupport.apk')
return test_args
......@@ -1193,18 +1147,23 @@ def generate_gtest_args(test_name):
]
def generate_performance_test(tester_config, test):
def generate_performance_test(tester_config, test, builder_name):
isolate_name = test['isolate']
test_name = test.get('name', isolate_name)
test_type = test.get('type', TEST_TYPES.TELEMETRY)
assert test_type in TEST_TYPES.ALL
shards = test.get('num_shards', None)
test_args = []
if test_type == TEST_TYPES.TELEMETRY:
test_args += generate_telemetry_args(tester_config)
platform = bot_platforms.PLATFORMS_BY_NAME[builder_name]
test_args += generate_telemetry_args(tester_config, platform)
assert shards is None
shards = platform.num_shards
elif test_type == TEST_TYPES.GTEST:
test_args += generate_gtest_args(test_name=test_name)
assert shards
# Append any additional args specific to an isolate
test_args += test.get('extra_args', [])
......@@ -1235,7 +1194,6 @@ def generate_performance_test(tester_config, test):
'script': '//tools/perf/process_perf_results.py',
}
shards = test.get('num_shards')
result['swarming'] = {
# Always say this is true regardless of whether the tester
# supports swarming. It doesn't hurt.
......@@ -1260,7 +1218,7 @@ def generate_performance_test(tester_config, test):
return result
def generate_builder_config(condensed_config):
def generate_builder_config(condensed_config, builder_name):
config = {}
if 'additional_compile_targets' in condensed_config:
......@@ -1273,7 +1231,8 @@ def generate_builder_config(condensed_config):
telemetry_tests = []
other_tests = []
for test in condensed_tests:
generated_script = generate_performance_test(condensed_config, test)
generated_script = generate_performance_test(
condensed_config, test, builder_name)
test_type = test.get('type', TEST_TYPES.TELEMETRY)
if test_type == TEST_TYPES.GTEST:
gtest_tests.append(generated_script)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment