Skip to content

GitLab

T
tangled
Commit 4475e480 authored by Emily Hanley's avatar Emily Hanley Committed by Commit Bot
Browse files
Options

Adding null checks if telemetry crashes and doesn't produce results 

Also updating what tests run on each shard to help debug the failing tests.

Bug: 758632
Change-Id: Ib0eb60f0b9ae7e3a2cd19ae8c1f8183d4d0b7d81
Reviewed-on: https://chromium-review.googlesource.com/955676Reviewed-by: default avatarAshley Enstad <ashleymarie@chromium.org>
Commit-Queue: Ashley Enstad <ashleymarie@chromium.org>
Cr-Commit-Position: refs/heads/master@{#542060}
parent ee59863e
Hide whitespace changes
Inline Side-by-side
Showing with 374 additions and 214 deletions
testing/scripts/run_performance_tests.py
View file @ 4475e480
...@@ -63,9 +63,10 @@ BENCHMARKS_TO_OUTPUT_HISTOGRAMS = [ ...@@ -63,9 +63,10 @@ BENCHMARKS_TO_OUTPUT_HISTOGRAMS = [
] ]
# We currently have two different sharding schemes for android # We currently have two different sharding schemes for android
# vs desktop. # vs desktop. When we are running at capacity we will have 26
CURRENT_DESKTOP_NUM_SHARDS = 5 # desktop shards and 39 android.
CURRENT_ANDROID_NUM_SHARDS = 21 CURRENT_DESKTOP_NUM_SHARDS = 26
CURRENT_ANDROID_NUM_SHARDS = 39
def get_sharding_map_path(total_shards, testing): def get_sharding_map_path(total_shards, testing):
# Determine if we want to do a test run of the benchmarks or run the # Determine if we want to do a test run of the benchmarks or run the
......
tools/perf/core/benchmark_android_bot_map.json
View file @ 4475e480
{ {
"0": { "0": {
"benchmarks": [ "benchmarks": [
"system_health.memory_desktop", "battor.steady_state",
"system_health.memory_mobile" "smoothness.desktop_tough_pinch_zoom_cases",
"tab_switching.typical_25"
] ]
}, },
"1": { "1": {
"benchmarks": [ "benchmarks": [
"loading.desktop", "battor.trivial_pages",
"smoothness.gpu_rasterization.polymer",
"thread_times.key_hit_test_cases"
]
},
"10": {
"benchmarks": [
"blink_perf.paint",
"smoothness.key_mobile_sites_smooth",
"v8.browsing_desktop"
]
},
"11": {
"benchmarks": [
"blink_perf.parser",
"smoothness.key_silk_cases", "smoothness.key_silk_cases",
"smoothness.simple_mobile_sites" "v8.browsing_desktop-future"
] ]
}, },
"2": { "12": {
"benchmarks": [ "benchmarks": [
"blink_perf.image_decoder", "blink_perf.shadow_dom",
"blink_perf.owp_storage", "smoothness.maps",
"wasm" "v8.browsing_mobile"
] ]
}, },
"3": { "13": {
"benchmarks": [ "benchmarks": [
"speedometer-future", "blink_perf.svg",
"speedometer2-future", "smoothness.pathological_mobile_sites",
"v8.browsing_desktop-future",
"v8.browsing_mobile-future" "v8.browsing_mobile-future"
] ]
}, },
"4": { "14": {
"benchmarks": [ "benchmarks": [
"system_health.common_desktop", "dromaeo",
"v8.browsing_mobile" "smoothness.simple_mobile_sites",
"v8.detached_context_age_in_gc"
] ]
}, },
"5": { "15": {
"benchmarks": [ "benchmarks": [
"dromaeo.domcoretraverse", "dromaeo.domcoreattr",
"memory.desktop", "smoothness.sync_scroll.key_mobile_sites_smooth",
"power.typical_10_mobile", "v8.runtime_stats.top_25"
"smoothness.tough_animation_cases",
"thread_times.key_mobile_sites_smooth",
"webrtc"
] ]
}, },
"6": { "16": {
"benchmarks": [ "benchmarks": [
"blink_perf.shadow_dom", "dromaeo.domcoremodify",
"dromaeo.domcorequery", "smoothness.top_25_smooth",
"power.idle_platform", "v8.runtimestats.browsing_desktop"
"smoothness.gpu_rasterization_and_decoding.image_decoding_cases"
] ]
}, },
"7": { "17": {
"benchmarks": [ "benchmarks": [
"battor.trivial_pages", "dromaeo.domcorequery",
"media.desktop", "smoothness.tough_ad_cases",
"octane",
"smoothness.gpu_rasterization.tough_scrolling_cases",
"smoothness.tough_filters_cases",
"thread_times.key_hit_test_cases",
"thread_times.key_idle_power_cases",
"v8.runtimestats.browsing_mobile" "v8.runtimestats.browsing_mobile"
] ]
}, },
"8": { "18": {
"benchmarks": [
"dromaeo.domcoretraverse",
"smoothness.tough_animation_cases",
"wasm"
]
},
"19": {
"benchmarks": [ "benchmarks": [
"dummy_benchmark.histogram_benchmark_1", "dummy_benchmark.histogram_benchmark_1",
"smoothness.tough_canvas_cases",
"webrtc"
]
},
"2": {
"benchmarks": [
"blink_perf.bindings",
"smoothness.gpu_rasterization.top_25_smooth",
"thread_times.key_idle_power_cases"
]
},
"20": {
"benchmarks": [
"dummy_benchmark.noisy_benchmark_1", "dummy_benchmark.noisy_benchmark_1",
"smoothness.tough_filters_cases"
]
},
"21": {
"benchmarks": [
"dummy_benchmark.stable_benchmark_1",
"smoothness.tough_image_decode_cases"
]
},
"22": {
"benchmarks": [
"jetstream", "jetstream",
"smoothness.tough_texture_upload_cases", "smoothness.tough_path_rendering_cases"
"speedometer", ]
},
"23": {
"benchmarks": [
"kraken",
"smoothness.tough_pinch_zoom_cases"
]
},
"24": {
"benchmarks": [
"loading.desktop",
"smoothness.tough_scrolling_cases"
]
},
"25": {
"benchmarks": [
"loading.mobile",
"smoothness.tough_texture_upload_cases"
]
},
"26": {
"benchmarks": [
"media.desktop",
"smoothness.tough_webgl_ad_cases"
]
},
"27": {
"benchmarks": [
"media.mobile",
"smoothness.tough_webgl_cases"
]
},
"28": {
"benchmarks": [
"memory.desktop",
"speedometer"
]
},
"29": {
"benchmarks": [
"memory.long_running_idle_gmail_background_tbmv2",
"speedometer-future"
]
},
"3": {
"benchmarks": [
"blink_perf.canvas",
"smoothness.gpu_rasterization.tough_filters_cases",
"thread_times.key_mobile_sites_smooth"
]
},
"30": {
"benchmarks": [
"memory.long_running_idle_gmail_tbmv2",
"speedometer2" "speedometer2"
] ]
}, },
"9": { "31": {
"benchmarks": [ "benchmarks": [
"memory.top_10_mobile", "memory.top_10_mobile",
"smoothness.gpu_rasterization.tough_filters_cases", "speedometer2-future"
"smoothness.image_decoding_cases",
"smoothness.pathological_mobile_sites",
"smoothness.tough_pinch_zoom_cases",
"v8.detached_context_age_in_gc"
] ]
}, },
"10": { "32": {
"benchmarks": [ "benchmarks": [
"blink_perf.bindings", "octane",
"blink_perf.canvas", "start_with_url.cold.startup_pages"
"rasterize_and_record_micro.partial_invalidation",
"smoothness.gpu_rasterization.polymer",
"start_with_url.cold.startup_pages",
"system_health.webview_startup"
] ]
}, },
"11": { "33": {
"benchmarks": [ "benchmarks": [
"blink_perf.layout",
"dromaeo",
"dromaeo.domcoreattr",
"oortonline_tbmv2", "oortonline_tbmv2",
"smoothness.maps", "start_with_url.warm.startup_pages"
"smoothness.tough_webgl_cases",
"thread_times.simple_mobile_sites"
] ]
}, },
"12": { "34": {
"benchmarks": [ "benchmarks": [
"battor.steady_state", "power.idle_platform",
"blink_perf.svg", "system_health.common_desktop"
"memory.long_running_idle_gmail_background_tbmv2",
"smoothness.gpu_rasterization.top_25_smooth",
"smoothness.gpu_rasterization.tough_path_rendering_cases",
"thread_times.key_noop_cases"
] ]
}, },
"13": { "35": {
"benchmarks": [ "benchmarks": [
"memory.long_running_idle_gmail_tbmv2", "power.typical_10_mobile",
"system_health.common_mobile" "system_health.common_mobile"
] ]
}, },
"14": { "36": {
"benchmarks": [ "benchmarks": [
"loading.mobile", "rasterize_and_record_micro.partial_invalidation",
"media.mobile", "system_health.memory_desktop"
"smoothness.tough_webgl_ad_cases",
"tab_switching.typical_25",
"v8.browsing_desktop"
] ]
}, },
"15": { "37": {
"benchmarks": [ "benchmarks": [
"blink_perf.events", "rasterize_and_record_micro.top_25",
"dummy_benchmark.stable_benchmark_1", "system_health.memory_mobile"
"smoothness.top_25_smooth",
"smoothness.tough_ad_cases"
] ]
}, },
"16": { "38": {
"benchmarks": [ "benchmarks": [
"dromaeo.domcoremodify", "scheduler.tough_scheduling_cases",
"smoothness.key_desktop_move_cases", "system_health.webview_startup"
"smoothness.sync_scroll.key_mobile_sites_smooth",
"thread_times.tough_compositor_cases",
"thread_times.tough_scrolling_cases"
] ]
}, },
"17": { "4": {
"benchmarks": [
"blink_perf.css",
"smoothness.gpu_rasterization.tough_path_rendering_cases",
"thread_times.key_noop_cases"
]
},
"5": {
"benchmarks": [ "benchmarks": [
"blink_perf.dom", "blink_perf.dom",
"rasterize_and_record_micro.top_25", "smoothness.gpu_rasterization.tough_pinch_zoom_cases",
"smoothness.tough_canvas_cases",
"thread_times.key_silk_cases" "thread_times.key_silk_cases"
] ]
}, },
"18": { "6": {
"benchmarks": [ "benchmarks": [
"blink_perf.css", "blink_perf.events",
"blink_perf.parser", "smoothness.gpu_rasterization.tough_scrolling_cases",
"scheduler.tough_scheduling_cases", "thread_times.simple_mobile_sites"
"smoothness.gpu_rasterization.tough_pinch_zoom_cases",
"smoothness.tough_image_decode_cases",
"start_with_url.warm.startup_pages"
] ]
}, },
"19": { "7": {
"benchmarks": [ "benchmarks": [
"blink_perf.paint", "blink_perf.image_decoder",
"kraken", "smoothness.gpu_rasterization_and_decoding.image_decoding_cases",
"smoothness.key_mobile_sites_smooth", "thread_times.tough_compositor_cases"
"tracing.tracing_with_background_memory_infra"
] ]
}, },
"20": { "8": {
"benchmarks": [ "benchmarks": [
"smoothness.desktop_tough_pinch_zoom_cases", "blink_perf.layout",
"smoothness.tough_path_rendering_cases", "smoothness.image_decoding_cases",
"smoothness.tough_scrolling_cases", "thread_times.tough_scrolling_cases"
"v8.runtime_stats.top_25", ]
"v8.runtimestats.browsing_desktop" },
"9": {
"benchmarks": [
"blink_perf.owp_storage",
"smoothness.key_desktop_move_cases",
"tracing.tracing_with_background_memory_infra"
] ]
} }
} }
tools/perf/core/benchmark_desktop_bot_map.json
View file @ 4475e480
{ {
"0": { "0": {
"benchmarks": [ "benchmarks": [
"blink_perf.dom", "battor.steady_state",
"blink_perf.paint", "media.desktop",
"dromaeo.domcoremodify", "smoothness.pathological_mobile_sites",
"dummy_benchmark.stable_benchmark_1", "tab_switching.typical_25"
]
},
"1": {
"benchmarks": [
"battor.trivial_pages",
"media.mobile", "media.mobile",
"memory.top_10_mobile", "smoothness.simple_mobile_sites",
"thread_times.key_hit_test_cases"
]
},
"10": {
"benchmarks": [
"blink_perf.paint",
"rasterize_and_record_micro.partial_invalidation",
"smoothness.tough_pinch_zoom_cases",
"v8.browsing_desktop"
]
},
"11": {
"benchmarks": [
"blink_perf.parser",
"rasterize_and_record_micro.top_25",
"smoothness.tough_scrolling_cases",
"v8.browsing_desktop-future"
]
},
"12": {
"benchmarks": [
"blink_perf.shadow_dom",
"scheduler.tough_scheduling_cases", "scheduler.tough_scheduling_cases",
"smoothness.key_desktop_move_cases", "smoothness.tough_texture_upload_cases",
"smoothness.key_mobile_sites_smooth", "v8.browsing_mobile"
"smoothness.tough_ad_cases",
"smoothness.tough_image_decode_cases",
"system_health.memory_desktop",
"thread_times.key_mobile_sites_smooth",
"thread_times.key_silk_cases",
"v8.runtime_stats.top_25"
] ]
}, },
"1": { "13": {
"benchmarks": [ "benchmarks": [
"blink_perf.bindings", "blink_perf.svg",
"blink_perf.canvas", "smoothness.desktop_tough_pinch_zoom_cases",
"blink_perf.image_decoder", "smoothness.tough_webgl_ad_cases",
"blink_perf.owp_storage", "v8.browsing_mobile-future"
"dummy_benchmark.histogram_benchmark_1",
"dummy_benchmark.noisy_benchmark_1",
"loading.desktop",
"octane",
"oortonline_tbmv2",
"power.idle_platform",
"smoothness.key_silk_cases",
"smoothness.pathological_mobile_sites",
"smoothness.top_25_smooth",
"smoothness.tough_webgl_cases",
"system_health.common_mobile",
"system_health.memory_mobile",
"thread_times.key_hit_test_cases",
"thread_times.key_noop_cases",
"thread_times.tough_scrolling_cases",
"tracing.tracing_with_background_memory_infra",
"v8.browsing_mobile",
"webrtc"
] ]
}, },
"2": { "14": {
"benchmarks": [ "benchmarks": [
"battor.trivial_pages",
"blink_perf.layout",
"dromaeo", "dromaeo",
"dromaeo.domcoreattr",
"memory.long_running_idle_gmail_background_tbmv2",
"rasterize_and_record_micro.top_25",
"smoothness.gpu_rasterization.polymer", "smoothness.gpu_rasterization.polymer",
"smoothness.tough_webgl_cases",
"v8.detached_context_age_in_gc"
]
},
"15": {
"benchmarks": [
"dromaeo.domcoreattr",
"smoothness.gpu_rasterization.top_25_smooth", "smoothness.gpu_rasterization.top_25_smooth",
"smoothness.gpu_rasterization_and_decoding.image_decoding_cases",
"smoothness.image_decoding_cases",
"smoothness.simple_mobile_sites",
"smoothness.tough_filters_cases",
"smoothness.tough_path_rendering_cases",
"smoothness.tough_scrolling_cases",
"speedometer", "speedometer",
"speedometer2", "v8.runtime_stats.top_25"
"tab_switching.typical_25",
"thread_times.tough_compositor_cases",
"v8.browsing_desktop-future",
"v8.browsing_mobile-future",
"wasm"
] ]
}, },
"3": { "16": {
"benchmarks": [
"dromaeo.domcoremodify",
"smoothness.gpu_rasterization.tough_filters_cases",
"speedometer-future",
"v8.runtimestats.browsing_desktop"
]
},
"17": {
"benchmarks": [ "benchmarks": [
"blink_perf.parser",
"blink_perf.svg",
"dromaeo.domcorequery", "dromaeo.domcorequery",
"smoothness.gpu_rasterization.tough_path_rendering_cases",
"speedometer2",
"v8.runtimestats.browsing_mobile"
]
},
"18": {
"benchmarks": [
"dromaeo.domcoretraverse", "dromaeo.domcoretraverse",
"jetstream",
"memory.desktop",
"power.typical_10_mobile",
"smoothness.gpu_rasterization.tough_filters_cases",
"smoothness.gpu_rasterization.tough_pinch_zoom_cases", "smoothness.gpu_rasterization.tough_pinch_zoom_cases",
"smoothness.tough_canvas_cases",
"smoothness.tough_pinch_zoom_cases",
"speedometer-future",
"speedometer2-future", "speedometer2-future",
"wasm"
]
},
"19": {
"benchmarks": [
"dummy_benchmark.histogram_benchmark_1",
"smoothness.gpu_rasterization.tough_scrolling_cases",
"start_with_url.cold.startup_pages", "start_with_url.cold.startup_pages",
"v8.browsing_desktop", "webrtc"
"v8.detached_context_age_in_gc",
"v8.runtimestats.browsing_desktop"
] ]
}, },
"4": { "2": {
"benchmarks": [
"blink_perf.bindings",
"memory.desktop",
"smoothness.sync_scroll.key_mobile_sites_smooth",
"thread_times.key_idle_power_cases"
]
},
"20": {
"benchmarks": [
"dummy_benchmark.noisy_benchmark_1",
"smoothness.gpu_rasterization_and_decoding.image_decoding_cases",
"start_with_url.warm.startup_pages"
]
},
"21": {
"benchmarks": [
"dummy_benchmark.stable_benchmark_1",
"smoothness.image_decoding_cases",
"system_health.common_desktop"
]
},
"22": {
"benchmarks": [
"jetstream",
"smoothness.key_desktop_move_cases",
"system_health.common_mobile"
]
},
"23": {
"benchmarks": [ "benchmarks": [
"battor.steady_state",
"blink_perf.css",
"blink_perf.events",
"blink_perf.shadow_dom",
"kraken", "kraken",
"smoothness.key_mobile_sites_smooth",
"system_health.memory_desktop"
]
},
"24": {
"benchmarks": [
"loading.desktop",
"smoothness.key_silk_cases",
"system_health.memory_mobile"
]
},
"25": {
"benchmarks": [
"loading.mobile", "loading.mobile",
"media.desktop",
"memory.long_running_idle_gmail_tbmv2",
"rasterize_and_record_micro.partial_invalidation",
"smoothness.desktop_tough_pinch_zoom_cases",
"smoothness.gpu_rasterization.tough_path_rendering_cases",
"smoothness.gpu_rasterization.tough_scrolling_cases",
"smoothness.maps", "smoothness.maps",
"smoothness.sync_scroll.key_mobile_sites_smooth", "system_health.webview_startup"
]
},
"3": {
"benchmarks": [
"blink_perf.canvas",
"memory.long_running_idle_gmail_background_tbmv2",
"smoothness.top_25_smooth",
"thread_times.key_mobile_sites_smooth"
]
},
"4": {
"benchmarks": [
"blink_perf.css",
"memory.long_running_idle_gmail_tbmv2",
"smoothness.tough_ad_cases",
"thread_times.key_noop_cases"
]
},
"5": {
"benchmarks": [
"blink_perf.dom",
"memory.top_10_mobile",
"smoothness.tough_animation_cases", "smoothness.tough_animation_cases",
"smoothness.tough_texture_upload_cases", "thread_times.key_silk_cases"
"smoothness.tough_webgl_ad_cases", ]
"start_with_url.warm.startup_pages", },
"system_health.common_desktop", "6": {
"system_health.webview_startup", "benchmarks": [
"thread_times.key_idle_power_cases", "blink_perf.events",
"thread_times.simple_mobile_sites", "octane",
"v8.runtimestats.browsing_mobile" "smoothness.tough_canvas_cases",
"thread_times.simple_mobile_sites"
]
},
"7": {
"benchmarks": [
"blink_perf.image_decoder",
"oortonline_tbmv2",
"smoothness.tough_filters_cases",
"thread_times.tough_compositor_cases"
]
},
"8": {
"benchmarks": [
"blink_perf.layout",
"power.idle_platform",
"smoothness.tough_image_decode_cases",
"thread_times.tough_scrolling_cases"
]
},
"9": {
"benchmarks": [
"blink_perf.owp_storage",
"power.typical_10_mobile",
"smoothness.tough_path_rendering_cases",
"tracing.tracing_with_background_memory_infra"
] ]
} }
} }
tools/perf/process_perf_results.py
View file @ 4475e480
...@@ -92,36 +92,39 @@ def _process_perf_results(output_json, configuration_name, ...@@ -92,36 +92,39 @@ def _process_perf_results(output_json, configuration_name,
# We need to keep track of disabled benchmarks so we don't try to # We need to keep track of disabled benchmarks so we don't try to
# upload the results. # upload the results.
disabled_benchmarks = []
test_results_list = [] test_results_list = []
tmpfile_dir = tempfile.mkdtemp('resultscache') tmpfile_dir = tempfile.mkdtemp('resultscache')
try: try:
for directory in benchmark_directory_list:
with open(join(directory, 'test_results.json')) as json_data:
json_results = json.load(json_data)
if json_results.get('version') == 3:
# Non-telemetry tests don't have written json results but
# if they are executing then they are enabled and will generate
# chartjson results.
if not bool(json_results.get('tests')):
disabled_benchmarks.append(directory)
if '.reference' in directory:
# We don't need to upload reference build data to the
# flakiness dashboard since we don't monitor the ref build
continue
test_results_list.append(json_results)
_merge_json_output(output_json, test_results_list)
with oauth_api.with_access_token(service_account_file) as oauth_file: with oauth_api.with_access_token(service_account_file) as oauth_file:
for directory in benchmark_directory_list: for directory in benchmark_directory_list:
print 'Uploading perf results from %s benchmark' % directory disabled = False
if directory in disabled_benchmarks: with open(join(directory, 'test_results.json')) as json_data:
json_results = json.load(json_data)
if not json_results:
# Output is null meaning the test didn't produce any results.
# Want to output an error and continue loading the rest of the
# test results.
print 'No results produced for %s, skipping upload' % directory
continue
if json_results.get('version') == 3:
# Non-telemetry tests don't have written json results but
# if they are executing then they are enabled and will generate
# chartjson results.
if not bool(json_results.get('tests')):
disabled = True
if not '.reference' in directory:
# We don't need to upload reference build data to the
# flakiness dashboard since we don't monitor the ref build
test_results_list.append(json_results)
if disabled:
# We don't upload disabled benchmarks # We don't upload disabled benchmarks
print 'Benchmark %s disabled' % directory print 'Benchmark %s disabled' % directory
continue continue
print 'Uploading perf results from %s benchmark' % directory
_upload_perf_results(join(directory, 'perf_results.json'), _upload_perf_results(join(directory, 'perf_results.json'),
directory, configuration_name, build_properties, directory, configuration_name, build_properties,
oauth_file, tmpfile_dir) oauth_file, tmpfile_dir)
_merge_json_output(output_json, test_results_list)
finally: finally:
shutil.rmtree(tmpfile_dir) shutil.rmtree(tmpfile_dir)
return 0 return 0
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment