Skip to content

GitLab

T
tangled
Commit fdff4bbd authored by Emily Hanley's avatar Emily Hanley Committed by Commit Bot
Browse files
Options

Triggering benchmarks by story index if present and merging results 

from each shard.

Updating bot map format to include an optional begin and end value
to indicate a subset of stories to run.

Bug: 713357
Change-Id: Id3188cc350e9df2c82177057e032301ddc8b851d
Reviewed-on: https://chromium-review.googlesource.com/1070310Reviewed-by: default avatarEmily Hanley <eyaich@chromium.org>
Reviewed-by: default avatarAshley Enstad <ashleymarie@chromium.org>
Reviewed-by: default avatarNed Nguyen <nednguyen@google.com>
Commit-Queue: Emily Hanley <eyaich@chromium.org>
Cr-Commit-Position: refs/heads/master@{#561946}
parent 61205133
Hide whitespace changes
Inline Side-by-side
Showing with 479 additions and 391 deletions
testing/scripts/run_performance_tests.py
View file @ fdff4bbd
......@@ -154,7 +154,7 @@ def write_results(
def execute_benchmark(benchmark, isolated_out_dir,
args, rest_args, is_reference):
args, rest_args, is_reference, stories=None):
# While we are between chartjson and histogram set we need
# to determine which output format to look for or see if it was
# already passed in in which case that format applies to all benchmarks
......@@ -179,6 +179,16 @@ def execute_benchmark(benchmark, isolated_out_dir,
per_benchmark_args.append('--output-trace-tag=_ref')
benchmark_name = benchmark + '.reference'
# If we are only running a subset of stories, add in the begin and end
# index.
if stories:
if 'begin' in stories.keys():
per_benchmark_args.append(
('--experimental-story-shard-begin-index=%d' % stories['begin']))
if 'end' in stories.keys():
per_benchmark_args.append(
('--experimental-story-shard-end-index=%d' % stories['end']))
# We don't care exactly what these are. In particular, the perf results
# could be any format (chartjson, legacy, histogram). We just pass these
# through, and expose these as results for this task.
......@@ -284,17 +294,18 @@ def main():
sharding_map_path = get_sharding_map_path(args)
with open(sharding_map_path) as f:
sharding_map = json.load(f)
sharding = None
sharding = sharding_map[shard_index]['benchmarks']
for benchmark in sharding:
for benchmark, stories in sharding.iteritems():
# Need to run the benchmark twice on browser and reference build
return_code = (execute_benchmark(
benchmark, isolated_out_dir, args, rest_args, False) or return_code)
benchmark, isolated_out_dir, args, rest_args,
False, stories=stories) or return_code)
# We ignore the return code of the reference build since we do not
# monitor it.
if args.run_ref_build:
execute_benchmark(benchmark, isolated_out_dir, args, rest_args, True)
execute_benchmark(benchmark, isolated_out_dir, args, rest_args, True,
stories=stories)
return return_code
......
tools/perf/core/benchmark_android_bot_map.json
View file @ fdff4bbd
{
"0": {
"benchmarks": [
"battor.steady_state",
"smoothness.desktop_tough_pinch_zoom_cases",
"tab_switching.typical_25"
]
"benchmarks": {
"battor.steady_state": {},
"smoothness.desktop_tough_pinch_zoom_cases": {},
"tab_switching.typical_25": {}
}
},
"1": {
"benchmarks": [
"battor.trivial_pages",
"smoothness.gpu_rasterization.polymer",
"thread_times.key_hit_test_cases"
]
"benchmarks": {
"battor.trivial_pages": {},
"smoothness.gpu_rasterization.polymer": {},
"thread_times.key_hit_test_cases": {}
}
},
"10": {
"benchmarks": [
"blink_perf.paint",
"smoothness.key_mobile_sites_smooth",
"v8.browsing_desktop"
]
"benchmarks": {
"blink_perf.paint": {},
"smoothness.key_mobile_sites_smooth": {},
"v8.browsing_desktop": {}
}
},
"11": {
"benchmarks": [
"blink_perf.parser",
"smoothness.key_silk_cases",
"v8.browsing_desktop-future"
]
"benchmarks": {
"blink_perf.parser": {},
"smoothness.key_silk_cases": {},
"v8.browsing_desktop-future": {}
}
},
"12": {
"benchmarks": [
"blink_perf.shadow_dom",
"smoothness.maps",
"v8.browsing_mobile"
]
"benchmarks": {
"blink_perf.shadow_dom": {},
"smoothness.maps": {},
"v8.browsing_mobile": {}
}
},
"13": {
"benchmarks": [
"blink_perf.svg",
"smoothness.pathological_mobile_sites",
"v8.browsing_mobile-future"
]
"benchmarks": {
"blink_perf.svg": {},
"smoothness.pathological_mobile_sites": {},
"v8.browsing_mobile-future": {}
}
},
"14": {
"benchmarks": [
"dromaeo",
"smoothness.simple_mobile_sites"
]
"benchmarks": {
"dromaeo": {},
"smoothness.simple_mobile_sites": {}
}
},
"15": {
"benchmarks": [
"smoothness.sync_scroll.key_mobile_sites_smooth",
"v8.runtime_stats.top_25"
]
"benchmarks": {
"smoothness.sync_scroll.key_mobile_sites_smooth": {},
"v8.runtime_stats.top_25": {}
}
},
"16": {
"benchmarks": [
"smoothness.top_25_smooth",
"rendering.mobile"
]
"benchmarks": {
"smoothness.top_25_smooth": {},
"rendering.mobile": {}
}
},
"17": {
"benchmarks": [
"smoothness.tough_ad_cases",
"rendering.desktop"
]
"benchmarks": {
"smoothness.tough_ad_cases": {},
"rendering.desktop": {}
}
},
"18": {
"benchmarks": [
"smoothness.tough_animation_cases",
"wasm"
]
"benchmarks": {
"smoothness.tough_animation_cases": {},
"wasm": {}
}
},
"19": {
"benchmarks": [
"dummy_benchmark.histogram_benchmark_1",
"smoothness.tough_canvas_cases",
"webrtc"
]
"benchmarks": {
"dummy_benchmark.histogram_benchmark_1": {},
"smoothness.tough_canvas_cases": {},
"webrtc": {}
}
},
"2": {
"benchmarks": [
"blink_perf.bindings",
"smoothness.gpu_rasterization.top_25_smooth",
"thread_times.key_idle_power_cases"
]
"benchmarks": {
"blink_perf.bindings": {},
"smoothness.gpu_rasterization.top_25_smooth": {},
"thread_times.key_idle_power_cases": {}
}
},
"20": {
"benchmarks": [
"dummy_benchmark.noisy_benchmark_1",
"smoothness.tough_filters_cases"
]
"benchmarks": {
"dummy_benchmark.noisy_benchmark_1": {},
"smoothness.tough_filters_cases": {}
}
},
"21": {
"benchmarks": [
"dummy_benchmark.stable_benchmark_1",
"smoothness.tough_image_decode_cases"
]
"benchmarks": {
"dummy_benchmark.stable_benchmark_1": {},
"smoothness.tough_image_decode_cases": {}
}
},
"22": {
"benchmarks": [
"jetstream",
"smoothness.tough_path_rendering_cases"
]
"benchmarks": {
"jetstream": {},
"smoothness.tough_path_rendering_cases": {}
}
},
"23": {
"benchmarks": [
"kraken",
"smoothness.tough_pinch_zoom_cases"
]
"benchmarks": {
"kraken": {},
"smoothness.tough_pinch_zoom_cases": {}
}
},
"24": {
"benchmarks": [
"loading.desktop",
"smoothness.tough_scrolling_cases"
]
"benchmarks": {
"loading.desktop": {},
"smoothness.tough_scrolling_cases": {}
}
},
"25": {
"benchmarks": [
"loading.mobile",
"smoothness.tough_texture_upload_cases"
]
"benchmarks": {
"loading.mobile": {},
"smoothness.tough_texture_upload_cases": {}
}
},
"26": {
"benchmarks": [
"media.desktop",
"smoothness.tough_webgl_ad_cases"
]
"benchmarks": {
"media.desktop": {},
"smoothness.tough_webgl_ad_cases": {}
}
},
"27": {
"benchmarks": [
"media.mobile",
"smoothness.tough_webgl_cases"
]
"benchmarks": {
"media.mobile": {},
"smoothness.tough_webgl_cases": {}
}
},
"28": {
"benchmarks": [
"memory.desktop",
"speedometer"
]
"benchmarks": {
"memory.desktop": {},
"speedometer": {}
}
},
"29": {
"benchmarks": [
"memory.long_running_idle_gmail_background_tbmv2",
"speedometer-future"
]
"benchmarks": {
"memory.long_running_idle_gmail_background_tbmv2": {},
"speedometer-future": {}
}
},
"3": {
"benchmarks": [
"blink_perf.canvas",
"smoothness.gpu_rasterization.tough_filters_cases",
"thread_times.key_mobile_sites_smooth"
]
"benchmarks": {
"blink_perf.canvas": {},
"smoothness.gpu_rasterization.tough_filters_cases": {},
"thread_times.key_mobile_sites_smooth": {}
}
},
"30": {
"benchmarks": [
"memory.long_running_idle_gmail_tbmv2",
"speedometer2"
]
"benchmarks": {
"memory.long_running_idle_gmail_tbmv2": {},
"speedometer2": {}
}
},
"31": {
"benchmarks": [
"memory.top_10_mobile",
"speedometer2-future"
]
"benchmarks": {
"memory.top_10_mobile": {},
"speedometer2-future": {}
}
},
"32": {
"benchmarks": [
"octane",
"start_with_url.cold.startup_pages"
]
"benchmarks": {
"octane": {},
"start_with_url.cold.startup_pages": {}
}
},
"33": {
"benchmarks": [
"oortonline_tbmv2",
"start_with_url.warm.startup_pages"
]
"benchmarks": {
"oortonline_tbmv2": {},
"start_with_url.warm.startup_pages": {}
}
},
"34": {
"benchmarks": [
"power.idle_platform",
"system_health.common_desktop"
]
"benchmarks": {
"power.idle_platform": {},
"system_health.common_desktop": {}
}
},
"35": {
"benchmarks": [
"power.typical_10_mobile",
"system_health.common_mobile"
]
"benchmarks": {
"power.typical_10_mobile": {},
"system_health.common_mobile": {}
}
},
"36": {
"benchmarks": [
"rasterize_and_record_micro.partial_invalidation",
"system_health.memory_desktop"
]
"benchmarks": {
"rasterize_and_record_micro.partial_invalidation": {},
"system_health.memory_desktop": {}
}
},
"37": {
"benchmarks": [
"rasterize_and_record_micro.top_25",
"system_health.memory_mobile"
]
"benchmarks": {
"rasterize_and_record_micro.top_25": {},
"system_health.memory_mobile": {}
}
},
"38": {
"benchmarks": [
"scheduler.tough_scheduling_cases",
"system_health.webview_startup"
]
"benchmarks": {
"scheduler.tough_scheduling_cases": {},
"system_health.webview_startup": {}
}
},
"4": {
"benchmarks": [
"blink_perf.css",
"smoothness.gpu_rasterization.tough_path_rendering_cases",
"thread_times.key_noop_cases"
]
"benchmarks": {
"blink_perf.css": {},
"smoothness.gpu_rasterization.tough_path_rendering_cases": {},
"thread_times.key_noop_cases": {}
}
},
"5": {
"benchmarks": [
"blink_perf.dom",
"smoothness.gpu_rasterization.tough_pinch_zoom_cases",
"thread_times.key_silk_cases"
]
"benchmarks": {
"blink_perf.dom": {},
"smoothness.gpu_rasterization.tough_pinch_zoom_cases": {},
"thread_times.key_silk_cases": {}
}
},
"6": {
"benchmarks": [
"blink_perf.events",
"smoothness.gpu_rasterization.tough_scrolling_cases",
"thread_times.simple_mobile_sites"
]
"benchmarks": {
"blink_perf.events": {},
"smoothness.gpu_rasterization.tough_scrolling_cases": {},
"thread_times.simple_mobile_sites": {}
}
},
"7": {
"benchmarks": [
"blink_perf.image_decoder",
"smoothness.gpu_rasterization_and_decoding.image_decoding_cases",
"thread_times.tough_compositor_cases"
]
"benchmarks": {
"blink_perf.image_decoder": {},
"smoothness.gpu_rasterization_and_decoding.image_decoding_cases": {},
"thread_times.tough_compositor_cases": {}
}
},
"8": {
"benchmarks": [
"blink_perf.layout",
"smoothness.image_decoding_cases",
"thread_times.tough_scrolling_cases"
]
"benchmarks": {
"blink_perf.layout": {},
"smoothness.image_decoding_cases": {},
"thread_times.tough_scrolling_cases": {}
}
},
"9": {
"benchmarks": [
"blink_perf.owp_storage",
"smoothness.key_desktop_move_cases",
"tracing.tracing_with_background_memory_infra"
]
"benchmarks": {
"blink_perf.owp_storage": {},
"smoothness.key_desktop_move_cases": {},
"tracing.tracing_with_background_memory_infra": {}
}
}
}
tools/perf/core/benchmark_bot_map.json
View file @ fdff4bbd
{
"0": {
"benchmarks": [
"dummy_benchmark.stable_benchmark_1",
"blink_perf.owp_storage",
"system_health.memory_mobile"
]
"benchmarks": {
"dummy_benchmark.stable_benchmark_1": {},
"blink_perf.owp_storage": {
"end": 3
},
"scheduler.tough_scheduling_cases": {
"end": 2
},
"system_health.memory_mobile": {},
"blink_perf.paint": {
"end": 4
}
}
},
"1": {
"benchmarks": [
"dummy_benchmark.histogram_benchmark_1",
"memory.top_10_mobile",
"scheduler.tough_scheduling_cases"
]
"benchmarks": {
"dummy_benchmark.histogram_benchmark_1": {},
"blink_perf.owp_storage": {
"begin":3
},
"memory.top_10_mobile": {},
"scheduler.tough_scheduling_cases": {
"begin": 2
},
"blink_perf.paint": {
"begin": 4,
"end": 10
}
}
},
"2": {
"benchmarks": [
"blink_perf.paint",
"rasterize_and_record_micro.partial_invalidation",
"smoothness.tough_pinch_zoom_cases"
]
"benchmarks": {
"blink_perf.paint": {
"begin": 10
},
"rasterize_and_record_micro.partial_invalidation": {},
"smoothness.tough_pinch_zoom_cases": {}
}
}
}
tools/perf/core/benchmark_desktop_bot_map.json
View file @ fdff4bbd
{
"0": {
"benchmarks": [
"battor.steady_state",
"media.desktop",
"smoothness.pathological_mobile_sites",
"tab_switching.typical_25"
]
"benchmarks": {
"battor.steady_state": {},
"media.desktop": {},
"smoothness.pathological_mobile_sites": {},
"tab_switching.typical_25": {}
}
},
"1": {
"benchmarks": [
"battor.trivial_pages",
"media.mobile",
"smoothness.simple_mobile_sites",
"thread_times.key_hit_test_cases"
]
"benchmarks": {
"battor.trivial_pages": {},
"media.mobile": {},
"smoothness.simple_mobile_sites": {},
"thread_times.key_hit_test_cases": {}
}
},
"10": {
"benchmarks": [
"blink_perf.paint",
"rasterize_and_record_micro.partial_invalidation",
"smoothness.tough_pinch_zoom_cases",
"v8.browsing_desktop"
]
"benchmarks": {
"blink_perf.paint": {},
"rasterize_and_record_micro.partial_invalidation": {},
"smoothness.tough_pinch_zoom_cases": {},
"v8.browsing_desktop": {}
}
},
"11": {
"benchmarks": [
"blink_perf.parser",
"rasterize_and_record_micro.top_25",
"smoothness.tough_scrolling_cases",
"v8.browsing_desktop-future"
]
"benchmarks": {
"blink_perf.parser": {},
"rasterize_and_record_micro.top_25": {},
"smoothness.tough_scrolling_cases": {},
"v8.browsing_desktop-future": {}
}
},
"12": {
"benchmarks": [
"blink_perf.shadow_dom",
"scheduler.tough_scheduling_cases",
"smoothness.tough_texture_upload_cases",
"v8.browsing_mobile"
]
"benchmarks": {
"blink_perf.shadow_dom": {},
"scheduler.tough_scheduling_cases": {},
"smoothness.tough_texture_upload_cases": {},
"v8.browsing_mobile": {}
}
},
"13": {
"benchmarks": [
"blink_perf.svg",
"smoothness.desktop_tough_pinch_zoom_cases",
"smoothness.tough_webgl_ad_cases",
"v8.browsing_mobile-future"
]
"benchmarks": {
"blink_perf.svg": {},
"smoothness.desktop_tough_pinch_zoom_cases": {},
"smoothness.tough_webgl_ad_cases": {},
"v8.browsing_mobile-future": {}
}
},
"14": {
"benchmarks": [
"dromaeo",
"smoothness.gpu_rasterization.polymer",
"smoothness.tough_webgl_cases"
]
"benchmarks": {
"dromaeo": {},
"smoothness.gpu_rasterization.polymer": {},
"smoothness.tough_webgl_cases": {}
}
},
"15": {
"benchmarks": [
"smoothness.gpu_rasterization.top_25_smooth",
"speedometer",
"v8.runtime_stats.top_25"
]
"benchmarks": {
"smoothness.gpu_rasterization.top_25_smooth": {},
"speedometer": {},
"v8.runtime_stats.top_25": {}
}
},
"16": {
"benchmarks": [
"smoothness.gpu_rasterization.tough_filters_cases",
"speedometer-future",
"rendering.mobile"
]
"benchmarks": {
"smoothness.gpu_rasterization.tough_filters_cases": {},
"speedometer-future": {},
"rendering.mobile": {}
}
},
"17": {
"benchmarks": [
"rendering.desktop",
"smoothness.gpu_rasterization.tough_path_rendering_cases",
"speedometer2"
]
"benchmarks": {
"rendering.desktop": {},
"smoothness.gpu_rasterization.tough_path_rendering_cases": {},
"speedometer2": {}
}
},
"18": {
"benchmarks": [
"smoothness.gpu_rasterization.tough_pinch_zoom_cases",
"speedometer2-future",
"wasm"
]
"benchmarks": {
"smoothness.gpu_rasterization.tough_pinch_zoom_cases": {},
"speedometer2-future": {},
"wasm": {}
}
},
"19": {
"benchmarks": [
"dummy_benchmark.histogram_benchmark_1",
"smoothness.gpu_rasterization.tough_scrolling_cases",
"start_with_url.cold.startup_pages",
"webrtc"
]
"benchmarks": {
"dummy_benchmark.histogram_benchmark_1": {},
"smoothness.gpu_rasterization.tough_scrolling_cases": {},
"start_with_url.cold.startup_pages": {},
"webrtc": {}
}
},
"2": {
"benchmarks": [
"blink_perf.bindings",
"memory.desktop",
"smoothness.sync_scroll.key_mobile_sites_smooth",
"thread_times.key_idle_power_cases"
]
"benchmarks": {
"blink_perf.bindings": {},
"memory.desktop": {},
"smoothness.sync_scroll.key_mobile_sites_smooth": {},
"thread_times.key_idle_power_cases": {}
}
},
"20": {
"benchmarks": [
"dummy_benchmark.noisy_benchmark_1",
"smoothness.gpu_rasterization_and_decoding.image_decoding_cases",
"start_with_url.warm.startup_pages"
]
"benchmarks": {
"dummy_benchmark.noisy_benchmark_1": {},
"smoothness.gpu_rasterization_and_decoding.image_decoding_cases": {},
"start_with_url.warm.startup_pages": {}
}
},
"21": {
"benchmarks": [
"dummy_benchmark.stable_benchmark_1",
"smoothness.image_decoding_cases",
"system_health.common_desktop"
]
"benchmarks": {
"dummy_benchmark.stable_benchmark_1": {},
"smoothness.image_decoding_cases": {},
"system_health.common_desktop": {}
}
},
"22": {
"benchmarks": [
"jetstream",
"smoothness.key_desktop_move_cases",
"system_health.common_mobile"
]
"benchmarks": {
"jetstream": {},
"smoothness.key_desktop_move_cases": {},
"system_health.common_mobile": {}
}
},
"23": {
"benchmarks": [
"kraken",
"smoothness.key_mobile_sites_smooth",
"system_health.memory_desktop"
]
"benchmarks": {
"kraken": {},
"smoothness.key_mobile_sites_smooth": {},
"system_health.memory_desktop": {}
}
},
"24": {
"benchmarks": [
"loading.desktop",
"smoothness.key_silk_cases",
"system_health.memory_mobile"
]
"benchmarks": {
"loading.desktop": {},
"smoothness.key_silk_cases": {},
"system_health.memory_mobile": {}
}
},
"25": {
"benchmarks": [
"loading.mobile",
"smoothness.maps",
"system_health.webview_startup"
]
"benchmarks": {
"loading.mobile": {},
"smoothness.maps": {},
"system_health.webview_startup": {}
}
},
"3": {
"benchmarks": [
"blink_perf.canvas",
"memory.long_running_idle_gmail_background_tbmv2",
"smoothness.top_25_smooth",
"thread_times.key_mobile_sites_smooth"
]
"benchmarks": {
"blink_perf.canvas": {},
"memory.long_running_idle_gmail_background_tbmv2": {},
"smoothness.top_25_smooth": {},
"thread_times.key_mobile_sites_smooth": {}
}
},
"4": {
"benchmarks": [
"blink_perf.css",
"memory.long_running_idle_gmail_tbmv2",
"smoothness.tough_ad_cases",
"thread_times.key_noop_cases"
]
"benchmarks": {
"blink_perf.css": {},
"memory.long_running_idle_gmail_tbmv2": {},
"smoothness.tough_ad_cases": {},
"thread_times.key_noop_cases": {}
}
},
"5": {
"benchmarks": [
"blink_perf.dom",
"memory.top_10_mobile",
"smoothness.tough_animation_cases",
"thread_times.key_silk_cases"
]
"benchmarks": {
"blink_perf.dom": {},
"memory.top_10_mobile": {},
"smoothness.tough_animation_cases": {},
"thread_times.key_silk_cases": {}
}
},
"6": {
"benchmarks": [
"blink_perf.events",
"octane",
"smoothness.tough_canvas_cases",
"thread_times.simple_mobile_sites"
]
"benchmarks": {
"blink_perf.events": {},
"octane": {},
"smoothness.tough_canvas_cases": {},
"thread_times.simple_mobile_sites": {}
}
},
"7": {
"benchmarks": [
"blink_perf.image_decoder",
"oortonline_tbmv2",
"smoothness.tough_filters_cases",
"thread_times.tough_compositor_cases"
]
"benchmarks": {
"blink_perf.image_decoder": {},
"oortonline_tbmv2": {},
"smoothness.tough_filters_cases": {},
"thread_times.tough_compositor_cases": {}
}
},
"8": {
"benchmarks": [
"blink_perf.layout",
"power.idle_platform",
"smoothness.tough_image_decode_cases",
"thread_times.tough_scrolling_cases"
]
"benchmarks": {
"blink_perf.layout": {},
"power.idle_platform": {},
"smoothness.tough_image_decode_cases": {},
"thread_times.tough_scrolling_cases": {}
}
},
"9": {
"benchmarks": [
"blink_perf.owp_storage",
"power.typical_10_mobile",
"smoothness.tough_path_rendering_cases",
"tracing.tracing_with_background_memory_infra"
]
"benchmarks": {
"blink_perf.owp_storage": {},
"power.typical_10_mobile": {},
"smoothness.tough_path_rendering_cases": {},
"tracing.tracing_with_background_memory_infra": {}
}
}
}
tools/perf/core/smoke_test_benchmark_shard_map.json
View file @ fdff4bbd
{
"0": {
"benchmarks": [
"dummy_benchmark.stable_benchmark_1"
]
"benchmarks": {
"dummy_benchmark.stable_benchmark_1": {}
}
},
"1": {
"benchmarks": [
"dummy_benchmark.histogram_benchmark_1"
]
"benchmarks": {
"dummy_benchmark.histogram_benchmark_1": {}
}
}
}
tools/perf/process_perf_results.py
View file @ fdff4bbd
......@@ -117,36 +117,36 @@ def _merge_json_output(output_json, jsons_to_merge, extra_links):
def _handle_perf_json_test_results(
benchmark_directory_list, test_results_list):
benchmark_directory_map, test_results_list):
benchmark_enabled_map = {}
for directory in benchmark_directory_list:
# Obtain the test name we are running
benchmark_name = _get_benchmark_name(directory)
is_ref = '.reference' in benchmark_name
enabled = True
with open(join(directory, 'test_results.json')) as json_data:
json_results = json.load(json_data)
if not json_results:
# Output is null meaning the test didn't produce any results.
# Want to output an error and continue loading the rest of the
# test results.
print 'No results produced for %s, skipping upload' % directory
continue
if json_results.get('version') == 3:
# Non-telemetry tests don't have written json results but
# if they are executing then they are enabled and will generate
# chartjson results.
if not bool(json_results.get('tests')):
enabled = False
if not is_ref:
# We don't need to upload reference build data to the
# flakiness dashboard since we don't monitor the ref build
test_results_list.append(json_results)
if not enabled:
# We don't upload disabled benchmarks or tests that are run
# as a smoke test
print 'Benchmark %s disabled' % benchmark_name
benchmark_enabled_map[benchmark_name] = enabled
for benchmark_name, directories in benchmark_directory_map.iteritems():
for directory in directories:
# Obtain the test name we are running
is_ref = '.reference' in benchmark_name
enabled = True
with open(join(directory, 'test_results.json')) as json_data:
json_results = json.load(json_data)
if not json_results:
# Output is null meaning the test didn't produce any results.
# Want to output an error and continue loading the rest of the
# test results.
print 'No results produced for %s, skipping upload' % directory
continue
if json_results.get('version') == 3:
# Non-telemetry tests don't have written json results but
# if they are executing then they are enabled and will generate
# chartjson results.
if not bool(json_results.get('tests')):
enabled = False
if not is_ref:
# We don't need to upload reference build data to the
# flakiness dashboard since we don't monitor the ref build
test_results_list.append(json_results)
if not enabled:
# We don't upload disabled benchmarks or tests that are run
# as a smoke test
print 'Benchmark %s disabled' % benchmark_name
benchmark_enabled_map[benchmark_name] = enabled
return benchmark_enabled_map
......@@ -155,19 +155,21 @@ def _generate_unique_logdog_filename(name_prefix):
return name_prefix + '_' + str(uuid.uuid4())
def _handle_perf_logs(benchmark_directory_list, extra_links):
def _handle_perf_logs(benchmark_directory_map, extra_links):
""" Upload benchmark logs to logdog and add a page entry for them. """
benchmark_logs_links = {}
for directory in benchmark_directory_list:
# Obtain the test name we are running
benchmark_name = _get_benchmark_name(directory)
with open(join(directory, 'benchmark_log.txt')) as f:
uploaded_link = logdog_helper.text(
name=_generate_unique_logdog_filename(benchmark_name),
data=f.read())
benchmark_logs_links[benchmark_name] = uploaded_link
for benchmark_name, directories in benchmark_directory_map.iteritems():
for directory in directories:
with open(join(directory, 'benchmark_log.txt')) as f:
uploaded_link = logdog_helper.text(
name=_generate_unique_logdog_filename(benchmark_name),
data=f.read())
if benchmark_name in benchmark_logs_links.keys():
benchmark_logs_links[benchmark_name].append(uploaded_link)
else:
benchmark_logs_links[benchmark_name] = [uploaded_link]
logdog_file_name = _generate_unique_logdog_filename('Benchmarks_Logs')
logdog_stream = logdog_helper.text(
......@@ -210,6 +212,16 @@ def _process_perf_results(output_json, configuration_name,
for f in listdir(join(task_output_dir, directory))
]
# Now create a map of benchmark name to the list of directories
# the lists were written to.
benchmark_directory_map = {}
for directory in benchmark_directory_list:
benchmark_name = _get_benchmark_name(directory)
if benchmark_name in benchmark_directory_map.keys():
benchmark_directory_map[benchmark_name].append(directory)
else:
benchmark_directory_map[benchmark_name] = [directory]
test_results_list = []
build_properties = json.loads(build_properties)
......@@ -221,16 +233,16 @@ def _process_perf_results(output_json, configuration_name,
# First, upload all the benchmark logs to logdog and add a page entry for
# those links in extra_links.
_handle_perf_logs(benchmark_directory_list, extra_links)
_handle_perf_logs(benchmark_directory_map, extra_links)
# Then try to obtain the list of json test results to merge
# and determine the status of each benchmark.
benchmark_enabled_map = _handle_perf_json_test_results(
benchmark_directory_list, test_results_list)
benchmark_directory_map, test_results_list)
if not smoke_test_mode:
return_code = _handle_perf_results(
benchmark_enabled_map, benchmark_directory_list,
benchmark_enabled_map, benchmark_directory_map,
configuration_name, build_properties, service_account_file, extra_links)
# Finally, merge all test results json, add the extra links and write out to
......@@ -238,9 +250,43 @@ def _process_perf_results(output_json, configuration_name,
_merge_json_output(output_json, test_results_list, extra_links)
return return_code
def _merge_chartjson_results(chartjson_dicts):
merged_results = chartjson_dicts[0]
for chartjson_dict in chartjson_dicts[1:]:
for key in chartjson_dict:
if key == 'charts':
for add_key in chartjson_dict[key]:
merged_results[key][add_key] = chartjson_dict[key][add_key]
return merged_results
def _merge_histogram_results(histogram_lists):
merged_results = []
for histogram_list in histogram_lists:
merged_results += histogram_list
return merged_results
def _merge_perf_results(results_filename, directories):
collected_results = []
for directory in directories:
filename = join(directory, 'perf_results.json')
with open(filename) as pf:
collected_results.append(json.load(pf))
# Assuming that multiple shards will only be chartjson or histogram set
# Non-telemetry benchmarks only ever run on one shard
merged_results = []
if isinstance(collected_results[0], dict):
merged_results = _merge_chartjson_results(collected_results)
elif isinstance(collected_results[0], list):
merged_results =_merge_histogram_results(collected_results)
with open(results_filename, 'w') as rf:
json.dump(merged_results, rf)
def _handle_perf_results(
benchmark_enabled_map, benchmark_directory_list, configuration_name,
benchmark_enabled_map, benchmark_directory_map, configuration_name,
build_properties, service_account_file, extra_links):
"""
Upload perf results to the perf dashboard.
......@@ -259,14 +305,27 @@ def _handle_perf_results(
logdog_label = 'Results Dashboard'
upload_fail = False
with oauth_api.with_access_token(service_account_file) as oauth_file:
for directory in benchmark_directory_list:
benchmark_name = _get_benchmark_name(directory)
for benchmark_name, directories in benchmark_directory_map.iteritems():
if not benchmark_enabled_map[benchmark_name]:
continue
# There are potentially multiple directores with results, re-write and
# merge them if necessary
results_filename = None
if len(directories) > 1:
merge_perf_dir = os.path.join(
os.path.abspath(tmpfile_dir), benchmark_name)
if not os.path.exists(merge_perf_dir):
os.makedirs(merge_perf_dir)
results_filename = os.path.join(
merge_perf_dir, 'merged_perf_results.json')
_merge_perf_results(results_filename, directories)
else:
# It was only written to one shard, use that shards data
results_filename = join(directories[0], 'perf_results.json')
print 'Uploading perf results from %s benchmark' % benchmark_name
upload_fail = _upload_and_write_perf_data_to_logfile(
benchmark_name, directory, configuration_name, build_properties,
oauth_file, tmpfile_dir, logdog_dict,
benchmark_name, results_filename, configuration_name,
build_properties, oauth_file, tmpfile_dir, logdog_dict,
('.reference' in benchmark_name))
logdog_file_name = _generate_unique_logdog_filename('Results_Dashboard_')
......@@ -284,7 +343,7 @@ def _handle_perf_results(
shutil.rmtree(tmpfile_dir)
def _upload_and_write_perf_data_to_logfile(benchmark_name, directory,
def _upload_and_write_perf_data_to_logfile(benchmark_name, results_file,
configuration_name, build_properties, oauth_file,
tmpfile_dir, logdog_dict, is_ref):
upload_failure = False
......@@ -293,7 +352,7 @@ def _upload_and_write_perf_data_to_logfile(benchmark_name, directory,
# upload results and write perf results to logdog file
upload_failure = _upload_perf_results(
join(directory, 'perf_results.json'),
results_file,
benchmark_name, configuration_name, build_properties,
oauth_file, tmpfile_dir, output_json_file)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment