Skip to content

GitLab

T
tangled
Commit 744cf4d7 authored by Ashley Enstad's avatar Ashley Enstad Committed by Commit Bot
Browse files
Options

Changing timing back to averaging from summing 

Bug: 713357
Change-Id: Iefd741be80323c322b63dd92f05b8758e96fb6ca
Reviewed-on: https://chromium-review.googlesource.com/1101490
Commit-Queue: Ashley Enstad <ashleymarie@chromium.org>
Reviewed-by: default avatarNed Nguyen <nednguyen@google.com>
Reviewed-by: default avatarEmily Hanley <eyaich@chromium.org>
Cr-Commit-Position: refs/heads/master@{#569791}
parent 23e3596c
Show whitespace changes
Inline Side-by-side
Showing with 15027 additions and 4376 deletions
tools/perf/core/retrieve_story_timing.py
View file @ 744cf4d7
......@@ -30,19 +30,19 @@ ORDER BY
QUERY_LAST_100_RUNS = """
SELECT
name,
AVG(run_durations) AS duration
AVG(time) AS duration
FROM (
SELECT
name,
start_time,
SUM(run.times) AS run_durations,
time,
ROW_NUMBER() OVER (PARTITION BY name ORDER BY start_time DESC)
AS row_num
FROM (
SELECT
run.name AS name,
start_time,
run.times
AVG(run.times) AS time
FROM
[test-results-hrd:events.test_results]
WHERE
......@@ -52,13 +52,9 @@ FROM (
AND run.is_unexpected IS FALSE
GROUP BY
name,
start_time,
run.times
start_time
ORDER BY
start_time DESC )
GROUP BY
name,
start_time)
start_time DESC ))
WHERE
row_num < 100
GROUP BY
......
tools/perf/core/shard_maps/linux_perf_shard_map.json
View file @ 744cf4d7
......@@ -92,30 +92,30 @@
"begin": 17
},
"rendering.desktop": {
"end": 58
"end": 57
}
}
},
"8": {
"benchmarks": {
"rendering.desktop": {
"begin": 58,
"end": 131
"begin": 57,
"end": 129
}
}
},
"9": {
"benchmarks": {
"rendering.desktop": {
"begin": 131,
"end": 214
"begin": 129,
"end": 211
}
}
},
"10": {
"benchmarks": {
"rendering.desktop": {
"begin": 214
"begin": 211
},
"rendering.mobile": {},
"scheduler.tough_scheduling_cases": {},
......@@ -136,25 +136,25 @@
"smoothness.simple_mobile_sites": {},
"smoothness.sync_scroll.key_mobile_sites_smooth": {},
"smoothness.top_25_smooth": {
"end": 24
"end": 17
}
}
},
"11": {
"benchmarks": {
"smoothness.top_25_smooth": {
"begin": 24
"begin": 17
},
"smoothness.tough_ad_cases": {},
"smoothness.tough_animation_cases": {
"end": 62
"end": 58
}
}
},
"12": {
"benchmarks": {
"smoothness.tough_animation_cases": {
"begin": 62
"begin": 58
},
"smoothness.tough_canvas_cases": {},
"smoothness.tough_filters_cases": {},
......@@ -164,14 +164,14 @@
"smoothness.tough_scrolling_cases": {},
"smoothness.tough_texture_upload_cases": {},
"smoothness.tough_webgl_ad_cases": {
"end": 4
"end": 1
}
}
},
"13": {
"benchmarks": {
"smoothness.tough_webgl_ad_cases": {
"begin": 4
"begin": 1
},
"smoothness.tough_webgl_cases": {},
"speedometer": {},
......@@ -181,33 +181,33 @@
"start_with_url.cold.startup_pages": {},
"start_with_url.warm.startup_pages": {},
"system_health.common_desktop": {
"end": 23
"end": 20
}
}
},
"14": {
"benchmarks": {
"system_health.common_desktop": {
"begin": 23,
"end": 61
"begin": 20,
"end": 59
}
}
},
"15": {
"benchmarks": {
"system_health.common_desktop": {
"begin": 61
"begin": 59
},
"system_health.common_mobile": {},
"system_health.memory_desktop": {
"end": 11
"end": 10
}
}
},
"16": {
"benchmarks": {
"system_health.memory_desktop": {
"begin": 11,
"begin": 10,
"end": 28
}
}
......@@ -244,14 +244,14 @@
"thread_times.simple_mobile_sites": {},
"thread_times.tough_compositor_cases": {},
"thread_times.tough_scrolling_cases": {
"end": 35
"end": 31
}
}
},
"20": {
"benchmarks": {
"thread_times.tough_scrolling_cases": {
"begin": 35
"begin": 31
},
"tracing.tracing_with_background_memory_infra": {},
"v8.browsing_desktop": {
......@@ -264,22 +264,27 @@
"v8.browsing_desktop": {
"begin": 18
},
"v8.browsing_desktop-future": {},
"v8.browsing_mobile": {},
"v8.browsing_mobile-future": {}
"v8.browsing_desktop-future": {
"end": 18
}
}
},
"22": {
"benchmarks": {
"v8.browsing_desktop-future": {
"begin": 18
},
"v8.browsing_mobile": {},
"v8.browsing_mobile-future": {},
"v8.runtime_stats.top_25": {
"end": 38
"end": 37
}
}
},
"23": {
"benchmarks": {
"v8.runtime_stats.top_25": {
"begin": 38,
"begin": 37,
"end": 80
}
}
......@@ -303,9 +308,9 @@
},
"extra_infos": {
"num_stories": 2401,
"predicted_min_shard_time": 2971.1511643131307,
"predicted_min_shard_index": 16,
"predicted_max_shard_time": 3198.222966333333,
"predicted_max_shard_index": 19
"predicted_min_shard_time": 2928.8240671919193,
"predicted_min_shard_index": 21,
"predicted_max_shard_time": 3281.755761525252,
"predicted_max_shard_index": 16
}
}
\ No newline at end of file
tools/perf/core/shard_maps/mac_1012_low_end_26_shard_map.json
View file @ 744cf4d7
......@@ -29,14 +29,14 @@
"jetstream": {},
"kraken": {},
"loading.desktop": {
"end": 12
"end": 13
}
}
},
"2": {
"benchmarks": {
"loading.desktop": {
"begin": 12,
"begin": 13,
"end": 51
}
}
......@@ -106,14 +106,14 @@
"benchmarks": {
"rendering.desktop": {
"begin": 126,
"end": 205
"end": 204
}
}
},
"10": {
"benchmarks": {
"rendering.desktop": {
"begin": 205
"begin": 204
},
"rendering.mobile": {},
"scheduler.tough_scheduling_cases": {},
......@@ -124,14 +124,14 @@
"smoothness.gpu_rasterization.tough_path_rendering_cases": {},
"smoothness.gpu_rasterization.tough_pinch_zoom_cases": {},
"smoothness.gpu_rasterization.tough_scrolling_cases": {
"end": 31
"end": 30
}
}
},
"11": {
"benchmarks": {
"smoothness.gpu_rasterization.tough_scrolling_cases": {
"begin": 31
"begin": 30
},
"smoothness.gpu_rasterization_and_decoding.image_decoding_cases": {},
"smoothness.image_decoding_cases": {},
......@@ -145,14 +145,14 @@
"smoothness.top_25_smooth": {},
"smoothness.tough_ad_cases": {},
"smoothness.tough_animation_cases": {
"end": 33
"end": 32
}
}
},
"12": {
"benchmarks": {
"smoothness.tough_animation_cases": {
"begin": 33
"begin": 32
},
"smoothness.tough_canvas_cases": {},
"smoothness.tough_filters_cases": {},
......@@ -160,14 +160,14 @@
"smoothness.tough_path_rendering_cases": {},
"smoothness.tough_pinch_zoom_cases": {},
"smoothness.tough_scrolling_cases": {
"end": 11
"end": 10
}
}
},
"13": {
"benchmarks": {
"smoothness.tough_scrolling_cases": {
"begin": 11
"begin": 10
},
"smoothness.tough_texture_upload_cases": {},
"smoothness.tough_webgl_ad_cases": {},
......@@ -270,14 +270,14 @@
"22": {
"benchmarks": {
"v8.runtime_stats.top_25": {
"end": 37
"end": 38
}
}
},
"23": {
"benchmarks": {
"v8.runtime_stats.top_25": {
"begin": 37,
"begin": 38,
"end": 78
}
}
......@@ -286,14 +286,14 @@
"benchmarks": {
"v8.runtime_stats.top_25": {
"begin": 78,
"end": 120
"end": 121
}
}
},
"25": {
"benchmarks": {
"v8.runtime_stats.top_25": {
"begin": 120
"begin": 121
},
"wasm": {},
"webrtc": {}
......@@ -301,9 +301,9 @@
},
"extra_infos": {
"num_stories": 2401,
"predicted_min_shard_time": 2776.9827897373734,
"predicted_min_shard_time": 2796.4444288080804,
"predicted_min_shard_index": 15,
"predicted_max_shard_time": 3109.875536161616,
"predicted_max_shard_time": 3146.344780525253,
"predicted_max_shard_index": 16
}
}
\ No newline at end of file
tools/perf/core/shard_maps/mac_1013_high_end_26_shard_map.json
View file @ 744cf4d7
......@@ -8,14 +8,14 @@
"blink_perf.events": {},
"blink_perf.image_decoder": {},
"blink_perf.layout": {
"end": 55
"end": 56
}
}
},
"1": {
"benchmarks": {
"blink_perf.layout": {
"begin": 55
"begin": 56
},
"blink_perf.owp_storage": {},
"blink_perf.paint": {},
......@@ -45,27 +45,22 @@
"benchmarks": {
"loading.desktop": {
"begin": 44,
"end": 80
"end": 81
}
}
},
"4": {
"benchmarks": {
"loading.desktop": {
"begin": 80
"begin": 81
},
"loading.mobile": {},
"media.desktop": {
"end": 25
}
"media.desktop": {},
"media.mobile": {}
}
},
"5": {
"benchmarks": {
"media.desktop": {
"begin": 25
},
"media.mobile": {},
"memory.desktop": {},
"memory.long_running_idle_gmail_background_tbmv2": {},
"memory.long_running_idle_gmail_tbmv2": {},
......@@ -73,52 +68,52 @@
"octane": {},
"oortonline_tbmv2": {},
"power.desktop": {
"end": 4
"end": 5
}
}
},
"6": {
"benchmarks": {
"power.desktop": {
"begin": 4
"begin": 5
},
"power.idle_platform": {},
"power.typical_10_mobile": {},
"rasterize_and_record_micro.partial_invalidation": {},
"rasterize_and_record_micro.top_25": {},
"rendering.desktop": {
"end": 7
"end": 9
}
}
},
"7": {
"benchmarks": {
"rendering.desktop": {
"begin": 7,
"end": 64
"begin": 9,
"end": 65
}
}
},
"8": {
"benchmarks": {
"rendering.desktop": {
"begin": 64,
"end": 127
"begin": 65,
"end": 129
}
}
},
"9": {
"benchmarks": {
"rendering.desktop": {
"begin": 127,
"end": 206
"begin": 129,
"end": 209
}
}
},
"10": {
"benchmarks": {
"rendering.desktop": {
"begin": 206
"begin": 209
},
"rendering.mobile": {},
"scheduler.tough_scheduling_cases": {},
......@@ -128,17 +123,12 @@
"smoothness.gpu_rasterization.tough_filters_cases": {},
"smoothness.gpu_rasterization.tough_path_rendering_cases": {},
"smoothness.gpu_rasterization.tough_pinch_zoom_cases": {},
"smoothness.gpu_rasterization.tough_scrolling_cases": {
"end": 34
}
"smoothness.gpu_rasterization.tough_scrolling_cases": {},
"smoothness.gpu_rasterization_and_decoding.image_decoding_cases": {}
}
},
"11": {
"benchmarks": {
"smoothness.gpu_rasterization.tough_scrolling_cases": {
"begin": 34
},
"smoothness.gpu_rasterization_and_decoding.image_decoding_cases": {},
"smoothness.image_decoding_cases": {},
"smoothness.key_desktop_move_cases": {},
"smoothness.key_mobile_sites_smooth": {},
......@@ -150,14 +140,14 @@
"smoothness.top_25_smooth": {},
"smoothness.tough_ad_cases": {},
"smoothness.tough_animation_cases": {
"end": 34
"end": 38
}
}
},
"12": {
"benchmarks": {
"smoothness.tough_animation_cases": {
"begin": 34
"begin": 38
},
"smoothness.tough_canvas_cases": {},
"smoothness.tough_filters_cases": {},
......@@ -165,14 +155,14 @@
"smoothness.tough_path_rendering_cases": {},
"smoothness.tough_pinch_zoom_cases": {},
"smoothness.tough_scrolling_cases": {
"end": 13
"end": 21
}
}
},
"13": {
"benchmarks": {
"smoothness.tough_scrolling_cases": {
"begin": 13
"begin": 21
},
"smoothness.tough_texture_upload_cases": {},
"smoothness.tough_webgl_ad_cases": {},
......@@ -184,41 +174,41 @@
"start_with_url.cold.startup_pages": {},
"start_with_url.warm.startup_pages": {},
"system_health.common_desktop": {
"end": 11
"end": 12
}
}
},
"14": {
"benchmarks": {
"system_health.common_desktop": {
"begin": 11,
"end": 57
"begin": 12,
"end": 58
}
}
},
"15": {
"benchmarks": {
"system_health.common_desktop": {
"begin": 57
"begin": 58
},
"system_health.common_mobile": {},
"system_health.memory_desktop": {
"end": 10
"end": 11
}
}
},
"16": {
"benchmarks": {
"system_health.memory_desktop": {
"begin": 10,
"end": 29
"begin": 11,
"end": 30
}
}
},
"17": {
"benchmarks": {
"system_health.memory_desktop": {
"begin": 29,
"begin": 30,
"end": 52
}
}
......@@ -247,55 +237,50 @@
"thread_times.simple_mobile_sites": {},
"thread_times.tough_compositor_cases": {},
"thread_times.tough_scrolling_cases": {
"end": 26
"end": 32
}
}
},
"20": {
"benchmarks": {
"thread_times.tough_scrolling_cases": {
"begin": 26
"begin": 32
},
"tracing.tracing_with_background_memory_infra": {},
"v8.browsing_desktop": {
"end": 17
"end": 18
}
}
},
"21": {
"benchmarks": {
"v8.browsing_desktop": {
"begin": 17
"begin": 18
},
"v8.browsing_desktop-future": {
"end": 17
}
"v8.browsing_desktop-future": {},
"v8.browsing_mobile": {},
"v8.browsing_mobile-future": {}
}
},
"22": {
"benchmarks": {
"v8.browsing_desktop-future": {
"begin": 17
},
"v8.browsing_mobile": {},
"v8.browsing_mobile-future": {},
"v8.runtime_stats.top_25": {
"end": 35
"end": 36
}
}
},
"23": {
"benchmarks": {
"v8.runtime_stats.top_25": {
"begin": 35,
"end": 75
"begin": 36,
"end": 76
}
}
},
"24": {
"benchmarks": {
"v8.runtime_stats.top_25": {
"begin": 75,
"begin": 76,
"end": 117
}
}
......@@ -311,9 +296,9 @@
},
"extra_infos": {
"num_stories": 2401,
"predicted_min_shard_time": 2614.9442498055555,
"predicted_min_shard_time": 2528.4318493333326,
"predicted_min_shard_index": 17,
"predicted_max_shard_time": 2931.3046755833334,
"predicted_max_shard_time": 2932.82558860606,
"predicted_max_shard_index": 18
}
}
\ No newline at end of file
tools/perf/core/shard_maps/timing_data/linux_perf_story_timing.json
View file @ 744cf4d7
This source diff could not be displayed because it is too large. You can view the blob instead.
tools/perf/core/shard_maps/timing_data/mac_1012_low_end_story_timing.json
View file @ 744cf4d7
This source diff could not be displayed because it is too large. You can view the blob instead.
tools/perf/core/shard_maps/timing_data/mac_1013_high_end_story_timing.json
View file @ 744cf4d7
This source diff could not be displayed because it is too large. You can view the blob instead.
tools/perf/core/sharding_map_generator.py
View file @ 744cf4d7
......@@ -74,7 +74,7 @@ from telemetry.internal.browser import browser_options
def main(args, benchmarks):
story_timing_ordered_dict = _load_timing_data_from_file(
benchmarks, args.timing_data)
benchmarks, args.timing_data, True)
all_stories = {}
for b in benchmarks:
......@@ -88,9 +88,15 @@ def main(args, benchmarks):
if args.test_data:
story_timing_ordered_dict = _load_timing_data_from_file(
benchmarks, args.test_data)
print test_sharding_map(
benchmarks, args.test_data, False)
test_results = test_sharding_map(
args.output_file, story_timing_ordered_dict, all_stories)
if args.test_data_output:
with open(args.test_data_output, 'w') as output_file:
json.dump(test_results, output_file, indent = 4, separators=(',', ': '))
else:
print test_results
def get_args():
......@@ -113,6 +119,10 @@ def get_args():
parser.add_argument(
'--test-data', action='store',
help='If specified, test the generated sharding map with this data.')
parser.add_argument(
'--test-data-output', action='store',
help='If specified with --test-data, file \
to output the tested timing data to.')
parser.add_argument(
'--debug', action='store',
help='If specified, the filename to write extra timing data to.')
......@@ -207,17 +217,30 @@ def _add_benchmarks_to_shard(sharding_map, shard_index, stories_in_shard,
sharding_map[str(shard_index)] = {'benchmarks': benchmarks_in_shard}
def _load_timing_data_from_file(benchmarks, timing_data_file):
def _load_timing_data_from_file(benchmarks, timing_data_file, repeat):
story_timing_ordered_dict = _init_timing_dict_for_benchmarks(benchmarks)
pageset_repeat_dict = _init_pageset_repeat_dict_for_benchmarks(benchmarks)
with open(timing_data_file, 'r') as timing_data_file:
story_timing = json.load(timing_data_file)
for run in story_timing:
benchmark = run['name'].split('/', 1)[0]
if run['name'] in story_timing_ordered_dict:
if run['duration']:
if repeat:
story_timing_ordered_dict[run['name']] = (float(run['duration'])
* pageset_repeat_dict[benchmark])
else:
story_timing_ordered_dict[run['name']] += float(run['duration'])
return story_timing_ordered_dict
def _init_pageset_repeat_dict_for_benchmarks(benchmarks):
pageset_repeat = {}
for b in benchmarks:
pageset_repeat[b.Name()] = b().options.get('pageset_repeat', 1)
return pageset_repeat
def _init_timing_dict_for_benchmarks(benchmarks):
timing_data = OrderedDict()
for b in benchmarks:
......
tools/perf/core/sharding_map_generator_unittest.py
View file @ 744cf4d7
......@@ -7,6 +7,9 @@ import copy
from core import sharding_map_generator
import json
import os
import subprocess
import sys
from telemetry import decorators
import tempfile
import unittest
......@@ -42,3 +45,30 @@ class TestShardingMapGenerator(unittest.TestCase):
self.assertEqual(results['2']['full_time'], 140)
finally:
os.remove(map_path)
@decorators.Disabled('android', 'win', 'linux')
def testGeneratePerfSharding(self):
path_output = tempfile.mkstemp(suffix='.json')[1]
path_results = tempfile.mkstemp(suffix='.json')[1]
try:
cmd = [sys.executable,
os.path.normpath('tools/perf/generate_perf_sharding')]
args = [
'--output-file', path_output,
'--timing-data', 'tools/perf/core/test_data/test_timing_data.json',
'--num-shards', '5',
'--test-data',
'tools/perf/core/test_data/test_timing_data_1_build.json',
'--test-data-output', path_results
]
subprocess.check_call(cmd + args)
with open(path_results, 'r') as test_results:
results = json.load(test_results)
shard_total_timing = []
for shard in results:
shard_total_timing.append(results[shard]['full_time'])
self.assertTrue(max(shard_total_timing) - min(shard_total_timing) < 400)
finally:
os.remove(path_output)
os.remove(path_results)
tools/perf/core/test_data/test_timing_data.json 0 → 100644
View file @ 744cf4d7
This source diff could not be displayed because it is too large. You can view the blob instead.
tools/perf/core/test_data/test_timing_data_1_build.json 0 → 100644
View file @ 744cf4d7
This source diff could not be displayed because it is too large. You can view the blob instead.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment