[Benchmarking] Add platforms cycletimes spreadsheet.

This is a first draft to get something out there.
I'm thinking I will rotation the chart to swap x and y later.

Bug: 1030840
Change-Id: I0b5c8204cb5be8f3d3708efc5588c382a8ce0d5b
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1974836Reviewed-by: John Chen <johnchen@chromium.org>
Commit-Queue: John Chen <johnchen@chromium.org>
Commit-Queue: Caleb Rouleau <crouleau@chromium.org>
Cr-Commit-Position: refs/heads/master@{#726180}

tools/perf/cycletime_contributions.csv

+AUTOGENERATED FILE DO NOT EDIT
+View a prettier version of this at,https://docs.google.com/spreadsheets/d/15pJY4cxtM2NVNFKQDgDnoT5PLo0Nm5Td-Ov-5PZefAw
+platform,shards,idealized cycle time (hrs),blink_perf.accessibility,blink_perf.bindings,blink_perf.canvas,blink_perf.css,blink_perf.display_locking,blink_perf.dom,blink_perf.events,blink_perf.image_decoder,blink_perf.layout,blink_perf.owp_storage,blink_perf.paint,blink_perf.parser,blink_perf.shadow_dom,blink_perf.svg,dromaeo,dummy_benchmark.noisy_benchmark_1,dummy_benchmark.stable_benchmark_1,jetstream,jetstream2,kraken,loading.desktop,loading.mobile,media.desktop,media.mobile,memory.desktop,octane,power.desktop,rasterize_and_record_micro.partial_invalidation,rasterize_and_record_micro.top_25,rendering.desktop,rendering.mobile,speedometer,speedometer-future,speedometer2,speedometer2-future,startup.mobile,system_health.common_desktop,system_health.common_mobile,system_health.memory_desktop,system_health.memory_mobile,system_health.webview_startup,tab_switching.typical_25,tracing.tracing_with_background_memory_infra,v8.browsing_desktop,v8.browsing_desktop-future,v8.browsing_mobile,v8.browsing_mobile-future,v8.runtime_stats.top_25,webrtc
+Android Nexus5 Perf,16,1.83,0.00,0.05,0.03,0.02,0.00,0.01,0.01,0.01,0.10,0.01,0.00,0.04,0.03,0.01,0.01,0.00,0.00,0.00,0.00,0.01,0.00,0.08,0.00,0.02,0.00,0.00,0.00,0.00,0.05,0.00,0.64,0.00,0.01,0.01,0.01,0.02,0.00,0.23,0.00,0.15,0.00,0.00,0.01,0.00,0.00,0.23,0.00,0.00,0.02
+Android Nexus5X WebView Perf,16,0.51,0.00,0.01,0.00,0.00,0.00,0.01,0.00,0.00,0.03,0.00,0.00,0.01,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.02,0.00,0.00,0.00,0.00,0.00,0.00,0.01,0.00,0.23,0.00,0.00,0.00,0.00,0.00,0.00,0.06,0.00,0.05,0.00,0.00,0.00,0.00,0.00,0.05,0.00,0.00,0.00
+Android Nexus6 WebView Perf,12,0.65,0.00,0.03,0.01,0.01,0.00,0.01,0.01,0.01,0.05,0.00,0.00,0.02,0.01,0.01,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.02,0.00,0.00,0.00,0.00,0.00,0.00,0.02,0.00,0.21,0.00,0.00,0.00,0.00,0.00,0.00,0.08,0.00,0.07,0.00,0.00,0.00,0.00,0.00,0.07,0.00,0.00,0.01
+Win 7 Nvidia GPU Perf,5,2.11,0.00,0.04,0.05,0.02,0.00,0.01,0.01,0.01,0.07,0.01,0.02,0.03,0.01,0.01,0.02,0.00,0.00,0.02,0.00,0.00,0.13,0.00,0.04,0.00,0.02,0.00,0.06,0.00,0.05,0.35,0.00,0.00,0.00,0.01,0.01,0.00,0.21,0.00,0.19,0.00,0.00,0.00,0.01,0.12,0.12,0.00,0.00,0.43,0.02
+Win 7 Perf,5,1.85,0.00,0.04,0.01,0.03,0.00,0.01,0.01,0.01,0.08,0.01,0.02,0.03,0.02,0.01,0.02,0.00,0.00,0.02,0.00,0.00,0.15,0.00,0.04,0.00,0.02,0.00,0.06,0.00,0.06,0.00,0.00,0.01,0.01,0.01,0.01,0.00,0.20,0.00,0.18,0.00,0.00,0.00,0.01,0.11,0.11,0.00,0.00,0.54,0.02
+android-go-perf,19,0.59,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.01,0.00,0.03,0.00,0.20,0.00,0.14,0.00,0.00,0.00,0.00,0.00,0.20,0.00,0.00,0.00
+android-go_webview-perf,13,0.22,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.01,0.00,0.00,0.00,0.12,0.00,0.09,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+android-nexus5x-perf,10,1.17,0.00,0.05,0.00,0.02,0.05,0.02,0.01,0.02,0.11,0.01,0.00,0.04,0.00,0.01,0.01,0.00,0.00,0.00,0.00,0.01,0.00,0.08,0.00,0.02,0.00,0.00,0.00,0.00,0.05,0.00,0.02,0.00,0.00,0.01,0.01,0.03,0.00,0.26,0.00,0.05,0.00,0.00,0.01,0.00,0.00,0.26,0.00,0.00,0.02
+android-pixel2-perf,35,0.47,0.00,0.01,0.01,0.01,0.00,0.00,0.00,0.01,0.02,0.00,0.01,0.01,0.01,0.00,0.00,0.00,0.00,0.00,0.01,0.00,0.00,0.02,0.00,0.01,0.00,0.00,0.00,0.00,0.01,0.00,0.16,0.00,0.00,0.00,0.00,0.01,0.00,0.05,0.00,0.05,0.00,0.00,0.00,0.00,0.00,0.05,0.00,0.00,0.00
+android-pixel2_weblayer-perf,4,0.11,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.01,0.00,0.05,0.00,0.05,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+android-pixel2_webview-perf,21,0.28,0.00,0.01,0.00,0.00,0.00,0.00,0.00,0.00,0.01,0.00,0.00,0.01,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.01,0.00,0.00,0.00,0.00,0.00,0.00,0.01,0.00,0.09,0.00,0.00,0.00,0.00,0.00,0.00,0.04,0.00,0.03,0.00,0.00,0.00,0.00,0.00,0.03,0.00,0.00,0.00
+linux-perf,26,0.34,0.00,0.01,0.01,0.00,0.00,0.00,0.00,0.00,0.01,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.02,0.00,0.01,0.00,0.00,0.00,0.01,0.00,0.01,0.06,0.00,0.00,0.00,0.00,0.00,0.00,0.04,0.00,0.03,0.00,0.00,0.00,0.00,0.02,0.02,0.00,0.00,0.07,0.00
+mac-10_12_laptop_low_end-perf,26,0.33,0.00,0.01,0.01,0.00,0.00,0.00,0.00,0.00,0.01,0.00,0.00,0.01,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.03,0.00,0.01,0.00,0.00,0.00,0.01,0.00,0.01,0.07,0.00,0.00,0.00,0.00,0.00,0.00,0.04,0.00,0.04,0.00,0.00,0.00,0.00,0.02,0.02,0.00,0.00,0.00,0.00
+mac-10_13_laptop_high_end-perf,26,0.30,0.00,0.01,0.01,0.00,0.00,0.00,0.00,0.00,0.01,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.02,0.00,0.01,0.00,0.00,0.00,0.01,0.00,0.01,0.06,0.00,0.00,0.00,0.00,0.00,0.00,0.04,0.00,0.03,0.00,0.00,0.00,0.00,0.02,0.02,0.00,0.00,0.00,0.00
+win-10-perf,26,0.42,0.00,0.01,0.01,0.01,0.00,0.00,0.00,0.00,0.01,0.00,0.00,0.01,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.03,0.00,0.01,0.00,0.00,0.00,0.01,0.00,0.01,0.07,0.00,0.00,0.00,0.00,0.00,0.00,0.04,0.00,0.04,0.00,0.00,0.00,0.00,0.02,0.02,0.00,0.00,0.08,0.00

tools/perf/generate_perf_sharding.py

@@ -42,6 +42,7 @@ files may not match with the true state of world.
 """
 
 _BENCHMARK_SCHEDULES = 'tools/perf/benchmark_schedules.csv'
+_CYCLETIME_CONTRIBUTION = 'tools/perf/cycletime_contributions.csv'
 
 
 def GetParser():
@@ -165,21 +166,63 @@ class _BenchmarkUsageRow(object):
             ', '.join(abridged_platforms)]
 
 
-def _UpdateWaterfallUsageData(output_path=None):
-  output_path = output_path or _source_filepath(_BENCHMARK_SCHEDULES)
-  builders = _GetBuilderPlatforms(builders=None, waterfall='perf')
+def _UpdateWaterfallUsageCsvs(schedules_output_path=None,
+                              cycletimes_output_path=None):
+  schedules_output_path = schedules_output_path or _source_filepath(
+      _BENCHMARK_SCHEDULES)
+  cycletimes_output_path = cycletimes_output_path or _source_filepath(
+      _CYCLETIME_CONTRIBUTION)
+  builders = sorted(_GetBuilderPlatforms(builders=None, waterfall='perf'),
+                    key=lambda b: b.name)
   # https://stackoverflow.com/questions/31838823
   EmptyBenchmarkUsageRow = lambda: _BenchmarkUsageRow(None, None, None)
   benchmark_waterfall_usage = collections.defaultdict(EmptyBenchmarkUsageRow)
+  platforms = collections.OrderedDict()
   for builder in builders:
+    platforms[builder] = collections.defaultdict(float)
     with open(builder.timing_file_path) as fh:
       timing_data = json.load(fh)
     for story in timing_data:
       benchmark_name = story['name'].split('/')[0]
+      platforms[builder][benchmark_name] += float(story['duration'])
       benchmark_waterfall_usage[benchmark_name].Add(_BenchmarkUsageRow(
           benchmark_name, set((builder,)), float(story['duration'])))
   benchmarks = benchmark_waterfall_usage.values()
+  benchmark_names = benchmark_waterfall_usage.keys()
   benchmarks.sort(key=lambda b: b.duration, reverse=True)
+  _UpdateSchedulesCsv(benchmarks, schedules_output_path)
+  _UpdateCycletimesCsv(platforms, benchmark_names, cycletimes_output_path)
+
+
+def _UpdateCycletimesCsv(platforms, benchmark_names, output_path):
+  benchmark_names = sorted(benchmark_names)
+  csv_data = [
+      ['AUTOGENERATED FILE DO NOT EDIT'],
+      ['View a prettier version of this at',
+       'https://docs.google.com/spreadsheets/d/'
+       '15pJY4cxtM2NVNFKQDgDnoT5PLo0Nm5Td-Ov-5PZefAw'],
+      ['platform',
+       'shards',
+       'idealized cycle time (hrs)'] + benchmark_names]
+  for platform, durations in platforms.iteritems():
+    multiplier = 2.0 if platform.run_reference_build else 1.0
+    estimated_cycle_time = (multiplier * sum(durations.values()) /
+                            60.0 / 60.0 / float(platform.num_shards))
+    # Double all cycle time stats by 2 to account for reference build.
+    csv_data.append([
+      platform.name,
+      platform.num_shards,
+      '%.2f' % estimated_cycle_time] + [
+          '%.2f' % (multiplier * durations[name] / 60.0 /
+                    60.0 / float(platform.num_shards))
+          for name in benchmark_names]
+    )
+  with open(output_path, 'w') as fh:
+    writer = csv.writer(fh, lineterminator='\n')
+    writer.writerows(csv_data)
+
+
+def _UpdateSchedulesCsv(benchmark_rows, output_path):
   csv_data = [
       ['AUTOGENERATED FILE DO NOT EDIT'],
       ['View a prettier version of this at',
@@ -191,7 +234,7 @@ def _UpdateWaterfallUsageData(output_path=None):
        'platforms count (abridged)',
        'platforms where unabridged',
        'platforms where abridged']]
-  for benchmark in benchmarks:
+  for benchmark in benchmark_rows:
     csv_data.append(benchmark.ToRow())
   with open(output_path, 'w') as fh:
     writer = csv.writer(fh, lineterminator='\n')
@@ -237,7 +280,7 @@ def _UpdateTimingDataCommand(args):
     _UpdateTimingData(builders)
   for builder in builders:
     _FilterTimingData(builder)
-  _UpdateWaterfallUsageData()
+  _UpdateWaterfallUsageCsvs()
 
 
 def _FilterTimingData(builder, output_path=None):
@@ -337,14 +380,22 @@ def _ValidateShardMaps(args):
 
   tempdir = tempfile.mkdtemp()
   try:
-    temp_filepath = os.path.join(tempdir, 'schedules.csv')
-    _UpdateWaterfallUsageData(temp_filepath)
-    real_path = _source_filepath(_BENCHMARK_SCHEDULES)
-    if not filecmp.cmp(temp_filepath, real_path):
+    temp_schedules_filepath = os.path.join(tempdir, 'schedules.csv')
+    temp_cycletime_filepath = os.path.join(tempdir, 'cycletime.csv')
+    _UpdateWaterfallUsageCsvs(temp_schedules_filepath,
+                              temp_cycletime_filepath)
+    real_schedules_path = _source_filepath(_BENCHMARK_SCHEDULES)
+    if not filecmp.cmp(temp_schedules_filepath, real_schedules_path):
       errors.append(
           '{benchmark_schedules} is not up to date. Please run '
           '`./generate_perf_sharding.py update-timing --filter-only` '
           'to regenerate it.'.format(benchmark_schedules=_BENCHMARK_SCHEDULES))
+    real_cycletime_path = _source_filepath(_CYCLETIME_CONTRIBUTION)
+    if not filecmp.cmp(temp_cycletime_filepath, real_cycletime_path):
+      errors.append(
+          '{cycletimes} is not up to date. Please run '
+          '`./generate_perf_sharding.py update-timing --filter-only` '
+          'to regenerate it.'.format(cycletimes=_CYCLETIME_CONTRIBUTION))
 
     builders = _GetBuilderPlatforms(builders=None, waterfall='perf')
     for builder in builders: