[Benchmarking] Filter timing data to only present current stories.

This means that if you abridge a benchmark or remove stories from
a benchmark then that information will be reflected in
http://go/chrome-benchmark-schedules as soon as you check in your
change.

Also, add a --filter-only option so that we can keep this up to
date without needing to run a slow bigquery query each time.
Also, add validation to presubmit so that this stays up to date.
Also, make the output deterministic by sorting it.


Bug: 1030840
Change-Id: I2cda6110a029953559874707852230693d0ac7b2
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1973232Reviewed-by: John Chen <johnchen@chromium.org>
Auto-Submit: Caleb Rouleau <crouleau@chromium.org>
Cr-Commit-Position: refs/heads/master@{#726030}

tools/perf/core/bot_platforms.py

@@ -43,6 +43,9 @@ class PerfPlatform(object):
         b for b in benchmark_configs if
           _IsPlatformSupported(b.benchmark, self._platform_os)])
     # pylint: enable=redefined-outer-name
+    benchmark_names = [config.name for config in self._benchmark_configs]
+    assert len(list(set(benchmark_names))) == len(benchmark_names), (
+        'Make sure that a benchmark does not appear twice.')
 
     base_file_name = name.replace(' ', '_').lower()
     self._timing_file_path = os.path.join(
@@ -184,13 +187,13 @@ _ANDROID_GO_BENCHMARK_CONFIGS = frozenset([
 _ANDROID_GO_WEBVIEW_BENCHMARK_CONFIGS = _ANDROID_GO_BENCHMARK_CONFIGS
 _ANDROID_NEXUS_5_BENCHMARK_CONFIGS = _OFFICIAL_EXCEPT_DISPLAY_LOCKING_JETSTREAM2
 _ANDROID_NEXUS_5X_BENCHMARK_CONFIGS = (
-    _OFFICIAL_EXCEPT_JETSTREAM2
+    (((_OFFICIAL_EXCEPT_JETSTREAM2
     # Remove unabridged rendering benchmark and replace with abridged benchmark.
-    - frozenset([_GetBenchmarkConfig('rendering.mobile')])
-    | frozenset([_GetBenchmarkConfig('rendering.mobile', True)])
+     - frozenset([_GetBenchmarkConfig('rendering.mobile')]))
+     | frozenset([_GetBenchmarkConfig('rendering.mobile', True)]))
     # Remove unabridged system health memory benchmark and replace with abridged
     # benchmark: crbug.com/1030788
-    - frozenset([_GetBenchmarkConfig('system_health.memory_mobile')])
+     - frozenset([_GetBenchmarkConfig('system_health.memory_mobile')]))
     | frozenset([_GetBenchmarkConfig('system_health.memory_mobile', True)]))
 _ANDROID_NEXUS_5X_WEBVIEW_BENCHMARK_CONFIGS = (
     _OFFICIAL_EXCEPT_DISPLAY_LOCKING_JETSTREAM2)

tools/perf/core/retrieve_story_timing.py

@@ -122,7 +122,7 @@ def FetchStoryTimingDataForSingleBuild(configurations, build_number):
       configurations_str, build_number))
 
 
-def FetchAverageStortyTimingData(configurations, num_last_days):
+def FetchAverageStoryTimingData(configurations, num_last_days):
   configurations_str = ','.join(repr(c) for c in configurations)
   return _run_query(QUERY_STORY_AVG_RUNTIME.format(
       configuration_names=configurations_str, num_last_days=num_last_days))
@@ -182,7 +182,7 @@ def main(args):
       data = FetchStoryTimingDataForSingleBuild(opts.configurations,
           opts.build_number)
     else:
-      data = FetchAverageStortyTimingData(opts.configurations, num_last_days=5)
+      data = FetchAverageStoryTimingData(opts.configurations, num_last_days=5)
 
   with open(opts.output_file, 'w') as output_file:
     json.dump(data, output_file, indent = 4, separators=(',', ': '))

tools/perf/core/shard_maps/timing_data/android-nexus5x-perf-fyi_timing.json

 [
-    {
-        "duration": "41.0",
-        "name": "heap_profiling.mobile.disabled/load:news:cnn:2018"
-    },
-    {
-        "duration": "29.0",
-        "name": "heap_profiling.mobile.disabled/load:search:google:2018"
-    },
-    {
-        "duration": "28.0",
-        "name": "heap_profiling.mobile.disabled/load:social:twitter"
-    },
-    {
-        "duration": "44.0",
-        "name": "heap_profiling.mobile.native/load:news:cnn:2018"
-    },
-    {
-        "duration": "30.0",
-        "name": "heap_profiling.mobile.native/load:search:google:2018"
-    },
-    {
-        "duration": "30.0",
-        "name": "heap_profiling.mobile.native/load:social:twitter"
-    },
-    {
-        "duration": "43.0",
-        "name": "heap_profiling.mobile.pseudo/load:news:cnn:2018"
-    },
-    {
-        "duration": "29.0",
-        "name": "heap_profiling.mobile.pseudo/load:search:google:2018"
-    },
-    {
-        "duration": "29.0",
-        "name": "heap_profiling.mobile.pseudo/load:social:twitter"
-    },
     {
         "duration": "14.0",
         "name": "tracing.tracing_with_background_memory_infra/Facebook"

tools/perf/generate_perf_sharding.py

@@ -8,10 +8,13 @@ from __future__ import print_function
 import argparse
 import collections
 import csv
+import filecmp
 import json
 import multiprocessing
 import os
+import shutil
 import sys
+import tempfile
 import textwrap
 
 from core import path_util
@@ -38,6 +41,8 @@ files may not match with the true state of world.
 
 """
 
+_BENCHMARK_SCHEDULES = 'tools/perf/benchmark_schedules.csv'
+
 
 def GetParser():
   parser = argparse.ArgumentParser(
@@ -68,6 +73,12 @@ def GetParser():
       help='Update the timing data that is used to create the shard maps, '
       'but don\'t update the shard maps themselves.')
   _AddBuilderPlatformSelectionArgs(parser_update_timing)
+  parser_update_timing.add_argument(
+      '--filter-only', action='store_true',
+      help='Do not grab new data from bigquery but instead simply filter '
+      'the existing data to reflect some change in the benchmark (for example '
+      'if the benchmark was switched to abridged mode on some platform or if '
+      'a story was removed from the benchmark.)')
   parser_update_timing.set_defaults(func=_UpdateTimingDataCommand)
 
   parser_deschedule = subparsers.add_parser(
@@ -107,10 +118,10 @@ def _DumpJson(data, output_path):
 
 def _LoadTimingData(args):
   builder_name, timing_file_path = args
-  data = retrieve_story_timing.FetchAverageStortyTimingData(
+  data = retrieve_story_timing.FetchAverageStoryTimingData(
       configurations=[builder_name], num_last_days=5)
   _DumpJson(data, timing_file_path)
-  print('Finish retrieve story timing data for %s' % repr(builder_name))
+  print('Finished retrieving story timing data for %s' % repr(builder_name))
 
 
 def _source_filepath(posix_path):
@@ -139,12 +150,12 @@ class _BenchmarkUsageRow(object):
         return benchmark_config
 
   def ToRow(self):
-    unabridged_platforms =[
+    unabridged_platforms = sorted([
         p.name for p in self.platforms
-        if not self._GetBenchmarkConfiguration(p).abridged]
-    abridged_platforms = [
+        if not self._GetBenchmarkConfiguration(p).abridged])
+    abridged_platforms = sorted([
         p.name for p in self.platforms
-        if self._GetBenchmarkConfiguration(p).abridged]
+        if self._GetBenchmarkConfiguration(p).abridged])
     unabridged_count = len(unabridged_platforms)
     abridged_count = len(abridged_platforms)
     platform_names = sorted([p.name for p in self.platforms])
@@ -154,7 +165,8 @@ class _BenchmarkUsageRow(object):
             ', '.join(abridged_platforms)]
 
 
-def _UpdateWaterfallUsageData():
+def _UpdateWaterfallUsageData(output_path=None):
+  output_path = output_path or _source_filepath(_BENCHMARK_SCHEDULES)
   builders = _GetBuilderPlatforms(builders=None, waterfall='perf')
   # https://stackoverflow.com/questions/31838823
   EmptyBenchmarkUsageRow = lambda: _BenchmarkUsageRow(None, None, None)
@@ -181,7 +193,7 @@ def _UpdateWaterfallUsageData():
        'platforms where abridged']]
   for benchmark in benchmarks:
     csv_data.append(benchmark.ToRow())
-  with open(_source_filepath('tools/perf/benchmark_schedules.csv'), 'w') as fh:
+  with open(output_path, 'w') as fh:
     writer = csv.writer(fh, lineterminator='\n')
     writer.writerows(csv_data)
 
@@ -220,7 +232,27 @@ def _PromptWarning():
 
 
 def _UpdateTimingDataCommand(args):
-  _UpdateTimingData(_GetBuilderPlatforms(args.builders, args.waterfall))
+  builders = _GetBuilderPlatforms(args.builders, args.waterfall)
+  if not args.filter_only:
+    _UpdateTimingData(builders)
+  for builder in builders:
+    _FilterTimingData(builder)
+  _UpdateWaterfallUsageData()
+
+
+def _FilterTimingData(builder, output_path=None):
+  output_path = output_path or builder.timing_file_path
+  with open(builder.timing_file_path) as f:
+    timing_dataset = json.load(f)
+  story_full_names = set()
+  for benchmark_config in builder.benchmark_configs:
+    for story in benchmark_config.stories:
+      story_full_names.add('/'.join([benchmark_config.name, story]))
+  # When benchmarks are abridged or stories are removed, we want that
+  # to be reflected in the timing data right away.
+  timing_dataset = [point for point in timing_dataset
+                    if str(point['name']) in story_full_names]
+  _DumpJson(timing_dataset, output_path)
 
 
 def _UpdateTimingData(builders):
@@ -231,7 +263,6 @@ def _UpdateTimingData(builders):
   p = multiprocessing.Pool(len(load_timing_args))
   # Use map_async to work around python bug. See crbug.com/1026004.
   p.map_async(_LoadTimingData, load_timing_args).get(12*60*60)
-  _UpdateWaterfallUsageData()
 
 
 def _GetBuilderPlatforms(builders, waterfall):
@@ -300,10 +331,34 @@ def _ParseBenchmarks(shard_map_path):
 
 
 def _ValidateShardMaps(args):
-  """Validate that the shard maps are consistent with the state of the repo."""
+  """Validate that the shard maps, csv files, etc. are consistent."""
   del args
   errors = []
 
+  tempdir = tempfile.mkdtemp()
+  try:
+    temp_filepath = os.path.join(tempdir, 'schedules.csv')
+    _UpdateWaterfallUsageData(temp_filepath)
+    real_path = _source_filepath(_BENCHMARK_SCHEDULES)
+    if not filecmp.cmp(temp_filepath, real_path):
+      errors.append(
+          '{benchmark_schedules} is not up to date. Please run '
+          '`./generate_perf_sharding.py update-timing --filter-only` '
+          'to regenerate it.'.format(benchmark_schedules=_BENCHMARK_SCHEDULES))
+
+    builders = _GetBuilderPlatforms(builders=None, waterfall='perf')
+    for builder in builders:
+      output_file = os.path.join(
+          tempdir, os.path.basename(builder.timing_file_path))
+      _FilterTimingData(builder, output_file)
+      if not filecmp.cmp(builder.timing_file_path, output_file):
+        errors.append(
+            '{timing_data} is not up to date. Please run '
+            '`./generate_perf_sharding.py update-timing --filter-only` '
+            'to regenerate it.'.format(timing_data=builder.timing_file_path))
+  finally:
+    shutil.rmtree(tempdir)
+
   # Check that bot_platforms.py matches the actual shard maps
   for platform in bot_platforms.ALL_PLATFORMS:
     platform_benchmark_names = set(