Add validator script that validates correctness of perf json configs

THis would help prevent issues like https://chromium-review.googlesource.com/c/chromium/src/+/1107213

Bug: 713357
Change-Id: Id43c20d5c232120232faf9568b5691721ccdbe47
Reviewed-on: https://chromium-review.googlesource.com/1107302
Commit-Queue: Ned Nguyen <nednguyen@google.com>
Reviewed-by: Emily Hanley <eyaich@chromium.org>
Cr-Commit-Position: refs/heads/master@{#571618}

testing/buildbot/chromium.perf.fyi.json

@@ -7,7 +7,7 @@
           "--browser=android-chromium",
           "--upload-results",
           "--run-ref-build",
-          "--test-shard-map-filename=mobile_39_shard_map.json"
+          "--test-shard-map-filename=android_go_14_shard_map.json"
         ],
         "isolate_name": "performance_test_suite",
         "merge": {

tools/perf/PRESUBMIT.py

@@ -23,7 +23,10 @@ def _CommonChecks(input_api, output_api, block_on_failure=False):
 
   results.extend(_CheckExpectations(input_api, output_api))
   results.extend(_CheckJson(input_api, output_api))
-  results.extend(_CheckPerfData(input_api, output_api, block_on_failure))
+  results.extend(
+      _CheckPerfDataCurrentness(input_api, output_api, block_on_failure))
+  results.extend(
+      _CheckPerfJsonConfigs(input_api, output_api, block_on_failure))
   results.extend(_CheckWprShaFiles(input_api, output_api))
   results.extend(input_api.RunTests(input_api.canned_checks.GetPylint(
       input_api, output_api, extra_paths_list=_GetPathsToPrepend(input_api),
@@ -75,7 +78,7 @@ def _CheckExpectations(input_api, output_api):
   return results
 
 
-def _CheckPerfData(input_api, output_api, block_on_failure):
+def _CheckPerfDataCurrentness(input_api, output_api, block_on_failure):
   results = []
   perf_dir = input_api.PresubmitLocalPath()
   vpython = 'vpython.bat' if input_api.is_windows else 'vpython'
@@ -86,10 +89,27 @@ def _CheckPerfData(input_api, output_api, block_on_failure):
   if return_code:
     if block_on_failure:
       results.append(output_api.PresubmitError(
-          'Validating perf data failed', long_text=out))
+          'Validating perf data currentness failed', long_text=out))
     else:
       results.append(output_api.PresubmitPromptWarning(
-          'Validating perf data failed', long_text=out))
+          'Validating perf data currentness failed', long_text=out))
+  return results
+
+
+def _CheckPerfJsonConfigs(input_api, output_api, block_on_failure):
+  results = []
+  perf_dir = input_api.PresubmitLocalPath()
+  vpython = 'vpython.bat' if input_api.is_windows else 'vpython'
+  out, return_code = _RunArgs([
+      vpython,
+      input_api.os_path.join(perf_dir, 'validate_perf_json_config')], input_api)
+  if return_code:
+    if block_on_failure:
+      results.append(output_api.PresubmitError(
+          'Validating perf data correctness failed', long_text=out))
+    else:
+      results.append(output_api.PresubmitPromptWarning(
+          'Validating perf data correctness failed', long_text=out))
   return results
 
 

tools/perf/core/perf_data_generator.py

@@ -60,29 +60,6 @@ def add_builder(waterfall, name, additional_compile_targets=None):
   return waterfall
 
 
-_VALID_SWARMING_DIMENSIONS = {
-    'gpu', 'device_ids', 'os', 'pool', 'perf_tests', 'perf_tests_with_args',
-    'device_os', 'device_type', 'device_os_flavor'}
-_VALID_PERF_POOLS = {
-    'Chrome-perf', 'chrome.tests.perf', 'chrome.tests.perf-webview'}
-
-
-def _ValidateSwarmingDimension(tester_name, swarming_dimensions):
-  for dimension in swarming_dimensions:
-    for k, v in dimension.iteritems():
-      if k not in _VALID_SWARMING_DIMENSIONS:
-        raise ValueError('Invalid swarming dimension in %s: %s' % (
-            tester_name, k))
-      if k == 'pool' and v not in _VALID_PERF_POOLS:
-        raise ValueError('Invalid perf pool %s in %s' % (v, tester_name))
-      if k == 'os' and v == 'Android':
-        if (not 'device_type' in dimension.keys() or
-            not 'device_os' in dimension.keys() or
-            not 'device_os_flavor' in dimension.keys()):
-          raise ValueError(
-              'Invalid android dimensions %s in %s' % (v, tester_name))
-
-
 def add_tester(waterfall, name, perf_id, platform, target_bits=64,
                num_host_shards=1, num_device_shards=1, swarming=None,
                replace_system_webview=False):
@@ -104,7 +81,6 @@ def add_tester(waterfall, name, perf_id, platform, target_bits=64,
   }
 
   if swarming:
-    _ValidateSwarmingDimension(name, swarming)
     waterfall['testers'][name]['swarming_dimensions'] = swarming
     waterfall['testers'][name]['swarming'] = True
 
@@ -1000,7 +976,7 @@ NEW_PERF_RECIPE_FYI_TESTERS = {
           'isolate': 'performance_test_suite',
           'extra_args': [
             '--run-ref-build',
-            '--test-shard-map-filename=mobile_39_shard_map.json',
+            '--test-shard-map-filename=android_go_14_shard_map.json',
           ],
           'num_shards': 14
         }
@@ -1289,15 +1265,11 @@ def generate_performance_test(tester_config, test):
   return result
 
 
-def load_and_update_new_recipe_fyi_json():
+def load_and_update_new_recipe_fyi_json(fyi_waterfall_file):
   tests = {}
-  filename = 'chromium.perf.fyi.json'
-  buildbot_dir = os.path.join(
-      path_util.GetChromiumSrcDir(), 'testing', 'buildbot')
-  fyi_filepath = os.path.join(buildbot_dir, filename)
-  with open(fyi_filepath) as fp_r:
+  with open(fyi_waterfall_file) as fp_r:
     tests = json.load(fp_r)
-  with open(fyi_filepath, 'w') as fp:
+  with open(fyi_waterfall_file, 'w') as fp:
     # We have loaded what is there, we want to update or add
     # what we have listed here
     get_new_recipe_testers(NEW_PERF_RECIPE_FYI_TESTERS, tests)
@@ -1359,6 +1331,9 @@ def main(args):
   waterfall_file = os.path.join(
       path_util.GetChromiumSrcDir(), 'testing', 'buildbot',
       'chromium.perf.json')
+  fyi_waterfall_file = os.path.join(
+      path_util.GetChromiumSrcDir(), 'testing', 'buildbot',
+      'chromium.perf.fyi.json')
 
   benchmark_file = os.path.join(
       path_util.GetChromiumSrcDir(), 'tools', 'perf', 'benchmark.csv')
@@ -1373,7 +1348,6 @@ def main(args):
              'configs and benchmark.csv.') % sys.argv[0]
       return 1
   else:
-    load_and_update_new_recipe_fyi_json()
+    load_and_update_new_recipe_fyi_json(fyi_waterfall_file)
     update_all_tests(get_waterfall_config(), waterfall_file)
-    update_benchmark_csv(benchmark_file)
   return 0

tools/perf/core/perf_json_config_validator.py

+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import os
+import json
+
+from core import path_util
+
+_VALID_SWARMING_DIMENSIONS = {
+    'gpu', 'device_ids', 'os', 'pool', 'perf_tests', 'perf_tests_with_args',
+    'device_os', 'device_type', 'device_os_flavor', 'id'}
+_VALID_PERF_POOLS = {
+    'Chrome-perf', 'Chrome-perf-fyi',
+    'chrome.tests.perf', 'chrome.tests.perf-webview',
+    'chrome.tests.perf-fyi', 'chrome.tests.perf-webview-fyi'}
+
+
+def _ValidateSwarmingDimension(builder_name, swarming_dimensions):
+  for dimension in swarming_dimensions:
+    for k, v in dimension.iteritems():
+      if k not in _VALID_SWARMING_DIMENSIONS:
+        raise ValueError('Invalid swarming dimension in %s: %s' % (
+            builder_name, k))
+      if k == 'pool' and v not in _VALID_PERF_POOLS:
+        raise ValueError('Invalid perf pool %s in %s' % (v, builder_name))
+      if k == 'os' and v == 'Android':
+        if (not 'device_type' in dimension.keys() or
+            not 'device_os' in dimension.keys() or
+            not 'device_os_flavor' in dimension.keys()):
+          raise ValueError(
+              'Invalid android dimensions %s in %s' % (v, builder_name))
+
+
+def _ParseShardMapFileName(args):
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--test-shard-map-filename', dest='shard_file')
+  options, _ = parser.parse_known_args(args)
+  return options.shard_file
+
+
+_SHARD_MAP_DIR = os.path.join(os.path.dirname(__file__), 'shard_maps')
+
+
+def _ValidateShardingData(builder_name, test_config):
+  shard_file_name = _ParseShardMapFileName(test_config['args'])
+  shard_file_path = os.path.join(_SHARD_MAP_DIR, shard_file_name)
+  if not os.path.exists(shard_file_path):
+    raise ValueError(
+        "shard test file %s in config of builder %s does not exist" % (
+          repr(shard_file_name), repr(builder_name)))
+
+  with open(shard_file_path) as f:
+    shard_map_data = json.load(f)
+
+  shard_map_data.pop('extra_infos', None)
+  shard_keys = set(shard_map_data.keys())
+  num_shards = test_config['swarming']['shards']
+  expected_shard_keys = set([str(i) for i in xrange(num_shards)])
+  if shard_keys != expected_shard_keys:
+    raise ValueError(
+        'The shard configuration of %s does not match the expected expected '
+        'number of shards (%d) in config of builder %s' % (
+            repr(shard_file_name), num_shards, repr(builder_name)))
+
+
+def ValidateTestingBuilder(builder_name, builder_data):
+  isolated_scripts = builder_data['isolated_scripts']
+  for test_config in isolated_scripts:
+    _ValidateSwarmingDimension(
+        builder_name,
+        swarming_dimensions=test_config['swarming'].get('dimension_sets', {}))
+    if test_config['isolate_name'] == 'performance_test_suite':
+      _ValidateShardingData(builder_name, test_config)
+
+
+def _IsBuilderName(name):
+  return not name.startswith('AAA')
+
+
+def _IsCompilingBuilder(builder_name, builder_data):
+  del builder_name  # unused
+  return 'isolated_scripts' not in builder_data
+
+
+def _IsTestingBuilder(builder_name, builder_data):
+  del builder_name  # unused
+  return 'isolated_scripts' in builder_data
+
+
+def ValidatePerfConfigFile(file_handle):
+  perf_data = json.load(file_handle)
+  for key, value in perf_data.iteritems():
+    if not _IsBuilderName(key):
+      continue
+    if _IsCompilingBuilder(builder_name=key, builder_data=value):
+      pass
+    elif _IsTestingBuilder(builder_name=key, builder_data=value):
+      ValidateTestingBuilder(builder_name=key, builder_data=value)
+    else:
+      raise ValueError('%s has unrecognizable type: %s' % key)
+
+
+def main(args):
+  del args  # unused
+  waterfall_file = os.path.join(
+      path_util.GetChromiumSrcDir(), 'testing', 'buildbot',
+      'chromium.perf.json')
+  fyi_waterfall_file = os.path.join(
+      path_util.GetChromiumSrcDir(), 'testing', 'buildbot',
+      'chromium.perf.fyi.json')
+
+
+  with open(fyi_waterfall_file) as f:
+    ValidatePerfConfigFile(f)
+
+  with open(waterfall_file) as f:
+    ValidatePerfConfigFile(f)

tools/perf/core/shard_maps/android_go_14_shard_map.json

+{
+    "0": {
+        "benchmarks": {
+            "blink_perf.bindings": {},
+            "blink_perf.canvas": {},
+            "blink_perf.css": {},
+            "blink_perf.dom": {
+                "end": 1
+            }
+        }
+    },
+    "1": {
+        "benchmarks": {
+            "blink_perf.dom": {
+                "begin": 1
+            },
+            "blink_perf.events": {},
+            "blink_perf.image_decoder": {},
+            "blink_perf.layout": {},
+            "blink_perf.owp_storage": {
+                "end": 3
+            }
+        }
+    },
+    "2": {
+        "benchmarks": {
+            "blink_perf.owp_storage": {
+                "begin": 3
+            },
+            "blink_perf.paint": {},
+            "blink_perf.parser": {},
+            "blink_perf.shadow_dom": {},
+            "blink_perf.svg": {
+                "end": 6
+            }
+        }
+    },
+    "3": {
+        "benchmarks": {
+            "blink_perf.svg": {
+                "begin": 6
+            },
+            "dromaeo": {},
+            "dummy_benchmark.histogram_benchmark_1": {},
+            "dummy_benchmark.noisy_benchmark_1": {},
+            "dummy_benchmark.stable_benchmark_1": {},
+            "jetstream": {},
+            "kraken": {},
+            "loading.desktop": {},
+            "loading.mobile": {
+                "end": 9
+            }
+        }
+    },
+    "4": {
+        "benchmarks": {
+            "loading.mobile": {
+                "begin": 9,
+                "end": 34
+            }
+        }
+    },
+    "5": {
+        "benchmarks": {
+            "loading.mobile": {
+                "begin": 34,
+                "end": 90
+            }
+        }
+    },
+    "6": {
+        "benchmarks": {
+            "loading.mobile": {
+                "begin": 90
+            },
+            "media.desktop": {},
+            "media.mobile": {},
+            "memory.desktop": {},
+            "memory.long_running_idle_gmail_background_tbmv2": {},
+            "memory.long_running_idle_gmail_tbmv2": {},
+            "memory.top_10_mobile": {
+                "end": 12
+            }
+        }
+    },
+    "7": {
+        "benchmarks": {
+            "memory.top_10_mobile": {
+                "begin": 12
+            },
+            "octane": {},
+            "oortonline_tbmv2": {},
+            "power.desktop": {},
+            "power.idle_platform": {},
+            "power.typical_10_mobile": {},
+            "rasterize_and_record_micro.partial_invalidation": {},
+            "rasterize_and_record_micro.top_25": {
+                "end": 16
+            }
+        }
+    },
+    "8": {
+        "benchmarks": {
+            "rasterize_and_record_micro.top_25": {
+                "begin": 16
+            },
+            "rendering.desktop": {},
+            "rendering.mobile": {
+                "end": 93
+            }
+        }
+    },
+    "9": {
+        "benchmarks": {
+            "rendering.mobile": {
+                "begin": 93
+            },
+            "scheduler.tough_scheduling_cases": {},
+            "smoothness.desktop_tough_pinch_zoom_cases": {},
+            "smoothness.gpu_rasterization.polymer": {},
+            "smoothness.gpu_rasterization.top_25_smooth": {},
+            "smoothness.gpu_rasterization.tough_filters_cases": {},
+            "smoothness.gpu_rasterization.tough_path_rendering_cases": {},
+            "smoothness.gpu_rasterization.tough_pinch_zoom_cases": {},
+            "smoothness.gpu_rasterization.tough_scrolling_cases": {
+                "end": 9
+            }
+        }
+    },
+    "10": {
+        "benchmarks": {
+            "smoothness.gpu_rasterization.tough_scrolling_cases": {
+                "begin": 9
+            },
+            "smoothness.gpu_rasterization_and_decoding.image_decoding_cases": {},
+            "smoothness.image_decoding_cases": {},
+            "smoothness.key_desktop_move_cases": {},
+            "smoothness.key_mobile_sites_smooth": {},
+            "smoothness.key_silk_cases": {},
+            "smoothness.maps": {},
+            "smoothness.pathological_mobile_sites": {},
+            "smoothness.simple_mobile_sites": {},
+            "smoothness.sync_scroll.key_mobile_sites_smooth": {},
+            "smoothness.top_25_smooth": {}
+        }
+    },
+    "11": {
+        "benchmarks": {
+            "smoothness.tough_ad_cases": {},
+            "smoothness.tough_animation_cases": {},
+            "smoothness.tough_canvas_cases": {},
+            "smoothness.tough_filters_cases": {},
+            "smoothness.tough_image_decode_cases": {},
+            "smoothness.tough_path_rendering_cases": {},
+            "smoothness.tough_pinch_zoom_cases": {},
+            "smoothness.tough_scrolling_cases": {},
+            "smoothness.tough_texture_upload_cases": {},
+            "smoothness.tough_webgl_ad_cases": {},
+            "smoothness.tough_webgl_cases": {},
+            "speedometer": {},
+            "speedometer-future": {},
+            "speedometer2": {},
+            "speedometer2-future": {},
+            "start_with_url.cold.startup_pages": {},
+            "start_with_url.warm.startup_pages": {},
+            "system_health.common_desktop": {},
+            "system_health.common_mobile": {},
+            "system_health.memory_desktop": {},
+            "system_health.memory_mobile": {},
+            "system_health.webview_startup": {},
+            "tab_switching.typical_25": {},
+            "thread_times.key_hit_test_cases": {},
+            "thread_times.key_idle_power_cases": {},
+            "thread_times.key_mobile_sites_smooth": {},
+            "thread_times.key_noop_cases": {},
+            "thread_times.key_silk_cases": {},
+            "thread_times.simple_mobile_sites": {},
+            "thread_times.tough_compositor_cases": {},
+            "thread_times.tough_scrolling_cases": {
+                "end": 32
+            }
+        }
+    },
+    "12": {
+        "benchmarks": {
+            "thread_times.tough_scrolling_cases": {
+                "begin": 32
+            },
+            "tracing.tracing_with_background_memory_infra": {},
+            "v8.browsing_desktop": {},
+            "v8.browsing_desktop-future": {},
+            "v8.browsing_mobile": {
+                "end": 24
+            }
+        }
+    },
+    "13": {
+        "benchmarks": {
+            "v8.browsing_mobile": {
+                "begin": 24
+            },
+            "v8.browsing_mobile-future": {},
+            "v8.runtime_stats.top_25": {},
+            "wasm": {},
+            "webrtc": {}
+        }
+    },
+    "extra_infos": {
+        "num_stories": 2401,
+        "predicted_min_shard_time": 6936.110622523241,
+        "predicted_min_shard_index": 4,
+        "predicted_max_shard_time": 7595.144805609757,
+        "predicted_max_shard_index": 5
+    }
+}

tools/perf/validate_perf_json_config

+#!/usr/bin/env vpython
+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import sys
+
+from core import perf_json_config_validator
+
+
+if __name__ == '__main__':
+  sys.exit(perf_json_config_validator.main(sys.argv[1:]))