Implement smoke test mode for performance_test_suite

1) Updates the script that executes the tests on the swarming bot by
accepting a flag for the benchmark to shard map.  We now have at least
three use cases for this: main perf waterfall android and desktop,
smoke testing, testing on our trybot and potentially android go.

2) Updates the custom merge script to not upload perf results when
we are in smoke test mode.

Bug:840427
Change-Id: Idec8f45cc2d1b5a3cd336ef3d1698d3d86161ba3

NOTRY=true  # win_7_chromium_rel_ng flaky

Change-Id: Idec8f45cc2d1b5a3cd336ef3d1698d3d86161ba3
Reviewed-on: https://chromium-review.googlesource.com/1055614
Commit-Queue: Ned Nguyen <nednguyen@google.com>
Reviewed-by: Ned Nguyen <nednguyen@google.com>
Cr-Commit-Position: refs/heads/master@{#558900}

testing/buildbot/chromium.perf.fyi.json

@@ -6,7 +6,8 @@
           "-v",
           "--browser=android-chromium",
           "--upload-results",
-          "--run-ref-build"
+          "--run-ref-build",
+          "--test-shard-map-filename=benchmark_android_bot_map.json"
         ],
         "isolate_name": "performance_test_suite",
         "merge": {
@@ -496,7 +497,8 @@
           "-v",
           "--browser=release",
           "--upload-results",
-          "--run-ref-build"
+          "--run-ref-build",
+          "--test-shard-map-filename=benchmark_desktop_bot_map.json"
         ],
         "isolate_name": "performance_test_suite",
         "merge": {
@@ -657,9 +659,9 @@
           "-v",
           "--browser=reference",
           "--upload-results",
-          "--testing=true",
           "--xvfb",
-          "--run-ref-build"
+          "--run-ref-build",
+          "--test-shard-map-filename=benchmark_bot_map.json"
         ],
         "isolate_name": "telemetry_perf_tests_without_chrome",
         "merge": {
@@ -706,7 +708,8 @@
           "-v",
           "--browser=android-chromium",
           "--upload-results",
-          "--run-ref-build"
+          "--run-ref-build",
+          "--test-shard-map-filename=benchmark_android_bot_map.json"
         ],
         "isolate_name": "performance_test_suite",
         "merge": {
@@ -754,7 +757,8 @@
           "-v",
           "--browser=android-webview",
           "--upload-results",
-          "--webview-embedder-apk=../../out/Release/apks/SystemWebViewShell.apk"
+          "--webview-embedder-apk=../../out/Release/apks/SystemWebViewShell.apk",
+          "--test-shard-map-filename=benchmark_android_bot_map.json"
         ],
         "isolate_name": "performance_webview_test_suite",
         "merge": {

testing/buildbot/chromium.perf.json

@@ -46719,7 +46719,8 @@
           "-v",
           "--browser=release",
           "--upload-results",
-          "--run-ref-build"
+          "--run-ref-build",
+          "--test-shard-map-filename=benchmark_desktop_bot_map.json"
         ],
         "isolate_name": "performance_test_suite",
         "merge": {
@@ -46854,7 +46855,8 @@
           "-v",
           "--browser=release",
           "--upload-results",
-          "--run-ref-build"
+          "--run-ref-build",
+          "--test-shard-map-filename=benchmark_desktop_bot_map.json"
         ],
         "isolate_name": "performance_test_suite",
         "merge": {

testing/scripts/run_performance_tests.py

@@ -80,30 +80,10 @@ BENCHMARKS_TO_OUTPUT_HISTOGRAMS = [
     'system_health.webview_startup',
 ]
 
-# We currently have two different sharding schemes for android
-# vs desktop.  When we are running at capacity we will have 26
-# desktop shards and 39 android.
-CURRENT_DESKTOP_NUM_SHARDS = 26
-CURRENT_ANDROID_NUM_SHARDS = 39
-
-def get_sharding_map_path(total_shards, testing):
-  # Determine if we want to do a test run of the benchmarks or run the
-  # full suite.
-  if not testing:
-    # Note: <= for testing purposes until we have all shards running
-    if int(total_shards) <= CURRENT_DESKTOP_NUM_SHARDS:
-      return os.path.join(
-          os.path.dirname(__file__), '..', '..', 'tools', 'perf', 'core',
-          'benchmark_desktop_bot_map.json')
-    else:
-      return os.path.join(
-          os.path.dirname(__file__), '..', '..', 'tools', 'perf', 'core',
-          'benchmark_android_bot_map.json')
-  else:
-    return os.path.join(
+def get_sharding_map_path(args):
+  return os.path.join(
       os.path.dirname(__file__), '..', '..', 'tools', 'perf', 'core',
-      'benchmark_bot_map.json')
-
+      args.test_shard_map_filename)
 
 def write_results(
     perf_test_name, perf_results, json_test_results, isolated_out_dir, encoded):
@@ -129,7 +109,7 @@ def execute_benchmark(benchmark, isolated_out_dir,
   is_histograms = append_output_format(benchmark, args, rest_args)
   # Insert benchmark name as first argument to run_benchmark call
   # which is the first argument in the rest_args.  Also need to append
-  # output format.
+  # output format and smoke test mode.
   per_benchmark_args = (rest_args[:1] + [benchmark] + rest_args[1:])
   benchmark_name = benchmark
   if is_reference:
@@ -200,12 +180,13 @@ def main():
   parser.add_argument('--xvfb', help='Start xvfb.', action='store_true')
   parser.add_argument('--non-telemetry',
                       help='Type of perf test', type=bool, default=False)
-  parser.add_argument('--testing', help='Test run, execute subset of tests',
-                      type=bool, default=False)
   parser.add_argument('--benchmarks',
                       help='Comma separated list of benchmark names'
                       ' to run in lieu of indexing into our benchmark bot maps',
                       required=False)
+  # Some executions may have a different sharding scheme and/or set of tests.
+  # These files must live in src/tools/perf/core/
+  parser.add_argument('--test-shard-map-filename', type=str, required=False)
   parser.add_argument('--output-format', action='append')
   parser.add_argument('--run-ref-build',
                       help='Run test on reference browser', action='store_true')
@@ -244,8 +225,7 @@ def main():
       if not (total_shards or shard_index):
         raise Exception('Shard indicators must be present for perf tests')
 
-      sharding_map_path = get_sharding_map_path(
-          total_shards, args.testing or False)
+      sharding_map_path = get_sharding_map_path(args)
       with open(sharding_map_path) as f:
         sharding_map = json.load(f)
       sharding = None

tools/perf/core/perf_data_generator.py

@@ -970,7 +970,10 @@ NEW_PERF_RECIPE_FYI_TESTERS = {
       'tests': [
         {
           'isolate': 'performance_test_suite',
-          'extra_args': ['--run-ref-build'],
+          'extra_args': [
+            '--run-ref-build',
+            '--test-shard-map-filename=benchmark_desktop_bot_map.json',
+          ],
           'num_shards': 26
         },
         {
@@ -997,8 +1000,11 @@ NEW_PERF_RECIPE_FYI_TESTERS = {
       'tests': [
         {
           'isolate': 'telemetry_perf_tests_without_chrome',
-          'extra_args': ['--xvfb',
-                         '--run-ref-build'],
+          'extra_args': [
+            '--xvfb',
+            '--run-ref-build',
+            '--test-shard-map-filename=benchmark_bot_map.json'
+          ],
           'num_shards': 3
         },
         {
@@ -1022,7 +1028,10 @@ NEW_PERF_RECIPE_FYI_TESTERS = {
         {
           'name': 'performance_test_suite',
           'isolate': 'performance_test_suite',
-          'extra_args': ['--run-ref-build'],
+          'extra_args': [
+            '--run-ref-build',
+            '--test-shard-map-filename=benchmark_android_bot_map.json',
+          ],
           'num_shards': 14
         }
       ],
@@ -1040,6 +1049,9 @@ NEW_PERF_RECIPE_FYI_TESTERS = {
       'tests': [
         {
           'isolate': 'performance_webview_test_suite',
+          'extra_args': [
+            '--test-shard-map-filename=benchmark_android_bot_map.json',
+          ],
           'num_shards': 7
         }
       ],
@@ -1057,7 +1069,10 @@ NEW_PERF_RECIPE_FYI_TESTERS = {
       'tests': [
         {
           'isolate': 'performance_test_suite',
-          'extra_args': ['--run-ref-build'],
+          'extra_args': [
+            '--run-ref-build',
+            '--test-shard-map-filename=benchmark_android_bot_map.json',
+          ],
           'num_shards': 7
         }
       ],
@@ -1081,7 +1096,10 @@ NEW_PERF_RECIPE_MIGRATED_TESTERS = {
       'tests': [
         {
           'isolate': 'performance_test_suite',
-          'extra_args': ['--run-ref-build'],
+          'extra_args': [
+              '--run-ref-build',
+              '--test-shard-map-filename=benchmark_desktop_bot_map.json',
+          ],
         },
         {
           'isolate': 'load_library_perf_tests',
@@ -1111,7 +1129,10 @@ NEW_PERF_RECIPE_MIGRATED_TESTERS = {
         # Add views_perftests, crbug.com/811766
         {
           'isolate': 'performance_test_suite',
-          'extra_args': ['--run-ref-build'],
+          'extra_args': [
+              '--run-ref-build',
+              '--test-shard-map-filename=benchmark_desktop_bot_map.json',
+          ],
         },
         {
           'isolate': 'load_library_perf_tests',
@@ -1213,11 +1234,6 @@ def generate_telemetry_args(tester_config):
     test_args.append(
         '--webview-embedder-apk=../../out/Release/apks/SystemWebViewShell.apk')
 
-  # Appending testing=true if we only want to run a subset of benchmarks
-  # for quicker testing
-  if tester_config.get('testing', False):
-    test_args.append('--testing=true')
-
   return test_args
 
 def generate_non_telemetry_args():

tools/perf/process_perf_results.py

@@ -103,9 +103,11 @@ def _merge_json_output(output_json, jsons_to_merge, perf_results_link,
   """
   merged_results = results_merger.merge_test_results(jsons_to_merge)
 
-  merged_results['links'] = {
-    perf_results_file_name: perf_results_link
-  }
+  # Only append the perf results link if present
+  if perf_results_link:
+    merged_results['links'] = {
+      perf_results_file_name: perf_results_link
+    }
 
   with open(output_json, 'w') as f:
     json.dump(merged_results, f)
@@ -113,9 +115,47 @@ def _merge_json_output(output_json, jsons_to_merge, perf_results_link,
   return 0
 
 
+def _handle_perf_json_test_results(
+    benchmark_directory_list, test_results_list):
+  benchmark_enabled_map = {}
+  for directory in benchmark_directory_list:
+    # Obtain the test name we are running
+    benchmark_name = _get_benchmark_name(directory)
+    is_ref = '.reference' in benchmark_name
+    enabled = True
+    with open(join(directory, 'test_results.json')) as json_data:
+      json_results = json.load(json_data)
+      if not json_results:
+        # Output is null meaning the test didn't produce any results.
+        # Want to output an error and continue loading the rest of the
+        # test results.
+        print 'No results produced for %s, skipping upload' % directory
+        continue
+      if json_results.get('version') == 3:
+        # Non-telemetry tests don't have written json results but
+        # if they are executing then they are enabled and will generate
+        # chartjson results.
+        if not bool(json_results.get('tests')):
+          enabled = False
+      if not is_ref:
+        # We don't need to upload reference build data to the
+        # flakiness dashboard since we don't monitor the ref build
+        test_results_list.append(json_results)
+    if not enabled:
+      # We don't upload disabled benchmarks or tests that are run
+      # as a smoke test
+      print 'Benchmark %s disabled' % benchmark_name
+    benchmark_enabled_map[benchmark_name] = enabled
+
+  return benchmark_enabled_map
+
+def _get_benchmark_name(directory):
+  return basename(directory).replace(" benchmark", "")
+
 def _process_perf_results(output_json, configuration_name,
                           service_account_file,
-                          build_properties, task_output_dir):
+                          build_properties, task_output_dir,
+                          smoke_test_mode):
   """Process one or more perf JSON results.
 
   Consists of merging the json-test-format output and uploading the perf test
@@ -139,8 +179,6 @@ def _process_perf_results(output_json, configuration_name,
       for f in listdir(join(task_output_dir, directory))
     ]
 
-  # We need to keep track of disabled benchmarks so we don't try to
-  # upload the results.
   test_results_list = []
   tmpfile_dir = tempfile.mkdtemp('resultscache')
   upload_failure = False
@@ -151,52 +189,38 @@ def _process_perf_results(output_json, configuration_name,
     configuration_name = build_properties['buildername']
 
   try:
+    # First obtain the list of json test results to merge
+    # and determine the status of each benchmark
+    benchmark_enabled_map = _handle_perf_json_test_results(
+        benchmark_directory_list, test_results_list)
+
+    # Upload all eligible benchmarks to the perf dashboard
     logdog_dict = {}
-    with oauth_api.with_access_token(service_account_file) as oauth_file:
-      for directory in benchmark_directory_list:
-        # Obtain the test name we are running
-        benchmark_name = basename(directory).replace(" benchmark", "")
-        is_ref = '.reference' in benchmark_name
-        disabled = False
-        with open(join(directory, 'test_results.json')) as json_data:
-          json_results = json.load(json_data)
-          if not json_results:
-            # Output is null meaning the test didn't produce any results.
-            # Want to output an error and continue loading the rest of the
-            # test results.
-            print 'No results produced for %s, skipping upload' % directory
+    logdog_stream = None
+    logdog_label = 'Results Dashboard'
+    if not smoke_test_mode:
+      with oauth_api.with_access_token(service_account_file) as oauth_file:
+        for directory in benchmark_directory_list:
+          benchmark_name = _get_benchmark_name(directory)
+          if not benchmark_enabled_map[benchmark_name]:
             continue
-          if json_results.get('version') == 3:
-            # Non-telemetry tests don't have written json results but
-            # if they are executing then they are enabled and will generate
-            # chartjson results.
-            if not bool(json_results.get('tests')):
-              disabled = True
-          if not is_ref:
-            # We don't need to upload reference build data to the
-            # flakiness dashboard since we don't monitor the ref build
-            test_results_list.append(json_results)
-        if disabled:
-          # We don't upload disabled benchmarks
-          print 'Benchmark %s disabled' % benchmark_name
-          continue
-
-        print 'Uploading perf results from %s benchmark' % benchmark_name
-
-        upload_fail = _upload_and_write_perf_data_to_logfile(
-            benchmark_name, directory, configuration_name, build_properties,
-            oauth_file, tmpfile_dir, logdog_dict, is_ref)
-        upload_failure = upload_failure or upload_fail
-
-      logdog_label = 'Results Dashboard'
+          print 'Uploading perf results from %s benchmark' % benchmark_name
+          upload_fail = _upload_and_write_perf_data_to_logfile(
+              benchmark_name, directory, configuration_name, build_properties,
+              oauth_file, tmpfile_dir, logdog_dict,
+              ('.reference' in benchmark_name))
+          upload_failure = upload_failure or upload_fail
+
       logdog_file_name = 'Results_Dashboard_' + str(uuid.uuid4())
+      logdog_stream = logdog_helper.text(logdog_file_name,
+          json.dumps(logdog_dict, sort_keys=True,
+              indent=4, separators=(',', ':')))
       if upload_failure:
         logdog_label += ' Upload Failure'
-      _merge_json_output(output_json, test_results_list,
-          logdog_helper.text(logdog_file_name,
-              json.dumps(logdog_dict, sort_keys=True,
-                  indent=4, separators=(',', ':'))),
-          logdog_label)
+
+    # Finally, merge all test results json and write out to output location
+    _merge_json_output(output_json, test_results_list,
+                       logdog_stream, logdog_label)
   finally:
     shutil.rmtree(tmpfile_dir)
   return upload_failure
@@ -259,13 +283,21 @@ def main():
   parser.add_argument('-o', '--output-json', required=True,
                       help=argparse.SUPPRESS)
   parser.add_argument('json_files', nargs='*', help=argparse.SUPPRESS)
+  parser.add_argument('--smoke-test-mode', required=False, default=False,
+                      help='This test should be run in smoke test mode'
+                      ' meaning it does not upload to the perf dashboard')
 
   args = parser.parse_args()
 
+  if not args.service_account_file and not args.smoke_test_mode:
+    raise Exception(
+        'Service account file must be specificed for dashboard upload')
+
   return _process_perf_results(
       args.output_json, args.configuration_name,
       args.service_account_file,
-      args.build_properties, args.task_output_dir)
+      args.build_properties, args.task_output_dir,
+      args.smoke_test_mode)
 
 
 if __name__ == '__main__':