[benchmarking] disable all reference builds on perf waterfall

This is the CL to disable reference builds on all testers from perf waterfall.

Bug: chrmoium:1108663
Change-Id: I26152451b80094fb301552fefe8f4270d7faa732
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2378786Reviewed-by: John Chen <johnchen@chromium.org>
Commit-Queue: John Chen <johnchen@chromium.org>
Cr-Commit-Position: refs/heads/master@{#803209}

testing/buildbot/chromium.perf.fyi.json

@@ -56,8 +56,7 @@
           "-v",
           "--browser=android-chrome-bundle",
           "--upload-results",
-          "--test-shard-map-filename=android-pixel2-perf-aab-fyi_map.json",
-          "--run-ref-build"
+          "--test-shard-map-filename=android-pixel2-perf-aab-fyi_map.json"
         ],
         "isolate_name": "performance_test_suite",
         "merge": {
@@ -271,7 +270,6 @@
           "--browser=release_x64",
           "--upload-results",
           "--test-shard-map-filename=win-10_laptop_low_end-perf_hp-candidate_map.json",
-          "--run-ref-build",
           "--output-format=histograms",
           "--experimental-tbmv3-metrics"
         ],

testing/buildbot/chromium.perf.json

@@ -9,7 +9,6 @@
           "--browser=android-chromium",
           "--upload-results",
           "--test-shard-map-filename=android_nexus5_perf_map.json",
-          "--run-ref-build",
           "--assert-gpu-compositing"
         ],
         "isolate_name": "performance_test_suite",
@@ -104,7 +103,6 @@
           "--browser=release_x64",
           "--upload-results",
           "--test-shard-map-filename=win_7_nvidia_gpu_perf_map.json",
-          "--run-ref-build",
           "--assert-gpu-compositing"
         ],
         "isolate_name": "performance_test_suite",
@@ -150,8 +148,7 @@
           "-v",
           "--browser=release",
           "--upload-results",
-          "--test-shard-map-filename=win_7_perf_map.json",
-          "--run-ref-build"
+          "--test-shard-map-filename=win_7_perf_map.json"
         ],
         "isolate_name": "performance_test_suite",
         "merge": {
@@ -388,8 +385,7 @@
           "-v",
           "--browser=android-chrome",
           "--upload-results",
-          "--test-shard-map-filename=android-go-perf_map.json",
-          "--run-ref-build"
+          "--test-shard-map-filename=android-go-perf_map.json"
         ],
         "isolate_name": "performance_test_suite",
         "merge": {
@@ -926,7 +922,6 @@
           "--browser=release",
           "--upload-results",
           "--test-shard-map-filename=linux-perf_map.json",
-          "--run-ref-build",
           "--assert-gpu-compositing"
         ],
         "isolate_name": "performance_test_suite",
@@ -985,7 +980,6 @@
           "--browser=release",
           "--upload-results",
           "--test-shard-map-filename=mac-10_12_laptop_low_end-perf_map.json",
-          "--run-ref-build",
           "--assert-gpu-compositing"
         ],
         "isolate_name": "performance_test_suite",
@@ -1044,7 +1038,6 @@
           "--browser=release",
           "--upload-results",
           "--test-shard-map-filename=mac-10_13_laptop_high_end-perf_map.json",
-          "--run-ref-build",
           "--assert-gpu-compositing"
         ],
         "isolate_name": "performance_test_suite",
@@ -1108,7 +1101,6 @@
           "--browser=release_x64",
           "--upload-results",
           "--test-shard-map-filename=win-10-perf_map.json",
-          "--run-ref-build",
           "--assert-gpu-compositing"
         ],
         "isolate_name": "performance_test_suite",
@@ -1167,7 +1159,6 @@
           "--browser=release_x64",
           "--upload-results",
           "--test-shard-map-filename=win-10_laptop_low_end-perf_map.json",
-          "--run-ref-build",
           "--assert-gpu-compositing"
         ],
         "isolate_name": "performance_test_suite",

tools/perf/core/bot_platforms.py

@@ -29,9 +29,15 @@ def _IsPlatformSupported(benchmark, platform):
 
 
 class PerfPlatform(object):
-  def __init__(self, name, description, benchmark_configs,
-               num_shards, platform_os, is_fyi=False,
-               run_reference_build=True, executables=None):
+  def __init__(self,
+               name,
+               description,
+               benchmark_configs,
+               num_shards,
+               platform_os,
+               is_fyi=False,
+               run_reference_build=False,
+               executables=None):
     benchmark_configs = benchmark_configs.Frozenset()
     self._name = name
     self._description = description
@@ -457,70 +463,68 @@ WIN_7_GPU = PerfPlatform(
 ANDROID_GO = PerfPlatform(
     'android-go-perf', 'Android O (gobo)', _ANDROID_GO_BENCHMARK_CONFIGS,
     19, 'android')
-ANDROID_GO_WEBVIEW = PerfPlatform(
-    'android-go_webview-perf', 'Android OPM1.171019.021 (gobo)',
-    _ANDROID_GO_WEBVIEW_BENCHMARK_CONFIGS, 13, 'android',
-    run_reference_build=False)
+ANDROID_GO_WEBVIEW = PerfPlatform('android-go_webview-perf',
+                                  'Android OPM1.171019.021 (gobo)',
+                                  _ANDROID_GO_WEBVIEW_BENCHMARK_CONFIGS, 13,
+                                  'android')
 ANDROID_NEXUS_5 = PerfPlatform(
     'Android Nexus5 Perf', 'Android KOT49H', _ANDROID_NEXUS_5_BENCHMARK_CONFIGS,
     8, 'android', executables=_ANDROID_NEXUS_5_EXECUTABLE_CONFIGS)
 ANDROID_NEXUS_5X_WEBVIEW = PerfPlatform(
     'Android Nexus5X WebView Perf', 'Android AOSP MOB30K',
-    _ANDROID_NEXUS_5X_WEBVIEW_BENCHMARK_CONFIGS, 16, 'android',
-    run_reference_build=False)
+    _ANDROID_NEXUS_5X_WEBVIEW_BENCHMARK_CONFIGS, 16, 'android')
 ANDROID_PIXEL2 = PerfPlatform('android-pixel2-perf',
                               'Android OPM1.171019.021',
                               _ANDROID_PIXEL2_BENCHMARK_CONFIGS,
                               28,
                               'android',
-                              executables=_ANDROID_PIXEL2_EXECUTABLE_CONFIGS,
-                              run_reference_build=False)
+                              executables=_ANDROID_PIXEL2_EXECUTABLE_CONFIGS)
 ANDROID_PIXEL2_WEBVIEW = PerfPlatform(
     'android-pixel2_webview-perf', 'Android OPM1.171019.021',
-    _ANDROID_PIXEL2_WEBVIEW_BENCHMARK_CONFIGS, 21, 'android',
-    run_reference_build=False)
+    _ANDROID_PIXEL2_WEBVIEW_BENCHMARK_CONFIGS, 21, 'android')
 ANDROID_PIXEL2_WEBLAYER = PerfPlatform(
     'android-pixel2_weblayer-perf', 'Android OPM1.171019.021',
-    _ANDROID_PIXEL2_WEBLAYER_BENCHMARK_CONFIGS, 4, 'android',
-    run_reference_build=False)
+    _ANDROID_PIXEL2_WEBLAYER_BENCHMARK_CONFIGS, 4, 'android')
 ANDROID_PIXEL4A_POWER = PerfPlatform('android-pixel4a_power-perf',
                                      'Android QD4A.200102.001.A1',
                                      _ANDROID_PIXEL4A_POWER_BENCHMARK_CONFIGS,
-                              1,
-                              'android',
-                              run_reference_build=False)
+                                     1, 'android')
 # FYI bots
 WIN_10_LOW_END_HP_CANDIDATE = PerfPlatform(
     'win-10_laptop_low_end-perf_HP-Candidate', 'HP 15-BS121NR Laptop Candidate',
     _WIN_10_LOW_END_HP_CANDIDATE_BENCHMARK_CONFIGS,
     1, 'win', is_fyi=True)
-ANDROID_NEXUS5X_PERF_FYI =  PerfPlatform(
-    'android-nexus5x-perf-fyi', 'Android MMB29Q',
+ANDROID_NEXUS5X_PERF_FYI = PerfPlatform('android-nexus5x-perf-fyi',
+                                        'Android MMB29Q',
                                         _ANDROID_NEXUS5X_FYI_BENCHMARK_CONFIGS,
-    3, 'android', is_fyi=True, run_reference_build=False)
+                                        3,
+                                        'android',
+                                        is_fyi=True)
 ANDROID_PIXEL2_PERF_AAB_FYI = PerfPlatform(
-    'android-pixel2-perf-aab-fyi', 'Android OPM1.171019.021',
+    'android-pixel2-perf-aab-fyi',
+    'Android OPM1.171019.021',
     _ANDROID_PIXEL2_AAB_FYI_BENCHMARK_CONFIGS,
-    1, 'android', is_fyi=True,
-    # TODO(crbug.com/612455): Enable ref builds once can pass both
-    # --browser=exact (used by this bot to have it run Monochrome6432)
-    # and --browser=reference together.
-    run_reference_build=False)
-ANDROID_PIXEL2_PERF_FYI = PerfPlatform(
-    'android-pixel2-perf-fyi', 'Android OPM1.171019.021',
+    1,
+    'android',
+    is_fyi=True)
+ANDROID_PIXEL2_PERF_FYI = PerfPlatform('android-pixel2-perf-fyi',
+                                       'Android OPM1.171019.021',
                                        _ANDROID_PIXEL2_FYI_BENCHMARK_CONFIGS,
-    4, 'android', is_fyi=True,
-    # TODO(crbug.com/612455): Enable ref builds once can pass both
-    # --browser=exact (used by this bot to have it run Monochrome6432)
-    # and --browser=reference together.
-    run_reference_build=False)
-CHROMEOS_KEVIN_PERF_FYI = PerfPlatform(
-    'chromeos-kevin-perf-fyi', '',
+                                       4,
+                                       'android',
+                                       is_fyi=True)
+CHROMEOS_KEVIN_PERF_FYI = PerfPlatform('chromeos-kevin-perf-fyi',
+                                       '',
                                        _CHROMEOS_KEVIN_FYI_BENCHMARK_CONFIGS,
-    4, 'chromeos', is_fyi=True, run_reference_build=False)
-LINUX_PERF_FYI = PerfPlatform(
-    'linux-perf-fyi', '', _LINUX_PERF_FYI_BENCHMARK_CONFIGS,
-    1, 'linux', is_fyi=True, run_reference_build=False)
+                                       4,
+                                       'chromeos',
+                                       is_fyi=True)
+LINUX_PERF_FYI = PerfPlatform('linux-perf-fyi',
+                              '',
+                              _LINUX_PERF_FYI_BENCHMARK_CONFIGS,
+                              1,
+                              'linux',
+                              is_fyi=True)
 
 ALL_PLATFORMS = {
     p for p in locals().values() if isinstance(p, PerfPlatform)

tools/perf/core/perf_data_generator.py

@@ -145,9 +145,6 @@ FYI_BUILDERS = {
     'android-pixel2-perf-aab-fyi': {
         'tests': [{
             'isolate': 'performance_test_suite',
-            'extra_args': [
-                '--run-ref-build',
-            ],
         }],
         'platform':
         'android-chrome-bundle',