Revert "infra: Generate source side spec by bucket"

This reverts commit 5c614415.

Reason for revert: The CL is missing an important piece of validation code. 'generate_buildboy_json.py --check' needs to validate the new bucket files, which it does not in this CL.

Original change's description:
> infra: Generate source side spec by bucket
> 
> Doesn't include internal builders.
> 
> Bug: 1028719
> Change-Id: Iec40cb84086ed23cd7d5ca60c317296460466497
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2033847
> Commit-Queue: Gregory Guterman <guterman@google.com>
> Reviewed-by: Stephen Martinis <martiniss@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#745574}

TBR=agable@chromium.org,martiniss@chromium.org,guterman@google.com

# Not skipping CQ checks because original CL landed > 1 day ago.

Bug: 1028719
Change-Id: I9f0beb4efc49f532eb8a25137aafc6ad35c36354
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2083621Reviewed-by: John Budorick <jbudorick@chromium.org>
Commit-Queue: Gregory Guterman <guterman@google.com>
Cr-Commit-Position: refs/heads/master@{#746110}

testing/buildbot/generate_buildbot_json.py

@@ -15,7 +15,6 @@ import difflib
 import itertools
 import json
 import os
-import re
 import string
 import sys
 import traceback
@@ -1154,53 +1153,35 @@ class BBJSONGenerator(object):
     new_test.update(mixin)
     return new_test
 
-  def generate_waterfall_tests(self, waterfall):
-    """Generates the tests for a waterfall.
-
-      Args:
-        waterfall: a dictionary parsed from a master pyl file
-      Returns:
-        A dictionary mapping builders to test specs
-      """
-    all_tests = {
-      name: self.get_tests_for_config(waterfall, name, config)
-        for name, config
-        in waterfall['machines'].iteritems()
-    }
-    all_tests['AAAAA1 AUTOGENERATED FILE DO NOT EDIT'] = {}
-    all_tests['AAAAA2 See generate_buildbot_json.py to make changes'] = {}
-    return all_tests
-
-  def get_tests_for_config(self, waterfall, name, config):
+  def generate_waterfall_json(self, waterfall):
+    all_tests = {}
     generator_map = self.get_test_generator_map()
     test_type_remapper = self.get_test_type_remapper()
-
-    tests = {}
-    # Copy only well-understood entries in the machine's configuration
-    # verbatim into the generated JSON.
-    if 'additional_compile_targets' in config:
-      tests['additional_compile_targets'] = config[
-        'additional_compile_targets']
-    for test_type, input_tests in config.get('test_suites', {}).iteritems():
-      if test_type not in generator_map:
-        raise self.unknown_test_suite_type(
-          test_type, name, waterfall['name']) # pragma: no cover
-      test_generator = generator_map[test_type]
-      # Let multiple kinds of generators generate the same kinds
-      # of tests. For example, gpu_telemetry_tests are a
-      # specialization of isolated_scripts.
-      new_tests = test_generator.generate(
-        waterfall, name, config, input_tests)
-      remapped_test_type = test_type_remapper.get(test_type, test_type)
-      tests[remapped_test_type] = test_generator.sort(
-        tests.get(remapped_test_type, []) + new_tests)
-
-    return tests
-
-  def jsonify(self, all_tests):
-    return json.dumps(
-        all_tests, indent=2, separators=(',', ': '),
-        sort_keys=True) + '\n'
+    for name, config in waterfall['machines'].iteritems():
+      tests = {}
+      # Copy only well-understood entries in the machine's configuration
+      # verbatim into the generated JSON.
+      if 'additional_compile_targets' in config:
+        tests['additional_compile_targets'] = config[
+          'additional_compile_targets']
+      for test_type, input_tests in config.get('test_suites', {}).iteritems():
+        if test_type not in generator_map:
+          raise self.unknown_test_suite_type(
+            test_type, name, waterfall['name']) # pragma: no cover
+        test_generator = generator_map[test_type]
+        # Let multiple kinds of generators generate the same kinds
+        # of tests. For example, gpu_telemetry_tests are a
+        # specialization of isolated_scripts.
+        new_tests = test_generator.generate(
+          waterfall, name, config, input_tests)
+        remapped_test_type = test_type_remapper.get(test_type, test_type)
+        tests[remapped_test_type] = test_generator.sort(
+          tests.get(remapped_test_type, []) + new_tests)
+      all_tests[name] = tests
+    all_tests['AAAAA1 AUTOGENERATED FILE DO NOT EDIT'] = {}
+    all_tests['AAAAA2 See generate_buildbot_json.py to make changes'] = {}
+    return json.dumps(all_tests, indent=2, separators=(',', ': '),
+                      sort_keys=True) + '\n'
 
   def generate_waterfalls(self): # pragma: no cover
     self.load_configuration_files()
@@ -1209,32 +1190,12 @@ class BBJSONGenerator(object):
     suffix = '.json'
     if self.args.new_files:
       suffix = '.new' + suffix
-
-    bucket_map = collections.defaultdict(dict)
     for waterfall in self.waterfalls:
-      if filters and waterfall['name'] not in filters:
-        continue
-
-      file_path = waterfall['name'] + suffix
-      all_tests = self.generate_waterfall_tests(waterfall)
-      waterfall_json = self.jsonify(all_tests)
-      self.write_file(self.pyl_file_path(file_path), waterfall_json)
-
-      # Assign to buckets
-      bucketname = waterfall['bucket']
-      if bucketname != 'NA':
-        # TODO(guterman): move the internal builders over and remove this
-        # Currently, the waterfalls have builders in the internal buckets,
-        # which we put 'NA' for. In the future, there shouldn't be any.
-        for buildername in waterfall['machines'].keys():
-          bucket_map[bucketname][buildername] = all_tests[buildername]
-
-    # Write bucket files
-    for bucketname, bucket in bucket_map.items():
-      bucket['AAAAA1 AUTOGENERATED FILE DO NOT EDIT'] = {}
-      bucket['AAAAA2 See generate_buildbot_json.py to make changes'] = {}
-      bucket_json = self.jsonify(bucket)
-      self.write_file(self.pyl_file_path(bucketname + '.json'), bucket_json)
+      should_gen = not filters or waterfall['name'] in filters
+      if should_gen:
+        file_path = waterfall['name'] + suffix
+        self.write_file(self.pyl_file_path(file_path),
+                        self.generate_waterfall_json(waterfall))
 
   def get_valid_bot_names(self):
     # Extract bot names from infra/config/luci-milo.cfg.
@@ -1677,8 +1638,7 @@ class BBJSONGenerator(object):
     self.resolve_configuration_files()
     ungenerated_waterfalls = set()
     for waterfall in self.waterfalls:
-      expected = self.jsonify(
-          self.generate_waterfall_tests(waterfall))
+      expected = self.generate_waterfall_json(waterfall)
       file_path = waterfall['name'] + '.json'
       current = self.read_file(self.pyl_file_path(file_path))
       if expected != current:
@@ -1860,9 +1820,9 @@ class BBJSONGenerator(object):
   def flatten_waterfalls_for_query(self, waterfalls):
     bots = {}
     for waterfall in waterfalls:
-      waterfall_tests = self.generate_waterfall_tests(waterfall)
-      for bot in waterfall_tests:
-        bot_info = waterfall_tests[bot]
+      waterfall_json = json.loads(self.generate_waterfall_json(waterfall))
+      for bot in waterfall_json:
+        bot_info = waterfall_json[bot]
         if 'AAAAA' not in bot:
           bots[bot] = bot_info
     return bots

testing/buildbot/waterfalls.pyl

@@ -11,8 +11,6 @@
 [
   {
     'name': 'chrome',
-    'bucket': 'NA',
-    'project': 'chromium',
     'machines': {
       'chromeos-arm-generic-cfi-thin-lto-chrome': {
         'additional_compile_targets': [
@@ -159,8 +157,6 @@
   },
   {
     'name': 'chromium',
-    'bucket': 'ci',
-    'project': 'chromium',
     'machines': {
       'android-archive-dbg': {
         'additional_compile_targets': [
@@ -232,8 +228,6 @@
   },
   {
     'name': 'chromium.android',
-    'bucket': 'ci',
-    'project': 'chromium',
     'machines': {
       'Android ASAN (dbg)': {
         'additional_compile_targets': [
@@ -713,8 +707,6 @@
   },
   {
     'name': 'chromium.android.fyi',
-    'bucket': 'ci',
-    'project': 'chromium',
     'machines': {
       'Android WebView P Blink-CORS FYI (rel)': {
         'mixins': [
@@ -780,8 +772,6 @@
   },
   {
     'name': 'chromium.chromiumos',
-    'bucket': 'ci',
-    'project': 'chromium',
     'machines': {
       'chromeos-amd64-generic-asan-rel': {
         'additional_compile_targets': [
@@ -874,8 +864,6 @@
   },
   {
     'name': 'chromium.clang',
-    'bucket': 'ci',
-    'project': 'chromium',
     'machines': {
       'CFI Linux ToT': {
         'mixins': [
@@ -1168,8 +1156,6 @@
   },
   {
     'name': 'chromium.dawn',
-    'bucket': 'ci',
-    'project': 'chromium',
     'machines': {
       'Dawn Linux x64 Builder' : {},
       'Dawn Linux x64 DEPS Builder' : {},
@@ -1359,8 +1345,6 @@
   },
   {
     'name': 'chromium.dev',
-    'bucket': 'NA',
-    'project': 'chromium',
     'machines': {
       'android-kitkat-arm-rel-swarming': {
         'mixins': [
@@ -1424,8 +1408,6 @@
   },
   {
     'name': 'chromium.devtools-frontend',
-    'bucket': 'NA',
-    'project': 'chromium',
     'machines': {
       'DevTools Linux (chromium)': {
         'mixins': [
@@ -1440,8 +1422,6 @@
   },
   {
     'name': 'chromium.fuzz',
-    'bucket': 'ci',
-    'project': 'chromium',
     'machines': {
       'ASAN Debug': {
         'additional_compile_targets': [
@@ -1532,8 +1512,6 @@
   },
   {
     'name': 'chromium.fyi',
-    'bucket': 'ci',
-    'project': 'chromium',
     'machines': {
       'Linux Viz': {
         'mixins': [
@@ -2103,8 +2081,6 @@
   },
   {
     'name': 'chromium.goma',
-    'bucket': 'goma',
-    'project': 'chromium',
     'machines': {
       'Chromium Android ARM 32-bit Goma RBE ToT': {
         'additional_compile_targets': [
@@ -2230,8 +2206,6 @@
   },
   {
     'name': 'chromium.goma.fyi',
-    'bucket': 'goma',
-    'project': 'chromium',
     'machines': {
       'Linux Builder Goma Canary': {
         'additional_compile_targets': [
@@ -2389,8 +2363,6 @@
     'mixins': [
       'swarming_containment_auto',
     ],
-    'bucket': 'ci',
-    'project': 'chromium',
     'machines': {
       'Android Release (Nexus 5X)': {
         'browser_config': 'android-chromium',
@@ -2557,8 +2529,6 @@
     'mixins': [
       'swarming_containment_auto',
     ],
-    'bucket': 'ci',
-    'project': 'chromium',
     'machines': {
       # BEGIN Fake builder used as mirror targets for ANGLE's GPU tryservers
       'ANGLE GPU Android Release (Nexus 5X)': {
@@ -3607,8 +3577,6 @@
   },
   {
     'name': 'chromium.linux',
-    'bucket': 'ci',
-    'project': 'chromium',
     'machines': {
       'Cast Audio Linux': {
         'mixins': [
@@ -3781,8 +3749,6 @@
   },
   {
     'name': 'chromium.lkgr',
-    'bucket': 'NA',
-    'project': 'chromium',
     'machines': {
       'ASAN Debug': {
         'additional_compile_targets': [
@@ -3873,8 +3839,6 @@
   },
   {
     'name': 'chromium.mac',
-    'bucket': 'ci',
-    'project': 'chromium',
     'machines': {
       'Mac Builder': {
         'additional_compile_targets': [
@@ -3975,8 +3939,6 @@
   },
   {
     'name': 'chromium.memory',
-    'bucket': 'ci',
-    'project': 'chromium',
     'machines': {
       'Android CFI': {
         'swarming': {
@@ -4127,8 +4089,6 @@
   },
   {
     'name': 'chromium.swangle',
-    'bucket': 'ci',
-    'project': 'chromium',
     'machines': {
       'linux-swangle-tot-angle-x64' : {
         'mixins': [
@@ -4254,8 +4214,6 @@
   },
   {
     'name': 'chromium.webrtc',
-    'bucket': 'webrtc',
-    'project': 'chromium',
     'machines': {
       'WebRTC Chromium Android Builder': {},
       'WebRTC Chromium Android Tester': {
@@ -4314,8 +4272,6 @@
   },
   {
     'name': 'chromium.webrtc.fyi',
-    'bucket': 'webrtc.fyi',
-    'project': 'chromium',
     'machines': {
       # For builders, specify targets if the builder has no associated
       # tester (if it does, it will build what the tester needs).
@@ -4428,8 +4384,6 @@
   },
   {
     'name': 'chromium.win',
-    'bucket': 'ci',
-    'project': 'chromium',
     'machines': {
       'WebKit Win10': {
         'mixins': [
@@ -4540,8 +4494,6 @@
   },
   {
     'name': 'client.devtools-frontend.integration',
-    'bucket': 'NA',
-    'project': 'chromium',
     'machines': {
       'DevTools Linux': {
         'mixins': [
@@ -4556,20 +4508,14 @@
   },
   {
     'name': 'client.openscreen.chromium',
-    'bucket': 'NA',
-    'project': 'chromium',
     'machines': {},
   },
   {
     'name': 'client.v8.branches',
-    'bucket': 'NA',
-    'project': 'chromium',
     'machines': {},
   },
   {
     'name': 'client.v8.chromium',
-    'bucket': 'NA',
-    'project': 'chromium',
     'machines': {
       'Linux - Future': {
         'mixins': [
@@ -4593,8 +4539,6 @@
   },
   {
     'name': 'client.v8.fyi',
-    'bucket': 'NA',
-    'project': 'chromium',
     'machines': {
       'Android V8 FYI Release (Nexus 5X)': {
         'browser_config': 'android-chromium',
@@ -4769,8 +4713,6 @@
   },
   {
     'name': 'tryserver.chromium.android',
-    'bucket': 'try',
-    'project': 'chromium',
     'machines': {
       'android-opus-kitkat-arm-rel': {
         'mixins': [
@@ -4797,8 +4739,6 @@
   },
   {
     'name': 'tryserver.chromium.linux',
-    'bucket': 'try',
-    'project': 'chromium',
     'machines': {
       'linux-layout-tests-fragment-item': {
         'additional_compile_targets': [
@@ -4826,8 +4766,6 @@
   },
   {
     'name': 'tryserver.chromium.mac',
-    'bucket': 'try',
-    'project': 'chromium',
     'machines': {
       'mac-rel': {
         'mixins': [
@@ -4844,8 +4782,6 @@
   },
   {
     'name': 'tryserver.devtools-frontend',
-    'bucket': 'NA',
-    'project': 'chromium',
     'machines': {
       'devtools_frontend_linux_blink_light_rel': {
         'mixins': [
@@ -4867,8 +4803,6 @@
   },
   {
     'name': 'tryserver.webrtc',
-    'bucket': 'NA',
-    'project': 'chromium',
     'machines': {
       'android_chromium_compile': {
         'additional_compile_targets': [