infra: Generate source side spec by bucket

Doesn't include internal builders.

Bug: 1028719
Change-Id: Iec40cb84086ed23cd7d5ca60c317296460466497
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2033847
Commit-Queue: Gregory Guterman <guterman@google.com>
Reviewed-by: Stephen Martinis <martiniss@chromium.org>
Cr-Commit-Position: refs/heads/master@{#745574}

testing/buildbot/generate_buildbot_json.py

@@ -15,6 +15,7 @@ import difflib
 import itertools
 import json
 import os
+import re
 import string
 import sys
 import traceback
@@ -1153,35 +1154,53 @@ class BBJSONGenerator(object):
     new_test.update(mixin)
     return new_test
 
-  def generate_waterfall_json(self, waterfall):
-    all_tests = {}
-    generator_map = self.get_test_generator_map()
-    test_type_remapper = self.get_test_type_remapper()
-    for name, config in waterfall['machines'].iteritems():
-      tests = {}
-      # Copy only well-understood entries in the machine's configuration
-      # verbatim into the generated JSON.
-      if 'additional_compile_targets' in config:
-        tests['additional_compile_targets'] = config[
-          'additional_compile_targets']
-      for test_type, input_tests in config.get('test_suites', {}).iteritems():
-        if test_type not in generator_map:
-          raise self.unknown_test_suite_type(
-            test_type, name, waterfall['name']) # pragma: no cover
-        test_generator = generator_map[test_type]
-        # Let multiple kinds of generators generate the same kinds
-        # of tests. For example, gpu_telemetry_tests are a
-        # specialization of isolated_scripts.
-        new_tests = test_generator.generate(
-          waterfall, name, config, input_tests)
-        remapped_test_type = test_type_remapper.get(test_type, test_type)
-        tests[remapped_test_type] = test_generator.sort(
-          tests.get(remapped_test_type, []) + new_tests)
-      all_tests[name] = tests
+  def generate_waterfall_tests(self, waterfall):
+    """Generates the tests for a waterfall.
+
+      Args:
+        waterfall: a dictionary parsed from a master pyl file
+      Returns:
+        A dictionary mapping builders to test specs
+      """
+    all_tests = {
+      name: self.get_tests_for_config(waterfall, name, config)
+        for name, config
+        in waterfall['machines'].iteritems()
+    }
     all_tests['AAAAA1 AUTOGENERATED FILE DO NOT EDIT'] = {}
     all_tests['AAAAA2 See generate_buildbot_json.py to make changes'] = {}
-    return json.dumps(all_tests, indent=2, separators=(',', ': '),
-                      sort_keys=True) + '\n'
+    return all_tests
+
+  def get_tests_for_config(self, waterfall, name, config):
+    generator_map = self.get_test_generator_map()
+    test_type_remapper = self.get_test_type_remapper()
+
+    tests = {}
+    # Copy only well-understood entries in the machine's configuration
+    # verbatim into the generated JSON.
+    if 'additional_compile_targets' in config:
+      tests['additional_compile_targets'] = config[
+        'additional_compile_targets']
+    for test_type, input_tests in config.get('test_suites', {}).iteritems():
+      if test_type not in generator_map:
+        raise self.unknown_test_suite_type(
+          test_type, name, waterfall['name']) # pragma: no cover
+      test_generator = generator_map[test_type]
+      # Let multiple kinds of generators generate the same kinds
+      # of tests. For example, gpu_telemetry_tests are a
+      # specialization of isolated_scripts.
+      new_tests = test_generator.generate(
+        waterfall, name, config, input_tests)
+      remapped_test_type = test_type_remapper.get(test_type, test_type)
+      tests[remapped_test_type] = test_generator.sort(
+        tests.get(remapped_test_type, []) + new_tests)
+
+    return tests
+
+  def jsonify(self, all_tests):
+    return json.dumps(
+        all_tests, indent=2, separators=(',', ': '),
+        sort_keys=True) + '\n'
 
   def generate_waterfalls(self): # pragma: no cover
     self.load_configuration_files()
@@ -1190,12 +1209,32 @@ class BBJSONGenerator(object):
     suffix = '.json'
     if self.args.new_files:
       suffix = '.new' + suffix
+
+    bucket_map = collections.defaultdict(dict)
     for waterfall in self.waterfalls:
-      should_gen = not filters or waterfall['name'] in filters
-      if should_gen:
-        file_path = waterfall['name'] + suffix
-        self.write_file(self.pyl_file_path(file_path),
-                        self.generate_waterfall_json(waterfall))
+      if filters and waterfall['name'] not in filters:
+        continue
+
+      file_path = waterfall['name'] + suffix
+      all_tests = self.generate_waterfall_tests(waterfall)
+      waterfall_json = self.jsonify(all_tests)
+      self.write_file(self.pyl_file_path(file_path), waterfall_json)
+
+      # Assign to buckets
+      bucketname = waterfall['bucket']
+      if bucketname != 'NA':
+        # TODO(guterman): move the internal builders over and remove this
+        # Currently, the waterfalls have builders in the internal buckets,
+        # which we put 'NA' for. In the future, there shouldn't be any.
+        for buildername in waterfall['machines'].keys():
+          bucket_map[bucketname][buildername] = all_tests[buildername]
+
+    # Write bucket files
+    for bucketname, bucket in bucket_map.items():
+      bucket['AAAAA1 AUTOGENERATED FILE DO NOT EDIT'] = {}
+      bucket['AAAAA2 See generate_buildbot_json.py to make changes'] = {}
+      bucket_json = self.jsonify(bucket)
+      self.write_file(self.pyl_file_path(bucketname + '.json'), bucket_json)
 
   def get_valid_bot_names(self):
     # Extract bot names from infra/config/luci-milo.cfg.
@@ -1638,7 +1677,8 @@ class BBJSONGenerator(object):
     self.resolve_configuration_files()
     ungenerated_waterfalls = set()
     for waterfall in self.waterfalls:
-      expected = self.generate_waterfall_json(waterfall)
+      expected = self.jsonify(
+          self.generate_waterfall_tests(waterfall))
       file_path = waterfall['name'] + '.json'
       current = self.read_file(self.pyl_file_path(file_path))
       if expected != current:
@@ -1820,9 +1860,9 @@ class BBJSONGenerator(object):
   def flatten_waterfalls_for_query(self, waterfalls):
     bots = {}
     for waterfall in waterfalls:
-      waterfall_json = json.loads(self.generate_waterfall_json(waterfall))
-      for bot in waterfall_json:
-        bot_info = waterfall_json[bot]
+      waterfall_tests = self.generate_waterfall_tests(waterfall)
+      for bot in waterfall_tests:
+        bot_info = waterfall_tests[bot]
         if 'AAAAA' not in bot:
           bots[bot] = bot_info
     return bots

testing/buildbot/waterfalls.pyl

@@ -11,6 +11,8 @@
 [
   {
     'name': 'chrome',
+    'bucket': 'NA',
+    'project': 'chromium',
     'machines': {
       'chromeos-arm-generic-cfi-thin-lto-chrome': {
         'additional_compile_targets': [
@@ -157,6 +159,8 @@
   },
   {
     'name': 'chromium',
+    'bucket': 'ci',
+    'project': 'chromium',
     'machines': {
       'android-archive-dbg': {
         'additional_compile_targets': [
@@ -228,6 +232,8 @@
   },
   {
     'name': 'chromium.android',
+    'bucket': 'ci',
+    'project': 'chromium',
     'machines': {
       'Android ASAN (dbg)': {
         'additional_compile_targets': [
@@ -707,6 +713,8 @@
   },
   {
     'name': 'chromium.android.fyi',
+    'bucket': 'ci',
+    'project': 'chromium',
     'machines': {
       'Android WebView P Blink-CORS FYI (rel)': {
         'mixins': [
@@ -772,6 +780,8 @@
   },
   {
     'name': 'chromium.chromiumos',
+    'bucket': 'ci',
+    'project': 'chromium',
     'machines': {
       'chromeos-amd64-generic-asan-rel': {
         'additional_compile_targets': [
@@ -864,6 +874,8 @@
   },
   {
     'name': 'chromium.clang',
+    'bucket': 'ci',
+    'project': 'chromium',
     'machines': {
       'CFI Linux ToT': {
         'mixins': [
@@ -1156,6 +1168,8 @@
   },
   {
     'name': 'chromium.dawn',
+    'bucket': 'ci',
+    'project': 'chromium',
     'machines': {
       'Dawn Linux x64 Builder' : {},
       'Dawn Linux x64 DEPS Builder' : {},
@@ -1345,6 +1359,8 @@
   },
   {
     'name': 'chromium.dev',
+    'bucket': 'NA',
+    'project': 'chromium',
     'machines': {
       'android-kitkat-arm-rel-swarming': {
         'mixins': [
@@ -1408,6 +1424,8 @@
   },
   {
     'name': 'chromium.devtools-frontend',
+    'bucket': 'NA',
+    'project': 'chromium',
     'machines': {
       'DevTools Linux (chromium)': {
         'mixins': [
@@ -1422,6 +1440,8 @@
   },
   {
     'name': 'chromium.fuzz',
+    'bucket': 'ci',
+    'project': 'chromium',
     'machines': {
       'ASAN Debug': {
         'additional_compile_targets': [
@@ -1512,6 +1532,8 @@
   },
   {
     'name': 'chromium.fyi',
+    'bucket': 'ci',
+    'project': 'chromium',
     'machines': {
       'Linux Viz': {
         'mixins': [
@@ -2081,6 +2103,8 @@
   },
   {
     'name': 'chromium.goma',
+    'bucket': 'goma',
+    'project': 'chromium',
     'machines': {
       'Chromium Android ARM 32-bit Goma RBE ToT': {
         'additional_compile_targets': [
@@ -2206,6 +2230,8 @@
   },
   {
     'name': 'chromium.goma.fyi',
+    'bucket': 'goma',
+    'project': 'chromium',
     'machines': {
       'Linux Builder Goma Canary': {
         'additional_compile_targets': [
@@ -2363,6 +2389,8 @@
     'mixins': [
       'swarming_containment_auto',
     ],
+    'bucket': 'ci',
+    'project': 'chromium',
     'machines': {
       'Android Release (Nexus 5X)': {
         'browser_config': 'android-chromium',
@@ -2529,6 +2557,8 @@
     'mixins': [
       'swarming_containment_auto',
     ],
+    'bucket': 'ci',
+    'project': 'chromium',
     'machines': {
       # BEGIN Fake builder used as mirror targets for ANGLE's GPU tryservers
       'ANGLE GPU Android Release (Nexus 5X)': {
@@ -3577,6 +3607,8 @@
   },
   {
     'name': 'chromium.linux',
+    'bucket': 'ci',
+    'project': 'chromium',
     'machines': {
       'Cast Audio Linux': {
         'mixins': [
@@ -3749,6 +3781,8 @@
   },
   {
     'name': 'chromium.lkgr',
+    'bucket': 'NA',
+    'project': 'chromium',
     'machines': {
       'ASAN Debug': {
         'additional_compile_targets': [
@@ -3839,6 +3873,8 @@
   },
   {
     'name': 'chromium.mac',
+    'bucket': 'ci',
+    'project': 'chromium',
     'machines': {
       'Mac Builder': {
         'additional_compile_targets': [
@@ -3939,6 +3975,8 @@
   },
   {
     'name': 'chromium.memory',
+    'bucket': 'ci',
+    'project': 'chromium',
     'machines': {
       'Android CFI': {
         'swarming': {
@@ -4089,6 +4127,8 @@
   },
   {
     'name': 'chromium.swangle',
+    'bucket': 'ci',
+    'project': 'chromium',
     'machines': {
       'linux-swangle-tot-angle-x64' : {
         'mixins': [
@@ -4214,6 +4254,8 @@
   },
   {
     'name': 'chromium.webrtc',
+    'bucket': 'webrtc',
+    'project': 'chromium',
     'machines': {
       'WebRTC Chromium Android Builder': {},
       'WebRTC Chromium Android Tester': {
@@ -4272,6 +4314,8 @@
   },
   {
     'name': 'chromium.webrtc.fyi',
+    'bucket': 'webrtc.fyi',
+    'project': 'chromium',
     'machines': {
       # For builders, specify targets if the builder has no associated
       # tester (if it does, it will build what the tester needs).
@@ -4384,6 +4428,8 @@
   },
   {
     'name': 'chromium.win',
+    'bucket': 'ci',
+    'project': 'chromium',
     'machines': {
       'WebKit Win10': {
         'mixins': [
@@ -4494,6 +4540,8 @@
   },
   {
     'name': 'client.devtools-frontend.integration',
+    'bucket': 'NA',
+    'project': 'chromium',
     'machines': {
       'DevTools Linux': {
         'mixins': [
@@ -4508,14 +4556,20 @@
   },
   {
     'name': 'client.openscreen.chromium',
+    'bucket': 'NA',
+    'project': 'chromium',
     'machines': {},
   },
   {
     'name': 'client.v8.branches',
+    'bucket': 'NA',
+    'project': 'chromium',
     'machines': {},
   },
   {
     'name': 'client.v8.chromium',
+    'bucket': 'NA',
+    'project': 'chromium',
     'machines': {
       'Linux - Future': {
         'mixins': [
@@ -4539,6 +4593,8 @@
   },
   {
     'name': 'client.v8.fyi',
+    'bucket': 'NA',
+    'project': 'chromium',
     'machines': {
       'Android V8 FYI Release (Nexus 5X)': {
         'browser_config': 'android-chromium',
@@ -4713,6 +4769,8 @@
   },
   {
     'name': 'tryserver.chromium.android',
+    'bucket': 'try',
+    'project': 'chromium',
     'machines': {
       'android-opus-kitkat-arm-rel': {
         'mixins': [
@@ -4739,6 +4797,8 @@
   },
   {
     'name': 'tryserver.chromium.linux',
+    'bucket': 'try',
+    'project': 'chromium',
     'machines': {
       'linux-layout-tests-fragment-item': {
         'additional_compile_targets': [
@@ -4766,6 +4826,8 @@
   },
   {
     'name': 'tryserver.chromium.mac',
+    'bucket': 'try',
+    'project': 'chromium',
     'machines': {
       'mac-rel': {
         'mixins': [
@@ -4782,6 +4844,8 @@
   },
   {
     'name': 'tryserver.devtools-frontend',
+    'bucket': 'NA',
+    'project': 'chromium',
     'machines': {
       'devtools_frontend_linux_blink_light_rel': {
         'mixins': [
@@ -4803,6 +4867,8 @@
   },
   {
     'name': 'tryserver.webrtc',
+    'bucket': 'NA',
+    'project': 'chromium',
     'machines': {
       'android_chromium_compile': {
         'additional_compile_targets': [