Auto-generate //testing/buildbot/chromium.json.

This CL converts the chromium waterfall config to be
auto-generated using the new generate_buildbot_json framework.

R=kbr@chromium.org, jbudorick@chromium.org
BUG=662541

Change-Id: I56c403369b4e0811fe604930d8da97048d434eda
Reviewed-on: https://chromium-review.googlesource.com/809958Reviewed-by: John Budorick <jbudorick@chromium.org>
Reviewed-by: Kenneth Russell <kbr@chromium.org>
Commit-Queue: Dirk Pranke <dpranke@chromium.org>
Cr-Commit-Position: refs/heads/master@{#522339}

testing/buildbot/chromium.json

 {
+  "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
+  "AAAAA2 See generate_buildbot_json.py to make changes": {},
   "Linux x64": {
     "additional_compile_targets": [
       "all"

testing/buildbot/generate_buildbot_json.py

@@ -309,6 +309,7 @@ class BBJSONGenerator(object):
       test = self.dictionary_merge(test, modifications)
     for k in self.get_test_key_removals(test_name, tester_name):
       del test[k]
+    if 'swarming' in test:
       self.clean_swarming_dictionary(test['swarming'])
     return test
 
@@ -378,6 +379,7 @@ class BBJSONGenerator(object):
       'name': test_name,
       'script': test_config['script']
     }
+    result = self.update_and_cleanup_test(result, test_name, tester_name)
     return result
 
   def generate_junit_test(self, waterfall, tester_name, tester_config,

testing/buildbot/generate_buildbot_json_unittest.py

@@ -300,6 +300,16 @@ BAD_COMPOSITION_TEST_SUITES = """\
 }
 """
 
+SCRIPT_SUITE = """\
+{
+  'foo_scripts': {
+    'foo_test': {
+      'script': 'foo.py',
+    },
+  },
+}
+"""
+
 UNREFED_TEST_SUITE = """\
 {
   'foo_tests': {},
@@ -326,6 +336,18 @@ EMPTY_EXCEPTIONS = """\
 }
 """
 
+SCRIPT_WITH_ARGS_EXCEPTIONS = """\
+{
+  'foo_test': {
+    'modifications': {
+      'Fake Tester': {
+        'args': ['--fake-arg'],
+      },
+    },
+  },
+}
+"""
+
 NO_BAR_TEST_EXCEPTIONS = """\
 {
   'bar_test': {
@@ -532,6 +554,24 @@ SCRIPT_OUTPUT = """\
 }
 """
 
+SCRIPT_WITH_ARGS_OUTPUT = """\
+{
+  "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
+  "AAAAA2 See generate_buildbot_json.py to make changes": {},
+  "Fake Tester": {
+    "scripts": [
+      {
+        "args": [
+          "--fake-arg"
+        ],
+        "name": "foo_test",
+        "script": "foo.py"
+      }
+    ]
+  }
+}
+"""
+
 JUNIT_OUTPUT = """\
 {
   "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
@@ -758,7 +798,15 @@ class UnitTest(unittest.TestCase):
     fbb.files['chromium.test.json'] = ISOLATED_SCRIPT_OUTPUT
     fbb.check_output_file_consistency(verbose=True)
 
-  def test_script_tests(self):
+  def test_script_with_args(self):
+    fbb = FakeBBGen(FOO_SCRIPT_WATERFALL,
+                    SCRIPT_SUITE,
+                    SCRIPT_WITH_ARGS_EXCEPTIONS,
+                    )
+    fbb.files['chromium.test.json'] = SCRIPT_WITH_ARGS_OUTPUT
+    fbb.check_output_file_consistency(verbose=True)
+
+  def test_script(self):
     fbb = FakeBBGen(FOO_SCRIPT_WATERFALL,
                     FOO_SCRIPT_SUITE,
                     NO_BAR_TEST_EXCEPTIONS)
@@ -818,6 +866,7 @@ class UnitTest(unittest.TestCase):
                             'The following nonexistent machines.*',
                             fbb.check_input_file_consistency)
 
+
 if __name__ == '__main__':
   unittest.main()
 

testing/buildbot/test_suite_exceptions.pyl

@@ -283,6 +283,12 @@
       },
     },
   },
+  'checkbins': {
+    'remove_from': [
+      'Linux x64',
+      'Mac',
+    ],
+  },
   'chrome_public_test_apk': {
     'remove_from': [
       # TODO(kbr): on chromium.android this removal looks like an accident.
@@ -1447,6 +1453,24 @@
       'Linux Tests (dbg)(1)(32)',
     ],
   },
+  'sizes': {
+    'remove_from': [
+      'Win',
+      'Win x64',
+    ],
+    'modifications': {
+      'Mac': {
+        'args': [
+          'mac-release/sizes',
+        ],
+      },
+      'Linux x64': {
+        'args': [
+          'linux-release-64/sizes',
+        ],
+      },
+    },
+  },
   'skia_unittests': {
     'remove_from': [
       # On chromium.android, unclear why these aren't run.

testing/buildbot/test_suites.pyl

@@ -484,6 +484,18 @@
     'ui_touch_selection_unittests': {},
   },
 
+  # TODO(dpranke): These are run on the p/chromium waterfall; they should
+  # probably be run on other builders, and we should get rid of the p/chromium
+  # waterfall.
+  'public_build_scripts': {
+    'checkbins': {
+      'script': 'checkbins.py',
+    },
+    'sizes': {
+      'script': 'sizes.py',
+    },
+  },
+
   'system_webview_shell_instrumentation_tests': {
     'system_webview_shell_layout_test_apk': {},
   },

testing/buildbot/waterfalls.pyl

@@ -9,6 +9,43 @@
 # * trailing commas are allowed.
 
 [
+  {
+    'name': 'chromium',
+    'machines': {
+      'Linux x64': {
+        'additional_compile_targets': [
+          'all',
+        ],
+        'test_suites': {
+          'scripts': 'public_build_scripts',
+        },
+      },
+      'Mac': {
+        'additional_compile_targets': [
+          'all',
+        ],
+        'test_suites': {
+          'scripts': 'public_build_scripts',
+        },
+      },
+      'Win': {
+        'additional_compile_targets': [
+          'all',
+        ],
+        'test_suites': {
+          'scripts': 'public_build_scripts',
+        },
+      },
+      'Win x64': {
+        'additional_compile_targets': [
+          'all',
+        ],
+        'test_suites': {
+          'scripts': 'public_build_scripts',
+        },
+      },
+    }
+  },
   {
     'name': 'chromium.android',
     'machines': {