Autogenerate chromium.{android,linux,mac,windows}.json.

Add a new generation script, and a description of these waterfalls and
the test suites that run on them. This CL subsumes the maintenance of
21,000+ lines of hand-written JSON.

Many simplifications are now possible. The initial goal here was to
replicate the existing JSON files verbatim, in order to guarantee that
the behavior of the waterfalls would be unmodified. Many exceptions have
been identified with various test suites and bots that can now be easily
removed.

The new script is unit tested and has 100% code coverage modulo a few
exceptions. New presubmit checks will enforce this rigor going forward.

Follow-on work will autogenerate the remainder of Chromium's waterfalls
and unify with the GPU bots' generation script.

BUG=662541
NOTRY=true

Change-Id: I9a916d6f341bbd1495f0a61761628d7f4fb227ff
Reviewed-on: https://chromium-review.googlesource.com/789793
Commit-Queue: Kenneth Russell <kbr@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Cr-Commit-Position: refs/heads/master@{#521634}

testing/buildbot/PRESUBMIT.py

@@ -10,10 +10,27 @@ for more details on the presubmit API built into depot_tools.
 
 
 def CommonChecks(input_api, output_api):
-  args = [input_api.python_executable, 'manage.py', '--check']
-  cmd = input_api.Command(
-      name='manage', cmd=args, kwargs={}, message=output_api.PresubmitError)
-  return input_api.RunTests([cmd])
+  return input_api.RunTests([
+    input_api.Command(
+      name='generate_buildbot_json', cmd=[
+        input_api.python_executable, 'generate_buildbot_json.py', '--check'],
+      kwargs={}, message=output_api.PresubmitError),
+
+    input_api.Command(
+      name='generate_buildbot_json_unittest', cmd=[
+        input_api.python_executable, 'generate_buildbot_json_unittest.py'],
+      kwargs={}, message=output_api.PresubmitError),
+
+    input_api.Command(
+      name='generate_buildbot_json_coveragetest', cmd=[
+        input_api.python_executable, 'generate_buildbot_json_coveragetest.py'],
+      kwargs={}, message=output_api.PresubmitError),
+
+    input_api.Command(
+      name='manage', cmd=[
+        input_api.python_executable, 'manage.py', '--check'],
+      kwargs={}, message=output_api.PresubmitError),
+  ])
 
 
 def CheckChangeOnUpload(input_api, output_api):

testing/buildbot/chromium.android.json

 {
+  "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
+  "AAAAA2 See generate_buildbot_json.py to make changes": {},
   "Android ASAN (dbg)": {
     "additional_compile_targets": [
       "all"

testing/buildbot/chromium.linux.json

 {
+  "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
+  "AAAAA2 See generate_buildbot_json.py to make changes": {},
   "Android Arm64 Builder (dbg)": {
     "additional_compile_targets": [
       "all"

testing/buildbot/chromium.mac.json

 {
+  "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
+  "AAAAA2 See generate_buildbot_json.py to make changes": {},
   "Mac Builder": {
     "additional_compile_targets": [
       "pdf_fuzzers"

testing/buildbot/chromium.win.json

 {
+  "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
+  "AAAAA2 See generate_buildbot_json.py to make changes": {},
   "Win 7 Tests x64 (1)": {
     "gtest_tests": [
       {

testing/buildbot/generate_buildbot_json_coveragetest.py

+#!/usr/bin/python
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import coverage
+import cStringIO
+import sys
+import unittest
+
+class FakeStream(object):
+  def write(self, value):
+    pass
+
+  def flush(self):
+    pass
+
+def main():
+  cov = coverage.coverage(include='*generate_buildbot_json.py')
+  cov.start()
+  import generate_buildbot_json_unittest
+  suite = unittest.TestLoader().loadTestsFromTestCase(
+    generate_buildbot_json_unittest.UnitTest)
+  unittest.TextTestRunner(stream=FakeStream()).run(suite)
+  cov.stop()
+  outf = cStringIO.StringIO()
+  percentage = cov.report(file=outf, show_missing=True)
+  if int(percentage) != 100:
+    print(outf.getvalue())
+    print('FATAL: Insufficient coverage (%.f%%)' % int(percentage))
+    return 1
+  return 0
+
+if __name__ == '__main__':
+  sys.exit(main())