Autogenerate chromium.fyi.json and chromium.android.fyi.json.

The latter was the first time where odd configuration differences were seen
between bots on different waterfalls with the same name, and between gtests
and instrumentation tests with the same names.

Functionality was added to the generation script to be able to target these
cases precisely, so that the existing JSON files could be autogenerated
exactly as they are. (A couple of shortcuts were made in the coverage test
in the meantime and left as TODOs.)

Includes refactorings to several test suites for better reuse.

BUG=662541

Change-Id: I91975c6b9ab0a7fa378de79f32c12709531c2bbe
Reviewed-on: https://chromium-review.googlesource.com/818210Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Commit-Queue: Dirk Pranke <dpranke@chromium.org>
Cr-Commit-Position: refs/heads/master@{#523128}

testing/buildbot/chromium.android.fyi.json

 {
+  "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
+  "AAAAA2 See generate_buildbot_json.py to make changes": {},
   "Android Cronet ARMv6 Builder": {
     "gtest_tests": [
       {

testing/buildbot/chromium.fyi.json

 {
+  "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
+  "AAAAA2 See generate_buildbot_json.py to make changes": {},
   "Android Builder (dbg) Goma Canary": {
     "additional_compile_targets": [
       "all"

testing/buildbot/generate_buildbot_json_unittest.py

@@ -249,6 +249,18 @@ FOO_TEST_SUITE = """\
 }
 """
 
+FOO_TEST_SUITE_WITH_ARGS = """\
+{
+  'foo_tests': {
+    'foo_test': {
+      'args': [
+        '--c_arg',
+      ],
+    },
+  },
+}
+"""
+
 FOO_SCRIPT_SUITE = """\
 {
   'foo_scripts': {
@@ -300,6 +312,19 @@ BAD_COMPOSITION_TEST_SUITES = """\
 }
 """
 
+INSTRUMENTATION_TESTS_WITH_DIFFERENT_NAMES = """\
+{
+  'composition_tests': {
+    'foo_tests': {
+      'test': 'foo_test',
+    },
+    'bar_tests': {
+      'test': 'foo_test',
+    },
+  },
+}
+"""
+
 SCRIPT_SUITE = """\
 {
   'foo_scripts': {
@@ -493,6 +518,33 @@ COMPOSITION_WATERFALL_FILTERED_OUTPUT = """\
 }
 """
 
+MERGED_ARGS_OUTPUT = """\
+{
+  "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
+  "AAAAA2 See generate_buildbot_json.py to make changes": {},
+  "Fake Tester": {
+    "gtest_tests": [
+      {
+        "args": [
+          "--bar",
+          "--c_arg"
+        ],
+        "swarming": {
+          "can_use_on_swarming_builders": true,
+          "dimension_sets": [
+            {
+              "kvm": "1"
+            }
+          ],
+          "hard_timeout": 600
+        },
+        "test": "foo_test"
+      }
+    ]
+  }
+}
+"""
+
 MODIFIED_OUTPUT = """\
 {
   "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
@@ -615,6 +667,25 @@ INSTRUMENTATION_TEST_OUTPUT = """\
 }
 """
 
+INSTRUMENTATION_TEST_DIFFERENT_NAMES_OUTPUT = """\
+{
+  "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
+  "AAAAA2 See generate_buildbot_json.py to make changes": {},
+  "Fake Tester": {
+    "instrumentation_tests": [
+      {
+        "name": "bar_tests",
+        "test": "foo_test"
+      },
+      {
+        "name": "foo_tests",
+        "test": "foo_test"
+      }
+    ]
+  }
+}
+"""
+
 ANDROID_WATERFALL_OUTPUT = """\
 {
   "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
@@ -777,6 +848,13 @@ class UnitTest(unittest.TestCase):
     fbb.files['chromium.test.json'] = COMPOSITION_WATERFALL_OUTPUT
     fbb.check_output_file_consistency(verbose=True)
 
+  def test_test_arg_merges(self):
+    fbb = FakeBBGen(FOO_GTESTS_WATERFALL,
+                    FOO_TEST_SUITE_WITH_ARGS,
+                    FOO_TEST_MODIFICATIONS)
+    fbb.files['chromium.test.json'] = MERGED_ARGS_OUTPUT
+    fbb.check_output_file_consistency(verbose=True)
+
   def test_test_filtering(self):
     fbb = FakeBBGen(COMPOSITION_GTEST_SUITE_WATERFALL,
                     GOOD_COMPOSITION_TEST_SUITES,
@@ -834,6 +912,14 @@ class UnitTest(unittest.TestCase):
     fbb.files['chromium.test.json'] = INSTRUMENTATION_TEST_OUTPUT
     fbb.check_output_file_consistency(verbose=True)
 
+  def test_instrumentation_tests_with_different_names(self):
+    fbb = FakeBBGen(FOO_INSTRUMENTATION_TEST_WATERFALL,
+                    INSTRUMENTATION_TESTS_WITH_DIFFERENT_NAMES,
+                    EMPTY_EXCEPTIONS)
+    fbb.files['chromium.test.json'] = \
+        INSTRUMENTATION_TEST_DIFFERENT_NAMES_OUTPUT
+    fbb.check_output_file_consistency(verbose=True)
+
   def test_ungenerated_output_files_are_caught(self):
     fbb = FakeBBGen(COMPOSITION_GTEST_SUITE_WATERFALL,
                     GOOD_COMPOSITION_TEST_SUITES,