Adds webdriver_tests_suite to linux-blink-rel bot and changes test_importer to...

Adds webdriver_tests_suite to linux-blink-rel bot and changes test_importer to retrieve WebDriver expectations from the same bot instead of linux-rel.

Bug: 881513
Change-Id: I955a00e6d643a6d74628ad7d4f19f77f2090bf99
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1613374Reviewed-by: Robert Ma <robertma@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Commit-Queue: Robert Ma <robertma@chromium.org>
Cr-Commit-Position: refs/heads/master@{#669019}

testing/buildbot/chromium.fyi.json

@@ -11015,6 +11015,25 @@
   },
   "linux-blink-rel-dummy": {
     "isolated_scripts": [
+      {
+        "isolate_name": "webdriver_wpt_tests",
+        "merge": {
+          "args": [
+            "--verbose"
+          ],
+          "script": "//third_party/blink/tools/merge_web_test_results.py"
+        },
+        "name": "webdriver_tests_suite",
+        "swarming": {
+          "can_use_on_swarming_builders": true,
+          "dimension_sets": [
+            {
+              "os": "Ubuntu-14.04"
+            }
+          ],
+          "shards": 4
+        }
+      },
       {
         "args": [
           "--num-retries=3"
@@ -11035,7 +11054,7 @@
               "os": "Ubuntu-14.04"
             }
           ],
-          "shards": 20
+          "shards": 12
         }
       }
     ]

testing/buildbot/test_suites.pyl

@@ -2439,6 +2439,40 @@
       },
     },
 
+    'chromium_web_tests_and_wpt_webdriver_isolated_scripts': {
+      'webkit_layout_tests': {
+        # layout test failures are retried 3 times when '--test-list' is not
+        # passed, but 0 times when '--test-list' is passed. We want to always
+        # retry 3 times, so we explicitly specify it.
+        'args': [
+          '--num-retries=3',
+        ],
+        'isolate_name': 'blink_web_tests_exparchive',
+        'merge': {
+          'args': [
+            '--verbose',
+          ],
+          'script': '//third_party/blink/tools/merge_web_test_results.py',
+        },
+        'results_handler': 'layout tests',
+        'swarming': {
+          'shards': 12,
+        }
+      },
+      'webdriver_tests_suite': {
+        'isolate_name': 'webdriver_wpt_tests',
+         'merge': {
+          'args': [
+            '--verbose',
+          ],
+          'script': '//third_party/blink/tools/merge_web_test_results.py',
+        },
+        'swarming': {
+          'shards': 4,
+        },
+      },
+    },
+
     'chromium_webkit_isolated_scripts': {
       'webkit_layout_tests': {
         # layout test failures are retried 3 times when '--test-list' is not

testing/buildbot/waterfalls.pyl

@@ -1759,7 +1759,7 @@
       },
       'linux-blink-rel-dummy': {
         'test_suites': {
-          'isolated_scripts': 'chromium_webkit_isolated_scripts',
+          'isolated_scripts': 'chromium_web_tests_and_wpt_webdriver_isolated_scripts',
         },
         'swarming': {
           'dimension_sets': [
@@ -1767,7 +1767,6 @@
               'os': 'Ubuntu-14.04',
             },
           ],
-          'shards': 20,
         },
       },
       'linux-chromeos-code-coverage': {

third_party/blink/tools/blinkpy/common/config/builders.json

@@ -72,6 +72,8 @@
     "linux-blink-rel": {
         "port_name": "linux-trusty",
         "specifiers": ["Trusty", "Release"],
+        "master": "tryserver.blink",
+        "has_webdriver_tests": true,
         "is_try_builder": true
     },
     "mac10.10-blink-rel": {
@@ -108,12 +110,5 @@
         "port_name": "win-win10",
         "specifiers": ["Win10", "Release"],
         "is_try_builder": true
-    },
-    "linux-rel": {
-        "bucket": "luci.chromium.try",
-        "port_name": "linux-trusty",
-        "specifiers": ["Trusty", "Release"],
-        "master": "tryserver.chromium.linux",
-        "is_try_builder": false
     }
 }

third_party/blink/tools/blinkpy/common/net/buildbot.py

@@ -110,11 +110,6 @@ class BuildBot(object):
     def accumulated_results_url_base(self, builder_name):
         return self.builder_results_url_base(builder_name) + '/results/layout-test-results'
 
-    def fetch_full_results(self, build, full=False, master=''):
-        if master:
-            return self.fetch_webdriver_test_results(build, master)
-        return self.fetch_results(build, full)
-
     @memoized
     def fetch_results(self, build, full=False):
         """Returns a WebTestResults object for results from a given Build.

third_party/blink/tools/blinkpy/common/net/buildbot_mock.py

@@ -36,7 +36,9 @@ class MockBuildBot(BuildBot):
         super(MockBuildBot, self).__init__()
         self._canned_results = {}
         self._canned_retry_summary_json = {}
+        self._webdriver_results = {}
         self.fetched_builds = []
+        self.fetched_webdriver_builds = []
         self._layout_test_step_name = 'webkit_layout_tests (with patch)'
 
     def set_results(self, build, results):
@@ -46,6 +48,13 @@ class MockBuildBot(BuildBot):
         self.fetched_builds.append(build)
         return self._canned_results.get(build)
 
+    def set_webdriver_test_results(self, build, master, results):
+        self._webdriver_results[(build, master)] = results
+
+    def fetch_webdriver_test_results(self, build, master):
+        self.fetched_webdriver_builds.append((build, master))
+        return self._webdriver_results.get((build, master))
+
     def set_retry_sumary_json(self, build, content):
         self._canned_retry_summary_json[build] = content
 

third_party/blink/tools/blinkpy/common/net/buildbot_unittest.py

@@ -128,24 +128,26 @@ class BuilderTest(LoggingTestCase):
         buildbot = BuildBot()
         self.assertIsNone(buildbot.fetch_webdriver_test_results(
             Build('builder', None), 'bar'))
-        self.assertLog(['DEBUG: Builder name or build number or master is None\n'])
+        self.assertLog(
+            ['DEBUG: Builder name or build number or master is None\n'])
 
     def test_fetch_webdriver_results_without_master(self):
         buildbot = BuildBot()
         self.assertIsNone(buildbot.fetch_webdriver_test_results(
             Build('builder', 1), ''))
-        self.assertLog(['DEBUG: Builder name or build number or master is None\n'])
+        self.assertLog(
+            ['DEBUG: Builder name or build number or master is None\n'])
 
     def test_fetch_webdriver_test_results_with_no_results(self):
         buildbot = BuildBot()
         buildbot.web = MockWeb()
         results = buildbot.fetch_webdriver_test_results(
-            Build('linux-rel', 123), 'tryserver.chromium.linux')
+            Build('bar-rel', 123), 'foo.chrome')
         self.assertIsNone(results)
         self.assertLog([
             'DEBUG: Got 404 response from:\n'
             'https://test-results.appspot.com/testfile?buildnumber=123&'
-            'master=tryserver.chromium.linux&builder=linux-rel&'
+            'master=foo.chrome&builder=bar-rel&'
             'testtype=webdriver_tests_suite+%28with+patch%29&name=full_results.json\n'
         ])
 
@@ -153,48 +155,13 @@ class BuilderTest(LoggingTestCase):
         buildbot = BuildBot()
         buildbot.web = MockWeb(urls={
             'https://test-results.appspot.com/testfile?buildnumber=123&'
-            'master=tryserver.chromium.linux&builder=linux-rel&'
+            'master=foo.chrome&builder=bar-rel&'
             'testtype=webdriver_tests_suite+%28with+patch%29&'
             'name=full_results.json':
                 json.dumps({'passed': True}),
         })
         results = buildbot.fetch_webdriver_test_results(
-            Build('linux-rel', 123), 'tryserver.chromium.linux')
-        self.assertEqual(results._results, {  # pylint: disable=protected-access
-            'passed': True
-        })
-        self.assertLog([])
-
-    def test_fetch_full_results_webdriver(self):
-        buildbot = BuildBot()
-        buildbot.web = MockWeb(urls={
-            'https://test-results.appspot.com/testfile?buildnumber=321&'
-            'master=tryserver.chromium.linux&builder=foo&'
-            'testtype=webdriver_tests_suite+%28with+patch%29&'
-            'name=full_results.json':
-                json.dumps({'passed': True}),
-        })
-        results = buildbot.fetch_full_results(
-            Build('foo', 321), master='tryserver.chromium.linux')
-        self.assertEqual(results._results, {  # pylint: disable=protected-access
-            'passed': True
-        })
-        self.assertLog([])
-
-    def test_fetch_full_results_web_tests(self):
-        buildbot = BuildBot()
-        buildbot.web = MockWeb(urls={
-            'https://test-results.appspot.com/testfile?buildnumber=123&'
-            'callback=ADD_RESULTS&builder=builder&name=full_results.json':
-                'ADD_RESULTS(%s);' % (json.dumps(
-                    [{"TestType": "webkit_layout_tests on Intel GPU (with patch)"},
-                     {"TestType": "base_unittests (with patch)"}])),
-            'https://test-results.appspot.com/data/layout_results/builder/123/'
-            'webkit_layout_tests%20on%20Intel%20GPU%20%28with%20patch%29/'
-            'layout-test-results/failing_results.json':
-                json.dumps({'passed': True}),
-        })
-        results = buildbot.fetch_full_results(Build('builder', 123), master='')
+            Build('bar-rel', 123), 'foo.chrome')
         self.assertEqual(results._results, {  # pylint: disable=protected-access
             'passed': True
         })

third_party/blink/tools/blinkpy/w3c/wpt_expectations_updater.py

@@ -131,13 +131,26 @@ class WPTExpectationsUpdater(object):
             # All tests passed, so there should be no failing results.
             return {}
 
-        master = self.host.builders.master_for_builder(build.builder_name)
-        web_test_results = self.host.buildbot.fetch_full_results(build, master=master)
-        if web_test_results is None:
+        test_result_list = [self.host.buildbot.fetch_results(build)]
+        has_webdriver_tests = self.host.builders.has_webdriver_tests_for_builder(
+            build.builder_name)
+        if has_webdriver_tests:
+            master = self.host.builders.master_for_builder(
+                build.builder_name)
+            test_result_list.append(
+                self.host.buildbot.fetch_webdriver_test_results(build, master))
+
+        test_result_list = filter(None, test_result_list)
+        if not test_result_list:
             _log.warning('No results for build %s', build)
             self.ports_with_no_results.add(self.port_name(build))
             return {}
-        failing_test_results = [result for result in web_test_results.didnt_run_as_expected_results() if not result.did_pass()]
+
+        failing_test_results = []
+        for test_result in test_result_list:
+            failing_test_results += [
+                result for result in test_result.didnt_run_as_expected_results() if not result.did_pass()]
+
         return self.generate_results_dict(self.port_name(build), failing_test_results)
 
     @memoized

third_party/blink/tools/blinkpy/w3c/wpt_expectations_updater_unittest.py

@@ -41,6 +41,8 @@ class WPTExpectationsUpdaterTest(LoggingTestCase):
             'MOCK Try Trusty': {
                 'port_name': 'test-linux-trusty',
                 'specifiers': ['Trusty', 'Release'],
+                'master': 'tryserver.blink',
+                'has_webdriver_tests': True,
                 'is_try_builder': True,
             },
             'MOCK Try Precise': {
@@ -229,6 +231,63 @@ class WPTExpectationsUpdaterTest(LoggingTestCase):
         results_dict = updater.get_failing_results_dict(Build('MOCK Try Mac10.10', 123))
         self.assertEqual(results_dict, {})
 
+    def test_get_failing_results_dict_webdriver_failing_results_(self):
+        host = self.mock_host()
+        host.buildbot.set_results(Build('MOCK Try Trusty', 123), WebTestResults({
+            'tests': {
+                'external': {
+                    'wpt': {
+                        'x': {
+                            'failing-test.html': {
+                                'expected': 'PASS',
+                                'actual': 'IMAGE',
+                                'is_unexpected': True,
+                            },
+                        },
+                    },
+                },
+            },
+        }))
+
+        host.buildbot.set_webdriver_test_results(Build('MOCK Try Trusty', 123), "tryserver.blink", WebTestResults({
+            'tests': {
+                'external': {
+                    'wpt': {
+                        'y': {
+                            'webdriver-fail.html': {
+                                'expected': 'PASS',
+                                'actual': 'FAIL',
+                                'is_unexpected': True,
+                            },
+                        },
+                    },
+                },
+            },
+        }))
+        updater = WPTExpectationsUpdater(host)
+        results_dict = updater.get_failing_results_dict(
+            Build('MOCK Try Trusty', 123))
+
+        self.assertEqual(len(results_dict.keys()), 2)
+        self.assertEqual(
+            results_dict,
+            {
+                'external/wpt/x/failing-test.html': {
+                    'test-linux-trusty': SimpleTestResult(
+                        actual='IMAGE',
+                        expected='PASS',
+                        bug='crbug.com/626703',
+                    ),
+                },
+                'external/wpt/y/webdriver-fail.html': {
+                    'test-linux-trusty': SimpleTestResult(
+                        actual='FAIL',
+                        expected='PASS',
+                        bug='crbug.com/626703',
+                    ),
+                },
+            })
+
     def test_merge_same_valued_keys_all_match(self):
         updater = WPTExpectationsUpdater(self.mock_host())
         self.assertEqual(

third_party/blink/tools/blinkpy/web_tests/builder_list.py

@@ -51,6 +51,7 @@ class BuilderList(object):
             "is_try_builder": Whether the builder is a try bot.
             "master": The master name of the builder. It is deprecated, but still required
                 by test-results.appspot.com API."
+            "has_webdriver_tests": Whether webdriver_tests_suite runs on this builder.
 
         Possible refactoring note: Potentially, it might make sense to use
         blinkpy.common.buildbot.Builder and add port_name and specifiers
@@ -87,6 +88,9 @@ class BuilderList(object):
     def master_for_builder(self, builder_name):
         return self._builders[builder_name].get('master', '')
 
+    def has_webdriver_tests_for_builder(self, builder_name):
+        return self._builders[builder_name].get('has_webdriver_tests')
+
     def port_name_for_builder_name(self, builder_name):
         return self._builders[builder_name]['port_name']
 

third_party/blink/tools/blinkpy/web_tests/builder_list_unittest.py

@@ -44,7 +44,7 @@ class BuilderListTest(unittest.TestCase):
             'Try B': {'port_name': 'port-b', 'specifiers': ['B', 'Release'], 'is_try_builder': True},
             'CQ Try A': {'bucket': 'bucket.a', 'port_name': 'port-a', 'specifiers': ['A', 'Release'], 'is_try_builder': True},
             'CQ Try B': {'bucket': 'bucket.b', 'port_name': 'port-b', 'specifiers': ['B', 'Release'], 'is_try_builder': True},
-            'CQ Try C': {'bucket': 'bucket.c', 'port_name': 'port-c', 'specifiers': ['c', 'Release'], 'is_try_builder': True, 'master': "luci"},
+            'CQ Try C': {'bucket': 'bucket.c', 'port_name': 'port-c', 'specifiers': ['c', 'Release'], 'is_try_builder': True, 'master': "luci", 'has_webdriver_tests': True},
         })
 
     def test_constructor_validates_list(self):
@@ -86,6 +86,16 @@ class BuilderListTest(unittest.TestCase):
         builders = self.sample_builder_list()
         self.assertEqual('luci', builders.master_for_builder('CQ Try C'))
 
+    def test_has_webdriver_tests_for_builder_default_value(self):
+        builders = self.sample_builder_list()
+        self.assertEqual(
+            None, builders.has_webdriver_tests_for_builder('Try A'))
+
+    def test_has_webdriver_tests_for_builder_configured_value(self):
+        builders = self.sample_builder_list()
+        self.assertEqual(
+            True, builders.has_webdriver_tests_for_builder('CQ Try C'))
+
     def test_port_name_for_builder_name(self):
         builders = self.sample_builder_list()
         self.assertEqual('port-b', builders.port_name_for_builder_name('Blink B'))