In wpt-importer, look only at the latest CQ job results.

Previously, the importer would sometimes abandon a potential import
even when it passed CQ if there were any failing (flaky) jobs.

This CL should make it so that when deciding whether the CQ passed,
importer looks only at the latest builds for each builder.

Bug: 739119
Change-Id: I126e33191ddc24797501f97ff0834e3f9e45d4bc
Reviewed-on: https://chromium-review.googlesource.com/578141
Commit-Queue: Quinten Yearsley <qyearsley@chromium.org>
Reviewed-by: Jeff Carpenter <jeffcarp@chromium.org>
Cr-Commit-Position: refs/heads/master@{#488453}

third_party/WebKit/Tools/Scripts/webkitpy/common/net/git_cl.py

@@ -82,7 +82,7 @@ class GitCL(object):
             self._host.sleep(poll_delay_seconds)
             try_results = self.try_job_results()
             _log.debug('Fetched try results: %s', try_results)
-            if self.all_jobs_finished(try_results):
+            if self.all_finished(try_results):
                 self._host.print_('All jobs finished.')
                 return try_results
             self._host.print_('Waiting. %d seconds passed.' % (self._host.time() - start))
@@ -91,7 +91,7 @@ class GitCL(object):
         return None
 
     def latest_try_jobs(self, builder_names=None):
-        """Returns a dict of Builds to results for the latest try jobs.
+        """Fetches a dict of Build to TryJobStatus for the latest try jobs.
 
         This includes jobs that are not yet finished and builds with infra
         failures, so if a build is in this list, that doesn't guarantee that
@@ -101,14 +101,15 @@ class GitCL(object):
             builder_names: Optional list of builders used to filter results.
 
         Returns:
-            A dict mapping Build objects to TryJobSTatus objects, with
+            A dict mapping Build objects to TryJobStatus objects, with
             only the latest jobs included.
         """
-        try_results = self.try_job_results(builder_names)
-        builds = try_results.keys()
-        if builder_names:
-            builds = [b for b in builds if b.builder_name in builder_names]
-        latest_builds = filter_latest_builds(builds)
+        return self.filter_latest(self.try_job_results(builder_names))
+
+    @staticmethod
+    def filter_latest(try_results):
+        """Returns the latest entries from from a Build to TryJobStatus dict."""
+        latest_builds = filter_latest_builds(try_results.keys())
         return {b: s for b, s in try_results.items() if b in latest_builds}
 
     def try_job_results(self, builder_names=None):
@@ -158,9 +159,14 @@ class GitCL(object):
         return TryJobStatus(result_dict['status'], result_dict['result'])
 
     @staticmethod
-    def all_jobs_finished(try_results):
+    def all_finished(try_results):
         return all(s.status == 'COMPLETED' for s in try_results.values())
 
     @staticmethod
-    def has_failing_try_results(try_results):
+    def all_success(try_results):
+        return all(s.status == 'COMPLETED' and s.result == 'SUCCESS'
+                   for s in try_results.values())
+
+    @staticmethod
+    def some_failed(try_results):
         return any(s.result == 'FAILURE' for s in try_results.values())

third_party/WebKit/Tools/Scripts/webkitpy/common/net/git_cl_mock.py

@@ -2,7 +2,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-from webkitpy.common.net.buildbot import filter_latest_builds
+from webkitpy.common.net.git_cl import GitCL
 
 
 # TODO(qyearsley): Use this class in wpt_expectations_updater_unittest and rebaseline_cl_unittest.
@@ -27,20 +27,27 @@ class MockGitCL(object):
     def get_issue_number(self):
         return '1234'
 
+    def try_job_results(self, **_):
+        return self._results
+
     def wait_for_try_jobs(self, **_):
         return self._results
 
     def latest_try_jobs(self, **_):
-        latest_builds = filter_latest_builds(self._results)
-        return {b: s for b, s in self._results.items() if b in latest_builds}
+        return self.filter_latest(self._results)
 
-    def try_job_results(self, **_):
-        return self._results
+    @staticmethod
+    def filter_latest(try_results):
+        return GitCL.filter_latest(try_results)
+
+    @staticmethod
+    def all_finished(try_results):
+        return GitCL.all_finished(try_results)
 
     @staticmethod
-    def all_jobs_finished(results):
-        return all(s.status == 'COMPLETED' for s in results.values())
+    def all_success(try_results):
+        return GitCL.all_success(try_results)
 
     @staticmethod
-    def has_failing_try_results(results):
-        return any(s.result == 'FAILURE' for s in results.values())
+    def some_failed(try_results):
+        return GitCL.some_failed(try_results)

third_party/WebKit/Tools/Scripts/webkitpy/common/net/git_cl_unittest.py

@@ -90,19 +90,33 @@ class GitCLTest(unittest.TestCase):
             'All jobs finished.\n')
 
     def test_has_failing_try_results_empty(self):
-        self.assertFalse(GitCL.has_failing_try_results({}))
+        self.assertFalse(GitCL.some_failed({}))
 
     def test_has_failing_try_results_only_success_and_started(self):
-        self.assertFalse(GitCL.has_failing_try_results({
+        self.assertFalse(GitCL.some_failed({
             Build('some-builder', 90): TryJobStatus('COMPLETED', 'SUCCESS'),
             Build('some-builder', 100): TryJobStatus('STARTED'),
         }))
 
     def test_has_failing_try_results_with_failing_results(self):
-        self.assertTrue(GitCL.has_failing_try_results({
+        self.assertTrue(GitCL.some_failed({
             Build('some-builder', 1): TryJobStatus('COMPLETED', 'FAILURE'),
         }))
 
+    def test_all_success_empty(self):
+        self.assertTrue(GitCL.all_success({}))
+
+    def test_all_success_true(self):
+        self.assertTrue(GitCL.all_success({
+            Build('some-builder', 1): TryJobStatus('COMPLETED', 'SUCCESS'),
+        }))
+
+    def test_all_success_with_started_build(self):
+        self.assertFalse(GitCL.all_success({
+            Build('some-builder', 1): TryJobStatus('COMPLETED', 'SUCCESS'),
+            Build('some-builder', 2): TryJobStatus('STARTED'),
+        }))
+
     def test_latest_try_builds(self):
         git_cl = GitCL(MockHost())
         git_cl.fetch_raw_try_job_results = lambda: [
@@ -177,6 +191,20 @@ class GitCLTest(unittest.TestCase):
                 Build('builder-b', 200): TryJobStatus('COMPLETED', 'FAILURE'),
             })
 
+    def test_filter_latest(self):
+        try_job_results = {
+            Build('builder-a', 100): TryJobStatus('COMPLETED', 'FAILURE'),
+            Build('builder-a', 200): TryJobStatus('COMPLETED', 'SUCCESS'),
+            Build('builder-b', 50): TryJobStatus('SCHEDULED'),
+        }
+        self.assertEqual(
+            GitCL.filter_latest(try_job_results),
+            {
+                Build('builder-a', 200): TryJobStatus('COMPLETED', 'SUCCESS'),
+                Build('builder-b', 50): TryJobStatus('SCHEDULED'),
+            })
+
+
     def test_try_job_results_with_task_id_in_url(self):
         git_cl = GitCL(MockHost())
         git_cl.fetch_raw_try_job_results = lambda: [

third_party/WebKit/Tools/Scripts/webkitpy/w3c/test_importer.py

@@ -63,6 +63,7 @@ class TestImporter(object):
         self.git_cl = GitCL(self.host, auth_refresh_token_json=options.auth_refresh_token_json)
 
         _log.debug('Noting the current Chromium commit.')
+        # TODO(qyearsley): Use Git (self.host.git) to run git commands.
         _, show_ref_output = self.run(['git', 'show-ref', 'HEAD'])
         chromium_commit = show_ref_output.split()[0]
 
@@ -90,8 +91,6 @@ class TestImporter(object):
             _log.info('Checking out %s', options.revision)
             self.run(['git', 'checkout', options.revision], cwd=local_wpt.path)
 
-        self.run(['git', 'submodule', 'update', '--init', '--recursive'], cwd=local_wpt.path)
-
         _log.info('Noting the revision we are importing.')
         _, show_ref_output = self.run(['git', 'show-ref', 'origin/master'], cwd=local_wpt.path)
         import_commit = 'wpt@%s' % show_ref_output.split()[0]
@@ -163,7 +162,7 @@ class TestImporter(object):
             self.git_cl.run(['set-close'])
             return False
 
-        if try_results and self.git_cl.has_failing_try_results(try_results):
+        if try_results and self.git_cl.some_failed(try_results):
             self.fetch_new_expectations_and_baselines()
             if self.host.git().has_working_directory_changes():
                 message = 'Update test expectations and baselines.'
@@ -178,14 +177,14 @@ class TestImporter(object):
         try_results = self.git_cl.wait_for_try_jobs(
             poll_delay_seconds=POLL_DELAY_SECONDS,
             timeout_seconds=TIMEOUT_SECONDS)
+        try_results = self.git_cl.filter_latest(try_results)
 
         if not try_results:
             self.git_cl.run(['set-close'])
             _log.error('No CQ try job results, aborting.')
             return False
 
-        # TODO(qyearsley): Change this to look only at the latest try jobs; crbug.com/739119
-        if try_results and all(s == TryJobStatus('COMPLETED', 'SUCCESS') for _, s in try_results.iteritems()):
+        if try_results and self.git_cl.all_success(try_results):
             _log.info('CQ appears to have passed; trying to commit.')
             self.git_cl.run(['upload', '-f', '--send-mail'])  # Turn off WIP mode.
             self.git_cl.run(['set-commit'])

third_party/WebKit/Tools/Scripts/webkitpy/w3c/test_importer_unittest.py

@@ -87,7 +87,7 @@ class TestImporterTest(LoggingTestCase):
             '/mock-checkout/third_party/WebKit/LayoutTests/W3CImportExpectations', '')
         importer = TestImporter(host)
         importer.git_cl = MockGitCL(host, results={
-            Build('builder-a', 123): TryJobStatus('COMPLETED', 'SUCCESS')
+            Build('builder-a', 123): TryJobStatus('COMPLETED', 'SUCCESS'),
         })
         success = importer.update_expectations_for_cl()
         self.assertTrue(success)
@@ -98,7 +98,7 @@ class TestImporterTest(LoggingTestCase):
             '/mock-checkout/third_party/WebKit/LayoutTests/W3CImportExpectations', '')
         importer = TestImporter(host)
         importer.git_cl = MockGitCL(host, results={
-            Build('builder-a', 123): TryJobStatus('COMPLETED', 'FAILURE')
+            Build('builder-a', 123): TryJobStatus('COMPLETED', 'FAILURE'),
         })
         importer.fetch_new_expectations_and_baselines = lambda: None
         success = importer.update_expectations_for_cl()
@@ -112,8 +112,10 @@ class TestImporterTest(LoggingTestCase):
         host.filesystem.write_text_file(
             '/mock-checkout/third_party/WebKit/LayoutTests/W3CImportExpectations', '')
         importer = TestImporter(host)
+        # Only the latest job for each builder is counted.
         importer.git_cl = MockGitCL(host, results={
-            Build('builder-a', 123): TryJobStatus('COMPLETED', 'SUCCESS')
+            Build('builder-a', 120): TryJobStatus('COMPLETED', 'FAILURE'),
+            Build('builder-a', 123): TryJobStatus('COMPLETED', 'SUCCESS'),
         })
         success = importer.run_commit_queue_for_cl()
         self.assertTrue(success)
@@ -134,7 +136,9 @@ class TestImporterTest(LoggingTestCase):
             '/mock-checkout/third_party/WebKit/LayoutTests/W3CImportExpectations', '')
         importer = TestImporter(host)
         importer.git_cl = MockGitCL(host, results={
-            Build('builder-a', 123): TryJobStatus('COMPLETED', 'FAILURE')
+            Build('builder-a', 120): TryJobStatus('COMPLETED', 'SUCCESS'),
+            Build('builder-a', 123): TryJobStatus('COMPLETED', 'FAILURE'),
+            Build('builder-b', 200): TryJobStatus('COMPLETED', 'SUCCESS'),
         })
         importer.fetch_new_expectations_and_baselines = lambda: None
         success = importer.run_commit_queue_for_cl()