Teach import notifier about harness errors

Bug: 1092520
Change-Id: Ifd801925303ffbf2694a9ba4a1f96a4e08cf058e
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2299520Reviewed-by: Robert Ma <robertma@chromium.org>
Commit-Queue: Stephen McGruer <smcgruer@chromium.org>
Cr-Commit-Position: refs/heads/master@{#788718}

third_party/blink/tools/blinkpy/w3c/import_notifier.py

@@ -129,14 +129,30 @@ class ImportNotifier(object):
                             gerrit_url_with_ps=gerrit_url_with_ps))
 
     def more_failures_in_baseline(self, baseline):
+        """Determines if a testharness.js baseline file has new failures.
+
+        The file is assumed to have been modified in the current git checkout,
+        and so has a diff we can parse.
+
+        We recognize two types of failures: FAIL lines, which are output for a
+        specific subtest failing, and harness errors, which indicate an uncaught
+        error in the test. Increasing numbers of either are considered new
+        failures - this includes going from FAIL to error or vice-versa.
+        """
+
         diff = self.git.run(['diff', '-U0', 'origin/master', '--', baseline])
         delta_failures = 0
+        delta_harness_errors = 0
         for line in diff.splitlines():
             if line.startswith('+FAIL'):
                 delta_failures += 1
             if line.startswith('-FAIL'):
                 delta_failures -= 1
-        return delta_failures > 0
+            if line.startswith('+Harness Error.'):
+                delta_harness_errors += 1
+            if line.startswith('-Harness Error.'):
+                delta_harness_errors -= 1
+        return delta_failures > 0 or delta_harness_errors > 0
 
     def examine_new_test_expectations(self, test_expectations):
         """Examines new test expectations to find new failures.

third_party/blink/tools/blinkpy/w3c/import_notifier_unittest.py

@@ -98,6 +98,66 @@ class ImportNotifierTest(unittest.TestCase):
         self.assertFalse(
             self.notifier.more_failures_in_baseline('foo-expected.txt'))
 
+    def test_more_failures_in_baseline_new_error(self):
+        executive = mock_git_commands({
+            'diff': ('diff --git a/foo-expected.txt b/foo-expected.txt\n'
+                     '--- a/foo-expected.txt\n'
+                     '+++ b/foo-expected.txt\n'
+                     '-PASS an existing pass\n'
+                     '+Harness Error. harness_status.status = 1 , harness_status.message = bad\n')
+        })
+        self.notifier.git = MockGit(executive=executive)
+        self.assertTrue(
+            self.notifier.more_failures_in_baseline('foo-expected.txt'))
+
+    def test_more_failures_in_baseline_remove_error(self):
+        executive = mock_git_commands({
+            'diff': ('diff --git a/foo-expected.txt b/foo-expected.txt\n'
+                     '--- a/foo-expected.txt\n'
+                     '+++ b/foo-expected.txt\n'
+                     '-Harness Error. harness_status.status = 1 , harness_status.message = bad\n'
+                     '+PASS a new pass\n')
+        })
+        self.notifier.git = MockGit(executive=executive)
+        self.assertFalse(
+            self.notifier.more_failures_in_baseline('foo-expected.txt'))
+
+    def test_more_failures_in_baseline_changing_error(self):
+        executive = mock_git_commands({
+            'diff': ('diff --git a/foo-expected.txt b/foo-expected.txt\n'
+                     '--- a/foo-expected.txt\n'
+                     '+++ b/foo-expected.txt\n'
+                     '-Harness Error. harness_status.status = 1 , harness_status.message = bad\n'
+                     '+Harness Error. new text, still an error\n')
+        })
+        self.notifier.git = MockGit(executive=executive)
+        self.assertFalse(
+            self.notifier.more_failures_in_baseline('foo-expected.txt'))
+
+    def test_more_failures_in_baseline_fail_to_error(self):
+        executive = mock_git_commands({
+            'diff': ('diff --git a/foo-expected.txt b/foo-expected.txt\n'
+                     '--- a/foo-expected.txt\n'
+                     '+++ b/foo-expected.txt\n'
+                     '-FAIL a previous failure\n'
+                     '+Harness Error. harness_status.status = 1 , harness_status.message = bad\n')
+        })
+        self.notifier.git = MockGit(executive=executive)
+        self.assertTrue(
+            self.notifier.more_failures_in_baseline('foo-expected.txt'))
+
+    def test_more_failures_in_baseline_error_to_fail(self):
+        executive = mock_git_commands({
+            'diff': ('diff --git a/foo-expected.txt b/foo-expected.txt\n'
+                     '--- a/foo-expected.txt\n'
+                     '+++ b/foo-expected.txt\n'
+                     '-Harness Error. harness_status.status = 1 , harness_status.message = bad\n'
+                     '+FAIL a new failure\n')
+        })
+        self.notifier.git = MockGit(executive=executive)
+        self.assertTrue(
+            self.notifier.more_failures_in_baseline('foo-expected.txt'))
+
     def test_examine_baseline_changes(self):
         self.host.filesystem.write_text_file(
             MOCK_WEB_TESTS + 'external/wpt/foo/OWNERS', 'test@chromium.org')