Add support for PRECONDITION_FAILED status in WPTMetadataBuilder

There is one known precondition failure that is added to WPTOverrideExpectations.
Others will be added in a follow-up Cl.

Also expands unit tests to check that we're parsing trailing comments
into annotations properly.

Bug: 937369
Change-Id: I325272eee364b8cae9ef7739ff9333b829f9f041
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2088234
Commit-Queue: Luke Z <lpz@chromium.org>
Reviewed-by: Robert Ma <robertma@chromium.org>
Cr-Commit-Position: refs/heads/master@{#747269}

third_party/blink/tools/blinkpy/w3c/wpt_metadata_builder.py

@@ -25,20 +25,20 @@ _log = logging.getLogger(__name__)
 # The test has a harness error in its baseline file.
 HARNESS_ERROR = 1
 # The test has at least one failing subtest in its baseline file.
-SUBTEST_FAIL = 1 << 1
+SUBTEST_FAIL = 1 << 1  # 2
 # The test should be skipped
-SKIP_TEST = 1 << 2
+SKIP_TEST = 1 << 2  # 4
 # The test passes - this typically appears alongside another status indicating
 # a flaky test.
-TEST_PASS = 1 << 3
+TEST_PASS = 1 << 3  # 8
 # The test fails
-TEST_FAIL = 1 << 4
+TEST_FAIL = 1 << 4  # 16
 # The test times out
-TEST_TIMEOUT = 1 << 5
+TEST_TIMEOUT = 1 << 5  # 32
 # The test crashes
-TEST_CRASH = 1 << 6
-
-# Next status: 1 << 7
+TEST_CRASH = 1 << 6  # 64
+# The test failed a precondition assertion
+TEST_PRECONDITION_FAILED = 1 << 7  # 128
 
 
 class WPTMetadataBuilder(object):
@@ -92,6 +92,8 @@ class WPTMetadataBuilder(object):
             statuses.append("TIMEOUT")
         if test_status_bitmap & TEST_CRASH:
             statuses.append("CRASH")
+        if test_status_bitmap & TEST_PRECONDITION_FAILED:
+            statuses.append("PRECONDITION_FAILED")
 
         if statuses:
             result += "  expected: [%s]\n" % ", ".join(statuses)
@@ -183,6 +185,8 @@ class WPTMetadataBuilder(object):
         if annotations:
             if "wpt_subtest_failure" in annotations:
                 status_bitmap |= SUBTEST_FAIL
+            if "wpt_precondition_failed" in annotations:
+                status_bitmap |= TEST_PRECONDITION_FAILED
         # Update status bitmap for this test
         status_dict[test_name] |= status_bitmap
 

third_party/blink/tools/blinkpy/w3c/wpt_metadata_builder_unittest.py

@@ -11,10 +11,18 @@ from blinkpy.common.host_mock import MockHost
 from blinkpy.web_tests.models.test_expectations import TestExpectations
 from blinkpy.web_tests.port.factory_mock import MockPortFactory
 from blinkpy.w3c.wpt_manifest import BASE_MANIFEST_NAME
-from blinkpy.w3c.wpt_metadata_builder import WPTMetadataBuilder, HARNESS_ERROR, SUBTEST_FAIL, SKIP_TEST, TEST_PASS, TEST_FAIL
+from blinkpy.w3c.wpt_metadata_builder import (
+    WPTMetadataBuilder,
+    HARNESS_ERROR,
+    SKIP_TEST,
+    SUBTEST_FAIL,
+    TEST_FAIL,
+    TEST_PASS,
+    TEST_PRECONDITION_FAILED,
+)
 
 
-def _make_expectation(port, test_path, test_statuses, test_names=[]):
+def _make_expectation(port, test_path, test_statuses, test_names=[], trailing_comments=""):
     """Creates an expectation object for a single test or directory.
 
     Args:
@@ -23,13 +31,14 @@ def _make_expectation(port, test_path, test_statuses, test_names=[]):
         test_status: the statuses of the test
         test_names: a set of tests under the 'test_path'. Should be non-empty
             when the 'test_path' is a directory instead of a single test
+        trailing_comments: comments at the end of the expectation line.
 
     Returns:
         An expectation object with the given test and statuses.
     """
     expectation_dict = OrderedDict()
-    expectation_dict["expectations"] = ("# results: [ %s ]\n%s [ %s ]" % (
-        test_statuses, test_path, test_statuses))
+    expectation_dict["expectations"] = (
+        "# results: [ %s ]\n%s [ %s ]%s" % (test_statuses, test_path, test_statuses, trailing_comments))
 
     # When test_path is a dir, we expect test_names to be provided.
     is_dir = test_path.endswith('/')
@@ -284,3 +293,34 @@ class WPTMetadataBuilderTest(unittest.TestCase):
         self.assertEqual(
             "[multiglob.https.any.window.html]\n  blink_expect_any_subtest_status: True # wpt_metadata_builder.py\n",
             contents)
+
+    def test_precondition_failed(self):
+        """A WPT that fails a precondition."""
+        test_name = "external/wpt/test.html"
+        expectations = TestExpectations(self.port)
+        metadata_builder = WPTMetadataBuilder(expectations, self.port)
+        filename, contents = metadata_builder.get_metadata_filename_and_contents(test_name, TEST_PRECONDITION_FAILED)
+        self.assertEqual("test.html.ini", filename)
+        # The PASS and FAIL expectations fan out to also include OK and ERROR
+        # to support reftest/testharness test differences.
+        self.assertEqual("[test.html]\n  expected: [PRECONDITION_FAILED]\n", contents)
+
+    def test_parse_subtest_failure_annotation(self):
+        """Check that we parse the wpt_subtest_failure annotation correctly."""
+        test_name = "external/wpt/test.html"
+        expectations = _make_expectation(self.port, test_name, "PASS", trailing_comments=" # wpt_subtest_failure")
+        metadata_builder = WPTMetadataBuilder(expectations, self.port)
+        test_and_status_dict = metadata_builder.get_tests_needing_metadata()
+        self.assertEqual(1, len(test_and_status_dict))
+        self.assertTrue(test_name in test_and_status_dict)
+        self.assertEqual(TEST_PASS | SUBTEST_FAIL, test_and_status_dict[test_name])
+
+    def test_parse_precondition_failure_annotation(self):
+        """Check that we parse the wpt_precondition_failed annotation correctly."""
+        test_name = "external/wpt/test.html"
+        expectations = _make_expectation(self.port, test_name, "PASS", trailing_comments=" # wpt_precondition_failed")
+        metadata_builder = WPTMetadataBuilder(expectations, self.port)
+        test_and_status_dict = metadata_builder.get_tests_needing_metadata()
+        self.assertEqual(1, len(test_and_status_dict))
+        self.assertTrue(test_name in test_and_status_dict)
+        self.assertEqual(TEST_PASS | TEST_PRECONDITION_FAILED, test_and_status_dict[test_name])

third_party/blink/web_tests/WPTOverrideExpectations

@@ -211,6 +211,9 @@ external/wpt/upgrade-insecure-requests/gen/top.meta/upgrade/sharedworker-module/
 external/wpt/websockets/cookies/third-party-cookie-accepted.https.html [ Pass ] # wpt_subtest_failure
 external/wpt/webxr/events_session_squeeze.https.html [ Pass Failure ] # wpt_subtest_failure
 
+# A precondition failure
+external/wpt/infrastructure/expected-fail/precondition-in-promise.html [ Failure ] # wpt_precondition_failed
+
 # This is a mass dump of all the current WPT failures seen on Waterfall. They are untriaged.
 external/wpt/2dcontext/building-paths/canvas_complexshapes_arcto_001.htm [ Pass Failure ]
 external/wpt/2dcontext/building-paths/canvas_complexshapes_beziercurveto_001.htm [ Pass Failure ]