Add support to wpt_metadata_builder for using checked-in wpt metadata.

This also includes a generated set of metadata for the payments API.

Bug: 1116091
Change-Id: Ie73e27eb79c3dbd7b9512418f2f3661ae5fe73dc
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2327138
Commit-Queue: Luke Z <lpz@chromium.org>
Reviewed-by: Robert Ma <robertma@chromium.org>
Reviewed-by: Stephen McGruer <smcgruer@chromium.org>
Cr-Commit-Position: refs/heads/master@{#799661}

testing/scripts/run_wpt_tests.py

@@ -21,7 +21,8 @@ import sys
 import common
 import wpt_common
 
-WPT_METADATA_DIR = "../../wpt_expectations_metadata/"
+WPT_CHECKED_IN_METADATA_DIR = "../../third_party/blink/web_tests/external/wpt"
+WPT_METADATA_OUTPUT_DIR = "../../wpt_expectations_metadata/"
 WPT_OVERRIDE_EXPECTATIONS_PATH = (
     "../../third_party/blink/web_tests/WPTOverrideExpectations")
 
@@ -57,7 +58,7 @@ class WPTTestAdapter(wpt_common.BaseWptScriptAdapter):
             # those that had non-zero exit codes).
             #"--no-fail-on-unexpected",
             "--metadata",
-            WPT_METADATA_DIR,
+            WPT_METADATA_OUTPUT_DIR,
             # By specifying metadata above, WPT will try to find manifest in the
             # metadata directory. So here we point it back to the correct path
             # for the manifest.
@@ -85,9 +86,11 @@ def main():
         sys.executable,
         os.path.join(wpt_common.BLINK_TOOLS_DIR, 'build_wpt_metadata.py'),
         "--metadata-output-dir",
-        WPT_METADATA_DIR,
+        WPT_METADATA_OUTPUT_DIR,
         "--additional-expectations",
-        WPT_OVERRIDE_EXPECTATIONS_PATH
+        WPT_OVERRIDE_EXPECTATIONS_PATH,
+        "--checked-in-metadata-dir",
+        WPT_CHECKED_IN_METADATA_DIR
     ])
 
     adapter = WPTTestAdapter()

third_party/blink/tools/blinkpy/w3c/wpt_metadata_builder.py

@@ -10,12 +10,13 @@ running the WPT test suite.
 """
 
 import argparse
+import fnmatch
 import logging
 import os
 import re
 
+from blinkpy.common.system.filesystem import FileSystem
 from blinkpy.common.system.log_utils import configure_logging
-from blinkpy.web_tests.models import test_expectations
 from blinkpy.web_tests.models.typ_types import ResultType
 from collections import defaultdict
 
@@ -50,8 +51,11 @@ class WPTMetadataBuilder(object):
         """
         self.expectations = expectations
         self.port = port
+        # TODO(lpz): Use self.fs everywhere in this class and add tests
+        self.fs = FileSystem()
         self.wpt_manifest = self.port.wpt_manifest("external/wpt")
         self.metadata_output_dir = ""
+        self.checked_in_metadata_dir = ""
 
     def run(self, args=None):
         """Main entry point to parse flags and execute the script."""
@@ -59,6 +63,11 @@ class WPTMetadataBuilder(object):
         parser.add_argument(
             "--metadata-output-dir",
             help="The directory to output the metadata files into.")
+        parser.add_argument(
+            "--checked-in-metadata-dir",
+            help="Root directory of any checked-in WPT metadata files to use. "
+            "If set, these files will take precedence over legacy expectations "
+            "and baselines when both exist for a test.")
         parser.add_argument(
             '-v',
             '--verbose',
@@ -70,6 +79,7 @@ class WPTMetadataBuilder(object):
         configure_logging(logging_level=log_level, include_time=True)
 
         self.metadata_output_dir = args.metadata_output_dir
+        self.checked_in_metadata_dir = args.checked_in_metadata_dir
         self._build_metadata_and_write()
 
         return 0
@@ -121,12 +131,40 @@ class WPTMetadataBuilder(object):
                 continue
             self._write_to_file(filename, file_contents)
 
+        if self.checked_in_metadata_dir and os.path.exists(
+                self.checked_in_metadata_dir):
+            _log.info("Copying checked-in WPT metadata on top of translated "
+                      "files.")
+            self._copy_checked_in_metadata()
+        else:
+            _log.warning("Not using checked-in WPT metadata, path is empty or "
+                         "does not exist: %s" % self.checked_in_metadata_dir)
+
         # Finally, output a stamp file with the same name as the output
         # directory. The stamp file is empty, it's only used for its mtime.
         # This makes the GN build system happy (see crbug.com/995112).
         with open(self.metadata_output_dir + ".stamp", "w"):
             pass
 
+    def _copy_checked_in_metadata(self):
+        """Copies checked-in metadata files to the metadata output directory."""
+        for filename in self.fs.files_under(self.checked_in_metadata_dir):
+            # We match any .ini files in the path. This will find .ini files
+            # other than just metadata (such as tox.ini), but that is ok
+            # since wptrunner will just ignore those.
+            if not fnmatch.fnmatch(filename, "*.ini"):
+                continue
+
+            # Found a checked-in .ini file. Copy it to the metadata output
+            # directory in the same sub-path as where it is checked in.
+            # So /checked/in/a/b/c.ini goes to /metadata/out/a/b/c.ini
+            output_path = filename.replace(self.checked_in_metadata_dir,
+                                           self.metadata_output_dir)
+            if not self.fs.exists(self.fs.dirname(output_path)):
+                self.fs.maybe_make_directory(self.fs.dirname(output_path))
+            _log.debug("Copying %s to %s" % (filename, output_path))
+            self.fs.copyfile(filename, output_path)
+
     def _write_to_file(self, filename, file_contents):
         # Write the contents to the file name
         if not os.path.exists(os.path.dirname(filename)):

third_party/blink/tools/blinkpy/w3c/wpt_metadata_builder_unittest.py

@@ -8,6 +8,7 @@ import os
 import unittest
 
 from blinkpy.common.host_mock import MockHost
+from blinkpy.common.system.filesystem_mock import MockFileSystem
 from blinkpy.web_tests.models.test_expectations import TestExpectations
 from blinkpy.web_tests.port.factory_mock import MockPortFactory
 from blinkpy.w3c.wpt_manifest import BASE_MANIFEST_NAME
@@ -412,3 +413,42 @@ class WPTMetadataBuilderTest(unittest.TestCase):
             "test.worker.html?variant=abc",
             mb._metadata_inline_test_name_from_test_name(
                 "dir/test.worker.html?variant=abc"))
+
+    def test_copy_checked_in_metadata(self):
+        # Ensure that ini metadata files are copied from the checked-in dir to
+        # the output dir as expected.
+        expectations = TestExpectations(self.port)
+        mb = WPTMetadataBuilder(expectations, self.port)
+        # Set the metadata builder to use mock filesystem populated with some
+        # test data
+        mb.checked_in_metadata_dir = "src"
+        mb.metadata_output_dir = "out"
+        mock_checked_in_files = {
+            "src/a/b/c.html": "",
+            "src/a/b/c.html.ini": "",
+            "src/a/d/e.html": "",
+            "src/a/d/e.html.ini": "checked-in",
+            "src/a/tox.ini": "",
+
+            # Put one duplicate file in the output directory to simulate a test
+            # with both legacy expectations and checked-in metadata
+            "out/a/d/e.html.ini": "legacy",
+        }
+        mb.fs = MockFileSystem(files=mock_checked_in_files)
+
+        # Ensure that the duplicate file starts out with the legacy content.
+        duplicate_ini_file = "out/a/d/e.html.ini"
+        self.assertEqual("legacy", mb.fs.read_text_file(duplicate_ini_file))
+
+        mb._copy_checked_in_metadata()
+
+        # Ensure only the ini files are copied, not the tests
+        self.assertEqual(3, len(mb.fs.written_files))
+        self.assertTrue("out/a/b/c.html.ini" in mb.fs.written_files)
+        self.assertTrue("out/a/d/e.html.ini" in mb.fs.written_files)
+        self.assertTrue("out/a/tox.ini" in mb.fs.written_files)
+
+        # Also ensure that the content of the duplicate file was overwritten
+        # with the checked-in contents.
+        self.assertEqual("checked-in",
+                         mb.fs.read_text_file(duplicate_ini_file))

third_party/blink/web_tests/external/wpt/payment-handler/can-make-payment-event.https.html.ini

+[can-make-payment-event.https.html]
+  [If CanMakePaymentEvent.respondWith(Promise.resolve(true)) is called, then the payment method is supported.]
+    expected: FAIL
+
+  [If an app supports "basic-card" in general and that's what merchant requests as well, then capability filtering should make the app available for use. CanMakePaymentEvent should not be fired for "basic-card".]
+    expected: FAIL
+
+  [If CanMakePaymentEvent.respondWith(false) is called, then the payment method is not supported.]
+    expected: FAIL
+
+  [If CanMakePaymentEvent.respondWith(Promise.reject(error)) is called, then the payment method is not supported.]
+    expected: FAIL
+
+  [If CanMakePaymentEvent.respondWith(true) is called, then the payment method is supported.]
+    expected: FAIL
+
+  [If an app has the exact "basic-card" capabilities that a merchant requested, capability filtering should make the app available for use. CanMakePaymentEvent should not be fired for "basic-card".]
+    expected: FAIL
+
+  [If a payment handler is not installed, then the payment method is not supported.]
+    expected: FAIL
+
+  [If CanMakePaymentEvent.respondWith(Promise.resolve(false)) is called, then the payment method is not supported.]
+    expected: FAIL
+
+  [If an app has less specific "basic-card" capabilites than merchant's request, capability filtering should not make the app available for use. CanMakePaymentEvent should not be fired for "basic-card". ]
+    expected: FAIL
+
+  [If an app has more specific "basic-card" capabilities than merchant's request, capability filtering should make the app available for use. CanMakePaymentEvent should not be fired for "basic-card".]
+    expected: FAIL
+

third_party/blink/web_tests/external/wpt/payment-handler/idlharness.https.any.js.ini

+[idlharness.https.any.html]
+
+[idlharness.https.any.worker.html]
+  [PaymentManager interface: attribute userHint]
+    expected: FAIL
+
+  [PaymentManager interface: existence and properties of interface prototype object's @@unscopables property]
+    expected: FAIL
+
+  [PaymentManager interface object length]
+    expected: FAIL
+
+  [PaymentManager interface: existence and properties of interface prototype object's "constructor" property]
+    expected: FAIL
+
+  [PaymentManager interface object name]
+    expected: FAIL
+
+  [PaymentManager interface: attribute instruments]
+    expected: FAIL
+
+  [PaymentManager interface: operation enableDelegations(sequence<PaymentDelegation>)]
+    expected: FAIL
+
+  [PaymentManager interface: existence and properties of interface prototype object]
+    expected: FAIL
+
+  [PaymentManager interface: existence and properties of interface object]
+    expected: FAIL
+
+
+[idlharness.https.any.serviceworker.html]
+  [CanMakePaymentEvent interface: new CanMakePaymentEvent("type") must inherit property "methodData" with the proper type]
+    expected: FAIL
+
+  [CanMakePaymentEvent interface: new CanMakePaymentEvent("type") must inherit property "respondWith(Promise<boolean>)" with the proper type]
+    expected: FAIL
+
+  [PaymentRequestEvent interface: new PaymentRequestEvent("type") must inherit property "instrumentKey" with the proper type]
+    expected: FAIL
+
+  [PaymentManager interface: attribute instruments]
+    expected: FAIL
+
+  [CanMakePaymentEvent interface: new CanMakePaymentEvent("type") must inherit property "paymentRequestOrigin" with the proper type]
+    expected: FAIL
+
+  [PaymentManager interface: attribute userHint]
+    expected: FAIL
+
+  [PaymentManager interface: existence and properties of interface prototype object's @@unscopables property]
+    expected: FAIL
+
+  [PaymentManager interface: existence and properties of interface prototype object's "constructor" property]
+    expected: FAIL
+
+  [PaymentRequestEvent interface: calling respondWith(Promise<PaymentHandlerResponse>) on new PaymentRequestEvent("type") with too few arguments must throw TypeError]
+    expected: FAIL
+
+  [PaymentManager must be primary interface of paymentManager]
+    expected: FAIL
+
+  [PaymentRequestEvent interface: calling changeShippingOption(DOMString) on new PaymentRequestEvent("type") with too few arguments must throw TypeError]
+    expected: FAIL
+
+  [PaymentRequestEvent interface object length]
+    expected: FAIL
+
+  [PaymentManager interface: operation enableDelegations(sequence<PaymentDelegation>)]
+    expected: FAIL
+
+  [Stringification of new PaymentRequestEvent("type")]
+    expected: FAIL
+
+  [PaymentRequestEvent interface: new PaymentRequestEvent("type") must inherit property "paymentOptions" with the proper type]
+    expected: FAIL
+
+  [PaymentRequestEvent interface: new PaymentRequestEvent("type") must inherit property "respondWith(Promise<PaymentHandlerResponse>)" with the proper type]
+    expected: FAIL
+
+  [PaymentRequestEvent interface: new PaymentRequestEvent("type") must inherit property "shippingOptions" with the proper type]
+    expected: FAIL
+
+  [PaymentRequestEvent interface: new PaymentRequestEvent("type") must inherit property "changeShippingAddress(optional AddressInit)" with the proper type]
+    expected: FAIL
+
+  [PaymentManager interface object length]
+    expected: FAIL
+
+  [PaymentManager interface object name]
+    expected: FAIL
+
+  [PaymentRequestEvent interface: calling changeShippingAddress(optional AddressInit) on new PaymentRequestEvent("type") with too few arguments must throw TypeError]
+    expected: FAIL
+
+  [CanMakePaymentEvent must be primary interface of new CanMakePaymentEvent("type")]
+    expected: FAIL
+
+  [PaymentRequestEvent interface: new PaymentRequestEvent("type") must inherit property "methodData" with the proper type]
+    expected: FAIL
+
+  [CanMakePaymentEvent interface: calling respondWith(Promise<boolean>) on new CanMakePaymentEvent("type") with too few arguments must throw TypeError]
+    expected: FAIL
+
+  [CanMakePaymentEvent interface: new CanMakePaymentEvent("type") must inherit property "topOrigin" with the proper type]
+    expected: FAIL
+
+  [PaymentRequestEvent interface: new PaymentRequestEvent("type") must inherit property "openWindow(USVString)" with the proper type]
+    expected: FAIL
+
+  [PaymentRequestEvent interface: calling changePaymentMethod(DOMString, optional object?) on new PaymentRequestEvent("type") with too few arguments must throw TypeError]
+    expected: FAIL
+
+  [PaymentManager interface: existence and properties of interface prototype object]
+    expected: FAIL
+
+  [PaymentRequestEvent interface: new PaymentRequestEvent("type") must inherit property "changeShippingOption(DOMString)" with the proper type]
+    expected: FAIL
+
+  [PaymentRequestEvent interface: new PaymentRequestEvent("type") must inherit property "modifiers" with the proper type]
+    expected: FAIL
+
+  [Stringification of new CanMakePaymentEvent("type")]
+    expected: FAIL
+
+  [PaymentRequestEvent interface: new PaymentRequestEvent("type") must inherit property "paymentRequestOrigin" with the proper type]
+    expected: FAIL
+
+  [PaymentRequestEvent interface: calling openWindow(USVString) on new PaymentRequestEvent("type") with too few arguments must throw TypeError]
+    expected: FAIL
+
+  [PaymentRequestEvent interface: new PaymentRequestEvent("type") must inherit property "topOrigin" with the proper type]
+    expected: FAIL
+
+  [PaymentRequestEvent interface: new PaymentRequestEvent("type") must inherit property "requestBillingAddress" with the proper type]
+    expected: FAIL
+
+  [PaymentRequestEvent interface: new PaymentRequestEvent("type") must inherit property "total" with the proper type]
+    expected: FAIL
+
+  [PaymentRequestEvent interface: new PaymentRequestEvent("type") must inherit property "changePaymentMethod(DOMString, optional object?)" with the proper type]
+    expected: FAIL
+
+  [PaymentRequestEvent must be primary interface of new PaymentRequestEvent("type")]
+    expected: FAIL
+
+  [PaymentRequestEvent interface: attribute requestBillingAddress]
+    expected: FAIL
+
+  [PaymentRequestEvent interface: new PaymentRequestEvent("type") must inherit property "paymentRequestId" with the proper type]
+    expected: FAIL
+
+  [PaymentRequestEvent interface: operation changeShippingAddress(optional AddressInit)]
+    expected: FAIL
+
+  [CanMakePaymentEvent interface object length]
+    expected: FAIL
+
+  [PaymentManager interface: existence and properties of interface object]
+    expected: FAIL
+
+
+[idlharness.https.any.sharedworker.html]
+  [PaymentManager interface: attribute userHint]
+    expected: FAIL
+
+  [PaymentManager interface: existence and properties of interface prototype object's @@unscopables property]
+    expected: FAIL
+
+  [PaymentManager interface object length]
+    expected: FAIL
+
+  [PaymentManager interface: existence and properties of interface prototype object's "constructor" property]
+    expected: FAIL
+
+  [PaymentManager interface object name]
+    expected: FAIL
+
+  [PaymentManager interface: attribute instruments]
+    expected: FAIL
+
+  [PaymentManager interface: operation enableDelegations(sequence<PaymentDelegation>)]
+    expected: FAIL
+
+  [PaymentManager interface: existence and properties of interface prototype object]
+    expected: FAIL
+
+  [PaymentManager interface: existence and properties of interface object]
+    expected: FAIL
+

third_party/blink/web_tests/external/wpt/payment-handler/payment-instruments.https.html.ini

+[payment-instruments.https.html]
+  [Resetting an existing instrument updates the instrument]
+    expected: FAIL
+
+  [Clearing the instruments]
+    expected: FAIL
+
+  [Don't crash when registering instruments with very long icon media type image/pngggggg...]
+    expected: FAIL
+
+  [Cannot register instruments with an existing non-https icon URL]
+    expected: FAIL
+
+  [Deleting an existing instrument returns true]
+    expected: FAIL
+
+  [Don't crash on very long key, name, method, and capability strings.]
+    expected: FAIL
+
+  [Don't crash on null characters in key, name, method, and capability strings.]
+    expected: FAIL
+
+  [Don't crash when registering an instrument with a very long icon size 888...x888...]
+    expected: FAIL
+
+  [Cannot register instruments with invalid icon media type image/jif]
+    expected: FAIL
+
+  [Instrument keys are returned in the original insertion order]
+    expected: FAIL
+
+  [Cannot register instruments with invalid icon URL (has a null character)]
+    expected: FAIL
+
+  [Deleting an existing instrument the second time returns false]
+    expected: FAIL
+
+  [Cannot register instruments with non-existing non-https icon URL]
+    expected: FAIL
+
+  [Getting an existing instrument returns the instrument]
+    expected: FAIL
+
+  [Cannot register instruments with invalid icon size "256 256" (missing "x")]
+    expected: FAIL
+

third_party/blink/web_tests/external/wpt/payment-handler/respond-with-minimal-ui.https.html.ini

+[respond-with-minimal-ui.https.html]
+  [respondWithMinimalUI({}) causes hasEnrolledInstrument() to return "false"]
+    expected: FAIL
+
+  [respondWithMinimalUI({canMakePayment: true, readyForMinimalUI: true, accountBalance: "6.78"}) does not crash]
+    expected: FAIL
+
+  [respondWithMinimalUI(0) causes hasEnrolledInstrument() to return "false"]
+    expected: FAIL
+
+  [respondWithMinimalUI({canMakePayment: true}) causes hasEnrolledInstrument() to return "true"]
+    expected: FAIL
+
+  [respondWithMinimalUI({canMakePayment: true, readyForMinimalUI: true, accountBalance: "--"}) does not crash]
+    expected: FAIL
+
+  [respondWithMinimalUI({canMakePayment: false}) causes hasEnrolledInstrument() to return "false"]
+    expected: FAIL
+
+  [respondWithMinimalUI({canMakePayment: true, readyForMinimalUI: true, accountBalance: ""}) does not crash]
+    expected: FAIL
+
+  [respondWithMinimalUI({canMakePayment: true, readyForMinimalUI: true}) does not crash]
+    expected: FAIL
+

third_party/blink/web_tests/external/wpt/payment-method-basic-card/apply_the_modifiers.html.ini

+[apply_the_modifiers.html]
+  expected: TIMEOUT

third_party/blink/web_tests/external/wpt/payment-method-basic-card/payment-request-canmakepayment-method.https.html.ini

+[payment-request-canmakepayment-method.https.html]
+  [Must return false when the PMI is not supported at by the user agent.]
+    expected: FAIL
+
+  [If basic-card is supported, then return a promise that resolves to true.]
+    expected: FAIL
+
+  [Must return true when basic-card is amongst unsupported PMIs.]
+    expected: FAIL
+

third_party/blink/web_tests/external/wpt/payment-method-basic-card/steps_for_selecting_the_payment_handler.html.ini

+[steps_for_selecting_the_payment_handler.html]
+  expected: TIMEOUT

third_party/blink/web_tests/external/wpt/payment-request/allowpaymentrequest/active-document-cross-origin.https.sub.html.ini

+[active-document-cross-origin.https.sub.html]
+  [PaymentRequest <iframe allowpaymentrequest> in non-active document (cross-origin)]
+    expected: FAIL
+

third_party/blink/web_tests/external/wpt/payment-request/allowpaymentrequest/active-document-same-origin.https.html.ini

+[active-document-same-origin.https.html]
+  [PaymentRequest <iframe allowpaymentrequest> in non-active document (same-origin)]
+    expected: FAIL
+

third_party/blink/web_tests/external/wpt/payment-request/constructor_convert_method_data.https.html.ini

+[constructor_convert_method_data.https.html]
+  [Converts PaymentMethodData's data to mandated IDL type during PaymentRequest construction.]
+    expected: FAIL
+

third_party/blink/web_tests/external/wpt/payment-request/idlharness.https.window.js.ini

+[idlharness.https.window.html]
+  [PaymentRequest interface: attribute onmerchantvalidation]
+    expected: FAIL
+
+  [PaymentRequest interface object length]
+    expected: FAIL
+
+  [PaymentRequest interface: paymentRequest must inherit property "onmerchantvalidation" with the proper type]
+    expected: FAIL
+

third_party/blink/web_tests/external/wpt/payment-request/onmerchantvalidation-attribute.https.html.ini

+[onmerchantvalidation-attribute.https.html]
+  [onmerchantvalidation attribute and listeners both work]
+    expected: FAIL
+
+  [Must have a onmerchantvalidation IDL attribute]
+    expected: FAIL
+
+  [onmerchantvalidation attribute is a generic handler for "merchantvalidation"]
+    expected: FAIL
+
+  [onmerchantvalidation attribute is a handler for MerchantValidationEvent]
+    expected: FAIL
+

third_party/blink/web_tests/external/wpt/payment-request/payment-is-showing.https.html.ini

+[payment-is-showing.https.html]
+  expected: ERROR
+  [Given multiple nested browsing contexts, and an iframe calls show() first, other nested browsing contexts can't show a request.]
+    expected: FAIL
+
+  [Using a popup window prevents the top-browsing context from showing a payment request]
+    expected: FAIL
+
+  [An iframe cannot show a payment request if the top-level window is already showing one.]
+    expected: FAIL
+
+  [Given multiple nested browsing contexts, and popup calls show() first, other nested browsing contexts can't show a request.]
+    expected: FAIL
+
+  [The top browsing context can only show one payment sheet at a time.]
+    expected: FAIL
+
+  [Navigating an iframe as a nested browsing context sets 'payment request is showing boolean' to false.]
+    expected: TIMEOUT
+
+  [Given multiple nested browsing contexts, and window calls show() first, other nested browsing contexts can't show a request.]
+    expected: FAIL
+
+  [If an iframe shows a payment request, the top-level browsing context can't also show one.]
+    expected: FAIL
+
+  [Navigating a popup as a nested browsing context sets 'payment request is showing boolean' to false.]
+    expected: NOTRUN
+

third_party/blink/web_tests/external/wpt/payment-request/payment-request-abort-method.https.html.ini

+[payment-request-abort-method.https.html]
+  [Aborting a request before it is shown doesn't prevent it from being shown later.]
+    expected: FAIL
+
+  [The same request cannot be shown multiple times.]
+    expected: FAIL
+

third_party/blink/web_tests/external/wpt/payment-request/payment-request-canmakepayment-method-protection.https.html.ini

+[payment-request-canmakepayment-method-protection.https.html]
+  [Optionally, at the user agent's discretion, return a promise rejected with a "NotAllowedError" DOMException.]
+    expected: FAIL
+

third_party/blink/web_tests/external/wpt/payment-request/payment-request-canmakepayment-method.https.html.ini

+[payment-request-canmakepayment-method.https.html]
+  expected: ERROR
+  [If payment method identifier are supported, resolve promise with true.]
+    expected: FAIL
+
+  [Mix of supported and unsupported methods, at least one method is supported.]
+    expected: FAIL
+
+  [If request.[[state\]\] is "closed", then return a promise rejected with an "InvalidStateError" DOMException.]
+    expected: FAIL
+
+  [If request.[[state\]\] is "interactive", then return a promise rejected with an "InvalidStateError" DOMException.]
+    expected: FAIL
+
+  [All methods are unsupported]
+    expected: FAIL
+
+  [If request.[[state\]\] is "created", then return a promise that resolves to true for known method.]
+    expected: FAIL
+

third_party/blink/web_tests/external/wpt/payment-request/payment-request-hasenrolledinstrument-method-protection.tentative.https.html.ini

+[payment-request-hasenrolledinstrument-method-protection.tentative.https.html]
+  [Optionally, at the user agent's discretion, return a promise rejected with a "NotAllowedError" DOMException.]
+    expected: FAIL
+

third_party/blink/web_tests/external/wpt/payment-request/payment-request-hasenrolledinstrument-method.tentative.https.html.ini

+[payment-request-hasenrolledinstrument-method.tentative.https.html]
+  [If request.[[state\]\] is "interactive", then return a promise rejected with an "InvalidStateError" DOMException.]
+    expected: FAIL
+
+  [If request.[[state\]\] is "closed", then return a promise rejected with an "InvalidStateError" DOMException.]
+    expected: FAIL
+
+  [hasEnrolledInstrument() resolves to false for unsupported payment methods.]
+    expected: FAIL
+

third_party/blink/web_tests/external/wpt/payment-request/payment-request-show-method.https.html.ini

+[payment-request-show-method.https.html]
+  [Throws if the promise [[state\]\] is not 'created'.]
+    expected: FAIL
+
+  [If payment method consultation produces no supported method of payment, then return a promise rejected with a "NotSupportedError" DOMException.]
+    expected: FAIL
+
+  [Calling show() multiple times always returns a new promise.]
+    expected: FAIL
+
+  [If the user agent's "payment request is showing" boolean is true, then return a promise rejected with an "AbortError" DOMException.]
+    expected: FAIL
+
+  [Calling show() without being triggered by user interaction throws]
+    expected: FAIL
+

third_party/blink/web_tests/external/wpt/payment-request/rejects_if_not_active.https.html.ini

+[rejects_if_not_active.https.html]
+  [If a payment request is showing, but its document is navigated away (so no longer fully active), the payment sheet is dismissed.]
+    expected: FAIL
+