Introduce a WPT-specific expectations file and update scripts to use it.

This allows us to put all WPT-specific failures in the new file and turn
waterfall builder green while we whittle away at the failures. The goal
would be to eventually remove this file as all failures get triaged.

The immediate benefit is that this builder starts to give meaningful signals
without being obscured by all of these failures. So we can use it 1) to gauge
flakiness/stability and 2) see results for tests that aren't affected by
failures (eg: some new testdriver APIs like Actions or Payments).

This change was tested by merging into crrev.com/c/1551486 and confirming
the new expectations file was used.

Change-Id: I7f2f83058804cee365e2082cbee9f6c23002d187
Bug: 937369
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1707065
Commit-Queue: Luke Z <lpz@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Reviewed-by: Robert Ma <robertma@chromium.org>
Cr-Commit-Position: refs/heads/master@{#683329}

testing/scripts/run_wpt_tests.py

@@ -81,6 +81,8 @@ class WPTTestAdapter(common.BaseIsolatedScriptArgsAdapter):
             self.options.old_json_output_file_path,
             '--new-json-output-dir', self.options.new_json_output_dir,
             '--new-json-output-filename', self.options.new_json_output_filename,
+            '--additional-expectations',
+            '../../third_party/blink/web_tests/WPTOverrideExpectations',
         ])
 
 

third_party/blink/tools/BUILD.gn

@@ -9,6 +9,8 @@ action("build_wpt_metadata") {
   args = [
     "--metadata-output-dir",
     rebase_path("$root_out_dir/wpt_expectations_metadata"),
+    "--additional-expectations",
+    "../../third_party/blink/web_tests/WPTOverrideExpectations",
   ]
   outputs = [
     "$root_out_dir/wpt_expectations_metadata/",

third_party/blink/tools/build_wpt_metadata.py

@@ -3,6 +3,8 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import argparse
+import optparse
 import sys
 
 from blinkpy.common.host import Host
@@ -11,11 +13,16 @@ from blinkpy.web_tests.models.test_expectations import TestExpectations
 
 
 def main(args):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--additional-expectations", action="append",
+                        help="Paths to additional expectations files for WPT.")
+    known_args, rest_args = parser.parse_known_args(args)
+
     host = Host()
-    port = host.port_factory.get()
+    port = host.port_factory.get(options=optparse.Values(vars(known_args)))
     expectations = TestExpectations(port)
     metadata_builder = WPTMetadataBuilder(expectations)
-    sys.exit(metadata_builder.run(args))
+    sys.exit(metadata_builder.run(rest_args))
 
 if __name__ == '__main__':
     main(sys.argv[1:])

third_party/blink/tools/update_wpt_output.py

@@ -3,6 +3,8 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import argparse
+import optparse
 import sys
 
 from blinkpy.common.host import Host
@@ -11,11 +13,16 @@ from blinkpy.web_tests.models.test_expectations import TestExpectations
 
 
 def main(args):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--additional-expectations", action="append",
+                        help="Paths to additional expectations files for WPT.")
+    known_args, rest_args = parser.parse_known_args(args)
+
     host = Host()
-    port = host.port_factory.get()
+    port = host.port_factory.get(options=optparse.Values(vars(known_args)))
     expectations = TestExpectations(port)
     output_updater = WPTOutputUpdater(expectations)
-    sys.exit(output_updater.run(args))
+    sys.exit(output_updater.run(rest_args))
 
 if __name__ == '__main__':
     main(sys.argv[1:])

third_party/blink/web_tests/WPTOverrideExpectations

+# This file lists test expectations for WPT tests running ONLY under the wptrunner code path in
+# Chromium. See go/wpt-in-chromium-ci for more info on the overall project.
+
+# There may be duplicate entries between this file and other expectations, which is acceptable.
+# Use of this file is temporary while wptrunner is in development.
+
+# Bluetooth not supported in Chrome as per
+# https://github.com/web-platform-tests/wpt/blob/master/bluetooth/README.md#web-bluetooth-testing
+crbug.com/lpz external/wpt/bluetooth [ Skip ]
+
+# background-fetch tests seem to crash the browser, which is reported as a FAIL
+crbug.com/lpz external/wpt/background-fetch/mixed-content-and-allowed-schemes.https.window.html [ Pass Timeout Failure ]