Commit 6e86d31b authored by Luke Zielinski's avatar Luke Zielinski Committed by Commit Bot

Disable (via flag) baselines translation and annotation handling in

wptMetadataBuilder for desktop wpt bots.

This will give us a clearer picture of what the true status of tests are
in preparation for triaging them.

Bug: 1142905
Change-Id: If0dc015ef3f7633f209ca2bd2a94c55f8ee77c76
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2503772Reviewed-by: default avatarRobert Ma <robertma@chromium.org>
Reviewed-by: default avatarRakib Hasan <rmhasan@google.com>
Reviewed-by: default avatarStephen McGruer <smcgruer@chromium.org>
Commit-Queue: Luke Z <lpz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#824588}
parent 3bd31fd1
...@@ -129,7 +129,9 @@ class WPTTestAdapter(wpt_common.BaseWptScriptAdapter): ...@@ -129,7 +129,9 @@ class WPTTestAdapter(wpt_common.BaseWptScriptAdapter):
"--additional-expectations", "--additional-expectations",
WPT_OVERRIDE_EXPECTATIONS_PATH, WPT_OVERRIDE_EXPECTATIONS_PATH,
"--checked-in-metadata-dir", "--checked-in-metadata-dir",
WPT_CHECKED_IN_METADATA_DIR WPT_CHECKED_IN_METADATA_DIR,
"--no-process-baselines",
"--no-handle-annotations"
]) ])
......
...@@ -56,6 +56,8 @@ class WPTMetadataBuilder(object): ...@@ -56,6 +56,8 @@ class WPTMetadataBuilder(object):
self.wpt_manifest = self.port.wpt_manifest("external/wpt") self.wpt_manifest = self.port.wpt_manifest("external/wpt")
self.metadata_output_dir = "" self.metadata_output_dir = ""
self.checked_in_metadata_dir = "" self.checked_in_metadata_dir = ""
self.process_baselines = True
self.handle_annotations = True
def run(self, args=None): def run(self, args=None):
"""Main entry point to parse flags and execute the script.""" """Main entry point to parse flags and execute the script."""
...@@ -73,6 +75,28 @@ class WPTMetadataBuilder(object): ...@@ -73,6 +75,28 @@ class WPTMetadataBuilder(object):
'--verbose', '--verbose',
action='store_true', action='store_true',
help='More verbose logging.') help='More verbose logging.')
parser.add_argument(
"--process-baselines",
action="store_true",
default=True,
dest="process_baselines",
help="Whether to translate baseline (-expected.txt) files into WPT "
"metadata files. This translation is lossy and results in any "
"subtest being accepted by wptrunner.")
parser.add_argument("--no-process-baselines",
action="store_false",
dest="process_baselines")
parser.add_argument(
"--handle-annotations",
action="store_true",
default=True,
dest="handle_annotations",
help="Whether to handle annotations in expectations files. These "
"are trailing comments that give additional details for how "
"to translate an expectation into WPT metadata.")
parser.add_argument("--no-handle-annotations",
action="store_false",
dest="handle_annotations")
args = parser.parse_args(args) args = parser.parse_args(args)
log_level = logging.DEBUG if args.verbose else logging.INFO log_level = logging.DEBUG if args.verbose else logging.INFO
...@@ -80,6 +104,8 @@ class WPTMetadataBuilder(object): ...@@ -80,6 +104,8 @@ class WPTMetadataBuilder(object):
self.metadata_output_dir = args.metadata_output_dir self.metadata_output_dir = args.metadata_output_dir
self.checked_in_metadata_dir = args.checked_in_metadata_dir self.checked_in_metadata_dir = args.checked_in_metadata_dir
self.process_baselines = args.process_baselines
self.handle_annotations = args.handle_annotations
self._build_metadata_and_write() self._build_metadata_and_write()
return 0 return 0
...@@ -198,11 +224,12 @@ class WPTMetadataBuilder(object): ...@@ -198,11 +224,12 @@ class WPTMetadataBuilder(object):
continue continue
# Check if the test has a baseline # Check if the test has a baseline
test_baseline = self.port.expected_text(test_name) if self.process_baselines:
if not test_baseline: test_baseline = self.port.expected_text(test_name)
continue if not test_baseline:
self._handle_test_with_baseline(test_name, test_baseline, continue
tests_needing_metadata) self._handle_test_with_baseline(test_name, test_baseline,
tests_needing_metadata)
return tests_needing_metadata return tests_needing_metadata
def _handle_test_with_expectation(self, test_name, expectation_line, def _handle_test_with_expectation(self, test_name, expectation_line,
...@@ -230,7 +257,7 @@ class WPTMetadataBuilder(object): ...@@ -230,7 +257,7 @@ class WPTMetadataBuilder(object):
status_bitmap |= TEST_TIMEOUT status_bitmap |= TEST_TIMEOUT
if ResultType.Crash in test_statuses: if ResultType.Crash in test_statuses:
status_bitmap |= TEST_CRASH status_bitmap |= TEST_CRASH
if annotations: if self.handle_annotations and annotations:
if "wpt_subtest_failure" in annotations: if "wpt_subtest_failure" in annotations:
status_bitmap |= SUBTEST_FAIL status_bitmap |= SUBTEST_FAIL
if "wpt_precondition_failed" in annotations: if "wpt_precondition_failed" in annotations:
......
[historical.https.html]
expected: OK
[allowPaymentRequest in HTMLIFrameElement]
expected: FAIL
[show-consume-activation.https.html]
expected: OK
[Calling share consumes user activation]
expected: FAIL
[show-method-optional-promise-rejects.https.html]
expected: OK
[Rejection of detailsPromise must abort the update with an 'AbortError' DOMException.]
expected: FAIL
[Total in the update is a string, so converting to IDL must abort the update with a TypeError.]
expected: FAIL
[Total is recursive, so converting to IDL must abort the update with a TypeError.]
expected: FAIL
[Updating with a negative total results in a TypeError.]
expected: FAIL
[Updating with a displayItem with an invalid currency results in RangeError.]
expected: FAIL
[Updating with duplicate shippingOptions (same IDs) results in a TypeError.]
expected: FAIL
[Updating with a shippingOption with an invalid currency value results in a RangError.]
expected: FAIL
[Must throw a RangeError when a modifier's total item has an invalid currency.]
expected: FAIL
[Must throw a RangeError when a modifier display item has an invalid currency.]
expected: FAIL
[Must throw as Modifier has a recursive dictionary.]
expected: FAIL
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment