Rolling new wpt tooling

New sha is 22901727d52297378d44b217af0b4c06d5b0a484

Change-Id: Ic1aa7c32de4f2057ce8d7c267774b64fb917a352
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2320749
Commit-Queue: Robert Ma <robertma@chromium.org>
Auto-Submit: Luke Z <lpz@chromium.org>
Reviewed-by: Robert Ma <robertma@chromium.org>
Cr-Commit-Position: refs/heads/master@{#792250}

third_party/blink/tools/blinkpy/third_party/README.chromium

@@ -22,7 +22,7 @@ Local Modifications: None
 Name: web-platform-tests - Test Suites for Web Platform specifications
 Short Name: wpt
 URL: https://github.com/web-platform-tests/wpt/
-Version: 635bf0a5f8fd80f4bbc8b6bef0046a64c9a55cf7
+Version: 22901727d52297378d44b217af0b4c06d5b0a484
 License: LICENSES FOR W3C TEST SUITES (https://www.w3.org/Consortium/Legal/2008/03-bsd-license.html)
 License File: wpt/wpt/LICENSE.md
 Security Critical: no

third_party/blink/tools/blinkpy/third_party/wpt/checkout.sh

@@ -9,7 +9,7 @@ cd $DIR
 
 TARGET_DIR=$DIR/wpt
 REMOTE_REPO="https://github.com/web-platform-tests/wpt.git"
-WPT_HEAD=635bf0a5f8fd80f4bbc8b6bef0046a64c9a55cf7
+WPT_HEAD=22901727d52297378d44b217af0b4c06d5b0a484
 
 function clone {
   # Remove existing repo if already exists.

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/lint/lint.py

@@ -206,6 +206,15 @@ def check_ahem_copy(repo_root, path):
     return []
 
 
+def check_tentative_directories(repo_root, path):
+    # type: (Text, Text) -> List[rules.Error]
+    path_parts = path.split(os.path.sep)
+    for directory in path_parts[:-1]:
+        if "tentative" in directory and directory != "tentative":
+            return [rules.TentativeDirectoryName.error(path)]
+    return []
+
+
 def check_git_ignore(repo_root, paths):
     # type: (Text, List[Text]) -> List[rules.Error]
     errors = []
@@ -987,7 +996,7 @@ def lint(repo_root, paths, output_format, ignore_glob=None):
     return sum(itervalues(error_count))
 
 path_lints = [check_file_type, check_path_length, check_worker_collision, check_ahem_copy,
-              check_gitignore_file]
+              check_tentative_directories, check_gitignore_file]
 all_paths_lints = [check_css_globally_unique, check_unique_testharness_basenames]
 file_lints = [check_regexp_line, check_parsed, check_python_ast, check_script_metadata,
               check_ahem_system_font]

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/lint/rules.py

@@ -341,6 +341,12 @@ class DuplicateBasenamePath(Rule):
     to_fix = "rename files so they have unique basename paths"
 
 
+class TentativeDirectoryName(Rule):
+    name = "TENTATIVE-DIRECTORY-NAME"
+    description = "Directories for tentative tests must be named exactly 'tentative'"
+    to_fix = "rename directory to be called 'tentative'"
+
+
 class Regexp(six.with_metaclass(abc.ABCMeta)):
     @abc.abstractproperty
     def pattern(self):

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/manifest/sourcefile.py

@@ -418,7 +418,7 @@ class SourceFile(object):
         tentative file.
 
         See https://web-platform-tests.org/writing-tests/file-names.html#test-features"""
-        return "tentative" in self.meta_flags
+        return "tentative" in self.meta_flags or "tentative" in self.dir_path.split(os.path.sep)
 
     @property
     def name_is_print_reftest(self):

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/webdriver/webdriver/error.py

@@ -3,14 +3,18 @@ import json
 
 from six import itervalues
 
+
 class WebDriverException(Exception):
     http_status = None
     status_code = None
 
     def __init__(self, http_status=None, status_code=None, message=None, stacktrace=None):
         super(WebDriverException, self)
-        self.http_status = http_status
-        self.status_code = status_code
+
+        if http_status is not None:
+            self.http_status = http_status
+        if status_code is not None:
+            self.status_code = status_code
         self.message = message
         self.stacktrace = stacktrace
 

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wpt/install.py

 import argparse
-import sys
 from . import browser
 
 latest_channels = {
     'firefox': 'nightly',
-    'chrome': 'dev',
+    'chrome': 'nightly',
     'chrome_android': 'dev',
     'edgechromium': 'dev',
     'safari': 'preview',
@@ -15,32 +14,34 @@ channel_by_name = {
     'stable': 'stable',
     'release': 'stable',
     'beta': 'beta',
+    'dev': 'dev',
+    'canary': 'canary',
     'nightly': latest_channels,
-    'dev': latest_channels,
     'preview': latest_channels,
     'experimental': latest_channels,
-    'canary': 'canary',
 }
 
+channel_args = argparse.ArgumentParser(add_help=False)
+channel_args.add_argument('--channel', choices=channel_by_name.keys(),
+                          default='nightly', action='store',
+                          help='''
+Name of browser release channel (default: nightly). "stable" and "release" are
+synonyms for the latest browser stable release; "beta" is the beta release;
+"dev" is only meaningful for Chrome (i.e. Chrome Dev); "nightly",
+"experimental", and "preview" are all synonyms for the latest available
+development or trunk release. (For WebDriver installs, we attempt to select an
+appropriate, compatible version for the latest browser release on the selected
+channel.) This flag overrides --browser-channel.''')
 
-def get_parser():
-    parser = argparse.ArgumentParser(description="""Install a given browser or webdriver frontend.
 
-    For convenience the release channel of the browser accepts various spellings,
-    but we actually support at most three variants; whatever the latest development
-    release is (e.g. Firefox nightly or Chrome dev), the latest beta release, and
-    the most recent stable release.""")
+def get_parser():
+    parser = argparse.ArgumentParser(
+        parents=[channel_args],
+        description="Install a given browser or webdriver frontend.")
     parser.add_argument('browser', choices=['firefox', 'chrome', 'servo'],
                         help='name of web browser product')
     parser.add_argument('component', choices=['browser', 'webdriver'],
                         help='name of component')
-    parser.add_argument('--channel', choices=channel_by_name.keys(),
-                        default="nightly", help='Name of browser release channel. '
-                        '"stable" and "release" are synonyms for the latest browser stable release,'
-                        '"nightly", "dev", "experimental", and "preview" are all synonyms for '
-                        'the latest available development release. For WebDriver installs, '
-                        'we attempt to select an appropriate, compatible, version for the '
-                        'latest browser release on the selected channel.')
     parser.add_argument('--download-only', action="store_true",
                         help="Download the selected component but don't install it")
     parser.add_argument('--rename', action="store", default=None,
@@ -59,13 +60,15 @@ def get_channel(browser, channel):
 
 
 def run(venv, **kwargs):
+    import logging
+    logger = logging.getLogger("install")
+
     browser = kwargs["browser"]
     destination = kwargs["destination"]
     channel = get_channel(browser, kwargs["channel"])
 
     if channel != kwargs["channel"]:
-        print("Interpreting channel '%s' as '%s'" % (kwargs["channel"],
-                                                     channel))
+        logger.info("Interpreting channel '%s' as '%s'", kwargs["channel"], channel)
 
     if destination is None:
         if venv:
@@ -77,7 +80,7 @@ def run(venv, **kwargs):
             raise argparse.ArgumentError(None,
                                          "No --destination argument, and no default for the environment")
 
-    install(browser, kwargs["component"], destination, channel,
+    install(browser, kwargs["component"], destination, channel, logger=logger,
             download_only=kwargs["download_only"], rename=kwargs["rename"])
 
 
@@ -92,11 +95,11 @@ def install(name, component, destination, channel="nightly", logger=None, downlo
 
     method = prefix + suffix
 
-    subclass = getattr(browser, name.title())
-    sys.stdout.write('Now installing %s %s...\n' % (name, component))
+    browser_cls = getattr(browser, name.title())
+    logger.info('Now installing %s %s...', name, component)
     kwargs = {}
     if download_only and rename:
         kwargs["rename"] = rename
-    path = getattr(subclass(logger), method)(dest=destination, channel=channel, **kwargs)
+    path = getattr(browser_cls(logger), method)(dest=destination, channel=channel, **kwargs)
     if path:
-        sys.stdout.write('Binary %s as %s\n' % ("downloaded" if download_only else "installed", path,))
+        logger.info('Binary %s as %s', "downloaded" if download_only else "installed", path)

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wpt/testfiles.py

@@ -35,6 +35,8 @@ if MYPY:
     from typing import Text
     from typing import Tuple
 
+DEFAULT_IGNORE_RULERS = ("resources/testharness*", "resources/testdriver*")
+
 here = ensure_text(os.path.dirname(__file__))
 wpt_root = os.path.abspath(os.path.join(here, os.pardir, os.pardir))
 
@@ -167,8 +169,8 @@ def repo_files_changed(revish, include_uncommitted=False, include_new=False):
 def exclude_ignored(files, ignore_rules):
     # type: (Iterable[Text], Optional[Sequence[Text]]) -> Tuple[List[Text], List[Text]]
     if ignore_rules is None:
-        ignore_rules = []
-    compiled_ignore_rules = [compile_ignore_rule(item) for item in ignore_rules]
+        ignore_rules = DEFAULT_IGNORE_RULERS
+    compiled_ignore_rules = [compile_ignore_rule(item) for item in set(ignore_rules)]
 
     changed = []
     ignored = []
@@ -334,14 +336,13 @@ def get_parser():
     parser = argparse.ArgumentParser()
     parser.add_argument("revish", default=None, help="Commits to consider. Defaults to the "
                         "commits on the current branch", nargs="?")
-    # TODO: Consolidate with `./wpt run --affected`:
-    # https://github.com/web-platform-tests/wpt/issues/14560
-    parser.add_argument("--ignore-rules", nargs="*", type=set,  # type: ignore
-                        default={"resources/testharness*"},
-                        help="Rules for paths to exclude from lists of changes. Rules are paths "
-                        "relative to the test root, with * before a separator or the end matching "
-                        "anything other than a path separator and ** in that position matching "
-                        "anything")
+    parser.add_argument("--ignore-rule", action="append",
+                        help="Override the rules for paths to exclude from lists of changes. "
+                        "Rules are paths relative to the test root, with * before a separator "
+                        "or the end matching anything other than a path separator and ** in that "
+                        "position matching anything. This flag can be used multiple times for "
+                        "multiple rules. Specifying this flag overrides the default: " +
+                        ", ".join(DEFAULT_IGNORE_RULERS))
     parser.add_argument("--modified", action="store_true",
                         help="Include files under version control that have been "
                         "modified or staged")
@@ -377,7 +378,7 @@ def run_changed_files(**kwargs):
     # type: (**Any) -> None
     revish = get_revish(**kwargs)
     changed, _ = files_changed(revish,
-                               kwargs["ignore_rules"],
+                               kwargs["ignore_rule"],
                                include_uncommitted=kwargs["modified"],
                                include_new=kwargs["new"])
 
@@ -391,7 +392,8 @@ def run_changed_files(**kwargs):
 def run_tests_affected(**kwargs):
     # type: (**Any) -> None
     revish = get_revish(**kwargs)
-    changed, _ = files_changed(revish, kwargs["ignore_rules"],
+    changed, _ = files_changed(revish,
+                               kwargs["ignore_rule"],
                                include_uncommitted=kwargs["modified"],
                                include_new=kwargs["new"])
     manifest_path = os.path.join(kwargs["metadata_root"], "MANIFEST.json")

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wpt/utils.py

+import errno
 import logging
 import os
+import shutil
+import stat
 import subprocess
 import tarfile
 import zipfile
@@ -95,3 +98,19 @@ def get(url):
     resp = requests.get(url, stream=True)
     resp.raise_for_status()
     return resp
+
+
+def rmtree(path):
+    # This works around two issues:
+    # 1. Cannot delete read-only files owned by us (e.g. files extracted from tarballs)
+    # 2. On Windows, we sometimes just need to retry in case the file handler
+    #    hasn't been fully released (a common issue).
+    def handle_remove_readonly(func, path, exc):
+        excvalue = exc[1]
+        if func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES:
+            os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)  # 0777
+            func(path)
+        else:
+            raise
+
+    return shutil.rmtree(path, onerror=handle_remove_readonly)

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/handlers.py

@@ -2,8 +2,9 @@ import json
 import os
 import sys
 import traceback
+from collections import defaultdict
 
-from six.moves.urllib.parse import parse_qs, quote, unquote, urljoin
+from six.moves.urllib.parse import quote, unquote, urljoin
 from six import iteritems
 
 from .constants import content_types
@@ -120,6 +121,24 @@ class DirectoryHandler(object):
                    {"link": link, "name": escape(item), "class": class_,
                     "headers": dot_headers_markup})
 
+def parse_qs(qs):
+    """Parse a query string given as a string argument (data of type
+    application/x-www-form-urlencoded). Data are returned as a dictionary. The
+    dictionary keys are the unique query variable names and the values are
+    lists of values for each name.
+
+    This implementation is used instead of Python's built-in `parse_qs` method
+    in order to support the semicolon character (which the built-in method
+    interprets as a parameter delimiter)."""
+    pairs = [item.split("=", 1) for item in qs.split('&') if item]
+    rv = defaultdict(list)
+    for pair in pairs:
+        if len(pair) == 1 or len(pair[1]) == 0:
+            continue
+        name = unquote(pair[0].replace('+', ' '))
+        value = unquote(pair[1].replace('+', ' '))
+        rv[name].append(value)
+    return dict(rv)
 
 def wrap_pipeline(path, request, response):
     query = parse_qs(request.url_parts.query)