Roll WPT internal tools to 9f6a7e3cfdae54795a0ca58e194b828537f9659e

Bug: None
Change-Id: I8dcbf7f1e583df2f20560a6f6dc2337847c42805
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2432629Reviewed-by: Robert Ma <robertma@chromium.org>
Commit-Queue: Stephen McGruer <smcgruer@chromium.org>
Cr-Commit-Position: refs/heads/master@{#811214}

third_party/blink/tools/blinkpy/third_party/README.chromium

@@ -22,7 +22,7 @@ Local Modifications: None
 Name: web-platform-tests - Test Suites for Web Platform specifications
 Short Name: wpt
 URL: https://github.com/web-platform-tests/wpt/
-Version: 358439343c2878d6a3784964a11e43644d49c452
+Version: 9f6a7e3cfdae54795a0ca58e194b828537f9659e
 License: LICENSES FOR W3C TEST SUITES (https://www.w3.org/Consortium/Legal/2008/03-bsd-license.html)
 License File: wpt/wpt/LICENSE.md
 Security Critical: no

third_party/blink/tools/blinkpy/third_party/wpt/checkout.sh

@@ -9,7 +9,7 @@ cd $DIR
 
 TARGET_DIR=$DIR/wpt
 REMOTE_REPO="https://github.com/web-platform-tests/wpt.git"
-WPT_HEAD=358439343c2878d6a3784964a11e43644d49c452
+WPT_HEAD=9f6a7e3cfdae54795a0ca58e194b828537f9659e
 
 function clone {
   # Remove existing repo if already exists.

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/lint/lint.py

@@ -6,6 +6,7 @@ import ast
 import io
 import json
 import logging
+import multiprocessing
 import os
 import re
 import subprocess
@@ -34,6 +35,7 @@ if MYPY:
     from typing import Callable
     from typing import Dict
     from typing import IO
+    from typing import Iterator
     from typing import Iterable
     from typing import List
     from typing import Optional
@@ -42,6 +44,7 @@ if MYPY:
     from typing import Text
     from typing import Tuple
     from typing import Type
+    from typing import TypeVar
 
     # The Ignorelist is a two level dictionary. The top level is indexed by
     # error names (e.g. 'TRAILING WHITESPACE'). Each of those then has a map of
@@ -50,12 +53,26 @@ if MYPY:
     # ignores the error.
     Ignorelist = Dict[Text, Dict[Text, Set[Optional[int]]]]
 
+    # Define an arbitrary typevar
+    T = TypeVar("T")
+
     try:
         from xml.etree import cElementTree as ElementTree
     except ImportError:
         from xml.etree import ElementTree as ElementTree  # type: ignore
 
 
+if sys.version_info >= (3, 7):
+    from contextlib import nullcontext
+else:
+    from contextlib import contextmanager
+
+    @contextmanager
+    def nullcontext(enter_result=None):
+        # type: (Optional[T]) -> Iterator[Optional[T]]
+        yield enter_result
+
+
 logger = None  # type: Optional[logging.Logger]
 
 
@@ -367,7 +384,7 @@ def check_unique_testharness_basenames(repo_root, paths):
     file_dict = defaultdict(list)
     for path in paths:
         source_file = SourceFile(repo_root, path, "/")
-        if source_file.type != "testharness":
+        if "testharness" not in source_file.possible_types:
             continue
         file_name, file_extension = os.path.splitext(path)
         file_dict[file_name].append(file_extension)
@@ -793,8 +810,8 @@ def check_all_paths(repo_root, paths):
     return errors
 
 
-def check_file_contents(repo_root, path, f):
-    # type: (Text, Text, IO[bytes]) -> List[rules.Error]
+def check_file_contents(repo_root, path, f=None):
+    # type: (Text, Text, Optional[IO[bytes]]) -> List[rules.Error]
     """
     Runs lints that check the file contents.
 
@@ -803,12 +820,18 @@ def check_file_contents(repo_root, path, f):
     :param f: a file-like object with the file contents
     :returns: a list of errors found in ``f``
     """
+    with io.open(os.path.join(repo_root, path), 'rb') if f is None else nullcontext(f) as real_f:
+        assert real_f is not None  # Py2: prod mypy -2 into accepting this isn't None
+        errors = []
+        for file_fn in file_lints:
+            errors.extend(file_fn(repo_root, path, real_f))
+            real_f.seek(0)
+        return errors
 
-    errors = []
-    for file_fn in file_lints:
-        errors.extend(file_fn(repo_root, path, f))
-        f.seek(0)
-    return errors
+
+def check_file_contents_apply(args):
+    # type: (Tuple[Text, Text]) -> List[rules.Error]
+    return check_file_contents(*args)
 
 
 def output_errors_text(log, errors):
@@ -964,6 +987,10 @@ def main(**kwargs_str):
     return lint(repo_root, paths, output_format, ignore_glob, github_checks_outputter)
 
 
+# best experimental guess at a decent cut-off for using the parallel path
+MIN_FILES_FOR_PARALLEL = 80
+
+
 def lint(repo_root, paths, output_format, ignore_glob=None, github_checks_outputter=None):
     # type: (Text, List[Text], Text, Optional[List[Text]], Optional[GitHubChecksOutputter]) -> int
     error_count = defaultdict(int)  # type: Dict[Text, int]
@@ -1005,26 +1032,47 @@ def lint(repo_root, paths, output_format, ignore_glob=None, github_checks_output
 
         return (errors[-1][0], path)
 
-    for path in paths[:]:
+    to_check_content = []
+    skip = set()
+
+    for path in paths:
         abs_path = os.path.join(repo_root, path)
         if not os.path.exists(abs_path):
-            paths.remove(path)
+            skip.add(path)
             continue
 
         if any(fnmatch.fnmatch(path, file_match) for file_match in skipped_files):
-            paths.remove(path)
+            skip.add(path)
             continue
 
         errors = check_path(repo_root, path)
         last = process_errors(errors) or last
 
         if not os.path.isdir(abs_path):
-            with io.open(abs_path, 'rb') as test_file:
-                errors = check_file_contents(repo_root, path, test_file)
-                last = process_errors(errors) or last
+            to_check_content.append((repo_root, path))
+
+    paths = [p for p in paths if p not in skip]
+
+    if len(to_check_content) >= MIN_FILES_FOR_PARALLEL:
+        pool = multiprocessing.Pool()
+        # submit this job first, as it's the longest running
+        all_paths_result = pool.apply_async(check_all_paths, (repo_root, paths))
+        # each item tends to be quick, so pass things in large chunks to avoid too much IPC overhead
+        errors_it = pool.imap_unordered(check_file_contents_apply, to_check_content, chunksize=40)
+        pool.close()
+        for errors in errors_it:
+            last = process_errors(errors) or last
+
+        errors = all_paths_result.get()
+        pool.join()
+        last = process_errors(errors) or last
+    else:
+        for item in to_check_content:
+            errors = check_file_contents(*item)
+            last = process_errors(errors) or last
 
-    errors = check_all_paths(repo_root, paths)
-    last = process_errors(errors) or last
+        errors = check_all_paths(repo_root, paths)
+        last = process_errors(errors) or last
 
     if output_format in ("normal", "markdown"):
         output_error_count(error_count)

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/manifest/sourcefile.py

@@ -865,9 +865,68 @@ class SourceFile(object):
     @property
     def type(self):
         # type: () -> Text
+        possible_types = self.possible_types
+        if len(possible_types) == 1:
+            return possible_types.pop()
+
         rv, _ = self.manifest_items()
         return rv
 
+    @property
+    def possible_types(self):
+        # type: () -> Set[Text]
+        """Determines the set of possible types without reading the file"""
+
+        if self.items_cache:
+            return {self.items_cache[0]}
+
+        if self.name_is_non_test:
+            return {SupportFile.item_type}
+
+        if self.name_is_manual:
+            return {ManualTest.item_type}
+
+        if self.name_is_conformance:
+            return {ConformanceCheckerTest.item_type}
+
+        if self.name_is_conformance_support:
+            return {SupportFile.item_type}
+
+        if self.name_is_webdriver:
+            return {WebDriverSpecTest.item_type}
+
+        if self.name_is_visual:
+            return {VisualTest.item_type}
+
+        if self.name_is_crashtest:
+            return {CrashTest.item_type}
+
+        if self.name_is_print_reftest:
+            return {PrintRefTest.item_type}
+
+        if self.name_is_multi_global:
+            return {TestharnessTest.item_type}
+
+        if self.name_is_worker:
+            return {TestharnessTest.item_type}
+
+        if self.name_is_window:
+            return {TestharnessTest.item_type}
+
+        if self.markup_type is None:
+            return {SupportFile.item_type}
+
+        if not self.name_is_reference:
+            return {ManualTest.item_type,
+                    TestharnessTest.item_type,
+                    RefTest.item_type,
+                    VisualTest.item_type,
+                    SupportFile.item_type}
+
+        return {TestharnessTest.item_type,
+                RefTest.item_type,
+                SupportFile.item_type}
+
     def manifest_items(self):
         # type: () -> Tuple[Text, List[ManifestItem]]
         """List of manifest items corresponding to the file. There is typically one
@@ -1070,6 +1129,7 @@ class SourceFile(object):
                     self.rel_path
                 )]
 
+        assert rv[0] in self.possible_types
         assert len(rv[1]) == len(set(rv[1]))
 
         self.items_cache = rv

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/serve/serve.py

@@ -418,6 +418,19 @@ class ServerProc(object):
 
     def create_daemon(self, init_func, host, port, paths, routes, bind_address,
                       config, **kwargs):
+        if sys.platform == "darwin":
+            # on Darwin, NOFILE starts with a very low limit (256), so bump it up a little
+            # by way of comparison, Debian starts with a limit of 1024, Windows 512
+            import resource  # local, as it only exists on Unix-like systems
+            maxfilesperproc = int(subprocess.check_output(
+                ["sysctl", "-n", "kern.maxfilesperproc"]
+            ).strip())
+            soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+            # 2048 is somewhat arbitrary, but gives us some headroom for wptrunner --parallel
+            # note that it's expected that 2048 will be the min here
+            new_soft = min(2048, maxfilesperproc, hard)
+            if soft < new_soft:
+                resource.setrlimit(resource.RLIMIT_NOFILE, (new_soft, hard))
         try:
             self.daemon = init_func(host, port, paths, routes, bind_address, config, **kwargs)
         except socket.error:

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/webdriver/webdriver/client.py

@@ -203,6 +203,32 @@ class ActionSequence(object):
             self.key_up(c)
         return self
 
+    def scroll(self, x, y, delta_x, delta_y, duration=None, origin=None):
+        """Queue a scroll action.
+
+        :param x: Destination x-axis coordinate of pointer in CSS pixels.
+        :param y: Destination y-axis coordinate of pointer in CSS pixels.
+        :param delta_x: scroll delta on x-axis in CSS pixels.
+        :param delta_y: scroll delta on y-axis in CSS pixels.
+        :param duration: Number of milliseconds over which to distribute the
+                         scroll. If None, remote end defaults to 0.
+        :param origin: Origin of coordinates, either "viewport" or an Element.
+                       If None, remote end defaults to "viewport".
+        """
+        action = {
+            "type": "scroll",
+            "x": x,
+            "y": y,
+            "deltaX": delta_x,
+            "deltaY": delta_y
+        }
+        if duration is not None:
+            action["duration"] = duration
+        if origin is not None:
+            action["origin"] = origin
+        self._actions.append(action)
+        return self
+
 
 class Actions(object):
     def __init__(self, session):
@@ -218,10 +244,6 @@ class Actions(object):
         """
         body = {"actions": [] if actions is None else actions}
         actions = self.session.send_session_command("POST", "actions", body)
-        """WebDriver window should be set to the top level window when wptrunner
-        processes the next event.
-        """
-        self.session.switch_frame(None)
         return actions
 
     @command
@@ -321,9 +343,7 @@ class Find(object):
         self.session = session
 
     @command
-    def css(self, element_selector, all=True, frame="window"):
-        if (frame != "window"):
-            self.session.switch_frame(frame)
+    def css(self, element_selector, all=True):
         elements = self._find_element("css selector", element_selector, all)
         return elements
 

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wpt/browser.py

@@ -656,9 +656,33 @@ class Chrome(Browser):
         self.logger.warning("Unable to find the browser binary.")
         return None
 
-    def find_webdriver(self, channel=None):
+    def find_webdriver(self, channel=None, browser_binary=None):
         return find_executable("chromedriver")
 
+    def webdriver_supports_browser(self, webdriver_binary, browser_binary):
+        chromedriver_version = self.webdriver_version(webdriver_binary)
+        if not chromedriver_version:
+            self.logger.warning(
+                "Unable to get version for ChromeDriver %s, rejecting it" %
+                webdriver_binary)
+            return False
+
+        browser_version = self.version(browser_binary)
+        if not browser_version:
+            # If we can't get the browser version, we just have to assume the
+            # ChromeDriver is good.
+            return True
+
+        # Check that the ChromeDriver version matches the Chrome version.
+        chromedriver_major = chromedriver_version.split('.')[0]
+        browser_major = browser_version.split('.')[0]
+        if chromedriver_major != browser_major:
+            self.logger.warning(
+                "ChromeDriver %s does not match Chrome/Chromium %s" %
+                (chromedriver_version, browser_version))
+            return False
+        return True
+
     def _official_chromedriver_url(self, chrome_version):
         # http://chromedriver.chromium.org/downloads/version-selection
         parts = chrome_version.split(".")
@@ -700,6 +724,17 @@ class Chrome(Browser):
     def install_webdriver_by_version(self, version, dest=None):
         if dest is None:
             dest = os.pwd
+
+        # There may be an existing chromedriver binary from a previous install.
+        # To provide a clean install experience, remove the old binary - this
+        # avoids tricky issues like unzipping over a read-only file.
+        existing_binary_path = find_executable("chromedriver", dest)
+        if existing_binary_path:
+            self.logger.info("Removing existing ChromeDriver binary: %s" %
+                existing_binary_path)
+            os.chmod(existing_binary_path, stat.S_IWUSR)
+            os.remove(existing_binary_path)
+
         url = self._latest_chromedriver_url(version) if version \
             else self._chromium_chromedriver_url(None)
         self.logger.info("Downloading ChromeDriver from %s" % url)
@@ -749,6 +784,21 @@ class Chrome(Browser):
             return None
         return m.group(1)
 
+    def webdriver_version(self, webdriver_binary):
+        if uname[0] == "Windows":
+            return _get_fileversion(webdriver_binary, self.logger)
+
+        try:
+            version_string = call(webdriver_binary, "--version").strip()
+        except subprocess.CalledProcessError:
+            self.logger.warning("Failed to call %s" % webdriver_binary)
+            return None
+        m = re.match(r"ChromeDriver ([0-9][0-9.]*)", version_string)
+        if not m:
+            self.logger.warning("Failed to extract version from: %s" % version_string)
+            return None
+        return m.group(1)
+
 
 class ChromeAndroidBase(Browser):
     """A base class for ChromeAndroid and AndroidWebView.
@@ -1016,6 +1066,29 @@ class EdgeChromium(Browser):
     def find_webdriver(self, channel=None):
         return find_executable("msedgedriver")
 
+    def webdriver_supports_browser(self, webdriver_binary, browser_binary):
+        edgedriver_version = self.webdriver_version(webdriver_binary)
+        if not edgedriver_version:
+            self.logger.warning(
+                "Unable to get version for EdgeDriver %s, rejecting it" %
+                webdriver_binary)
+            return False
+
+        browser_version = self.version(browser_binary)
+        if not browser_version:
+            # If we can't get the browser version, we just have to assume the
+            # EdgeDriver is good.
+            return True
+
+        # Check that the EdgeDriver version matches the Edge version.
+        edgedriver_major = edgedriver_version.split('.')[0]
+        browser_major = browser_version.split('.')[0]
+        if edgedriver_major != browser_major:
+            self.logger.warning("EdgeDriver %s does not match Edge %s" %
+                                (edgedriver_version, browser_version))
+            return False
+        return True
+
     def install_webdriver(self, dest=None, channel=None, browser_binary=None):
         if self.platform != "win" and self.platform != "macos":
             raise ValueError("Only Windows and Mac platforms are currently supported")
@@ -1074,6 +1147,21 @@ class EdgeChromium(Browser):
             self.logger.warning("Failed to find Edge binary.")
             return None
 
+    def webdriver_version(self, webdriver_binary):
+        if self.platform == "win":
+            return _get_fileversion(webdriver_binary, self.logger)
+
+        try:
+            version_string = call(webdriver_binary, "--version").strip()
+        except subprocess.CalledProcessError:
+            self.logger.warning("Failed to call %s" % webdriver_binary)
+            return None
+        m = re.match(r"MSEdgeDriver ([0-9][0-9.]*)", version_string)
+        if not m:
+            self.logger.warning("Failed to extract version from: %s" % version_string)
+            return None
+        return m.group(1)
+
 
 class Edge(Browser):
     """Edge-specific interface."""

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wpt/run.py

@@ -334,16 +334,19 @@ class Chrome(BrowserSetup):
                 logger.info("MojoJS enabled")
             except Exception as e:
                 logger.error("Cannot enable MojoJS: %s" % e)
+
         if kwargs["webdriver_binary"] is None:
             webdriver_binary = None
             if not kwargs["install_webdriver"]:
                 webdriver_binary = self.browser.find_webdriver()
+                if webdriver_binary and not self.browser.webdriver_supports_browser(
+                        webdriver_binary, kwargs["binary"]):
+                    webdriver_binary = None
 
             if webdriver_binary is None:
                 install = self.prompt_install("chromedriver")
 
                 if install:
-                    logger.info("Downloading chromedriver")
                     webdriver_binary = self.browser.install_webdriver(
                         dest=self.venv.bin_path,
                         channel=browser_channel,
@@ -355,7 +358,7 @@ class Chrome(BrowserSetup):
             if webdriver_binary:
                 kwargs["webdriver_binary"] = webdriver_binary
             else:
-                raise WptrunError("Unable to locate or install chromedriver binary")
+                raise WptrunError("Unable to locate or install matching ChromeDriver binary")
         if browser_channel in self.experimental_channels:
             logger.info("Automatically turning on experimental features for Chrome Dev/Canary or Chromium trunk")
             kwargs["binary_args"].append("--enable-experimental-web-platform-features")
@@ -513,13 +516,16 @@ class EdgeChromium(BrowserSetup):
                 kwargs["binary"] = binary
             else:
                 raise WptrunError("Unable to locate Edge binary")
+
         if kwargs["webdriver_binary"] is None:
             webdriver_binary = None
             if not kwargs["install_webdriver"]:
                 webdriver_binary = self.browser.find_webdriver()
+                if (webdriver_binary and not self.browser.webdriver_supports_browser(
+                    webdriver_binary, kwargs["binary"])):
+                    webdriver_binary = None
 
-            # Install browser if none are found or if it's found in venv path
-            if webdriver_binary is None or webdriver_binary in self.venv.bin_path:
+            if webdriver_binary is None:
                 install = self.prompt_install("msedgedriver")
 
                 if install: