Roll internal WPT tools

This rolls up to 4b8a64638b4c06fb38eb562b754eae389b381ec0

Assorted changes, including webkitgtk_minobrowser support and mojojs
improvements

Change-Id: I2a806f1a0040394541fbe52f749f3ab69d3820a8
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2453794Reviewed-by: Robert Ma <robertma@chromium.org>
Commit-Queue: Luke Z <lpz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#814352}

third_party/blink/tools/blinkpy/third_party/README.chromium

@@ -22,7 +22,7 @@ Local Modifications: None
 Name: web-platform-tests - Test Suites for Web Platform specifications
 Short Name: wpt
 URL: https://github.com/web-platform-tests/wpt/
-Version: 9f6a7e3cfdae54795a0ca58e194b828537f9659e
+Version: 4b8a64638b4c06fb38eb562b754eae389b381ec0
 License: LICENSES FOR W3C TEST SUITES (https://www.w3.org/Consortium/Legal/2008/03-bsd-license.html)
 License File: wpt/wpt/LICENSE.md
 Security Critical: no

third_party/blink/tools/blinkpy/third_party/wpt/checkout.sh

@@ -9,7 +9,7 @@ cd $DIR
 
 TARGET_DIR=$DIR/wpt
 REMOTE_REPO="https://github.com/web-platform-tests/wpt.git"
-WPT_HEAD=9f6a7e3cfdae54795a0ca58e194b828537f9659e
+WPT_HEAD=4b8a64638b4c06fb38eb562b754eae389b381ec0
 
 function clone {
   # Remove existing repo if already exists.

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/webdriver/webdriver/client.py

@@ -581,7 +581,7 @@ class Session(object):
         return self.send_session_command("GET", "source")
 
     @command
-    def new_window(self, type_hint=None):
+    def new_window(self, type_hint="tab"):
         body = {"type": type_hint}
         value = self.send_session_command("POST", "window/new", body)
 

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/webdriver/webdriver/transport.py

@@ -2,6 +2,7 @@ import json
 import select
 
 from six import text_type, PY3
+from six.moves.collections_abc import Mapping
 from six.moves.http_client import HTTPConnection
 from six.moves.urllib import parse as urlparse
 
@@ -10,6 +11,56 @@ from . import error
 """Implements HTTP transport for the WebDriver wire protocol."""
 
 
+missing = object()
+
+
+class ResponseHeaders(Mapping):
+    """Read-only dictionary-like API for accessing response headers.
+
+    This class:
+      * Normalizes the header keys it is built with to lowercase (such that
+        iterating the items will return lowercase header keys).
+      * Has case-insensitive header lookup.
+      * Always returns all header values that have the same name, separated by
+        commas.
+
+    It does not ensure header types (e.g. binary vs string).
+    """
+    def __init__(self, items):
+        self.headers_dict = {}
+        for key, value in items:
+            key = key.lower()
+            if key not in self.headers_dict:
+                self.headers_dict[key] = []
+            self.headers_dict[key].append(value)
+
+    def __getitem__(self, key):
+        """Get all headers of a certain (case-insensitive) name. If there is
+        more than one, the values are returned comma separated"""
+        values = self.headers_dict[key.lower()]
+        if len(values) == 1:
+            return values[0]
+        else:
+            return ", ".join(values)
+
+    def get_list(self, key, default=missing):
+        """Get all the header values for a particular field name as a list"""
+        try:
+            return self.headers_dict[key.lower()]
+        except KeyError:
+            if default is not missing:
+                return default
+            else:
+                raise
+
+    def __iter__(self):
+        for item in self.headers_dict:
+            yield item
+
+    def __len__(self):
+        return len(self.headers_dict)
+
+
 class Response(object):
     """
     Describes an HTTP response received from a remote end whose
@@ -40,7 +91,7 @@ class Response(object):
     def from_http(cls, http_response, decoder=json.JSONDecoder, **kwargs):
         try:
             body = json.load(http_response, cls=decoder, **kwargs)
-            headers = dict(http_response.getheaders())
+            headers = ResponseHeaders(http_response.getheaders())
         except ValueError:
             raise ValueError("Failed to decode response body as JSON:\n" +
                 http_response.read())

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wpt/browser.py

@@ -12,7 +12,7 @@ from distutils.spawn import find_executable
 from six.moves.urllib.parse import urlsplit
 import requests
 
-from .utils import call, get, rmtree, untar, unzip
+from .utils import call, get, rmtree, untar, unzip, get_download_to_descriptor, sha256sum
 
 uname = platform.uname()
 
@@ -110,7 +110,7 @@ class Browser(object):
         return NotImplemented
 
     @abstractmethod
-    def find_webdriver(self, channel=None):
+    def find_webdriver(self, venv_path=None, channel=None):
         """Find the binary of the WebDriver."""
         return NotImplemented
 
@@ -297,7 +297,7 @@ class Firefox(Browser):
             return None
         return path
 
-    def find_webdriver(self, channel=None):
+    def find_webdriver(self, venv_path=None, channel=None):
         return find_executable("geckodriver")
 
     def get_version_and_channel(self, binary):
@@ -506,7 +506,7 @@ class FirefoxAndroid(Browser):
     def find_binary(self, venv_path=None, channel=None):
         return self.apk_path
 
-    def find_webdriver(self, channel=None):
+    def find_webdriver(self, venv_path=None, channel=None):
         raise NotImplementedError
 
     def install_webdriver(self, dest=None, channel=None, browser_binary=None):
@@ -570,19 +570,21 @@ class Chrome(Browser):
             chrome_version = chrome_version.split(' ')[0]
             url = "https://storage.googleapis.com/chrome-wpt-mojom/%s/linux64/mojojs.zip" % chrome_version
 
-        last_url_file = os.path.join(dest, "mojojs", "gen", "DOWNLOADED_FROM")
+        extracted = os.path.join(dest, "mojojs", "gen")
+        last_url_file = os.path.join(extracted, "DOWNLOADED_FROM")
         if os.path.exists(last_url_file):
             with open(last_url_file, "rt") as f:
                 last_url = f.read().strip()
             if last_url == url:
                 self.logger.info("Mojo bindings already up to date")
-                return
-            rmtree(os.path.join(dest, "mojojs", "gen"))
+                return extracted
+            rmtree(extracted)
 
         self.logger.info("Downloading Mojo bindings from %s" % url)
         unzip(get(url).raw, dest)
         with open(last_url_file, "wt") as f:
             f.write(url)
+        return extracted
 
     def _chromedriver_platform_string(self):
         platform = self.platforms.get(uname[0])
@@ -656,7 +658,7 @@ class Chrome(Browser):
         self.logger.warning("Unable to find the browser binary.")
         return None
 
-    def find_webdriver(self, channel=None, browser_binary=None):
+    def find_webdriver(self, venv_path=None, channel=None, browser_binary=None):
         return find_executable("chromedriver")
 
     def webdriver_supports_browser(self, webdriver_binary, browser_binary):
@@ -824,7 +826,7 @@ class ChromeAndroidBase(Browser):
     def find_binary(self, venv_path=None, channel=None):
         raise NotImplementedError
 
-    def find_webdriver(self, channel=None):
+    def find_webdriver(self, venv_path=None, channel=None):
         return find_executable("chromedriver")
 
     def install_webdriver(self, dest=None, channel=None, browser_binary=None):
@@ -931,7 +933,7 @@ class ChromeiOS(Browser):
     def find_binary(self, venv_path=None, channel=None):
         raise NotImplementedError
 
-    def find_webdriver(self, channel=None):
+    def find_webdriver(self, venv_path=None, channel=None):
         raise NotImplementedError
 
     def install_webdriver(self, dest=None, channel=None, browser_binary=None):
@@ -986,7 +988,7 @@ class Opera(Browser):
     def find_binary(self, venv_path=None, channel=None):
         raise NotImplementedError
 
-    def find_webdriver(self, channel=None):
+    def find_webdriver(self, venv_path=None, channel=None):
         return find_executable("operadriver")
 
     def install_webdriver(self, dest=None, channel=None, browser_binary=None):
@@ -1063,7 +1065,7 @@ class EdgeChromium(Browser):
                 return find_executable("Microsoft Edge Canary", os.pathsep.join(macpaths))
         return binary
 
-    def find_webdriver(self, channel=None):
+    def find_webdriver(self, venv_path=None, channel=None):
         return find_executable("msedgedriver")
 
     def webdriver_supports_browser(self, webdriver_binary, browser_binary):
@@ -1178,7 +1180,7 @@ class Edge(Browser):
     def find_binary(self, venv_path=None, channel=None):
         raise NotImplementedError
 
-    def find_webdriver(self, channel=None):
+    def find_webdriver(self, venv_path=None, channel=None):
         return find_executable("MicrosoftWebDriver")
 
     def install_webdriver(self, dest=None, channel=None, browser_binary=None):
@@ -1212,7 +1214,7 @@ class InternetExplorer(Browser):
     def find_binary(self, venv_path=None, channel=None):
         raise NotImplementedError
 
-    def find_webdriver(self, channel=None):
+    def find_webdriver(self, venv_path=None, channel=None):
         return find_executable("IEDriverServer.exe")
 
     def install_webdriver(self, dest=None, channel=None, browser_binary=None):
@@ -1240,7 +1242,7 @@ class Safari(Browser):
     def find_binary(self, venv_path=None, channel=None):
         raise NotImplementedError
 
-    def find_webdriver(self, channel=None):
+    def find_webdriver(self, venv_path=None, channel=None):
         path = None
         if channel == "preview":
             path = "/Applications/Safari Technology Preview.app/Contents/MacOS"
@@ -1332,7 +1334,7 @@ class Servo(Browser):
             path = find_executable("servo")
         return path
 
-    def find_webdriver(self, channel=None):
+    def find_webdriver(self, venv_path=None, channel=None):
         return None
 
     def install_webdriver(self, dest=None, channel=None, browser_binary=None):
@@ -1365,7 +1367,7 @@ class Sauce(Browser):
     def find_binary(self, venev_path=None, channel=None):
         raise NotImplementedError
 
-    def find_webdriver(self, channel=None):
+    def find_webdriver(self, venv_path=None, channel=None):
         raise NotImplementedError
 
     def install_webdriver(self, dest=None, channel=None, browser_binary=None):
@@ -1390,7 +1392,7 @@ class WebKit(Browser):
     def find_binary(self, venv_path=None, channel=None):
         return None
 
-    def find_webdriver(self, channel=None):
+    def find_webdriver(self, venv_path=None, channel=None):
         return None
 
     def install_webdriver(self, dest=None, channel=None, browser_binary=None):
@@ -1402,7 +1404,101 @@ class WebKit(Browser):
 
 class WebKitGTKMiniBrowser(WebKit):
 
+
+    def _get_osidversion(self):
+        with open('/etc/os-release', 'r') as osrelease_handle:
+            for line in osrelease_handle.readlines():
+                if line.startswith('ID='):
+                    os_id = line.split('=')[1].strip().strip('"')
+                if line.startswith('VERSION_ID='):
+                    version_id = line.split('=')[1].strip().strip('"')
+        assert(os_id)
+        assert(version_id)
+        osidversion = os_id + '-' + version_id
+        assert(' ' not in osidversion)
+        assert(len(osidversion) > 3)
+        return osidversion.capitalize()
+
+
+    def download(self, dest=None, channel=None, rename=None):
+        base_dowload_uri = "https://webkitgtk.org/built-products/"
+        base_download_dir = base_dowload_uri + "x86_64/release/" + channel + "/" + self._get_osidversion() + "/MiniBrowser/"
+        try:
+            response = get(base_download_dir + "LAST-IS")
+        except requests.exceptions.HTTPError as e:
+            if e.response.status_code == 404:
+                raise RuntimeError("Can't find a WebKitGTK MiniBrowser %s bundle for %s at %s"
+                                   % (channel, self._get_osidversion(), base_dowload_uri))
+            raise
+
+        bundle_filename = response.text.strip()
+        bundle_url = base_download_dir + bundle_filename
+
+        if dest is None:
+            dest = self._get_dest(None, channel)
+        bundle_file_path = os.path.join(dest, bundle_filename)
+
+        self.logger.info("Downloading WebKitGTK MiniBrowser bundle from %s" % bundle_url)
+        with open(bundle_file_path, "w+b") as f:
+            get_download_to_descriptor(f, bundle_url)
+
+        bundle_filename_no_ext, _ = os.path.splitext(bundle_filename)
+        bundle_hash_url = base_download_dir + bundle_filename_no_ext + ".sha256sum"
+        bundle_expected_hash = get(bundle_hash_url).text.strip().split(" ")[0]
+        bundle_computed_hash = sha256sum(bundle_file_path)
+
+        if bundle_expected_hash != bundle_computed_hash:
+            self.logger.error("Calculated SHA256 hash is %s but was expecting %s" % (bundle_computed_hash,bundle_expected_hash))
+            raise RuntimeError("The WebKitGTK MiniBrowser bundle at %s has incorrect SHA256 hash." % bundle_file_path)
+        return bundle_file_path
+
+    def install(self, dest=None, channel=None, prompt=True):
+        dest = self._get_dest(dest, channel)
+        bundle_path = self.download(dest, channel)
+        bundle_uncompress_directory = os.path.join(dest, "webkitgtk_minibrowser")
+
+        # Clean it from previous runs
+        if os.path.exists(bundle_uncompress_directory):
+            rmtree(bundle_uncompress_directory)
+        os.mkdir(bundle_uncompress_directory)
+
+        with open(bundle_path, "rb") as f:
+            unzip(f, bundle_uncompress_directory)
+
+        install_dep_script = os.path.join(bundle_uncompress_directory, "install-dependencies.sh")
+        if os.path.isfile(install_dep_script):
+            self.logger.info("Executing install-dependencies.sh script from bundle.")
+            install_dep_cmd = [install_dep_script]
+            if not prompt:
+                install_dep_cmd.append("--autoinstall")
+            # use subprocess.check_call() directly to display unbuffered stdout/stderr in real-time.
+            subprocess.check_call(install_dep_cmd)
+
+        minibrowser_path = os.path.join(bundle_uncompress_directory, "MiniBrowser")
+        if not os.path.isfile(minibrowser_path):
+            raise RuntimeError("Can't find a MiniBrowser binary at %s" % minibrowser_path)
+
+        os.remove(bundle_path)
+        install_ok_file = os.path.join(bundle_uncompress_directory, ".installation-ok")
+        open(install_ok_file, "w").close()  # touch
+        self.logger.info("WebKitGTK MiniBrowser bundle for channel %s installed." % channel)
+        return minibrowser_path
+
+    def _find_executable_in_channel_bundle(self, binary, venv_path=None, channel=None):
+        if venv_path:
+            venv_base_path = self._get_dest(venv_path, channel)
+            bundle_dir = os.path.join(venv_base_path, "webkitgtk_minibrowser")
+            install_ok_file = os.path.join(bundle_dir, ".installation-ok")
+            if os.path.isfile(install_ok_file):
+                return find_executable(binary, bundle_dir)
+        return None
+
+
     def find_binary(self, venv_path=None, channel=None):
+        minibrowser_path = self._find_executable_in_channel_bundle("MiniBrowser", venv_path, channel)
+        if minibrowser_path:
+            return minibrowser_path
+
         libexecpaths = ["/usr/libexec/webkit2gtk-4.0"]  # Fedora path
         triplet = "x86_64-linux-gnu"
         # Try to use GCC to detect this machine triplet
@@ -1414,15 +1510,13 @@ class WebKitGTKMiniBrowser(WebKit):
                 pass
         # Add Debian/Ubuntu path
         libexecpaths.append("/usr/lib/%s/webkit2gtk-4.0" % triplet)
-        if channel == "nightly":
-            libexecpaths.append("/opt/webkitgtk/nightly")
         return find_executable("MiniBrowser", os.pathsep.join(libexecpaths))
 
-    def find_webdriver(self, channel=None):
-        path = os.environ['PATH']
-        if channel == "nightly":
-            path = "%s:%s" % (path, "/opt/webkitgtk/nightly")
-        return find_executable("WebKitWebDriver", path)
+    def find_webdriver(self, venv_path=None, channel=None):
+        webdriver_path = self._find_executable_in_channel_bundle("WebKitWebDriver", venv_path, channel)
+        if not webdriver_path:
+            webdriver_path = find_executable("WebKitWebDriver")
+        return webdriver_path
 
     def version(self, binary=None, webdriver_binary=None):
         if binary is None:
@@ -1456,7 +1550,7 @@ class Epiphany(Browser):
     def find_binary(self, venv_path=None, channel=None):
         return find_executable("epiphany")
 
-    def find_webdriver(self, channel=None):
+    def find_webdriver(self, venv_path=None, channel=None):
         return find_executable("WebKitWebDriver")
 
     def install_webdriver(self, dest=None, channel=None, browser_binary=None):

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wpt/install.py

@@ -7,7 +7,8 @@ latest_channels = {
     'chrome_android': 'dev',
     'edgechromium': 'dev',
     'safari': 'preview',
-    'servo': 'nightly'
+    'servo': 'nightly',
+    'webkitgtk_minibrowser': 'nightly'
 }
 
 channel_by_name = {

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wpt/run.py

@@ -322,16 +322,21 @@ class Chrome(BrowserSetup):
                 kwargs["binary"] = binary
             else:
                 raise WptrunError("Unable to locate Chrome binary")
+
+        if kwargs["mojojs_path"]:
+            kwargs["enable_mojojs"] = True
+            logger.info("--mojojs-path is provided, enabling MojoJS")
         # TODO(Hexcles): Enable this everywhere when Chrome 86 becomes stable.
-        if browser_channel in self.experimental_channels:
+        elif browser_channel in self.experimental_channels:
             try:
-                self.browser.install_mojojs(
+                path = self.browser.install_mojojs(
                     dest=self.venv.path,
                     channel=browser_channel,
                     browser_binary=kwargs["binary"],
                 )
+                kwargs["mojojs_path"] = path
                 kwargs["enable_mojojs"] = True
-                logger.info("MojoJS enabled")
+                logger.info("MojoJS enabled automatically (mojojs_path: %s)" % path)
             except Exception as e:
                 logger.error("Cannot enable MojoJS: %s" % e)
 
@@ -662,18 +667,19 @@ class WebKitGTKMiniBrowser(BrowserSetup):
     browser_cls = browser.WebKitGTKMiniBrowser
 
     def install(self, channel=None):
-        raise NotImplementedError
+        if self.prompt_install(self.name):
+            return self.browser.install(self.venv.path, channel, self.prompt)
 
     def setup_kwargs(self, kwargs):
         if kwargs["binary"] is None:
-            binary = self.browser.find_binary(channel=kwargs["browser_channel"])
+            binary = self.browser.find_binary(venv_path=self.venv.path, channel=kwargs["browser_channel"])
 
             if binary is None:
                 raise WptrunError("Unable to find MiniBrowser binary")
             kwargs["binary"] = binary
 
         if kwargs["webdriver_binary"] is None:
-            webdriver_binary = self.browser.find_webdriver(channel=kwargs["browser_channel"])
+            webdriver_binary = self.browser.find_webdriver(venv_path=self.venv.path, channel=kwargs["browser_channel"])
 
             if webdriver_binary is None:
                 raise WptrunError("Unable to find WebKitWebDriver in PATH")

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wpt/utils.py

@@ -5,8 +5,11 @@ import shutil
 import stat
 import subprocess
 import tarfile
+import time
 import zipfile
 from io import BytesIO
+from socket import error as SocketError  # NOQA: N812
+from six.moves.urllib.request import urlopen
 
 MYPY = False
 if MYPY:
@@ -100,6 +103,41 @@ def get(url):
     return resp
 
 
+def get_download_to_descriptor(fd, url, max_retries=5):
+    """Download an URL in chunks and saves it to a file descriptor (truncating it)
+    It doesn't close the descriptor, but flushes it on success.
+    It retries the download in case of ECONNRESET up to max_retries.
+    This function is meant to download big files directly to the disk without
+    caching the whole file in memory.
+    """
+    if max_retries < 1:
+        max_retries = 1
+    wait = 2
+    for current_retry in range(1, max_retries+1):
+        try:
+            logger.info("Downloading %s Try %d/%d" % (url, current_retry, max_retries))
+            resp = urlopen(url)
+            # We may come here in a retry, ensure to truncate fd before start writing.
+            fd.seek(0)
+            fd.truncate(0)
+            while True:
+                chunk = resp.read(16*1024)
+                if not chunk:
+                    break  # Download finished
+                fd.write(chunk)
+            fd.flush()
+            # Success
+            return
+        except SocketError as e:
+            if current_retry < max_retries and e.errno == errno.ECONNRESET:
+                # Retry
+                logger.error("Connection reset by peer. Retrying after %ds..." % wait)
+                time.sleep(wait)
+                wait *= 2
+            else:
+                # Maximum retries or unknown error
+                raise
+
 def rmtree(path):
     # This works around two issues:
     # 1. Cannot delete read-only files owned by us (e.g. files extracted from tarballs)
@@ -114,3 +152,13 @@ def rmtree(path):
             raise
 
     return shutil.rmtree(path, onerror=handle_remove_readonly)
+
+
+def sha256sum(file_path):
+    """Computes the SHA256 hash sum of a file"""
+    from hashlib import sha256
+    hash = sha256()
+    with open(file_path, 'rb') as f:
+        for chunk in iter(lambda: f.read(4096), b''):
+            hash.update(chunk)
+    return hash.hexdigest()