Roll internal WPT tools

This rolls up to c808aa3d15a42648d8b25a838024813990959e37

Bug: None
Change-Id: I46407cabbe41fff8dc0b8aef766fb66baf59f9a3
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2468897
Auto-Submit: Stephen McGruer <smcgruer@chromium.org>
Reviewed-by: Luke Z <lpz@chromium.org>
Reviewed-by: Robert Ma <robertma@chromium.org>
Commit-Queue: Robert Ma <robertma@chromium.org>
Cr-Commit-Position: refs/heads/master@{#816991}

third_party/blink/tools/blinkpy/third_party/README.chromium

@@ -22,7 +22,7 @@ Local Modifications: None
 Name: web-platform-tests - Test Suites for Web Platform specifications
 Short Name: wpt
 URL: https://github.com/web-platform-tests/wpt/
-Version: 4b8a64638b4c06fb38eb562b754eae389b381ec0
+Version: c808aa3d15a42648d8b25a838024813990959e37
 License: LICENSES FOR W3C TEST SUITES (https://www.w3.org/Consortium/Legal/2008/03-bsd-license.html)
 License File: wpt/wpt/LICENSE.md
 Security Critical: no

third_party/blink/tools/blinkpy/third_party/wpt/checkout.sh

@@ -9,7 +9,7 @@ cd $DIR
 
 TARGET_DIR=$DIR/wpt
 REMOTE_REPO="https://github.com/web-platform-tests/wpt.git"
-WPT_HEAD=4b8a64638b4c06fb38eb562b754eae389b381ec0
+WPT_HEAD=c808aa3d15a42648d8b25a838024813990959e37
 
 function clone {
   # Remove existing repo if already exists.

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/lint/lint.py

@@ -957,6 +957,8 @@ def create_parser():
                         "working directory, not just files that changed")
     parser.add_argument("--github-checks-text-file", type=ensure_text,
                         help="Path to GitHub checks output file for Taskcluster runs")
+    parser.add_argument("-j", "--jobs", type=int, default=0,
+                        help="Level to parallelism to use (defaults to 0, which detects the number of CPUs)")
     return parser
 
 
@@ -984,18 +986,23 @@ def main(**kwargs_str):
 
     github_checks_outputter = get_gh_checks_outputter(kwargs["github_checks_text_file"])
 
-    return lint(repo_root, paths, output_format, ignore_glob, github_checks_outputter)
+    jobs = kwargs.get("jobs", 0)
+
+    return lint(repo_root, paths, output_format, ignore_glob, github_checks_outputter, jobs)
 
 
 # best experimental guess at a decent cut-off for using the parallel path
 MIN_FILES_FOR_PARALLEL = 80
 
 
-def lint(repo_root, paths, output_format, ignore_glob=None, github_checks_outputter=None):
-    # type: (Text, List[Text], Text, Optional[List[Text]], Optional[GitHubChecksOutputter]) -> int
+def lint(repo_root, paths, output_format, ignore_glob=None, github_checks_outputter=None, jobs=0):
+    # type: (Text, List[Text], Text, Optional[List[Text]], Optional[GitHubChecksOutputter], int) -> int
     error_count = defaultdict(int)  # type: Dict[Text, int]
     last = None
 
+    if jobs == 0:
+        jobs = multiprocessing.cpu_count()
+
     with io.open(os.path.join(repo_root, "lint.ignore"), "r") as f:
         ignorelist, skipped_files = parse_ignorelist(f)
 
@@ -1053,8 +1060,8 @@ def lint(repo_root, paths, output_format, ignore_glob=None, github_checks_output
 
     paths = [p for p in paths if p not in skip]
 
-    if len(to_check_content) >= MIN_FILES_FOR_PARALLEL:
-        pool = multiprocessing.Pool()
+    if jobs > 1 and len(to_check_content) >= MIN_FILES_FOR_PARALLEL:
+        pool = multiprocessing.Pool(jobs)
         # submit this job first, as it's the longest running
         all_paths_result = pool.apply_async(check_all_paths, (repo_root, paths))
         # each item tends to be quick, so pass things in large chunks to avoid too much IPC overhead

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/serve/serve.py

@@ -73,7 +73,9 @@ class WrapperHandler(object):
         self.handler(request, response)
 
     def handle_request(self, request, response):
-        for header_name, header_value in self.headers:
+        headers = self.headers + handlers.load_headers(
+            request, self._get_filesystem_path(request))
+        for header_name, header_value in headers:
             response.headers.set(header_name, header_value)
 
         self.check_exposure(request)
@@ -111,13 +113,17 @@ class WrapperHandler(object):
                 path = replace_end(path, src, dest)
         return path
 
+    def _get_filesystem_path(self, request):
+        """Get the path of the underlying resource file on disk."""
+        return self._get_path(filesystem_path(self.base_path, request, self.url_base), False)
+
     def _get_metadata(self, request):
         """Get an iterator over script metadata based on // META comments in the
         associated js file.
 
         :param request: The Request being processed.
         """
-        path = self._get_path(filesystem_path(self.base_path, request, self.url_base), False)
+        path = self._get_filesystem_path(request)
         try:
             with open(path, "rb") as f:
                 for key, value in read_script_metadata(f, js_meta_re):
@@ -532,7 +538,7 @@ def start_servers(host, ports, paths, routes, bind_address, config, **kwargs):
         # If trying to start HTTP/2.0 server, check compatibility
         if scheme == 'h2' and not http2_compatible():
             logger.error('Cannot start HTTP/2.0 server as the environment is not compatible. ' +
-                         'Requires Python 2.7.10+ (< 3.0) and OpenSSL 1.0.2+')
+                         'Requires Python 2.7.10+ or 3.6+ and OpenSSL 1.0.2+')
             continue
 
         for port in ports:

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wpt/browser.py

@@ -661,7 +661,7 @@ class Chrome(Browser):
     def find_webdriver(self, venv_path=None, channel=None, browser_binary=None):
         return find_executable("chromedriver")
 
-    def webdriver_supports_browser(self, webdriver_binary, browser_binary):
+    def webdriver_supports_browser(self, webdriver_binary, browser_binary, browser_channel):
         chromedriver_version = self.webdriver_version(webdriver_binary)
         if not chromedriver_version:
             self.logger.warning(
@@ -676,9 +676,17 @@ class Chrome(Browser):
             return True
 
         # Check that the ChromeDriver version matches the Chrome version.
-        chromedriver_major = chromedriver_version.split('.')[0]
-        browser_major = browser_version.split('.')[0]
+        chromedriver_major = int(chromedriver_version.split('.')[0])
+        browser_major = int(browser_version.split('.')[0])
         if chromedriver_major != browser_major:
+            # There is no official ChromeDriver release for the dev channel -
+            # it switches between beta and tip-of-tree, so we accept version+1
+            # too for dev.
+            if browser_channel == "dev" and chromedriver_major == (browser_major + 1):
+                self.logger.debug(
+                    "Accepting ChromeDriver %s for Chrome/Chromium Dev %s" %
+                    (chromedriver_version, browser_version))
+                return True
             self.logger.warning(
                 "ChromeDriver %s does not match Chrome/Chromium %s" %
                 (chromedriver_version, browser_version))

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wpt/run.py

@@ -345,7 +345,7 @@ class Chrome(BrowserSetup):
             if not kwargs["install_webdriver"]:
                 webdriver_binary = self.browser.find_webdriver()
                 if webdriver_binary and not self.browser.webdriver_supports_browser(
-                        webdriver_binary, kwargs["binary"]):
+                        webdriver_binary, kwargs["binary"], browser_channel):
                     webdriver_binary = None
 
             if webdriver_binary is None:

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/handlers.py

@@ -32,7 +32,6 @@ def guess_content_type(path):
     return "application/octet-stream"
 
 
-
 def filesystem_path(base_path, request, url_base="/"):
     if base_path is None:
         base_path = request.doc_root
@@ -53,6 +52,7 @@ def filesystem_path(base_path, request, url_base="/"):
 
     return new_path
 
+
 class DirectoryHandler(object):
     def __init__(self, base_path=None, url_base="/"):
         self.base_path = base_path
@@ -121,6 +121,7 @@ class DirectoryHandler(object):
                    {"link": link, "name": escape(item), "class": class_,
                     "headers": dot_headers_markup})
 
+
 def parse_qs(qs):
     """Parse a query string given as a string argument (data of type
     application/x-www-form-urlencoded). Data are returned as a dictionary. The
@@ -140,7 +141,12 @@ def parse_qs(qs):
         rv[name].append(value)
     return dict(rv)
 
+
 def wrap_pipeline(path, request, response):
+    """Applies pipelines to a response.
+
+    Pipelines are specified in the filename (.sub.) or the query param (?pipe).
+    """
     query = parse_qs(request.url_parts.query)
     pipe_string = ""
 
@@ -161,6 +167,36 @@ def wrap_pipeline(path, request, response):
     return response
 
 
+def load_headers(request, path):
+    """Loads headers from files for a given path.
+
+    Attempts to load both the neighbouring __dir__{.sub}.headers and
+    PATH{.sub}.headers (applying template substitution if needed); results are
+    concatenated in that order.
+    """
+    def _load(request, path):
+        headers_path = path + ".sub.headers"
+        if os.path.exists(headers_path):
+            use_sub = True
+        else:
+            headers_path = path + ".headers"
+            use_sub = False
+
+        try:
+            with open(headers_path, "rb") as headers_file:
+                data = headers_file.read()
+        except IOError:
+            return []
+        else:
+            if use_sub:
+                data = template(request, data, escape_type="none")
+            return [tuple(item.strip() for item in line.split(b":", 1))
+                    for line in data.splitlines() if line]
+
+    return (_load(request, os.path.join(os.path.dirname(path), "__dir__")) +
+            _load(request, path))
+
+
 class FileHandler(object):
     def __init__(self, base_path=None, url_base="/"):
         self.base_path = base_path
@@ -197,33 +233,13 @@ class FileHandler(object):
             raise HTTPException(404)
 
     def get_headers(self, request, path):
-        rv = (self.load_headers(request, os.path.join(os.path.dirname(path), "__dir__")) +
-              self.load_headers(request, path))
+        rv = load_headers(request, path)
 
         if not any(key.lower() == b"content-type" for (key, _) in rv):
             rv.insert(0, (b"Content-Type", guess_content_type(path).encode("ascii")))
 
         return rv
 
-    def load_headers(self, request, path):
-        headers_path = path + ".sub.headers"
-        if os.path.exists(headers_path):
-            use_sub = True
-        else:
-            headers_path = path + ".headers"
-            use_sub = False
-
-        try:
-            with open(headers_path, "rb") as headers_file:
-                data = headers_file.read()
-        except IOError:
-            return []
-        else:
-            if use_sub:
-                data = template(request, data, escape_type="none")
-            return [tuple(item.strip() for item in line.split(b":", 1))
-                    for line in data.splitlines() if line]
-
     def get_data(self, response, path, byte_ranges):
         """Return either the handle to a file, or a string containing
         the content of a chunk of the file, if we have a range request."""
@@ -312,7 +328,6 @@ class PythonScriptHandler(object):
 
         self._set_path_and_load_file(request, response, func)
 
-
     def frame_handler(self, request):
         """
         This creates a FunctionHandler with one or more of the handling functions.
@@ -340,8 +355,10 @@ class PythonScriptHandler(object):
             return handler
         return self._set_path_and_load_file(request, None, func)
 
+
 python_script_handler = PythonScriptHandler()
 
+
 class FunctionHandler(object):
     def __init__(self, func):
         self.func = func
@@ -370,10 +387,11 @@ class FunctionHandler(object):
             wrap_pipeline('', request, response)
 
 
-#The generic name here is so that this can be used as a decorator
+# The generic name here is so that this can be used as a decorator
 def handler(func):
     return FunctionHandler(func)
 
+
 class JsonHandler(object):
     def __init__(self, func):
         self.func = func
@@ -395,9 +413,11 @@ class JsonHandler(object):
         response.headers.set("Content-Length", length)
         return value
 
+
 def json_handler(func):
     return JsonHandler(func)
 
+
 class AsIsHandler(object):
     def __init__(self, base_path=None, url_base="/"):
         self.base_path = base_path
@@ -414,8 +434,10 @@ class AsIsHandler(object):
         except IOError:
             raise HTTPException(404)
 
+
 as_is_handler = AsIsHandler()
 
+
 class BasicAuthHandler(object):
     def __init__(self, handler, user, password):
         """
@@ -442,8 +464,10 @@ class BasicAuthHandler(object):
                 return response
             return self.handler(request, response)
 
+
 basic_auth_handler = BasicAuthHandler(file_handler, None, None)
 
+
 class ErrorHandler(object):
     def __init__(self, status):
         self.status = status
@@ -454,7 +478,7 @@ class ErrorHandler(object):
 
 class StringHandler(object):
     def __init__(self, data, content_type, **headers):
-        """Hander that reads a file from a path and substitutes some fixed data
+        """Handler that returns a fixed data string and headers
 
         :param data: String to use
         :param content_type: Content type header to server the response with
@@ -478,7 +502,9 @@ class StringHandler(object):
 
 class StaticHandler(StringHandler):
     def __init__(self, path, format_args, content_type, **headers):
-        """Hander that reads a file from a path and substitutes some fixed data
+        """Handler that reads a file from a path and substitutes some fixed data
+
+        Note that *.headers files have no effect in this handler.
 
         :param path: Path to the template file to use
         :param format_args: Dictionary of values to substitute into the template file

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/response.py

@@ -413,9 +413,9 @@ class H2Response(Response):
             item = None
             item_iter = self.iter_content()
             try:
-                item = item_iter.next()
+                item = next(item_iter)
                 while True:
-                    check_last = item_iter.next()
+                    check_last = next(item_iter)
                     self.writer.write_data(item, last=False)
                     item = check_last
             except StopIteration:
@@ -451,6 +451,13 @@ class H2ResponseWriter(object):
         secondary_headers = []  # Non ':' prefixed headers are to be added afterwards
 
         for header, value in headers:
+            # h2_headers are native strings
+            # header field names are strings of ASCII
+            if isinstance(header, binary_type):
+                header = header.decode('ascii')
+            # value in headers can be either string or integer
+            if isinstance(value, binary_type):
+                value = self.decode(value)
             if header in h2_headers:
                 header = ':' + header
                 formatted_headers.append((header, str(value)))
@@ -635,6 +642,15 @@ class H2ResponseWriter(object):
         self.content_written = True
         self.socket.sendall(raw_data)
 
+    def decode(self, data):
+        """Convert bytes to unicode according to response.encoding."""
+        if isinstance(data, binary_type):
+            return data.decode(self._response.encoding)
+        elif isinstance(data, text_type):
+            return data
+        else:
+            raise ValueError(type(data))
+
     def encode(self, data):
         """Convert unicode to bytes according to response.encoding."""
         if isinstance(data, binary_type):

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/server.py

@@ -26,7 +26,7 @@ from .logger import get_logger
 from .request import Server, Request, H2Request
 from .response import Response, H2Response
 from .router import Router
-from .utils import HTTPException
+from .utils import HTTPException, isomorphic_decode
 from .constants import h2_headers
 
 # We need to stress test that browsers can send/receive many headers (there is
@@ -506,6 +506,8 @@ class H2Headers(dict):
     def __init__(self, headers):
         self.raw_headers = OrderedDict()
         for key, val in headers:
+            key = isomorphic_decode(key)
+            val = isomorphic_decode(val)
             self.raw_headers[key] = val
             dict.__setitem__(self, self._convert_h2_header_to_h1(key), val)
 

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/utils.py

@@ -152,8 +152,9 @@ def get_port(host=''):
     return port
 
 def http2_compatible():
-    # Currently, the HTTP/2.0 server is only working in python 2.7.10+ and OpenSSL 1.0.2+
+    # Currently, the HTTP/2.0 server is only working in python 2.7.10+ or 3.6+ and OpenSSL 1.0.2+
     import ssl
     ssl_v = ssl.OPENSSL_VERSION_INFO
-    return ((sys.version_info[0] == 2 and sys.version_info[1] == 7 and sys.version_info[2] >= 10) and
+    py_v = sys.version_info
+    return (((py_v[0] == 2 and py_v[1] == 7 and py_v[2] >= 10) or (py_v[0] == 3 and py_v[1] >= 6)) and
         (ssl_v[0] == 1 and (ssl_v[1] == 1 or (ssl_v[1] == 0 and ssl_v[2] >= 2))))