Roll in new version of WPT tools

Bug: 749879
Change-Id: Iff4a6e047b029bb12fa225bbf2cdeffc9172a1b7
Reviewed-on: https://chromium-review.googlesource.com/941966Reviewed-by: Philip Jägenstedt <foolip@chromium.org>
Commit-Queue: Robert Ma <robertma@chromium.org>
Cr-Commit-Position: refs/heads/master@{#540311}

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/README.chromium

@@ -51,7 +51,7 @@ Local Modifications: None
 Name: web-platform-tests - Test Suites for Web Platform specifications
 Short Name: wpt
 URL: https://github.com/w3c/web-platform-tests/
-Version: 60a9eb2ca37127deaeb7a979f27a9d0c30970786
+Version: 38612167f54c475836c122013756e92b9a8859cc
 License: LICENSES FOR W3C TEST SUITES (http://www.w3.org/Consortium/Legal/2008/04-testsuite-copyright.html)
 License File: wpt/wpt/LICENSE.md
 Security Critical: no

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/checkout.sh

@@ -9,7 +9,7 @@ cd $DIR
 
 TARGET_DIR=$DIR/wpt
 REMOTE_REPO="https://chromium.googlesource.com/external/w3c/web-platform-tests.git"
-WPT_HEAD=60a9eb2ca37127deaeb7a979f27a9d0c30970786
+WPT_HEAD=38612167f54c475836c122013756e92b9a8859cc
 
 function clone {
   # Remove existing repo if already exists.

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/lint/lint.py

@@ -430,7 +430,7 @@ def check_parsed(repo_root, path, f):
     for reftest_node in source_file.reftest_nodes:
         href = reftest_node.attrib.get("href", "").strip(space_chars)
         parts = urlsplit(href)
-        if parts.scheme or parts.netloc:
+        if (parts.scheme or parts.netloc) and parts != urlsplit("about:blank"):
             errors.append(("ABSOLUTE-URL-REF",
                      "Reference test with a reference file specified via an absolute URL: '%s'" % href, path, None))
             continue
@@ -722,7 +722,8 @@ def changed_files(wpt_root):
 
 def lint_paths(kwargs, wpt_root):
     if kwargs.get("paths"):
-        paths = kwargs["paths"]
+        r = os.path.realpath(wpt_root)
+        paths = [os.path.relpath(os.path.realpath(x), r) for x in kwargs["paths"]]
     elif kwargs["all"]:
         paths = list(all_filesystem_paths(wpt_root))
     else:

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/localpaths.py

@@ -9,7 +9,10 @@ sys.path.insert(0, os.path.join(here, "six"))
 sys.path.insert(0, os.path.join(here, "html5lib"))
 sys.path.insert(0, os.path.join(here, "wptserve"))
 sys.path.insert(0, os.path.join(here, "pywebsocket", "src"))
-sys.path.insert(0, os.path.join(here, "py"))
-sys.path.insert(0, os.path.join(here, "pytest"))
+sys.path.insert(0, os.path.join(here, "third_party", "attrs", "src"))
+sys.path.insert(0, os.path.join(here, "third_party", "funcsigs"))
+sys.path.insert(0, os.path.join(here, "third_party", "pluggy"))
+sys.path.insert(0, os.path.join(here, "third_party", "py"))
+sys.path.insert(0, os.path.join(here, "third_party", "pytest"))
 sys.path.insert(0, os.path.join(here, "webdriver"))
 sys.path.insert(0, os.path.join(here, "wptrunner"))

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/manifest/XMLParser.py

@@ -70,8 +70,8 @@ class XMLParser(object):
     def _end(self, tag):
         return self._target.end(_fixname(tag))
 
-    def _external(self, context, base, systemId, publicId):
-        if publicId in {
+    def _external(self, context, base, system_id, public_id):
+        if public_id in {
                 "-//W3C//DTD XHTML 1.0 Transitional//EN",
                 "-//W3C//DTD XHTML 1.1//EN",
                 "-//W3C//DTD XHTML 1.0 Strict//EN",

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/manifest/manifest.py

@@ -2,7 +2,8 @@ import json
 import os
 import re
 from collections import defaultdict
-from six import iteritems, itervalues, viewkeys
+from six import iteritems, itervalues, viewkeys, string_types
+from tempfile import mkstemp
 
 from .item import ManualTest, WebdriverSpecTest, Stub, RefTestNode, RefTest, TestharnessTest, SupportFile, ConformanceCheckerTest, VisualTest
 from .log import get_logger
@@ -20,14 +21,6 @@ class ManifestVersionMismatch(ManifestError):
     pass
 
 
-def sourcefile_items(args):
-    tests_root, url_base, rel_path, status = args
-    source_file = SourceFile(tests_root,
-                             rel_path,
-                             url_base)
-    return rel_path, source_file.manifest_items()
-
-
 class Manifest(object):
     def __init__(self, url_base="/"):
         assert url_base is not None
@@ -221,7 +214,7 @@ def load(tests_root, manifest):
     logger = get_logger()
 
     # "manifest" is a path or file-like object.
-    if isinstance(manifest, basestring):
+    if isinstance(manifest, string_types):
         if os.path.exists(manifest):
             logger.debug("Opening manifest at %s" % manifest)
         else:
@@ -240,6 +233,11 @@ def write(manifest, manifest_path):
     dir_name = os.path.dirname(manifest_path)
     if not os.path.exists(dir_name):
         os.makedirs(dir_name)
-    with open(manifest_path, "wb") as f:
-        json.dump(manifest.to_json(), f, sort_keys=True, indent=1, separators=(',', ': '))
-        f.write("\n")
+
+    fd, temp_manifest_path = mkstemp(dir=dir_name)
+    temp_manifest = open(temp_manifest_path, "wb")
+    json.dump(manifest.to_json(), temp_manifest,
+              sort_keys=True, indent=1, separators=(',', ': '))
+    temp_manifest.write("\n")
+    os.rename(temp_manifest_path, manifest_path)
+    os.close(fd)

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/serve/serve.py

@@ -203,6 +203,8 @@ subdomains = [u"www",
               u"天気の良い日",
               u"élève"]
 
+not_subdomains = [u"nonexistent-origin"]
+
 class RoutesBuilder(object):
     def __init__(self):
         self.forbidden_override = [("GET", "/tools/runner/*", handlers.file_handler),
@@ -468,6 +470,24 @@ def get_subdomains(host):
             for subdomain in subdomains}
 
 
+def get_not_subdomains(host):
+    #This assumes that the tld is ascii-only or already in punycode
+    return {subdomain: (subdomain.encode("idna"), host)
+            for subdomain in not_subdomains}
+
+
+def make_hosts_file(config, host):
+    rv = []
+
+    for domain in config["domains"].values():
+        rv.append("%s\t%s\n" % (host, domain))
+
+    for not_domain in config.get("not_domains", {}).values():
+        rv.append("0.0.0.0\t%s\n" % not_domain)
+
+    return "".join(rv)
+
+
 def start_servers(host, ports, paths, routes, bind_hostname, config, ssl_config,
                   **kwargs):
     servers = defaultdict(list)
@@ -626,6 +646,7 @@ def get_ports(config, ssl_environment):
 def normalise_config(config, ports):
     host = config["external_host"] if config["external_host"] else config["host"]
     domains = get_subdomains(host)
+    not_domains = get_not_subdomains(host)
     ports_ = {}
     for scheme, ports_used in ports.iteritems():
         ports_[scheme] = ports_used
@@ -633,6 +654,9 @@ def normalise_config(config, ports):
     for key, value in domains.iteritems():
         domains[key] = ".".join(value)
 
+    for key, value in not_domains.iteritems():
+        not_domains[key] = ".".join(value)
+
     domains[""] = host
 
     ports_ = {}
@@ -644,6 +668,7 @@ def normalise_config(config, ports):
     config_ = config.copy()
     config_["host"] = host
     config_["domains"] = domains
+    config_["not_domains"] = not_domains
     config_["ports"] = ports_
     return config_
 

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/sslutils/base.py

-def get_logger(name="ssl"):
-    logger = structured.get_default_logger(name)
-    if logger is None:
-        logger = structured.structuredlog.StructuredLogger(name)
-    return logger
-
 class NoSSLEnvironment(object):
     ssl_enabled = False
 

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/wpt/browser.py

@@ -4,9 +4,13 @@ import platform
 import re
 import shutil
 import stat
+import subprocess
+import sys
 from abc import ABCMeta, abstractmethod
 from ConfigParser import RawConfigParser
+from datetime import datetime, timedelta
 from distutils.spawn import find_executable
+from io import BytesIO
 
 from utils import call, get, untar, unzip
 
@@ -155,9 +159,15 @@ class Firefox(Browser):
         dest = os.path.join(dest, "profiles")
         if not os.path.exists(dest):
             os.makedirs(dest)
-        with open(os.path.join(dest, "prefs_general.js"), "wb") as f:
-            resp = get("https://hg.mozilla.org/mozilla-central/raw-file/tip/testing/profiles/prefs_general.js")
-            f.write(resp.content)
+        prefs_path = os.path.join(dest, "prefs_general.js")
+
+        now = datetime.now()
+        if (not os.path.exists(prefs_path) or
+            (datetime.fromtimestamp(os.stat(prefs_path).st_mtime) <
+             now - timedelta(days=2))):
+            with open(prefs_path, "wb") as f:
+                resp = get("https://hg.mozilla.org/mozilla-central/raw-file/tip/testing/profiles/prefs_general.js")
+                f.write(resp.content)
 
         return dest
 
@@ -279,6 +289,29 @@ class Chrome(Browser):
                 sys.exit(1)
 
 
+class ChromeAndroid(Browser):
+    """Chrome-specific interface for android.
+
+    Includes installation, webdriver installation, and wptrunner setup methods.
+    """
+
+    product = "chrome_android"
+    requirements = "requirements_chrome_android.txt"
+
+    def install(self, dest=None):
+        raise NotImplementedError
+
+    def find_webdriver(self):
+        return find_executable("chromedriver")
+
+    def install_webdriver(self, dest=None):
+        chrome = Chrome()
+        return chrome.install_webdriver(dest)
+
+    def version(self, root):
+        raise NotImplementedError
+
+
 class Opera(Browser):
     """Opera-specific interface.
 

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/wpt/install.py

@@ -39,8 +39,3 @@ def install(name, component, destination):
     subclass = getattr(browser, name.title())
     sys.stdout.write('Now installing %s %s...\n' % (name, component))
     getattr(subclass(), method)(dest=destination)
-
-
-if __name__ == '__main__':
-    args = parser.parse_args()
-    run(None, **vars(args))

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/wpt/markdown.py

@@ -38,18 +38,3 @@ def table(headings, data, log):
     for row in data:
         log("|%s|" % "|".join(" %s" % row[i].ljust(max_widths[i] - 1) for i in cols))
     log("")
-
-
-def err_string(results_dict, iterations):
-    """Create and return string with errors from test run."""
-    rv = []
-    total_results = sum(results_dict.values())
-    for key, value in sorted(results_dict.items()):
-        rv.append("%s%s" %
-                  (key, ": %s/%s" % (value, iterations) if value != iterations else ""))
-    if total_results < iterations:
-        rv.append("MISSING: %s/%s" % (iterations - total_results, iterations))
-    rv = ", ".join(rv)
-    if is_inconsistent(results_dict, iterations):
-        rv = "**%s**" % rv
-    return rv

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/wpt/run.py

@@ -11,6 +11,8 @@ wpt_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os
 sys.path.insert(0, os.path.abspath(os.path.join(wpt_root, "tools")))
 
 from . import browser, utils, virtualenv
+from ..serve import serve
+
 logger = None
 
 
@@ -94,13 +96,10 @@ otherwise install OpenSSL and ensure that it's on your $PATH.""")
 
 def check_environ(product):
     if product not in ("firefox", "servo"):
-        expected_hosts = ["web-platform.test",
-                          "www.web-platform.test",
-                          "www1.web-platform.test",
-                          "www2.web-platform.test",
-                          "xn--n8j6ds53lwwkrqhv28a.web-platform.test",
-                          "xn--lve-6lad.web-platform.test",
-                          "nonexistent-origin.web-platform.test"]
+        expected_hosts = {".".join(x)
+                          for x in serve.get_subdomains("web-platform.test").values()}
+        expected_hosts |= {".".join(x)
+                           for x in serve.get_not_subdomains("web-platform.test").values()}
         missing_hosts = set(expected_hosts)
         if platform.uname()[0] != "Windows":
             hosts_path = "/etc/hosts"
@@ -114,13 +113,17 @@ def check_environ(product):
                 for host in hosts:
                     missing_hosts.discard(host)
             if missing_hosts:
-                raise WptrunError("""Missing hosts file configuration. Expected entries like:
+                if platform.uname()[0] != "Windows":
+                    message = """Missing hosts file configuration. Run
+
+python wpt make-hosts-file >> %s
 
-%s
+from a shell with Administrator privileges.""" % hosts_path
+                else:
+                    message = """Missing hosts file configuration. Run
 
-See README.md for more details.""" % "\n".join("%s\t%s" %
-                                               ("127.0.0.1" if "nonexistent" not in host else "0.0.0.0", host)
-                                               for host in expected_hosts))
+./wpt make-hosts-file | sudo tee -a %s""" % hosts_path
+                raise WptrunError(message)
 
 
 class BrowserSetup(object):
@@ -223,6 +226,28 @@ class Chrome(BrowserSetup):
             else:
                 raise WptrunError("Unable to locate or install chromedriver binary")
 
+class ChromeAndroid(BrowserSetup):
+    name = "chrome_android"
+    browser_cls = browser.ChromeAndroid
+
+    def setup_kwargs(self, kwargs):
+        if kwargs["webdriver_binary"] is None:
+            webdriver_binary = self.browser.find_webdriver()
+
+            if webdriver_binary is None:
+                install = self.prompt_install("chromedriver")
+
+                if install:
+                    print("Downloading chromedriver")
+                    webdriver_binary = self.browser.install_webdriver(dest=self.venv.bin_path)
+            else:
+                print("Using webdriver binary %s" % webdriver_binary)
+
+            if webdriver_binary:
+                kwargs["webdriver_binary"] = webdriver_binary
+            else:
+                raise WptrunError("Unable to locate or install chromedriver binary")
+
 
 class Opera(BrowserSetup):
     name = "opera"
@@ -321,6 +346,7 @@ class Servo(BrowserSetup):
 product_setup = {
     "firefox": Firefox,
     "chrome": Chrome,
+    "chrome_android": ChromeAndroid,
     "edge": Edge,
     "ie": InternetExplorer,
     "servo": Servo,

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/wpt/utils.py

@@ -27,7 +27,7 @@ class Kwargs(dict):
                 value = value()
             if not value:
                 if err_fn is not None:
-                    return err_fn(kwargs, "Failed to find %s" % desc)
+                    return err_fn(self, "Failed to find %s" % desc)
                 else:
                     return
             self[name] = value

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/wpt/virtualenv.py

 import os
+import shutil
 import sys
 import logging
 from distutils.spawn import find_executable
@@ -21,7 +22,7 @@ class Virtualenv(object):
     def create(self):
         if os.path.exists(self.path):
             shutil.rmtree(self.path)
-        call(self.virtualenv, self.path)
+        call(self.virtualenv, self.path, "-p", sys.executable)
 
     @property
     def bin_path(self):

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/wptserve/wptserve/handlers.py

@@ -237,6 +237,7 @@ class PythonScriptHandler(object):
             if "main" in environ:
                 handler = FunctionHandler(environ["main"])
                 handler(request, response)
+                wrap_pipeline(path, request, response)
             else:
                 raise HTTPException(500, "No main function in script %s" % path)
         except IOError:
@@ -267,6 +268,7 @@ class FunctionHandler(object):
             else:
                 content = rv
             response.content = content
+            wrap_pipeline('', request, response)
 
 
 #The generic name here is so that this can be used as a decorator
@@ -309,6 +311,7 @@ class AsIsHandler(object):
         try:
             with open(path) as f:
                 response.writer.write_content(f.read())
+            wrap_pipeline(path, request, response)
             response.close_connection = True
         except IOError:
             raise HTTPException(404)

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/wptserve/wptserve/pipes.py

@@ -6,6 +6,7 @@ import types
 import uuid
 from cStringIO import StringIO
 
+from six import text_type
 
 def resolve_content(response):
     return b"".join(item for item in response.iter_content(read_file=True))
@@ -276,18 +277,18 @@ def slice(request, response, start, end=None):
 
 
 class ReplacementTokenizer(object):
-    def ident(scanner, token):
+    def ident(self, token):
         return ("ident", token)
 
-    def index(scanner, token):
+    def index(self, token):
         token = token[1:-1]
         try:
             token = int(token)
         except ValueError:
-            token = unicode(token, "utf8")
+            token = token.decode('utf8')
         return ("index", token)
 
-    def var(scanner, token):
+    def var(self, token):
         token = token[:-1]
         return ("var", token)
 
@@ -391,6 +392,12 @@ def template(request, content, escape_type="html"):
             value = request.headers
         elif field == "GET":
             value = FirstWrapper(request.GET)
+        elif field == "domains":
+            if ('not_domains' in request.server.config and
+                    tokens[1][1] in request.server.config['not_domains']):
+                value = request.server.config['not_domains']
+            else:
+                value = request.server.config['domains']
         elif field in request.server.config:
             value = request.server.config[tokens[0][1]]
         elif field == "location":
@@ -425,7 +432,7 @@ def template(request, content, escape_type="html"):
 
         #Should possibly support escaping for other contexts e.g. script
         #TODO: read the encoding of the response
-        return escape_func(unicode(value)).encode("utf-8")
+        return escape_func(text_type(value)).encode("utf-8")
 
     template_regexp = re.compile(r"{{([^}]*)}}")
     new_content = template_regexp.sub(config_replacement, content)

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/wptserve/wptserve/response.py

@@ -9,6 +9,8 @@ import socket
 from .constants import response_codes
 from .logger import get_logger
 
+from six import string_types, binary_type, text_type
+
 missing = object()
 
 class Response(object):
@@ -400,7 +402,7 @@ class ResponseWriter(object):
             if name.lower() not in self._headers_seen:
                 self.write_header(name, f())
 
-        if (type(self._response.content) in (str, unicode) and
+        if (isinstance(self._response.content, string_types) and
             "content-length" not in self._headers_seen):
             #Would be nice to avoid double-encoding here
             self.write_header("Content-Length", len(self.encode(self._response.content)))
@@ -457,9 +459,9 @@ class ResponseWriter(object):
 
     def encode(self, data):
         """Convert unicode to bytes according to response.encoding."""
-        if isinstance(data, str):
+        if isinstance(data, binary_type):
             return data
-        elif isinstance(data, unicode):
+        elif isinstance(data, text_type):
             return data.encode(self._response.encoding)
         else:
             raise ValueError

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/wptserve/wptserve/server.py

@@ -110,14 +110,15 @@ class WebTestServer(ThreadingMixIn, BaseHTTPServer.HTTPServer):
     # Ensure that we don't hang on shutdown waiting for requests
     daemon_threads = True
 
-    def __init__(self, server_address, RequestHandlerClass, router, rewriter, bind_hostname,
+    def __init__(self, server_address, request_handler_cls,
+                 router, rewriter, bind_hostname,
                  config=None, use_ssl=False, key_file=None, certificate=None,
                  encrypt_after_connect=False, latency=None, **kwargs):
         """Server for HTTP(s) Requests
 
         :param server_address: tuple of (server_name, port)
 
-        :param RequestHandlerClass: BaseHTTPRequestHandler-like class to use for
+        :param request_handler_cls: BaseHTTPRequestHandler-like class to use for
                                     handling requests.
 
         :param router: Router instance to use for matching requests to handler
@@ -161,7 +162,7 @@ class WebTestServer(ThreadingMixIn, BaseHTTPServer.HTTPServer):
             hostname_port = ("",server_address[1])
 
         #super doesn't work here because BaseHTTPServer.HTTPServer is old-style
-        BaseHTTPServer.HTTPServer.__init__(self, hostname_port, RequestHandlerClass, **kwargs)
+        BaseHTTPServer.HTTPServer.__init__(self, hostname_port, request_handler_cls, **kwargs)
 
         if config is not None:
             Server.config = config