Roll in new version of WPT tools again to include a Windows fix

The previous checkout contains a bug that would cause manifest update to
fail on Windows sometimes, so we are rolling a new version again to
include a high-priority fix:
https://github.com/w3c/web-platform-tests/pull/9737

Bug: 749879
Change-Id: I36495316d5ddea288c7422c2dbed30c1fed6106d
Reviewed-on: https://chromium-review.googlesource.com/944720Reviewed-by: Philip Jägenstedt <foolip@chromium.org>
Commit-Queue: Robert Ma <robertma@chromium.org>
Cr-Commit-Position: refs/heads/master@{#540364}

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/README.chromium

@@ -51,7 +51,7 @@ Local Modifications: None
 Name: web-platform-tests - Test Suites for Web Platform specifications
 Short Name: wpt
 URL: https://github.com/w3c/web-platform-tests/
-Version: 38612167f54c475836c122013756e92b9a8859cc
+Version: 5122353fba91c9d97599cb94a02fb8ec7ec1384e
 License: LICENSES FOR W3C TEST SUITES (http://www.w3.org/Consortium/Legal/2008/04-testsuite-copyright.html)
 License File: wpt/wpt/LICENSE.md
 Security Critical: no

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/checkout.sh

@@ -9,7 +9,7 @@ cd $DIR
 
 TARGET_DIR=$DIR/wpt
 REMOTE_REPO="https://chromium.googlesource.com/external/w3c/web-platform-tests.git"
-WPT_HEAD=38612167f54c475836c122013756e92b9a8859cc
+WPT_HEAD=5122353fba91c9d97599cb94a02fb8ec7ec1384e
 
 function clone {
   # Remove existing repo if already exists.

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/localpaths.py

@@ -8,7 +8,7 @@ sys.path.insert(0, os.path.join(here))
 sys.path.insert(0, os.path.join(here, "six"))
 sys.path.insert(0, os.path.join(here, "html5lib"))
 sys.path.insert(0, os.path.join(here, "wptserve"))
-sys.path.insert(0, os.path.join(here, "pywebsocket", "src"))
+sys.path.insert(0, os.path.join(here, "pywebsocket"))
 sys.path.insert(0, os.path.join(here, "third_party", "attrs", "src"))
 sys.path.insert(0, os.path.join(here, "third_party", "funcsigs"))
 sys.path.insert(0, os.path.join(here, "third_party", "pluggy"))

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/manifest/manifest.py

@@ -3,7 +3,6 @@ import os
 import re
 from collections import defaultdict
 from six import iteritems, itervalues, viewkeys, string_types
-from tempfile import mkstemp
 
 from .item import ManualTest, WebdriverSpecTest, Stub, RefTestNode, RefTest, TestharnessTest, SupportFile, ConformanceCheckerTest, VisualTest
 from .log import get_logger
@@ -91,6 +90,8 @@ class Manifest(object):
                     hash_changed = True
                 else:
                     new_type, manifest_items = old_type, self._data[old_type][rel_path]
+                if old_type in ("reftest", "reftest_node") and new_type != old_type:
+                    reftest_changes = True
             else:
                 new_type, manifest_items = source_file.manifest_items()
 
@@ -233,11 +234,6 @@ def write(manifest, manifest_path):
     dir_name = os.path.dirname(manifest_path)
     if not os.path.exists(dir_name):
         os.makedirs(dir_name)
-
-    fd, temp_manifest_path = mkstemp(dir=dir_name)
-    temp_manifest = open(temp_manifest_path, "wb")
-    json.dump(manifest.to_json(), temp_manifest,
-              sort_keys=True, indent=1, separators=(',', ': '))
-    temp_manifest.write("\n")
-    os.rename(temp_manifest_path, manifest_path)
-    os.close(fd)
+    with open(manifest_path, "wb") as f:
+        json.dump(manifest.to_json(), f, sort_keys=True, indent=1, separators=(',', ': '))
+        f.write("\n")

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/manifest/vcs.py

@@ -14,7 +14,11 @@ class Git(object):
     def get_func(repo_path):
         def git(cmd, *args):
             full_cmd = ["git", cmd] + list(args)
-            return subprocess.check_output(full_cmd, cwd=repo_path, stderr=subprocess.STDOUT)
+            try:
+                return subprocess.check_output(full_cmd, cwd=repo_path, stderr=subprocess.STDOUT)
+            except WindowsError:
+                full_cmd[0] = "git.bat"
+                return subprocess.check_output(full_cmd, cwd=repo_path, stderr=subprocess.STDOUT)
         return git
 
     @classmethod

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/serve/serve.py

@@ -234,8 +234,10 @@ class RoutesBuilder(object):
     def add_handler(self, method, route, handler):
         self.extra.append((str(method), str(route), handler))
 
-    def add_static(self, path, format_args, content_type, route):
-        handler = handlers.StaticHandler(path, format_args, content_type)
+    def add_static(self, path, format_args, content_type, route, headers=None):
+        if headers is None:
+            headers = {}
+        handler = handlers.StaticHandler(path, format_args, content_type, **headers)
         self.add_handler(b"GET", str(route), handler)
 
     def add_mount_point(self, url_base, path):

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/wpt/browser.py

@@ -18,26 +18,38 @@ logger = logging.getLogger(__name__)
 
 uname = platform.uname()
 
-def path(path, exe):
-    path = path.replace("/", os.path.sep)
-    if exe and uname[0] == "Windows":
-        path += ".exe"
-    return path
-
 
 class Browser(object):
     __metaclass__ = ABCMeta
 
     @abstractmethod
     def install(self, dest=None):
+        """Install the browser."""
         return NotImplemented
 
     @abstractmethod
-    def install_webdriver(self):
+    def install_webdriver(self, dest=None):
+        """Install the WebDriver implementation for this browser."""
+        return NotImplemented
+
+    @abstractmethod
+    def find_binary(self):
+        """Find the binary of the browser.
+
+        If the WebDriver for the browser is able to find the binary itself, this
+        method doesn't need to be implemented, in which case NotImplementedError
+        is suggested to be raised to prevent accidental use.
+        """
+        return NotImplemented
+
+    @abstractmethod
+    def find_webdriver(self):
+        """Find the binary of the WebDriver."""
         return NotImplemented
 
     @abstractmethod
-    def version(self):
+    def version(self, root):
+        """Retrieve the release version of the installed browser."""
         return NotImplemented
 
     @abstractmethod
@@ -45,12 +57,6 @@ class Browser(object):
         """Name of the browser-specific wptrunner requirements file"""
         return NotImplemented
 
-    def prepare_environment(self):
-        """Do any additional setup of the environment required to start the
-           browser successfully
-        """
-        pass
-
 
 class Firefox(Browser):
     """Firefox-specific interface.
@@ -63,7 +69,6 @@ class Firefox(Browser):
     platform_ini = "firefox/platform.ini"
     requirements = "requirements_firefox.txt"
 
-
     def platform_string(self):
         platform = {
             "Linux": "linux",
@@ -101,25 +106,39 @@ class Firefox(Browser):
         return "%s%s" % (platform, bits)
 
     def latest_nightly_listing(self):
-        return get("https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central/")
-
-    def get_from_nightly(self, pattern):
-        index = self.latest_nightly_listing()
-        filename = re.compile(pattern).search(index.text).group(1)
-        return get("https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central/%s" %
-                   filename)
+        resp = get("https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central/")
+        resp.raise_for_status()
+        return resp.text
+
+    def get_nightly_link(self, index, platform):
+        pattern = re.compile("<a[^>]*>(firefox-(\d+)\.\d(?:\w\d)?.en-US.%s\.tar\.bz2)" % platform)
+        max_version = None
+        for match in pattern.finditer(index):
+            try:
+                version = int(match.group(2))
+            except ValueError:
+                continue
+            if max_version is None or version > max_version[0]:
+                max_version = (version, match.group(1))
+        if not max_version:
+            raise ValueError("Failed to find version to download")
+        return ("https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central/%s" %
+                max_version[1])
 
     def install(self, dest=None):
         """Install Firefox."""
         if dest is None:
             dest = os.getcwd()
 
-        resp = self.get_from_nightly("<a[^>]*>(firefox-\d+\.\d(?:\w\d)?.en-US.%s\.tar\.bz2)" % self.platform_string())
+        nightly_link = self.get_nightly_link(self.latest_nightly_listing(),
+                                             self.platform_string())
+        resp = get(nightly_link)
+        resp.raise_for_status()
         untar(resp.raw, dest=dest)
         return find_executable("firefox", os.path.join(dest, "firefox"))
 
-    def find_binary(self, path=None):
-        return find_executable("firefox", path)
+    def find_binary(self):
+        return find_executable("firefox")
 
     def find_certutil(self):
         path = find_executable("certutil")
@@ -132,26 +151,6 @@ class Firefox(Browser):
     def find_webdriver(self):
         return find_executable("geckodriver")
 
-    def install_certutil(self, dest=None):
-        # TODO: this doesn't really work because it just gets the binary, and is missing the
-        # libnss3 library. Getting that means either downloading the corresponding Firefox
-        # and extracting the library (which is hard on mac becase DMG), or maybe downloading from
-        # nss's treeherder builds?
-        if dest is None:
-            dest = os.pwd
-
-        # Don't create a path like bin/bin/certutil
-        split = os.path.split(dest)
-        if split[1] == "bin":
-            dest = split[0]
-
-        resp = self.get_from_nightly(
-            "<a[^>]*>(firefox-\d+\.\d(?:\w\d)?.en-US.%s\.common\.tests.zip)</a>" % self.platform_string())
-        bin_path = path("bin/certutil", exe=True)
-        unzip(resp.raw, dest=dest, limit=[bin_path])
-
-        return os.path.join(dest, bin_path)
-
     def install_prefs(self, dest=None):
         if dest is None:
             dest = os.pwd
@@ -217,7 +216,7 @@ class Firefox(Browser):
 class Chrome(Browser):
     """Chrome-specific interface.
 
-    Includes installation, webdriver installation, and wptrunner setup methods.
+    Includes webdriver installation, and wptrunner setup methods.
     """
 
     product = "chrome"
@@ -246,11 +245,13 @@ class Chrome(Browser):
 
         return "%s%s" % (platform, bits)
 
+    def find_binary(self):
+        raise NotImplementedError
+
     def find_webdriver(self):
         return find_executable("chromedriver")
 
     def install_webdriver(self, dest=None):
-        """Install latest Webdriver."""
         if dest is None:
             dest = os.pwd
         latest = get("http://chromedriver.storage.googleapis.com/LATEST_RELEASE").text.strip()
@@ -264,35 +265,14 @@ class Chrome(Browser):
         return path
 
     def version(self, root):
-        """Retrieve the release version of the installed browser."""
         output = call(self.binary, "--version")
         return re.search(r"[0-9\.]+( [a-z]+)?$", output.strip()).group(0)
 
-    def prepare_environment(self):
-        # https://bugs.chromium.org/p/chromium/issues/detail?id=713947
-        logger.debug("DBUS_SESSION_BUS_ADDRESS %s" % os.environ.get("DBUS_SESSION_BUS_ADDRESS"))
-        if "DBUS_SESSION_BUS_ADDRESS" not in os.environ:
-            if find_executable("dbus-launch"):
-                logger.debug("Attempting to start dbus")
-                dbus_conf = subprocess.check_output(["dbus-launch"])
-                logger.debug(dbus_conf)
-
-                # From dbus-launch(1):
-                #
-                # > When dbus-launch prints bus information to standard output,
-                # > by default it is in a simple key-value pairs format.
-                for line in dbus_conf.strip().split("\n"):
-                    key, _, value = line.partition("=")
-                    os.environ[key] = value
-            else:
-                logger.critical("dbus not running and can't be started")
-                sys.exit(1)
-
 
 class ChromeAndroid(Browser):
-    """Chrome-specific interface for android.
+    """Chrome-specific interface for Android.
 
-    Includes installation, webdriver installation, and wptrunner setup methods.
+    Includes webdriver installation.
     """
 
     product = "chrome_android"
@@ -301,6 +281,9 @@ class ChromeAndroid(Browser):
     def install(self, dest=None):
         raise NotImplementedError
 
+    def find_binary(self):
+        raise NotImplementedError
+
     def find_webdriver(self):
         return find_executable("chromedriver")
 
@@ -315,7 +298,7 @@ class ChromeAndroid(Browser):
 class Opera(Browser):
     """Opera-specific interface.
 
-    Includes installation, webdriver installation, and wptrunner setup methods.
+    Includes webdriver installation, and wptrunner setup methods.
     """
 
     product = "opera"
@@ -344,11 +327,13 @@ class Opera(Browser):
 
         return "%s%s" % (platform, bits)
 
+    def find_binary(self):
+        raise NotImplementedError
+
     def find_webdriver(self):
         return find_executable("operadriver")
 
     def install_webdriver(self, dest=None):
-        """Install latest Webdriver."""
         if dest is None:
             dest = os.pwd
         latest = get("https://api.github.com/repos/operasoftware/operachromiumdriver/releases/latest").json()["tag_name"]
@@ -370,32 +355,9 @@ class Opera(Browser):
         output = call(self.binary, "--version")
         return re.search(r"[0-9\.]+( [a-z]+)?$", output.strip()).group(0)
 
-    def prepare_environment(self):
-        # https://bugs.chromium.org/p/chromium/issues/detail?id=713947
-        logger.debug("DBUS_SESSION_BUS_ADDRESS %s" % os.environ.get("DBUS_SESSION_BUS_ADDRESS"))
-        if "DBUS_SESSION_BUS_ADDRESS" not in os.environ:
-            if find_executable("dbus-launch"):
-                logger.debug("Attempting to start dbus")
-                dbus_conf = subprocess.check_output(["dbus-launch"])
-                logger.debug(dbus_conf)
-
-                # From dbus-launch(1):
-                #
-                # > When dbus-launch prints bus information to standard output,
-                # > by default it is in a simple key-value pairs format.
-                for line in dbus_conf.strip().split("\n"):
-                    key, _, value = line.partition("=")
-                    os.environ[key] = value
-            else:
-                logger.critical("dbus not running and can't be started")
-                sys.exit(1)
-
 
 class Edge(Browser):
-    """Edge-specific interface.
-
-    Includes installation, webdriver installation, and wptrunner setup methods.
-    """
+    """Edge-specific interface."""
 
     product = "edge"
     requirements = "requirements_edge.txt"
@@ -403,22 +365,21 @@ class Edge(Browser):
     def install(self, dest=None):
         raise NotImplementedError
 
+    def find_binary(self):
+        raise NotImplementedError
+
     def find_webdriver(self):
         return find_executable("MicrosoftWebDriver")
 
     def install_webdriver(self, dest=None):
-        """Install latest Webdriver."""
         raise NotImplementedError
 
-    def version(self):
+    def version(self, root):
         raise NotImplementedError
 
 
 class InternetExplorer(Browser):
-    """Internet Explorer-specific interface.
-
-    Includes installation, webdriver installation, and wptrunner setup methods.
-    """
+    """Internet Explorer-specific interface."""
 
     product = "ie"
     requirements = "requirements_ie.txt"
@@ -426,36 +387,60 @@ class InternetExplorer(Browser):
     def install(self, dest=None):
         raise NotImplementedError
 
+    def find_binary(self):
+        raise NotImplementedError
+
     def find_webdriver(self):
         return find_executable("IEDriverServer.exe")
 
     def install_webdriver(self, dest=None):
-        """Install latest Webdriver."""
         raise NotImplementedError
 
-    def version(self):
+    def version(self, root):
         raise NotImplementedError
 
 
-class Servo(Browser):
-    """Servo-specific interface.
+class Safari(Browser):
+    """Safari-specific interface.
 
     Includes installation, webdriver installation, and wptrunner setup methods.
     """
 
+    product = "safari"
+    requirements = "requirements_safari.txt"
+
+    def install(self, dest=None):
+        raise NotImplementedError
+
+    def find_binary(self):
+        raise NotImplementedError
+
+    def find_webdriver(self):
+        return find_executable("safaridriver")
+
+    def install_webdriver(self):
+        raise NotImplementedError
+
+    def version(self, root):
+        raise NotImplementedError
+
+
+class Servo(Browser):
+    """Servo-specific interface."""
+
     product = "servo"
     requirements = "requirements_servo.txt"
 
     def install(self, dest=None):
         raise NotImplementedError
 
-    def find_binary(self, path=None):
+    def find_binary(self):
         return find_executable("servo")
 
     def find_webdriver(self):
         return None
 
-    def install_webdriver(self):
+    def install_webdriver(self, dest=None):
         raise NotImplementedError
 
     def version(self, root):
@@ -463,10 +448,7 @@ class Servo(Browser):
 
 
 class Sauce(Browser):
-    """Sauce-specific interface.
-
-    Includes installation, webdriver installation, and wptrunner setup methods.
-    """
+    """Sauce-specific interface."""
 
     product = "sauce"
     requirements = "requirements_sauce.txt"
@@ -474,13 +456,13 @@ class Sauce(Browser):
     def install(self, dest=None):
         raise NotImplementedError
 
-    def find_binary(self, path=None):
-        return None
+    def find_binary(self):
+        raise NotImplementedError
 
     def find_webdriver(self):
-        return None
+        raise NotImplementedError
 
-    def install_webdriver(self):
+    def install_webdriver(self, dest=None):
         raise NotImplementedError
 
     def version(self, root):

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/wpt/commands.json

 {
     "run": {"path": "run.py", "script": "run", "parser": "create_parser", "help": "Run tests in a browser",
             "virtualenv": true, "install": ["requests"], "requirements": ["../wptrunner/requirements.txt"]},
+    "update-expectations": {"path": "update.py", "script": "update_expectations",
+                            "parser": "create_parser_update", "help": "Update expectations files from raw logs.",
+                            "virtualenv": true, "install": ["requests"],
+                            "requirements": ["../wptrunner/requirements.txt"]},
     "files-changed": {"path": "testfiles.py", "script": "run_changed_files", "parser": "get_parser",
                       "help": "Get a list of files that have changed", "virtualenv": false},
     "tests-affected": {"path": "testfiles.py", "script": "run_tests_affected", "parser": "get_parser_affected",

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/wpt/run.py

@@ -226,6 +226,7 @@ class Chrome(BrowserSetup):
             else:
                 raise WptrunError("Unable to locate or install chromedriver binary")
 
+
 class ChromeAndroid(BrowserSetup):
     name = "chrome_android"
     browser_cls = browser.ChromeAndroid
@@ -314,6 +315,23 @@ https://selenium-release.storage.googleapis.com/index.html
             kwargs["webdriver_binary"] = webdriver_binary
 
 
+class Safari(BrowserSetup):
+    name = "safari"
+    browser_cls = browser.Safari
+
+    def install(self, venv):
+        raise NotImplementedError
+
+    def setup_kwargs(self, kwargs):
+        if kwargs["webdriver_binary"] is None:
+            webdriver_binary = self.browser.find_webdriver()
+
+            if webdriver_binary is None:
+                raise WptrunError("Unable to locate safaridriver binary")
+
+            kwargs["webdriver_binary"] = webdriver_binary
+
+
 class Sauce(BrowserSetup):
     name = "sauce"
     browser_cls = browser.Sauce
@@ -349,6 +367,7 @@ product_setup = {
     "chrome_android": ChromeAndroid,
     "edge": Edge,
     "ie": InternetExplorer,
+    "safari": Safari,
     "servo": Servo,
     "sauce": Sauce,
     "opera": Opera,

third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/wptserve/wptserve/handlers.py

@@ -366,7 +366,7 @@ class StringHandler(object):
 
         self.resp_headers = [("Content-Type", content_type)]
         for k, v in headers.iteritems():
-            resp_headers.append((k.replace("_", "-"), v))
+            self.resp_headers.append((k.replace("_", "-"), v))
 
         self.handler = handler(self.handle_request)