Commit a0627fa7 authored by Robert Ma's avatar Robert Ma Committed by Commit Bot

Roll WPT tools to fix a manifest issue

Bug: https://github.com/web-platform-tests/wpt/issues/17749
Fix: https://github.com/web-platform-tests/wpt/pull/17785

This bug would prevent the manifest from updating when local changes are
made in the working tree, which severely affects the daily workflow of
Blink engineers (bots are not affected).

Change-Id: Icf080553a8c8626c10d7973eb21ec4214c889d16
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1697371Reviewed-by: default avatarLuke Z <lpz@chromium.org>
Commit-Queue: Robert Ma <robertma@chromium.org>
Cr-Commit-Position: refs/heads/master@{#676570}
parent 322f256e
......@@ -32,7 +32,7 @@ Local Modifications: None
Name: web-platform-tests - Test Suites for Web Platform specifications
Short Name: wpt
URL: https://github.com/web-platform-tests/wpt/
Version: 7edf9eabfae3550a4dd2a48625e68b02a734de15
Version: 3fb0150bb0a53b5a6630e8eda7f43bf75d8a6bbe
License: LICENSES FOR W3C TEST SUITES (http://www.w3.org/Consortium/Legal/2008/04-testsuite-copyright.html)
License File: wpt/wpt/LICENSE.md
Security Critical: no
......
......@@ -9,7 +9,7 @@ cd $DIR
TARGET_DIR=$DIR/wpt
REMOTE_REPO="https://github.com/web-platform-tests/wpt.git"
WPT_HEAD=7edf9eabfae3550a4dd2a48625e68b02a734de15
WPT_HEAD=3fb0150bb0a53b5a6630e8eda7f43bf75d8a6bbe
function clone {
# Remove existing repo if already exists.
......
......@@ -146,6 +146,13 @@ def check_path_length(repo_root, path):
return []
def check_file_type(repo_root, path):
# type: (str, str) -> List[rules.Error]
if os.path.islink(path):
return [rules.FileType.error(path, (path, "symlink"))]
return []
def check_worker_collision(repo_root, path):
# type: (str, str) -> List[rules.Error]
endings = [(".any.html", ".any.js"),
......@@ -913,7 +920,8 @@ def lint(repo_root, paths, output_format, ignore_glob):
logger.info(line)
return sum(itervalues(error_count))
path_lints = [check_path_length, check_worker_collision, check_ahem_copy, check_gitignore_file]
path_lints = [check_file_type, check_path_length, check_worker_collision, check_ahem_copy,
check_gitignore_file]
all_paths_lints = [check_css_globally_unique]
file_lints = [check_regexp_line, check_parsed, check_python_ast, check_script_metadata]
......
......@@ -57,6 +57,11 @@ class PathLength(Rule):
description = "/%s longer than maximum path length (%d > 150)"
class FileType(Rule):
name = "FILE TYPE"
description = "/%s is an unsupported file type (%s)"
class WorkerCollision(Rule):
name = "WORKER COLLISION"
description = ("path ends with %s which collides with generated tests "
......
......@@ -2,7 +2,7 @@ import itertools
import json
import os
from collections import MutableMapping, defaultdict
from six import iteritems, iterkeys, itervalues, string_types
from six import iteritems, iterkeys, itervalues, string_types, binary_type, text_type
from . import vcs
from .item import (ConformanceCheckerTest, ManifestItem, ManualTest, RefTest, RefTestNode, Stub,
......@@ -323,7 +323,7 @@ class Manifest(object):
for source_file, update in tree:
if not update:
assert isinstance(source_file, (bytes, str))
assert isinstance(source_file, (binary_type, text_type))
rel_path = source_file # type: Text
seen_files.add(rel_path)
assert rel_path in path_hash
......
......@@ -61,24 +61,11 @@ class GitHasher(object):
# type: () -> Set[bytes]
"""get a set of files which have changed between HEAD and working copy"""
assert self.git is not None
changes = set() # type: Set[bytes]
cmd = [b"status", b"-z", b"--ignore-submodules=all"]
data = self.git(*cmd) # type: bytes
in_rename = False
for line in data.split(b"\0")[:-1]:
if in_rename:
changes.add(line)
in_rename = False
else:
status = line[:2]
if b"R" in status or b"C" in status:
in_rename = True
changes.add(line[3:])
return changes
# note that git runs the command with tests_root as the cwd, which may
# not be the root of the git repo (e.g., within a browser repo)
cmd = [b"diff-index", b"--relative", b"--no-renames", b"--name-only", b"-z", b"HEAD"]
data = self.git(*cmd)
return set(data.split(b"\0"))
def hash_cache(self):
# type: () -> Dict[bytes, Optional[bytes]]
......@@ -90,7 +77,9 @@ class GitHasher(object):
if self.git is None:
return hash_cache
cmd = [b"ls-tree", b"-r", b"-z", b"HEAD"]
# note that git runs the command with tests_root as the cwd, which may
# not be the root of the git repo (e.g., within a browser repo)
cmd = ["ls-tree", "-r", "-z", "HEAD"]
local_changes = self._local_changes()
for result in self.git(*cmd).split(b"\0")[:-1]: # type: bytes
data, rel_path = result.rsplit(b"\t", 1)
......@@ -168,7 +157,10 @@ class CacheFile(with_metaclass(abc.ABCMeta)):
try:
if not rebuild:
with open(self.path, 'r') as f:
data = json.load(f)
try:
data = json.load(f)
except ValueError:
pass
data = self.check_valid(data)
except IOError:
pass
......
......@@ -8,6 +8,7 @@ import json
import logging
import os
import platform
import signal
import socket
import sys
import threading
......@@ -574,13 +575,11 @@ def start_http2_server(host, port, paths, routes, bind_address, config, **kwargs
class WebSocketDaemon(object):
def __init__(self, host, port, doc_root, handlers_root, log_level, bind_address,
ssl_config):
def __init__(self, host, port, doc_root, handlers_root, bind_address, ssl_config):
self.host = host
cmd_args = ["-p", port,
"-d", doc_root,
"-w", handlers_root,
"--log-level", log_level]
"-w", handlers_root]
if ssl_config is not None:
# This is usually done through pywebsocket.main, however we're
......@@ -604,17 +603,6 @@ class WebSocketDaemon(object):
opts, args = pywebsocket._parse_args_and_config(cmd_args)
opts.cgi_directories = []
opts.is_executable_method = None
# Logging needs to be configured both before and after reloading,
# because some modules store loggers as global variables.
pywebsocket._configure_logging(opts)
# Ensure that when we start this in a new process we have the global
# lock in the logging module unlocked.
reload_module(logging)
release_mozlog_lock()
pywebsocket._configure_logging(opts)
# DO NOT LOG BEFORE THIS LINE.
self.server = pywebsocket.WebSocketServer(opts)
ports = [item[0].getsockname()[1] for item in self.server._sockets]
assert all(item == ports[0] for item in ports)
......@@ -661,21 +649,27 @@ def release_mozlog_lock():
def start_ws_server(host, port, paths, routes, bind_address, config, **kwargs):
# Ensure that when we start this in a new process we have the global lock
# in the logging module unlocked
reload_module(logging)
release_mozlog_lock()
return WebSocketDaemon(host,
str(port),
repo_root,
config.paths["ws_doc_root"],
config.log_level.lower(),
bind_address,
ssl_config=None)
def start_wss_server(host, port, paths, routes, bind_address, config, **kwargs):
# Ensure that when we start this in a new process we have the global lock
# in the logging module unlocked
reload_module(logging)
release_mozlog_lock()
return WebSocketDaemon(host,
str(port),
repo_root,
config.paths["ws_doc_root"],
config.log_level.lower(),
bind_address,
config.ssl_config)
......@@ -841,11 +835,19 @@ def get_parser():
def run(**kwargs):
received_signal = threading.Event()
with build_config(os.path.join(repo_root, "config.json"),
**kwargs) as config:
global logger
logger = config.logger
set_logger(logger)
# Configure the root logger to cover third-party libraries.
logging.getLogger().setLevel(config.log_level)
def handle_signal(signum, frame):
logger.debug("Received signal %s. Shutting down.", signum)
received_signal.set()
bind_address = config["bind_address"]
......@@ -868,20 +870,19 @@ def run(**kwargs):
with stash.StashServer(stash_address, authkey=str(uuid.uuid4())):
servers = start(config, build_routes(config["aliases"]), **kwargs)
signal.signal(signal.SIGTERM, handle_signal)
signal.signal(signal.SIGINT, handle_signal)
try:
while all(item.is_alive() for item in iter_procs(servers)):
for item in iter_procs(servers):
item.join(1)
exited = [item for item in iter_procs(servers) if not item.is_alive()]
subject = "subprocess" if len(exited) == 1 else "subprocesses"
while all(item.is_alive() for item in iter_procs(servers)) and not received_signal.is_set():
for item in iter_procs(servers):
item.join(1)
exited = [item for item in iter_procs(servers) if not item.is_alive()]
subject = "subprocess" if len(exited) == 1 else "subprocesses"
logger.info("%s %s exited:" % (len(exited), subject))
logger.info("%s %s exited:" % (len(exited), subject))
for item in iter_procs(servers):
logger.info("Status of %s:\t%s" % (item.name, "running" if item.is_alive() else "not running"))
except KeyboardInterrupt:
logger.info("Shutting down")
for item in iter_procs(servers):
logger.info("Status of %s:\t%s" % (item.name, "running" if item.is_alive() else "not running"))
def main():
......
......@@ -218,7 +218,12 @@ class Actions(object):
``ActionSequence.dict``.
"""
body = {"actions": [] if actions is None else actions}
return self.session.send_session_command("POST", "actions", body)
actions = self.session.send_session_command("POST", "actions", body)
"""WebDriver window should be set to the top level window when wptrunner
processes the next event.
"""
self.session.switch_frame(None)
return actions
@command
def release(self):
......@@ -308,8 +313,11 @@ class Find(object):
self.session = session
@command
def css(self, selector, all=True):
return self._find_element("css selector", selector, all)
def css(self, element_selector, all=True, frame="window"):
if (frame != "window"):
self.session.switch_frame(frame)
elements = self._find_element("css selector", element_selector, all)
return elements
def _find_element(self, strategy, selector, all):
route = "elements" if all else "element"
......@@ -413,7 +421,7 @@ class Session(object):
if self.session_id is not None:
return
body = {}
body = {"capabilities": {}}
if self.requested_capabilities is not None:
body["capabilities"] = self.requested_capabilities
......
......@@ -3,6 +3,7 @@ import platform
import re
import shutil
import stat
import errno
import subprocess
import tempfile
import urlparse
......@@ -17,6 +18,24 @@ from utils import call, get, untar, unzip
uname = platform.uname()
def _get_fileversion(binary, logger=None):
command = "(Get-Item '%s').VersionInfo.FileVersion" % binary.replace("'", "''")
try:
return call("powershell.exe", command).strip()
except (subprocess.CalledProcessError, OSError):
if logger is not None:
logger.warning("Failed to call %s in PowerShell" % command)
return None
def handle_remove_readonly(func, path, exc):
excvalue = exc[1]
if func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES:
os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777
func(path)
else:
raise
class Browser(object):
__metaclass__ = ABCMeta
......@@ -108,7 +127,7 @@ class Firefox(Browser):
("linux", "x86"): "linux",
("linux", "x86_64"): "linux64",
("win", "x86"): "win",
("win", "x86_64"): "win64",
("win", "AMD64"): "win64",
("macos", "x86_64"): "osx",
}
os_key = (self.platform, uname[4])
......@@ -149,7 +168,7 @@ class Firefox(Browser):
installer_path = os.path.join(dest, filename)
with open(installer_path, "w") as f:
with open(installer_path, "wb") as f:
f.write(resp.content)
try:
......@@ -197,6 +216,14 @@ class Firefox(Browser):
path = os.path.join(venv_path, "browsers", channel)
binary = self.find_binary_path(path, channel)
if not binary and self.platform == "win":
winpaths = [os.path.expandvars("$SYSTEMDRIVE\\Program Files\\Mozilla Firefox"),
os.path.expandvars("$SYSTEMDRIVE\\Program Files (x86)\\Mozilla Firefox")]
for winpath in winpaths:
binary = self.find_binary_path(winpath, channel)
if binary is not None:
break
if not binary and self.platform == "macos":
macpaths = ["/Applications/Firefox Nightly.app/Contents/MacOS",
os.path.expanduser("~/Applications/Firefox Nightly.app/Contents/MacOS"),
......@@ -215,7 +242,7 @@ class Firefox(Browser):
path = find_executable("certutil")
if path is None:
return None
if os.path.splitdrive(path)[1].split(os.path.sep) == ["", "Windows", "system32", "certutil.exe"]:
if os.path.splitdrive(os.path.normcase(path))[1].split(os.path.sep) == ["", "windows", "system32", "certutil.exe"]:
return None
return path
......@@ -418,7 +445,8 @@ class Chrome(Browser):
return "/usr/bin/google-chrome"
if uname[0] == "Darwin":
return "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
# TODO Windows?
if uname[0] == "Windows":
return os.path.expandvars("$SYSTEMDRIVE\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe")
self.logger.warning("Unable to find the browser binary.")
return None
......@@ -521,19 +549,19 @@ class Chrome(Browser):
def version(self, binary=None, webdriver_binary=None):
binary = binary or self.binary
if uname[0] != "Windows":
try:
version_string = call(binary, "--version").strip()
except subprocess.CalledProcessError:
self.logger.warning("Failed to call %s" % binary)
return None
m = re.match(r"(?:Google Chrome|Chromium) (.*)", version_string)
if not m:
self.logger.warning("Failed to extract version from: %s" % version_string)
return None
return m.group(1)
self.logger.warning("Unable to extract version from binary on Windows.")
return None
if uname[0] == "Windows":
return _get_fileversion(binary, self.logger)
try:
version_string = call(binary, "--version").strip()
except subprocess.CalledProcessError:
self.logger.warning("Failed to call %s" % binary)
return None
m = re.match(r"(?:Google Chrome|Chromium) (.*)", version_string)
if not m:
self.logger.warning("Failed to extract version from: %s" % version_string)
return None
return m.group(1)
class ChromeAndroid(Browser):
......@@ -644,6 +672,7 @@ class EdgeChromium(Browser):
"Darwin": "macos"
}.get(uname[0])
product = "edgechromium"
edgedriver_name = "msedgedriver"
requirements = "requirements_edge_chromium.txt"
def install(self, dest=None, channel=None):
......@@ -678,22 +707,40 @@ class EdgeChromium(Browser):
return find_executable("msedgedriver")
def install_webdriver(self, dest=None, channel=None, browser_binary=None):
if self.platform == "win":
raise ValueError("Only Windows platform is currently supported")
if self.platform != "win" and self.platform != "macos":
raise ValueError("Only Windows and Mac platforms are currently supported")
if dest is None:
dest = os.pwd
platform = "x64" if uname[4] == "x86_64" else "x86"
url = "https://az813057.vo.msecnd.net/webdriver/msedgedriver_%s/msedgedriver.exe" % platform
if channel is None:
version_url = "https://msedgedriver.azureedge.net/LATEST_DEV"
else:
version_url = "https://msedgedriver.azureedge.net/LATEST_%s" % channel.upper()
version = get(version_url).text.strip()
self.logger.info("Downloading MSEdgeDriver from %s" % url)
resp = get(url)
installer_path = os.path.join(dest, "msedgedriver.exe")
with open(installer_path, "wb") as f:
f.write(resp.content)
if self.platform == "macos":
bits = "mac64"
edgedriver_path = os.path.join(dest, self.edgedriver_name)
else:
bits = "win64" if uname[4] == "x86_64" else "win32"
edgedriver_path = os.path.join(dest, "%s.exe" % self.edgedriver_name)
url = "https://msedgedriver.azureedge.net/%s/edgedriver_%s.zip" % (version, bits)
# cleanup existing Edge driver files to avoid access_denied errors when unzipping
if os.path.isfile(edgedriver_path):
# remove read-only attribute
os.chmod(edgedriver_path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777
os.remove(edgedriver_path)
driver_notes_path = os.path.join(dest, "Driver_notes")
if os.path.isdir(driver_notes_path):
shutil.rmtree(driver_notes_path, ignore_errors=False, onerror=handle_remove_readonly)
return find_executable("msedgedriver", dest)
self.logger.info("Downloading MSEdgeDriver from %s" % url)
unzip(get(url).raw, dest)
if os.path.isfile(edgedriver_path):
self.logger.info("Successfully downloaded MSEdgeDriver to %s" % edgedriver_path)
return find_executable(self.edgedriver_name, dest)
def version(self, binary=None, webdriver_binary=None):
if binary is None:
......@@ -711,12 +758,7 @@ class EdgeChromium(Browser):
return m.group(1)
else:
if binary is not None:
command = "(Get-Item '%s').VersionInfo.FileVersion" % binary
try:
return call("powershell.exe", command).strip()
except (subprocess.CalledProcessError, OSError):
self.logger.warning("Failed to call %s in PowerShell" % command)
return None
return _get_fileversion(binary, self.logger)
self.logger.warning("Failed to find Edge binary.")
return None
......
......@@ -6,6 +6,7 @@ import sys
latest_channels = {
'firefox': 'nightly',
'chrome': 'dev',
'edgechromium': 'dev',
'safari': 'preview',
'servo': 'nightly'
}
......@@ -18,6 +19,7 @@ channel_by_name = {
'dev': latest_channels,
'preview': latest_channels,
'experimental': latest_channels,
'canary': 'canary',
}
......
......@@ -338,15 +338,23 @@ class EdgeChromium(BrowserSetup):
browser_cls = browser.EdgeChromium
def setup_kwargs(self, kwargs):
browser_channel = kwargs["browser_channel"]
if kwargs["binary"] is None:
binary = self.browser.find_binary(channel=browser_channel)
if binary:
kwargs["binary"] = self.browser.find_binary()
else:
raise WptrunError("Unable to locate Edge binary")
if kwargs["webdriver_binary"] is None:
webdriver_binary = self.browser.find_webdriver()
if webdriver_binary is None:
# Install browser if none are found or if it's found in venv path
if webdriver_binary is None or webdriver_binary in self.venv.bin_path:
install = self.prompt_install("msedgedriver")
if install:
logger.info("Downloading msedgedriver")
webdriver_binary = self.browser.install_webdriver(dest=self.venv.bin_path)
webdriver_binary = self.browser.install_webdriver(dest=self.venv.bin_path, channel=browser_channel)
else:
logger.info("Using webdriver binary %s" % webdriver_binary)
......@@ -354,6 +362,9 @@ class EdgeChromium(BrowserSetup):
kwargs["webdriver_binary"] = webdriver_binary
else:
raise WptrunError("Unable to locate or install msedgedriver binary")
if browser_channel == "dev":
logger.info("Automatically turning on experimental features for Edge Dev")
kwargs["binary_args"].append("--enable-experimental-web-platform-features")
class Edge(BrowserSetup):
......@@ -576,7 +587,8 @@ def setup_wptrunner(venv, prompt=True, install_browser=False, **kwargs):
kwargs["browser_channel"] = channel
else:
logger.info("Valid channels for %s not known; using argument unmodified" % kwargs["product"])
del kwargs["channel"]
kwargs["browser_channel"] = kwargs["channel"]
del kwargs["channel"]
if install_browser:
logger.info("Installing browser")
......
......@@ -34,6 +34,11 @@ class Virtualenv(object):
def exists(self):
return os.path.isdir(self.path)
@property
def broken_link(self):
python_link = os.path.join(self.path, ".Python")
return os.path.lexists(python_link) and not os.path.exists(python_link)
def create(self):
if os.path.exists(self.path):
shutil.rmtree(self.path)
......@@ -88,7 +93,7 @@ class Virtualenv(object):
execfile(path, {"__file__": path}) # noqa: F821
def start(self):
if not self.exists:
if not self.exists or self.broken_link:
self.create()
self.activate()
......
......@@ -342,23 +342,25 @@ def sub(request, response, escape_type="html"):
A dictionary of parts of the request URL. Valid keys are
'server, 'scheme', 'host', 'hostname', 'port', 'path' and 'query'.
'server' is scheme://host:port, 'host' is hostname:port, and query
includes the leading '?', but other delimiters are omitted.
includes the leading '?', but other delimiters are omitted.
headers
A dictionary of HTTP headers in the request.
header_or_default(header, default)
The value of an HTTP header, or a default value if it is absent.
For example:
For example::
{{header_or_default(X-Test, test-header-absent)}}
GET
A dictionary of query parameters supplied with the request.
uuid()
A pesudo-random UUID suitable for usage with stash
file_hash(algorithm, filepath)
The cryptographic hash of a file. Supported algorithms: md5, sha1,
sha224, sha256, sha384, and sha512. For example:
sha224, sha256, sha384, and sha512. For example::
{{file_hash(md5, dom/interfaces.html)}}
fs_path(filepath)
The absolute path to a file inside the wpt document root
......@@ -369,16 +371,15 @@ def sub(request, response, escape_type="html"):
{{domains[www]}} => www.localhost
{{ports[http][1]}} => 81
It is also possible to assign a value to a variable name, which must start
with the $ character, using the ":" syntax e.g.::
It is also possible to assign a value to a variable name, which must start with
the $ character, using the ":" syntax e.g.
{{$id:uuid()}}
{{$id:uuid()}}
Later substitutions in the same file may then refer to the variable
by name e.g.
by name e.g.::
{{$id}}
{{$id}}
"""
content = resolve_content(response)
......
......@@ -179,11 +179,13 @@ class Response(object):
If any part of the content is a function, this will be called
and the resulting value (if any) returned.
:param read_file: - boolean controlling the behaviour when content
is a file handle. When set to False the handle will be returned directly
allowing the file to be passed to the output in small chunks. When set to
True, the entire content of the file will be returned as a string facilitating
non-streaming operations like template substitution.
:param read_file: boolean controlling the behaviour when content is a
file handle. When set to False the handle will be
returned directly allowing the file to be passed to
the output in small chunks. When set to True, the
entire content of the file will be returned as a
string facilitating non-streaming operations like
template substitution.
"""
if isinstance(self.content, binary_type):
yield self.content
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment