Update WPT in blink web_tests to use pywebsocket3

This commit updates WPT located in blinkpy to use pywebsocket3. Since
the current blinkpy does not support version 8 of MANIFEST.json, we
cherry-pick the corresponding patch in the upstream WPT, and apply it
locally.

Change-Id: I78125f49974c6cbc03160e6efa42ebc0d7bf6a43
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2094906Reviewed-by: Yutaka Hirano <yhirano@chromium.org>
Reviewed-by: Robert Ma <robertma@chromium.org>
Reviewed-by: Adam Rice <ricea@chromium.org>
Commit-Queue: Robert Ma <robertma@chromium.org>
Cr-Commit-Position: refs/heads/master@{#749225}

third_party/blink/tools/blinkpy/third_party/README.chromium

@@ -43,6 +43,7 @@ Description: This includes code for the manifest tool, lint tool, and wptserve.
     for more details on maintenance.
 Local Modifications:
 - Removed all files except for those listed in wpt/WPTWhiteList.
-- Removed a few unimported subcommands from tools/wpt/path.
 - Cherry-picked the server part of https://github.com/web-platform-tests/wpt/pull/21705
   (we cannot yet directly roll to that revision because of MANIFEST v8 changes).
+- Cherry-picked the server part of https://github.com/web-platform-tests/wpt/pull/21845
+  (Same reason with the previous cherry-pick. This enables pywebsocket3 usage).

third_party/blink/tools/blinkpy/third_party/wpt/WPTWhiteList

@@ -5,6 +5,7 @@
 ./tools/__init__.py
 ./tools/ci/commands.json
 ./tools/conftest.py
+./tools/docker/commands.json
 ./tools/gitignore/__init__.py
 ./tools/gitignore/gitignore.py
 ./tools/lint/__init__.py

third_party/blink/tools/blinkpy/third_party/wpt/checkout.sh

@@ -21,9 +21,8 @@ function clone {
   echo "WPTHead: " `git rev-parse HEAD`
 
   # Apply local changes.
-  git apply $DIR/chromium.patch
-  # Chromium presubmit requires scripts with shebang to be executable.
-  chmod 755 tools/manifest/update.py
+  git cherry-pick 644a206e8ace488eac7e2b2a58a4b5354b02363a
+  git cherry-pick 7e52ecb9b61b73425093d39dbadceb9c6e10b754
 }
 
 function reduce {

third_party/blink/tools/blinkpy/third_party/wpt/chromium.patch

-diff --git a/tools/wpt/paths b/tools/wpt/paths
-index 4528222fbf..93c97dc02b 100644
---- a/tools/wpt/paths
-+++ b/tools/wpt/paths
-@@ -1,5 +1,3 @@
--tools/ci/
--tools/docker/
- tools/lint/
- tools/manifest/
- tools/serve/
- tools/wpt/
-diff --git a/tools/serve/serve.py b/tools/serve/serve.py
-index 0985810cc5..055b60f1e7 100644
---- a/tools/serve/serve.py
-+++ b/tools/serve/serve.py
-@@ -366,6 +366,7 @@ class RoutesBuilder(object):
-             ("GET", "*.any.serviceworker.html", ServiceWorkersHandler),
-             ("GET", "*.any.worker.js", AnyWorkerHandler),
-             ("GET", "*.asis", handlers.AsIsHandler),
-+            ("GET", "/.well-known/origin-policy", handlers.PythonScriptHandler),
-             ("*", "*.py", handlers.PythonScriptHandler),
-             ("GET", "*", handlers.FileHandler)
-         ]
-@@ -742,6 +743,9 @@ def build_config(override_path=None, **kwargs):
- def _make_subdomains_product(s, depth=2):
-     return {u".".join(x) for x in chain(*(product(s, repeat=i) for i in range(1, depth+1)))}
- 
-+def _make_origin_policy_subdomains(limit):
-+    return {u"op%d" % x for x in range(1,limit+1)}
-+
- 
- _subdomains = {u"www",
-                u"www1",
-@@ -753,6 +757,12 @@ _not_subdomains = {u"nonexistent"}
- 
- _subdomains = _make_subdomains_product(_subdomains)
- 
-+# Origin policy subdomains need to not be reused by any other tests, since origin policies have
-+# origin-wide impacts like installing a CSP or Feature Policy that could interfere with features
-+# under test.
-+# See https://github.com/web-platform-tests/rfcs/pull/44.
-+_subdomains |= _make_origin_policy_subdomains(99)
-+
- _not_subdomains = _make_subdomains_product(_not_subdomains)
- 
- 

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/docker/commands.json

+{
+    "docker-run": {"path": "frontend.py", "script": "run", "parser": "parser_run", "help": "Run wpt docker image",
+                   "virtualenv": false},
+    "docker-build": {"path": "frontend.py", "script": "build", "help": "Build wpt docker image",
+                     "virtualenv": false}
+}

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/localpaths.py

@@ -6,7 +6,7 @@ repo_root = os.path.abspath(os.path.join(here, os.pardir))
 
 sys.path.insert(0, os.path.join(here))
 sys.path.insert(0, os.path.join(here, "wptserve"))
-sys.path.insert(0, os.path.join(here, "pywebsocket"))
+sys.path.insert(0, os.path.join(here, "third_party", "pywebsocket3"))
 sys.path.insert(0, os.path.join(here, "third_party", "atomicwrites"))
 sys.path.insert(0, os.path.join(here, "third_party", "attrs", "src"))
 sys.path.insert(0, os.path.join(here, "third_party", "funcsigs"))

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/serve/serve.py

@@ -592,21 +592,9 @@ class WebSocketDaemon(object):
                     "-w", handlers_root]
 
         if ssl_config is not None:
-            # This is usually done through pywebsocket.main, however we're
-            # working around that to get the server instance and manually
-            # setup the wss server.
-            if pywebsocket._import_ssl():
-                tls_module = pywebsocket._TLS_BY_STANDARD_MODULE
-            elif pywebsocket._import_pyopenssl():
-                tls_module = pywebsocket._TLS_BY_PYOPENSSL
-            else:
-                print("No SSL module available")
-                sys.exit(1)
-
             cmd_args += ["--tls",
                          "--private-key", ssl_config["key_path"],
-                         "--certificate", ssl_config["cert_path"],
-                         "--tls-module", tls_module]
+                         "--certificate", ssl_config["cert_path"]]
 
         if (bind_address):
             cmd_args = ["-H", host] + cmd_args

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wpt/paths

+tools/ci/
+tools/docker/
 tools/lint/
 tools/manifest/
 tools/serve/

third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/config.py

@@ -69,6 +69,31 @@ class Config(Mapping):
     def as_dict(self):
         return json_types(self.__dict__)
 
+    # Environment variables are limited in size so we need to prune the most egregious contributors
+    # to size, the origin policy subdomains.
+    def as_dict_for_wd_env_variable(self):
+        result = self.as_dict()
+
+        for key in [
+            ("subdomains",),
+            ("domains", "alt"),
+            ("domains", ""),
+            ("all_domains", "alt"),
+            ("all_domains", ""),
+            ("domains_set",),
+            ("all_domains_set",)
+        ]:
+            target = result
+            for part in key[:-1]:
+                target = target[part]
+            value = target[key[-1]]
+            if isinstance(value, dict):
+                target[key[-1]] = {k:v for (k,v) in iteritems(value) if not k.startswith("op")}
+            else:
+                target[key[-1]] = [x for x in value if not x.startswith("op")]
+
+        return result
+
 
 def json_types(obj):
     if isinstance(obj, dict):

third_party/blink/tools/blinkpy/web_tests/servers/wptserve.py

@@ -36,7 +36,7 @@ class WPTServe(server_base.ServerBase):
         self._config_file = fs.join(self._runtime_path, 'wpt.config.json')
 
         finder = PathFinder(fs)
-        path_to_pywebsocket = finder.path_from_chromium_base('third_party', 'pywebsocket', 'src')
+        path_to_pywebsocket = finder.path_from_chromium_base('third_party', 'pywebsocket3', 'src')
         self.path_to_wpt_support = finder.path_from_blink_tools('blinkpy', 'third_party', 'wpt')
         path_to_wpt_root = fs.join(self.path_to_wpt_support, 'wpt')
         path_to_wpt_tests = fs.abspath(fs.join(self._port_obj.web_tests_dir(), 'external', 'wpt'))

third_party/blink/tools/blinkpy/web_tests/servers/wptserve_unittest.py

@@ -59,7 +59,7 @@ class TestWPTServe(LoggingTestCase):
         self.assertEqual(server._env, {
             'MOCK_ENVIRON_COPY': '1',
             'PATH': '/bin:/mock/bin',
-            'PYTHONPATH': '/mock-checkout/third_party/pywebsocket/src'
+            'PYTHONPATH': '/mock-checkout/third_party/pywebsocket3/src'
         })
 
     def test_prepare_config(self):