Run WPT .any.js and .worker.js variations in run-webkit-tests

When .any.js and .worker.js files are seen during MANIFEST generation
they are mapped to .any.html, .any.worker.html, and .worker.html files
as appropriate which WPTServe will generate on-the-fly. This allows
for easy testing of [Exposed=(Window,Worker)] APIs and tests that are
simply script with no boilerplate HTML. These are now supported in
run-webkit-tests.

When specifying tests to run, the source name should be specified,
for example:

  $ run-webkit-test external/wpt/storage/interfaces.worker.js

But expectations will use the generated name, e.g.:

  external/wpt/storage/interfaces.worker-expected.txt

And failures will be reported under the generated name:

  external/wpt/storage/interfaces.worker.html

This change is based on jsbell's crrev.com/2767673002.

Bug: 703837
Change-Id: I063b18f44e0460932c3fe8114420a327f1b1e088
Reviewed-on: https://chromium-review.googlesource.com/574645
Commit-Queue: Jeff Carpenter <jeffcarp@chromium.org>
Reviewed-by: Quinten Yearsley <qyearsley@chromium.org>
Reviewed-by: Jeff Carpenter <jeffcarp@chromium.org>
Reviewed-by: Joshua Bell <jsbell@chromium.org>
Cr-Commit-Position: refs/heads/master@{#487304}

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py

@@ -145,7 +145,22 @@ class TestResultWriter(object):
         """
         fs = self._filesystem
         output_filename = fs.join(self._root_output_dir, self._test_name)
-        return fs.splitext(output_filename)[0] + modifier
+        base, extension = fs.splitext(output_filename)
+
+        # Below is an affordance for WPT test files that become multiple tests using different URL params,
+        # For example,
+        # - html/syntax/parsing/html5lib_adoption01.html
+        # Becomes two tests:
+        # - html/syntax/parsing/html5lib_adoption01.html?run_type=write
+        # - html/syntax/parsing/html5lib_adoption01.html?run_type=uri
+        # But previously their result file would be the same, since everything after the extension
+        # is removed. Instead, for files with URL params, we use the whole filename for writing results.
+        if '?' in extension:
+            # Question marks are reserved characters in Windows filenames.
+            sanitized_filename = output_filename.replace('?', '_')
+            return sanitized_filename + modifier
+
+        return base + modifier
 
     def _write_file(self, path, contents):
         if contents is not None:

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/lint_test_expectations.py

@@ -35,6 +35,7 @@ from webkitpy.common.host import Host
 from webkitpy.common import exit_codes
 from webkitpy.layout_tests.models import test_expectations
 from webkitpy.layout_tests.port.factory import platform_options
+from webkitpy.w3c.wpt_manifest import WPTManifest
 
 _log = logging.getLogger(__name__)
 
@@ -151,6 +152,11 @@ def main(argv, _, stderr):
     else:
         host = Host()
 
+    # Need to generate MANIFEST.json since some expectations correspond to WPT
+    # tests that aren't files and only exist in the manifest.
+    _log.info('Generating MANIFEST.json for web-platform-tests ...')
+    WPTManifest.ensure_manifest(host)
+
     try:
         exit_status = run_checks(host, options, stderr)
     except KeyboardInterrupt:

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/base.py

@@ -34,7 +34,6 @@ in the layout test infrastructure.
 
 import collections
 import errno
-import functools
 import json
 import logging
 import optparse
@@ -667,22 +666,43 @@ class Port(object):
         return reftest_list
 
     def tests(self, paths):
-        """Returns the list of tests found matching paths."""
+        """Returns all tests or tests matching supplied paths.
+
+        Args:
+            paths: Array of paths to match. If supplied, this function will only
+                return tests matching at least one path in paths.
+
+        Returns:
+            An array of test paths and test names. The latter are web platform
+            tests that don't correspond to file paths but are valid tests,
+            for instance a file path test.any.js could correspond to two test
+            names: test.any.html and test.any.worker.html.
+        """
         tests = self.real_tests(paths)
 
         suites = self.virtual_test_suites()
         if paths:
             tests.extend(self._virtual_tests_matching_paths(paths, suites))
+            tests.extend(self._wpt_test_urls_matching_paths(paths))
         else:
             tests.extend(self._all_virtual_tests(suites))
+            tests.extend(['external/wpt' + test for test in self._wpt_manifest().all_urls()])
+
         return tests
 
     def real_tests(self, paths):
         # When collecting test cases, skip these directories.
-        skipped_directories = set(['platform', 'resources', 'support', 'script-tests', 'reference', 'reftest'])
-        files = find_files.find(self._filesystem, self.layout_tests_dir(), paths,
-                                skipped_directories, functools.partial(Port.is_test_file, self), self.test_key)
-        return self._convert_wpt_file_paths_to_url_paths([self.relative_test_filename(f) for f in files])
+        skipped_directories = set([
+            'platform', 'resources', 'support', 'script-tests',
+            'reference', 'reftest', 'external'
+        ])
+        is_non_wpt_real_test_file = lambda fs, dirname, filename: (
+            self.is_test_file(fs, dirname, filename)
+            and not re.search(r'[/\\]external[/\\]wpt([/\\].*)?$', dirname)
+        )
+        files = find_files.find(self._filesystem, self.layout_tests_dir(), paths, skipped_directories,
+                                is_non_wpt_real_test_file, self.test_key)
+        return [self.relative_test_filename(f) for f in files]
 
     @staticmethod
     # If any changes are made here be sure to update the isUsedInReftest method in old-run-webkit-tests as well.
@@ -722,17 +742,15 @@ class Port(object):
         return Port._has_supported_extension(
             filesystem, filename) and not Port.is_reference_html_file(filesystem, dirname, filename)
 
-    def _convert_wpt_file_paths_to_url_paths(self, files):
+    def _convert_wpt_file_path_to_url_paths(self, file_path):
         tests = []
-        for file_path in files:
-            # Path separators are normalized by relative_test_filename().
-            match = re.search(r'external/wpt/(.*)$', file_path)
-            if not match:
-                tests.append(file_path)
-                continue
-            urls = self._wpt_manifest().file_path_to_url_paths(match.group(1))
-            for url in urls:
-                tests.append(file_path[0:match.start(1)] + url)
+        # Path separators are normalized by relative_test_filename().
+        match = re.search(r'external/wpt/(.*)$', file_path)
+        if not match:
+            return [file_path]
+        urls = self._wpt_manifest().file_path_to_url_paths(match.group(1))
+        for url in urls:
+            tests.append(file_path[0:match.start(1)] + url)
         return tests
 
     @memoized
@@ -828,7 +846,7 @@ class Port(object):
         """Returns True if the test name refers to an existing test or baseline."""
         # Used by test_expectations.py to determine if an entry refers to a
         # valid test and by printing.py to determine if baselines exist.
-        return self.test_isfile(test_name) or self.test_isdir(test_name)
+        return self.is_wpt_test(test_name) or self.test_isfile(test_name) or self.test_isdir(test_name)
 
     def split_test(self, test_name):
         """Splits a test name into the 'directory' part and the 'basename' part."""
@@ -1518,9 +1536,42 @@ class Port(object):
                     tests.append(test)
         return tests
 
+    def _wpt_test_urls_matching_paths(self, paths):
+        tests = []
+
+        for test_url_path in self._wpt_manifest().all_urls():
+            if test_url_path[0] == '/':
+                test_url_path = test_url_path[1:]
+
+            full_test_url_path = 'external/wpt/' + test_url_path
+
+            for path in paths:
+                if 'external' not in path:
+                    continue
+
+                wpt_path = path.replace('external/wpt/', '')
+
+                # When `test_url_path` is test.any.html or test.any.worker.html and path is test.any.js
+                matches_any_js_test = (
+                    self._wpt_manifest().is_test_file(wpt_path)
+                    and test_url_path.startswith(re.sub(r'\.js$', '', wpt_path))
+                )
+
+                # Get a list of directories for both paths, filter empty strings
+                full_test_url_directories = filter(None, full_test_url_path.split(self._filesystem.sep))
+                path_directories = filter(None, path.split(self._filesystem.sep))
+
+                # For all other path matches within WPT
+                if matches_any_js_test or path_directories == full_test_url_directories[0:len(path_directories)]:
+                    wpt_file_paths = self._convert_wpt_file_path_to_url_paths(test_url_path)
+                    tests.extend('external/wpt/' + wpt_file_path for wpt_file_path in wpt_file_paths)
+
+        return tests
+
     def _populate_virtual_suite(self, suite):
         if not suite.tests:
             base_tests = self.real_tests([suite.base])
+            base_tests.extend(self._wpt_test_urls_matching_paths([suite.base]))
             suite.tests = {}
             for test in base_tests:
                 suite.tests[test.replace(suite.base, suite.name, 1)] = test

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py

@@ -281,14 +281,72 @@ class PortTest(unittest.TestCase):
         PortTest._add_manifest_to_mock_file_system(port.host.filesystem)
         self.assertIn('external/wpt/dom/ranges/Range-attributes.html', port.tests([]))
         self.assertNotIn('external/wpt/console/console-is-a-namespace.any.js', port.tests([]))
-        self.assertEqual(port.tests(['external']), ['external/wpt/dom/ranges/Range-attributes.html'])
-        self.assertEqual(port.tests(['external/']), ['external/wpt/dom/ranges/Range-attributes.html'])
+
+        # test.any.js shows up on the filesystem as one file but it effectively becomes two test files:
+        # test.any.html and test.any.worker.html. We should support running test.any.js by name and
+        # indirectly by specifying a parent directory.
+        self.assertEqual(port.tests(['external']),
+                         ['external/wpt/html/dom/elements/global-attributes/dir_auto-EN-L.html',
+                          'external/wpt/dom/ranges/Range-attributes.html',
+                          'external/wpt/console/console-is-a-namespace.any.worker.html',
+                          'external/wpt/console/console-is-a-namespace.any.html'])
+        self.assertEqual(port.tests(['external/']),
+                         ['external/wpt/html/dom/elements/global-attributes/dir_auto-EN-L.html',
+                          'external/wpt/dom/ranges/Range-attributes.html',
+                          'external/wpt/console/console-is-a-namespace.any.worker.html',
+                          'external/wpt/console/console-is-a-namespace.any.html'])
         self.assertEqual(port.tests(['external/csswg-test']), [])
-        self.assertEqual(port.tests(['external/wpt']), ['external/wpt/dom/ranges/Range-attributes.html'])
-        self.assertEqual(port.tests(['external/wpt/']), ['external/wpt/dom/ranges/Range-attributes.html'])
+        self.assertEqual(port.tests(['external/wpt']),
+                         ['external/wpt/html/dom/elements/global-attributes/dir_auto-EN-L.html',
+                          'external/wpt/dom/ranges/Range-attributes.html',
+                          'external/wpt/console/console-is-a-namespace.any.worker.html',
+                          'external/wpt/console/console-is-a-namespace.any.html'])
+        self.assertEqual(port.tests(['external/wpt/']),
+                         ['external/wpt/html/dom/elements/global-attributes/dir_auto-EN-L.html',
+                          'external/wpt/dom/ranges/Range-attributes.html',
+                          'external/wpt/console/console-is-a-namespace.any.worker.html',
+                          'external/wpt/console/console-is-a-namespace.any.html'])
+        self.assertEqual(port.tests(['external/wpt/console']),
+                         ['external/wpt/console/console-is-a-namespace.any.worker.html',
+                          'external/wpt/console/console-is-a-namespace.any.html'])
+        self.assertEqual(port.tests(['external/wpt/console/']),
+                         ['external/wpt/console/console-is-a-namespace.any.worker.html',
+                          'external/wpt/console/console-is-a-namespace.any.html'])
+        self.assertEqual(port.tests(['external/wpt/console/console-is-a-namespace.any.js']),
+                         ['external/wpt/console/console-is-a-namespace.any.worker.html',
+                          'external/wpt/console/console-is-a-namespace.any.html'])
+        self.assertEqual(port.tests(['external/wpt/console/console-is-a-namespace.any.html']),
+                         ['external/wpt/console/console-is-a-namespace.any.html'])
+        self.assertEqual(port.tests(['external/wpt/dom']),
+                         ['external/wpt/dom/ranges/Range-attributes.html'])
+        self.assertEqual(port.tests(['external/wpt/dom/']),
+                         ['external/wpt/dom/ranges/Range-attributes.html'])
         self.assertEqual(port.tests(['external/wpt/dom/ranges/Range-attributes.html']),
                          ['external/wpt/dom/ranges/Range-attributes.html'])
 
+    def test_virtual_wpt_tests_paths(self):
+        port = self.make_port(with_tests=True)
+        PortTest._add_manifest_to_mock_file_system(port.host.filesystem)
+        all_wpt = [
+            'virtual/virtual_wpt/external/wpt/console/console-is-a-namespace.any.html',
+            'virtual/virtual_wpt/external/wpt/html/dom/elements/global-attributes/dir_auto-EN-L.html',
+            'virtual/virtual_wpt/external/wpt/console/console-is-a-namespace.any.worker.html',
+            'virtual/virtual_wpt/external/wpt/dom/ranges/Range-attributes.html',
+        ]
+        dom_wpt = [
+            'virtual/virtual_wpt_dom/external/wpt/dom/ranges/Range-attributes.html',
+        ]
+
+        self.assertEqual(port.tests(['virtual/virtual_wpt/external/']), all_wpt)
+        self.assertEqual(port.tests(['virtual/virtual_wpt/external/wpt/']), all_wpt)
+        self.assertEqual(port.tests(['virtual/virtual_wpt/external/wpt/console']),
+                         ['virtual/virtual_wpt/external/wpt/console/console-is-a-namespace.any.html',
+                          'virtual/virtual_wpt/external/wpt/console/console-is-a-namespace.any.worker.html'])
+
+        self.assertEqual(port.tests(['virtual/virtual_wpt_dom/external/wpt/dom/']), dom_wpt)
+        self.assertEqual(port.tests(['virtual/virtual_wpt_dom/external/wpt/dom/ranges/']), dom_wpt)
+        self.assertEqual(port.tests(['virtual/virtual_wpt_dom/external/wpt/dom/ranges/Range-attributes.html']), dom_wpt)
+
     def test_is_test_file(self):
         port = self.make_port(with_tests=True)
         is_test_file = functools.partial(Port.is_test_file, port, port.host.filesystem)
@@ -329,6 +387,16 @@ class PortTest(unittest.TestCase):
         # A file in external/wpt_automation.
         self.assertTrue(port.is_test_file(filesystem, LAYOUT_TEST_DIR + '/external/wpt_automation', 'foo.html'))
 
+    def test_is_wpt_test(self):
+        port = self.make_port(with_tests=True)
+        filesystem = port.host.filesystem
+        PortTest._add_manifest_to_mock_file_system(filesystem)
+
+        self.assertTrue(port.is_wpt_test('external/wpt/dom/ranges/Range-attributes.html'))
+        self.assertTrue(port.is_wpt_test('external/wpt/html/dom/elements/global-attributes/dir_auto-EN-L.html'))
+        self.assertFalse(port.is_wpt_test('dom/domparsing/namespaces-1.html'))
+        self.assertFalse(port.is_wpt_test('rutabaga'))
+
     def test_is_slow_wpt_test(self):
         port = self.make_port(with_tests=True)
         filesystem = port.host.filesystem

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/test.py

@@ -533,6 +533,8 @@ class TestPort(Port):
             VirtualTestSuite(prefix='skipped', base='failures/expected', args=['--virtual-arg2']),
             VirtualTestSuite(prefix='references_use_default_args', base='passes/reftest.html',
                              args=['--virtual-arg'], references_use_default_args=True),
+            VirtualTestSuite(prefix='virtual_wpt', base='external/wpt', args=['--virtual-arg']),
+            VirtualTestSuite(prefix='virtual_wpt_dom', base='external/wpt/dom', args=['--virtual-arg']),
         ]
 
 

third_party/WebKit/Tools/Scripts/webkitpy/tool/commands/rebaseline_cl_unittest.py

@@ -101,6 +101,8 @@ class RebaselineCLTest(BaseTestCase, LoggingTestCase):
                 self.mac_port.layout_tests_dir(), test)
             self._write(path, 'contents')
 
+        self.mac_port.host.filesystem.write_text_file('/test.checkout/LayoutTests/external/wpt/MANIFEST.json', '{}')
+
     def tearDown(self):
         BaseTestCase.tearDown(self)
         LoggingTestCase.tearDown(self)

third_party/WebKit/Tools/Scripts/webkitpy/w3c/wpt_manifest.py

@@ -13,6 +13,7 @@ import json
 import logging
 
 from webkitpy.common.path_finder import PathFinder
+from webkitpy.common.memoized import memoized
 
 _log = logging.getLogger(__file__)
 
@@ -23,6 +24,7 @@ class WPTManifest(object):
         # TODO(tkent): Create a Manifest object by Manifest.from_json().
         # See ../thirdparty/wpt/wpt/tools/manifest/manifest.py.
         self.raw_dict = json.loads(json_content)
+        self.test_types = ('manual', 'reftest', 'testharness')
 
     def _items_for_path(self, path_in_wpt):
         """Returns manifest items for the given WPT path, or None if not found.
@@ -35,28 +37,50 @@ class WPTManifest(object):
         it will be a list with three items ([url, references, extras]).
         """
         items = self.raw_dict['items']
-        if path_in_wpt in items['manual']:
-            return items['manual'][path_in_wpt]
-        elif path_in_wpt in items['reftest']:
-            return items['reftest'][path_in_wpt]
-        elif path_in_wpt in items['testharness']:
-            return items['testharness'][path_in_wpt]
+        for test_type in self.test_types:
+            if path_in_wpt in items[test_type]:
+                return items[test_type][path_in_wpt]
         return None
 
+    @memoized
+    def all_urls(self):
+        """Returns a set of the urls for all items in the manifest."""
+        urls = set()
+        if 'items' in self.raw_dict:
+            items = self.raw_dict['items']
+            for category in self.test_types:
+                if category in items:
+                    for records in items[category].values():
+                        urls.update([item[0] for item in records])
+        return urls
+
+    @memoized
+    def all_wpt_tests(self):
+        if 'items' not in self.raw_dict:
+            return []
+
+        all_tests = []
+        for test_type in self.test_types:
+            for path_in_wpt in self.raw_dict['items'][test_type]:
+                all_tests.append(path_in_wpt)
+        return all_tests
+
     def is_test_file(self, path_in_wpt):
         return self._items_for_path(path_in_wpt) is not None
 
+    def is_test_url(self, url):
+        """Checks if url is a valid test in the manifest.
+
+        The url must be the WPT test name with a leading slash (/).
+        """
+        if url[0] != '/':
+            raise Exception('Test url missing leading /: %s' % url)
+        return url in self.all_urls()
+
     def file_path_to_url_paths(self, path_in_wpt):
         manifest_items = self._items_for_path(path_in_wpt)
         assert manifest_items is not None
-        if len(manifest_items) != 1:
-            return []
-        url = manifest_items[0][0]
-        if url[1:] != path_in_wpt:
-            # TODO(tkent): foo.any.js and bar.worker.js should be accessed
-            # as foo.any.html, foo.any.worker, and bar.worker with WPTServe.
-            return []
-        return [path_in_wpt]
+        return [item[0][1:] for item in manifest_items]
 
     @staticmethod
     def _get_extras_from_item(item):