Avoid computing wpt manifests when running non-WPT virtual tests

Generating the manifests can take 10's of seconds. This CL avoids that
cost unless it is necessary by computing the bases for a virtual suite
based on the passed in paths. Using those bases, we can determine
whether or not a WPT test is specified before computing the manifests.

Bug: 1091307
Change-Id: I3bb37266ca929edb7ce4e52c8118b1cc25608c11
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2255778
Commit-Queue: Daniel Libby <dlibby@microsoft.com>
Reviewed-by: Robert Ma <robertma@chromium.org>
Cr-Commit-Position: refs/heads/master@{#783756}

third_party/blink/tools/blinkpy/web_tests/port/base.py

@@ -1894,25 +1894,67 @@ class Port(object):
                                                    suite_prefixes))
         return tests
 
-    def _all_virtual_tests_for_suite(self, suite):
+    def _get_bases_for_suite_with_paths(self, suite, paths):
+        """Returns a set of bases of the virutual suite that are referenced by
+        paths. E.g. given a virtual test suite `foo` with the following bases:
+          bar/baz
+          bar/quu
+          qux
+        and given paths of [virtual/foo/bar], this method would return
+          [bar/baz, bar/quu]
+
+        Given paths of [virtual/foo/bar/baz/test.html], the return would be
+        [bar/baz]
+        """
+
+        real_paths = [p.replace(suite.full_prefix, '', 1) for p in paths \
+            if p.startswith(suite.full_prefix)]
+        # Test for paths that are under the suite's bases, so that we don't run
+        # a non-existent test.
+        bases = set()
+        for real_path in real_paths:
+            for base in suite.bases:
+                if real_path.startswith(base) or base.startswith(real_path):
+                    bases.add(base)
+
+        return list(bases)
+
+    def _virtual_tests_for_suite_with_paths(self, suite, paths):
         if not suite.bases:
             return []
+
+        bases = self._get_bases_for_suite_with_paths(suite, paths)
+
+        if not bases:
+            return []
+
         tests = []
         tests.extend(
-            map(lambda x: suite.full_prefix + x, self.real_tests(suite.bases)))
-        tests.extend(
-            self._wpt_test_urls_matching_paths(
-                suite.bases, [suite.full_prefix] * len(suite.bases)))
+            map(lambda x: suite.full_prefix + x, self.real_tests(bases)))
+
+        wpt_bases = []
+        for base in bases:
+            if any(base.startswith(wpt_dir) for wpt_dir in self.WPT_DIRS):
+                wpt_bases.append(base)
+
+        if wpt_bases:
+            tests.extend(
+                self._wpt_test_urls_matching_paths(
+                    wpt_bases, [suite.full_prefix] * len(wpt_bases)))
+
         return tests
 
     def _virtual_tests_matching_paths(self, paths):
         tests = []
         normalized_paths = [self.normalize_test_name(p) for p in paths]
         for suite in self.virtual_test_suites():
-            if not any(
-                    p.startswith(suite.full_prefix) for p in normalized_paths):
+            virtual_paths = [
+                p for p in normalized_paths if p.startswith(suite.full_prefix)
+            ]
+            if not virtual_paths:
                 continue
-            for test in self._all_virtual_tests_for_suite(suite):
+            for test in self._virtual_tests_for_suite_with_paths(
+                    suite, virtual_paths):
                 if any(test.startswith(p) for p in normalized_paths):
                     tests.append(test)
 

third_party/blink/tools/blinkpy/web_tests/port/base_unittest.py

@@ -1024,6 +1024,40 @@ class PortTest(LoggingTestCase):
             sorted(port.tests(['virtual/virtual_wpt_dom/'])),
             dom_wpt + ['virtual/virtual_wpt_dom/wpt_internal/dom/bar.html'])
 
+    def test_virtual_test_paths(self):
+        port = self.make_port(with_tests=True)
+        PortTest._add_manifest_to_mock_file_system(port)
+        ssl_tests = [
+            'virtual/mixed_wpt/http/tests/ssl/text.html',
+        ]
+        http_passes_tests = [
+            'virtual/mixed_wpt/http/tests/passes/image.html',
+            'virtual/mixed_wpt/http/tests/passes/text.html',
+        ]
+        dom_tests = [
+            'virtual/mixed_wpt/external/wpt/dom/ranges/Range-attributes-slow.html',
+            'virtual/mixed_wpt/external/wpt/dom/ranges/Range-attributes.html',
+        ]
+
+        #  The full set of tests must be returned when running the entire suite.
+        self.assertEqual(sorted(port.tests(['virtual/mixed_wpt/'])),
+                         dom_tests + http_passes_tests + ssl_tests)
+
+        self.assertEqual(sorted(port.tests(['virtual/mixed_wpt/external'])),
+                         dom_tests)
+
+        self.assertEqual(sorted(port.tests(['virtual/mixed_wpt/http'])),
+                         http_passes_tests + ssl_tests)
+        self.assertEqual(
+            sorted(
+                port.tests([
+                    'virtual/mixed_wpt/http/tests/ssl',
+                    'virtual/mixed_wpt/external/wpt/dom'
+                ])), dom_tests + ssl_tests)
+
+        # Make sure we don't run a non-existent test.
+        self.assertEqual(sorted(port.tests(['virtual/mixed_wpt/passes'])), [])
+
     def test_is_non_wpt_test_file(self):
         port = self.make_port(with_tests=True)
         self.assertTrue(port.is_non_wpt_test_file('', 'foo.html'))

third_party/blink/tools/blinkpy/web_tests/port/test.py

@@ -135,7 +135,7 @@ class TestList(object):
 #
 # These numbers may need to be updated whenever we add or delete tests. This includes virtual tests.
 #
-TOTAL_TESTS = 171
+TOTAL_TESTS = 174
 TOTAL_WONTFIX = 3
 TOTAL_SKIPS = 26 + TOTAL_WONTFIX
 TOTAL_CRASHES = 78
@@ -737,6 +737,9 @@ class TestPort(Port):
             VirtualTestSuite(prefix='virtual_empty_bases',
                              bases=[],
                              args=['--virtual-arg-empty-bases']),
+            VirtualTestSuite(prefix='mixed_wpt',
+                             bases=['http', 'external/wpt/dom'],
+                             args=['--virtual-arg']),
         ]
 
 

third_party/blink/tools/blinkpy/web_tests/run_web_tests_unittest.py

@@ -43,6 +43,7 @@ from blinkpy.common.path_finder import WEB_TESTS_LAST_COMPONENT
 from blinkpy.common.system.path import abspath_to_uri
 from blinkpy.common.system.system_host import SystemHost
 
+from blinkpy.w3c.wpt_manifest import MANIFEST_NAME
 from blinkpy.web_tests import run_web_tests
 from blinkpy.web_tests.models import test_expectations
 from blinkpy.web_tests.models import test_failures
@@ -2296,6 +2297,14 @@ class RebaselineTest(unittest.TestCase, StreamTestingMixin):
             baseline_full_path = '%s/%s' % (test.WEB_TEST_DIR, baseline)
             self.assertIsNone(written_files.get(baseline_full_path))
 
+    def assert_wpt_manifests_not_written(self, host, written_files):
+        external_manifest = host.filesystem.join(test.WEB_TEST_DIR,
+                                                 'external/wpt', MANIFEST_NAME)
+        internal_manifest = host.filesystem.join(test.WEB_TEST_DIR,
+                                                 'wpt_internal', MANIFEST_NAME)
+        self.assertNotIn(external_manifest, written_files)
+        self.assertNotIn(internal_manifest, written_files)
+
     def test_reset_results_basic(self):
         # Test that we update baselines in place when the test fails
         # (text and image mismatch).
@@ -2310,6 +2319,7 @@ class RebaselineTest(unittest.TestCase, StreamTestingMixin):
         # baselines, it's OK for actual results to not match baselines.
         self.assertEqual(details.exit_code, 0)
         self.assertEqual(len(written_files.keys()), 7)
+        self.assert_wpt_manifests_not_written(host, written_files)
         self.assert_baselines(
             written_files,
             log_stream,
@@ -2703,7 +2713,7 @@ class RebaselineTest(unittest.TestCase, StreamTestingMixin):
                                              host=host)
         written_files = host.filesystem.written_files
         self.assertEqual(details.exit_code, 0)
-        self.assertEqual(len(written_files.keys()), 9)
+        self.assertEqual(len(written_files.keys()), 7)
         # We should create new image baseline only.
         self.assert_baselines(
             written_files,
@@ -2792,7 +2802,8 @@ class RebaselineTest(unittest.TestCase, StreamTestingMixin):
         self.assertEqual(details.exit_code, 0)
         self.assertFalse(host.filesystem.exists(virtual_baseline_txt))
         written_files = host.filesystem.written_files
-        self.assertEqual(len(written_files.keys()), 10)
+        self.assertEqual(len(written_files.keys()), 8)
+        self.assert_wpt_manifests_not_written(host, written_files)
         # We should create new image baseline only.
         self.assert_baselines(
             written_files,