Make rebaseline-cl support run_wpt_tests.py

Breakdown of this big change:
1. Add two WPT builders to builders.json, and a new method to
   builder_list to check whether a builder runs wptrunner, which is
   needed to create special cases for WPT (see below).
2. Relax conditions in results_fetcher to include wpt_tests_suite, too.
3. Add a new helper method get_file_path_for_wpt_test to base.py with
   some cleanup to make param names more consistent.
4. Add a "non-prefix mode" to TestBaselineSet, where added tests are
   treated as the full test paths without lookup via the port object.
   This is needed because test paths on WPT builders are not relative to
   web_tests (but instead to web_tests/external/wpt); it also makes
   sense as rebaseline-cl always adds specific tests instead of test
   prefixes.
5. Two special cases for WPT builders in rebaseline.py that will stick:
   a. _file_name_for_expected_result looks up the real test file of a
      WPT test (to work with variants, etc.) and add ".ini" instead of
      "-expected.txt" to the filename.
   b. _suffixes_for_actual_failures unconditionally returns "txt" for a
      WPT test, ignoring other artifacts (e.g. png for reftests).
6. Three special cases for WPT builders that need to be reverted when
   we have full-platform coverage (marked with TODO), all related to the
   optimizer:
   a: Skip copy-existing-baselines-internal in rebaseline.py.
   b: Skip _optimize_baselines in rebaseline.py.
   c. Directly download to the generic baseline path instead of the
      platform-specific one in rebaseline_test.py.

Drive-by:
1. To make test setup easier, _add_manifest_to_mock_file_system is moved
   from base.py to test.py as a new public helper.
2. Add or tweak the __repr__ methods of some classes for clearer debug
   logging.
3. Reformat using "git cl format".

Bug: 1154085
Change-Id: I4c4434f73f68d9848267579f1ef544a6b05903cd
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2575859
Commit-Queue: Robert Ma <robertma@chromium.org>
Reviewed-by: Luke Z <lpz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#835872}

third_party/blink/tools/blinkpy/common/config/builders.json

@@ -70,6 +70,18 @@
         "specifiers": ["Fuchsia", "Release"],
         "is_try_builder": true
     },
+    "linux-wpt-identity-fyi-rel": {
+        "master": "tryserver.chromium.linux",
+        "port_name": "linux-trusty",
+        "specifiers": ["Trusty", "Release"],
+        "is_try_builder": true
+    },
+    "linux-wpt-input-fyi-rel": {
+        "master": "tryserver.chromium.linux",
+        "port_name": "linux-trusty",
+        "specifiers": ["Trusty", "Release"],
+        "is_try_builder": true
+    },
     "linux-blink-rel": {
         "master": "tryserver.blink",
         "port_name": "linux-trusty",

third_party/blink/tools/blinkpy/common/net/results_fetcher.py

@@ -160,8 +160,8 @@ class TestResultsFetcher(object):
             # patch)'. Only make sure it starts with blink_web_tests and
             # runs with a patch. This should be changed eventually to use actual
             # structured data from the test results server.
-            if (entry['TestType'].startswith('blink_web_tests')
-                and entry['TestType'].endswith('(with patch)'))
+            if re.match(r'(blink_web_tests|wpt_tests_suite).*\(with patch\)$',
+                        entry['TestType'])
         ]
         # In manual testing, I sometimes saw results where the same suite was
         # repeated twice. De-duplicate here to try to catch this.

third_party/blink/tools/blinkpy/common/net/results_fetcher_test.py

@@ -122,23 +122,41 @@ class BuilderTest(LoggingTestCase):
             urls={
                 'https://test-results.appspot.com/testfile?buildnumber=5&'
                 'callback=ADD_RESULTS&builder=foo&name=full_results.json':
-                'ADD_RESULTS(%s);' % (json.dumps(
-                    [{
-                        "TestType": "blink_web_tests (with patch)"
-                    },
-                     {
-                         "TestType":
-                         "not_site_per_process_blink_web_tests (with patch)"
-                     }, {
-                         "TestType": "blink_web_tests (retry with patch)"
-                     }, {
-                         "TestType": "base_unittests (with patch)"
-                     }]))
+                'ADD_RESULTS(%s);' %
+                (json.dumps([{
+                    "TestType": "blink_web_tests (with patch)"
+                }, {
+                    "TestType":
+                    "not_site_per_process_blink_web_tests (with patch)"
+                }, {
+                    "TestType": "blink_web_tests (retry with patch)"
+                }, {
+                    "TestType": "base_unittests (with patch)"
+                }]))
             })
         step_name = fetcher.get_layout_test_step_name(Build('foo', 5))
         self.assertEqual(step_name, 'blink_web_tests (with patch)')
         self.assertLog([])
 
+    def test_get_step_name_for_wpt(self):
+        fetcher = TestResultsFetcher()
+        fetcher.web = MockWeb(
+            urls={
+                'https://test-results.appspot.com/testfile?buildnumber=5&'
+                'callback=ADD_RESULTS&builder=foo&name=full_results.json':
+                'ADD_RESULTS(%s);' %
+                (json.dumps([{
+                    "TestType": "wpt_tests_suite (with patch)"
+                }, {
+                    "TestType": "wpt_tests_suite (retry with patch)"
+                }, {
+                    "TestType": "base_unittests (with patch)"
+                }]))
+            })
+        step_name = fetcher.get_layout_test_step_name(Build('foo', 5))
+        self.assertEqual(step_name, 'wpt_tests_suite (with patch)')
+        self.assertLog([])
+
     def test_get_step_name_without_build_number(self):
         fetcher = TestResultsFetcher()
         self.assertIsNone(

third_party/blink/tools/blinkpy/common/net/web_test_results.py

@@ -37,6 +37,10 @@ class WebTestResult(object):
         self._test_name = test_name
         self._result_dict = result_dict
 
+    def __repr__(self):
+        return "WebTestResult(test_name=%s, result_dict=%s)" % \
+            (repr(self._test_name), repr(self._result_dict))
+
     def suffixes_for_test_result(self):
         suffixes = set()
         artifact_names = self._result_dict.get('artifacts', {}).keys()

third_party/blink/tools/blinkpy/tool/commands/rebaseline.py

@@ -32,8 +32,6 @@ import logging
 import optparse
 import re
 
-from collections import defaultdict
-
 from blinkpy.common.path_finder import WEB_TESTS_LAST_COMPONENT
 from blinkpy.common.memoized import memoized
 from blinkpy.common.net.results_fetcher import Build
@@ -113,7 +111,16 @@ class AbstractRebaseliningCommand(Command):
         return self._host_port.output_filename(
             test_name, test_failures.FILENAME_SUFFIX_ACTUAL, '.' + suffix)
 
-    def _file_name_for_expected_result(self, test_name, suffix):
+    def _file_name_for_expected_result(self, test_name, suffix, is_wpt=False):
+        if is_wpt:
+            # *-actual.txt produced by wptrunner are actually manifest files
+            # that can make the test pass if renamed to *.ini.
+            # WPT bots do not include "external/wpt" in test names.
+            file_name = self._host_port.get_file_path_for_wpt_test(
+                'external/wpt/' + test_name)
+            assert file_name, ('Cannot find %s in WPT' % test_name)
+            return file_name + '.ini'
+
         # output_filename takes extensions starting with '.'.
         return self._host_port.output_filename(
             test_name, test_failures.FILENAME_SUFFIX_EXPECTED, '.' + suffix)
@@ -169,10 +176,18 @@ class TestBaselineSet(object):
     about where to fetch the baselines from.
     """
 
-    def __init__(self, host):
+    def __init__(self, host, prefix_mode=True):
+        """Args:
+            host: A Host object.
+            prefix_mode: (Optional, default to True) Whether the collection
+                contains test prefixes or specific tests.
+        """
         self._host = host
-        self._port = self._host.port_factory.get()
+        # Set self._port to None to avoid accidentally calling port.tests when
+        # we are not in prefix mode.
+        self._port = self._host.port_factory.get() if prefix_mode else None
         self._builder_names = set()
+        self._prefix_mode = prefix_mode
         self._test_prefix_map = collections.defaultdict(list)
 
     def __iter__(self):
@@ -184,6 +199,11 @@ class TestBaselineSet(object):
     def _iter_combinations(self):
         """Iterates through (test, build, port) combinations."""
         for test_prefix, build_port_pairs in self._test_prefix_map.iteritems():
+            if not self._prefix_mode:
+                for build, port_name in build_port_pairs:
+                    yield (test_prefix, build, port_name)
+                continue
+
             for test in self._port.tests([test_prefix]):
                 for build, port_name in build_port_pairs:
                     yield (test, build, port_name)
@@ -191,18 +211,21 @@ class TestBaselineSet(object):
     def __str__(self):
         if not self._test_prefix_map:
             return '<Empty TestBaselineSet>'
-        return ('<TestBaselineSet with:\n  ' + '\n  '.join('%s: %s, %s' % \
-            triple for triple in self._iter_combinations()) + '>')
+        return '<TestBaselineSet with:\n  %s>' % '\n  '.join(
+            '%s: %s, %s' % triple for triple in self._iter_combinations())
 
     def test_prefixes(self):
-        """Returns a sorted list of test prefixes."""
+        """Returns a sorted list of test prefixes (or tests) added thus far."""
         return sorted(self._test_prefix_map)
 
     def all_tests(self):
         """Returns a sorted list of all tests without duplicates."""
         tests = set()
         for test_prefix in self._test_prefix_map:
-            tests.update(self._port.tests([test_prefix]))
+            if self._prefix_mode:
+                tests.update(self._port.tests([test_prefix]))
+            else:
+                tests.add(test_prefix)
         return sorted(tests)
 
     def build_port_pairs(self, test_prefix):
@@ -213,7 +236,9 @@ class TestBaselineSet(object):
         """Adds an entry for baselines to download for some set of tests.
 
         Args:
-            test_prefix: This can be a full test path, or directory of tests, or a path with globs.
+            test_prefix: This can be a full test path; if the instance was
+                constructed in prefix mode (the default), this can also be a
+                directory of tests or a path with globs.
             build: A Build object. This specifies where to fetch baselines from.
             port_name: This specifies what platform the baseline is for.
         """
@@ -321,11 +346,14 @@ class AbstractParallelRebaselineCommand(AbstractRebaseliningCommand):
                 port_name,
             ])
 
-            copy_command = [
-                self._tool.executable, path_to_blink_tool,
-                'copy-existing-baselines-internal'
-            ] + args
-            copy_baseline_commands.append(tuple([copy_command, cwd]))
+            # TODO(crbug.com/1154085): Undo this special case when we have WPT
+            # bots on more ports.
+            if not self._tool.builders.is_wpt_builder(build.builder_name):
+                copy_command = [
+                    self._tool.executable, path_to_blink_tool,
+                    'copy-existing-baselines-internal'
+                ] + args
+                copy_baseline_commands.append(tuple([copy_command, cwd]))
 
             args.extend(['--builder', build.builder_name])
             if build.build_number:
@@ -372,6 +400,12 @@ class AbstractParallelRebaselineCommand(AbstractRebaseliningCommand):
         for test, build, _ in test_baseline_set:
             if build.builder_name not in builders_to_fetch_from:
                 continue
+
+            # TODO(crbug.com/1154085): Undo this special case when we have WPT
+            # bots on more ports.
+            if self._tool.builders.is_wpt_builder(build.builder_name):
+                continue
+
             tests_to_suffixes[test].update(
                 self._suffixes_for_actual_failures(test, build))
 
@@ -398,7 +432,7 @@ class AbstractParallelRebaselineCommand(AbstractRebaseliningCommand):
 
     def _update_expectations_files(self, lines_to_remove):
         tests = lines_to_remove.keys()
-        to_remove = defaultdict(set)
+        to_remove = collections.defaultdict(set)
         all_versions = frozenset([
             config.version.lower() for config in self._tool.port_factory.get().
             all_test_configurations()
@@ -535,6 +569,11 @@ class AbstractParallelRebaselineCommand(AbstractRebaseliningCommand):
         test_result = self._result_for_test(test, build)
         if not test_result:
             return set()
+        # Regardless of the test type, we only need the text output (i.e. the
+        # INI manifest) on a WPT bot (a reftest produces both text and image
+        # output, but the image is only informative).
+        if self._tool.builders.is_wpt_builder(build.builder_name):
+            return {'txt'}
         return test_result.suffixes_for_test_result()
 
     def _test_passed_unexpectedly(self, test, build, port_name):

third_party/blink/tools/blinkpy/tool/commands/rebaseline_cl.py

@@ -343,7 +343,8 @@ class RebaselineCL(AbstractParallelRebaselineCommand):
                 if f.startswith(test_base)
             ]
 
-        test_baseline_set = TestBaselineSet(self._tool)
+        # Here we have a concrete list of tests so we don't need prefix lookup.
+        test_baseline_set = TestBaselineSet(self._tool, prefix_mode=False)
         for build, tests in builds_to_tests.iteritems():
             for test in tests:
                 if only_changed_tests and test not in tests_in_cl:

third_party/blink/tools/blinkpy/tool/commands/rebaseline_test.py

@@ -2,7 +2,6 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import json
 import logging
 
 from blinkpy.tool.commands.rebaseline import AbstractRebaseliningCommand
@@ -44,9 +43,16 @@ class RebaselineTest(AbstractRebaseliningCommand):
             options.builder)
         test_name = options.test
         for suffix in self._baseline_suffix_list:
-            self._rebaseline_test(port_name, test_name, suffix, results_url)
-
-    def _rebaseline_test(self, port_name, test_name, suffix, results_url):
+            self._rebaseline_test(
+                port_name, test_name, suffix, results_url,
+                self._tool.builders.is_wpt_builder(options.builder))
+
+    def _rebaseline_test(self,
+                         port_name,
+                         test_name,
+                         suffix,
+                         results_url,
+                         is_wpt=False):
         """Downloads a baseline file and saves it to the filesystem.
 
         Args:
@@ -56,16 +62,22 @@ class RebaselineTest(AbstractRebaseliningCommand):
             suffix: The baseline file extension (e.g. png); together with the
                 test name and results_url this determines what file to download.
             results_url: Base URL to download the actual result from.
+            is_wpt: (Optional, default to False) Whether this is a WPT builder.
         """
         port = self._tool.port_factory.get(port_name)
 
-        baseline_directory = port.baseline_version_dir()
+        # TODO(crbug.com/1154085): Undo this special case when we have WPT bots
+        # on more ports.
+        if is_wpt:
+            baseline_directory = port.web_tests_dir()
+        else:
+            baseline_directory = port.baseline_version_dir()
 
         source_baseline = '%s/%s' % (
             results_url, self._file_name_for_actual_result(test_name, suffix))
         target_baseline = self._tool.filesystem.join(
             baseline_directory,
-            self._file_name_for_expected_result(test_name, suffix))
+            self._file_name_for_expected_result(test_name, suffix, is_wpt))
 
         if suffix == 'png' and port.reference_files(test_name):
             _log.warning('Cannot rebaseline image result for reftest: %s',

third_party/blink/tools/blinkpy/tool/commands/rebaseline_unittest.py

@@ -2,7 +2,6 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import json
 import optparse
 import unittest
 
@@ -11,10 +10,12 @@ from blinkpy.common.net.web_test_results import WebTestResults
 from blinkpy.common.path_finder import RELATIVE_WEB_TESTS
 from blinkpy.common.system.executive_mock import MockExecutive
 from blinkpy.tool.commands.rebaseline import (
-    AbstractParallelRebaselineCommand, Rebaseline, TestBaselineSet)
+    AbstractRebaseliningCommand, AbstractParallelRebaselineCommand, Rebaseline,
+    TestBaselineSet)
 from blinkpy.tool.mock_tool import MockBlinkTool
 from blinkpy.web_tests.builder_list import BuilderList
 from blinkpy.web_tests.port.factory_mock import MockPortFactory
+from blinkpy.web_tests.port.test import add_manifest_to_mock_filesystem
 
 
 class BaseTestCase(unittest.TestCase):
@@ -161,6 +162,25 @@ class BaseTestCase(unittest.TestCase):
                 }))
 
 
+class TestAbstractRebaselineCommand(BaseTestCase):
+    """Tests for the base class of a rebaseline command.
+
+    This class only contains test cases for utility methods.
+    """
+
+    command_constructor = AbstractRebaseliningCommand
+
+    def test_file_name_for_expected_result(self):
+        # pylint: disable=protected-access
+        add_manifest_to_mock_filesystem(self.tool.port_factory.get())
+        self.assertEqual(
+            self.command._file_name_for_expected_result(
+                'console/console-is-a-namespace.any.worker.html',
+                'txt',
+                is_wpt=True),
+            'external/wpt/console/console-is-a-namespace.any.js.ini')
+
+
 class TestAbstractParallelRebaselineCommand(BaseTestCase):
     """Tests for the base class of multiple rebaseline commands.
 
@@ -206,6 +226,45 @@ class TestAbstractParallelRebaselineCommand(BaseTestCase):
             '/mock-checkout/' + RELATIVE_WEB_TESTS + 'x/foo-expected.txt',
         ])
 
+    def test_suffixes_for_actual_failures_for_wpt(self):
+        self.tool.results_fetcher.set_results(
+            Build('some-wpt-bot'),
+            WebTestResults({
+                'tests': {
+                    'wpt.html': {
+                        'expected': 'PASS',
+                        'actual': 'FAIL',
+                    }
+                }
+            }))
+        self.assertEqual(
+            # pylint: disable=protected-access
+            self.command._suffixes_for_actual_failures('wpt.html',
+                                                       Build('some-wpt-bot')),
+            {'txt'},
+        )
+
+    def test_suffixes_for_actual_failures_for_non_wpt(self):
+        # pylint: disable=protected-access
+        build = Build('MOCK Win7')
+        self.tool.results_fetcher.set_results(
+            build,
+            WebTestResults({
+                'tests': {
+                    'pixel.html': {
+                        'expected': 'PASS',
+                        'actual': 'FAIL',
+                        'artifacts': {
+                            'actual_image': ['pixel-actual.png'],
+                        },
+                    }
+                }
+            }))
+        self.assertEqual(
+            self.command._suffixes_for_actual_failures('pixel.html', build),
+            {'png'},
+        )
+
 
 class TestRebaseline(BaseTestCase):
     """Tests for the blink_tool.py rebaseline command.
@@ -975,6 +1034,10 @@ class TestBaselineSetTest(unittest.TestCase):
                 'port_name': 'test-win-win10',
                 'specifiers': ['Win10', 'Release']
             },
+            'some-wpt-bot': {
+                'port_name': 'linux-trusty',
+                'specifiers': ['Trusty', 'Release']
+            },
         })
         self.host = host
 
@@ -1004,12 +1067,11 @@ class TestBaselineSetTest(unittest.TestCase):
         test_baseline_set = TestBaselineSet(host=self.host)
         test_baseline_set.add('a/x.html', Build('MOCK Mac10.12'))
         test_baseline_set.add('a/x.html', Build('MOCK Win10'))
-        self.assertEqual(
-            str(test_baseline_set),
-            ('<TestBaselineSet with:\n'
-             '  a/x.html: Build(builder_name=\'MOCK Mac10.12\', build_number=None), test-mac-mac10.12\n'
-             '  a/x.html: Build(builder_name=\'MOCK Win10\', build_number=None), test-win-win10>'
-             ))
+        self.assertEqual(str(test_baseline_set), (
+            '<TestBaselineSet with:\n'
+            '  a/x.html: Build(builder_name=\'MOCK Mac10.12\', build_number=None), test-mac-mac10.12\n'
+            '  a/x.html: Build(builder_name=\'MOCK Win10\', build_number=None), test-win-win10>'
+        ))
 
     def test_getters(self):
         test_baseline_set = TestBaselineSet(host=self.host)
@@ -1020,3 +1082,17 @@ class TestBaselineSetTest(unittest.TestCase):
             test_baseline_set.build_port_pairs('a/x.html'),
             [(Build(builder_name='MOCK Mac10.12'), 'test-mac-mac10.12'),
              (Build(builder_name='MOCK Win10'), 'test-win-win10')])
+
+    def test_non_prefix_mode(self):
+        test_baseline_set = TestBaselineSet(host=self.host, prefix_mode=False)
+        # This test does not exist in setUp.
+        test_baseline_set.add('wpt/foo.html', Build('some-wpt-bot'))
+        # But it should still appear in various getters since no test lookup is
+        # done when prefix_mode=False.
+        self.assertEqual(
+            list(test_baseline_set),
+            [('wpt/foo.html', Build('some-wpt-bot'), 'linux-trusty')])
+        self.assertEqual(test_baseline_set.all_tests(), ['wpt/foo.html'])
+        self.assertEqual(test_baseline_set.test_prefixes(), ['wpt/foo.html'])
+        self.assertEqual(test_baseline_set.build_port_pairs('wpt/foo.html'),
+                         [(Build('some-wpt-bot'), 'linux-trusty')])

third_party/blink/tools/blinkpy/web_tests/builder_list.py

@@ -126,6 +126,9 @@ class BuilderList(object):
     def is_try_server_builder(self, builder_name):
         return self._builders[builder_name].get('is_try_builder', False)
 
+    def is_wpt_builder(self, builder_name):
+        return 'wpt' in builder_name
+
     def platform_specifier_for_builder(self, builder_name):
         return self.specifiers_for_builder(builder_name)[0]
 
@@ -173,7 +176,7 @@ class BuilderList(object):
             specifiers = set(spec.lower() for spec in info['specifiers'])
             is_try_builder_info = info.get('is_try_builder', False)
             if (version.lower() in specifiers
-                    and build_type.lower() in specifiers 
+                    and build_type.lower() in specifiers
                     and is_try_builder_info == is_try_builder):
                 return builder_name
         return ''

third_party/blink/tools/blinkpy/web_tests/builder_list_unittest.py

@@ -35,6 +35,11 @@ class BuilderListTest(unittest.TestCase):
     @staticmethod
     def sample_builder_list():
         return BuilderList({
+            'some-wpt-bot': {
+                'port_name': 'port-c',
+                'specifiers': ['C', 'Release'],
+                'is_try_builder': True,
+            },
             'Blink A': {
                 'port_name': 'port-a',
                 'specifiers': ['A', 'Release']
@@ -95,8 +100,16 @@ class BuilderListTest(unittest.TestCase):
     def test_all_builder_names(self):
         builders = self.sample_builder_list()
         self.assertEqual([
-            'Blink A', 'Blink B', 'Blink B (dbg)', 'Blink C (dbg)', 'CQ Try A',
-            'CQ Try B', 'CQ Try C', 'Try A', 'Try B'
+            'Blink A',
+            'Blink B',
+            'Blink B (dbg)',
+            'Blink C (dbg)',
+            'CQ Try A',
+            'CQ Try B',
+            'CQ Try C',
+            'Try A',
+            'Try B',
+            'some-wpt-bot',
         ], builders.all_builder_names())
 
     def test_all_continuous_builder_names(self):
@@ -107,9 +120,10 @@ class BuilderListTest(unittest.TestCase):
 
     def test_all_try_builder_names(self):
         builders = self.sample_builder_list()
-        self.assertEqual(
-            ['CQ Try A', 'CQ Try B', 'CQ Try C', 'Try A', 'Try B'],
-            builders.all_try_builder_names())
+        self.assertEqual([
+            'CQ Try A', 'CQ Try B', 'CQ Try C', 'Try A', 'Try B',
+            'some-wpt-bot'
+        ], builders.all_try_builder_names())
 
     def test_all_cq_try_builder_names(self):
         builders = self.sample_builder_list()
@@ -195,3 +209,8 @@ class BuilderListTest(unittest.TestCase):
         self.assertEqual('B',
                          builders.version_specifier_for_port_name('port-b'))
         self.assertIsNone(builders.version_specifier_for_port_name('port-x'))
+
+    def test_is_wpt_builder(self):
+        builders = self.sample_builder_list()
+        self.assertFalse(builders.is_wpt_builder('Blink A'))
+        self.assertTrue(builders.is_wpt_builder('some-wpt-bot'))

third_party/blink/tools/blinkpy/web_tests/port/base.py

@@ -981,36 +981,36 @@ class Port(object):
             WPTManifest.ensure_manifest(self, path)
         return WPTManifest(self.host, manifest_path)
 
-    def is_wpt_crash_test(self, test_file):
+    def is_wpt_crash_test(self, test_name):
         """Returns whether a WPT test is a crashtest.
 
         See https://web-platform-tests.org/writing-tests/crashtest.html.
         """
-        match = self.WPT_REGEX.match(test_file)
+        match = self.WPT_REGEX.match(test_name)
         if not match:
             return False
         wpt_path = match.group(1)
         path_in_wpt = match.group(2)
         return self.wpt_manifest(wpt_path).is_crash_test(path_in_wpt)
 
-    def is_slow_wpt_test(self, test_file):
+    def is_slow_wpt_test(self, test_name):
         # When DCHECK is enabled, idlharness tests run 5-6x slower due to the
         # amount of JavaScript they use (most web_tests run very little JS).
         # This causes flaky timeouts for a lot of them, as a 0.5-1s test becomes
         # close to the default 6s timeout.
-        if (self.is_wpt_idlharness_test(test_file)
+        if (self.is_wpt_idlharness_test(test_name)
                 and self._build_has_dcheck_always_on()):
             return True
 
-        match = self.WPT_REGEX.match(test_file)
+        match = self.WPT_REGEX.match(test_name)
         if not match:
             return False
         wpt_path = match.group(1)
         path_in_wpt = match.group(2)
         return self.wpt_manifest(wpt_path).is_slow_test(path_in_wpt)
 
-    def get_wpt_fuzzy_metadata(self, test_file):
-        """Returns the WPT fuzzy metadata for the given test.
+    def get_wpt_fuzzy_metadata(self, test_name):
+        """Returns the fuzzy metadata for the given WPT test.
 
         The metadata is a pair of lists, (maxDifference, totalPixels), where
         each list is a [min, max] range, inclusive. If the test is not a WPT
@@ -1018,13 +1018,29 @@ class Port(object):
 
         See https://web-platform-tests.org/writing-tests/reftests.html#fuzzy-matching
         """
-        match = self.WPT_REGEX.match(test_file)
+        match = self.WPT_REGEX.match(test_name)
         if not match:
             return None, None
         wpt_path = match.group(1)
         path_in_wpt = match.group(2)
         return self.wpt_manifest(wpt_path).extract_fuzzy_metadata(path_in_wpt)
 
+    def get_file_path_for_wpt_test(self, test_name):
+        """Returns the real file path for the given WPT test.
+
+        Or None if the test is not a WPT.
+        """
+        match = self.WPT_REGEX.match(test_name)
+        if not match:
+            return None
+        wpt_path = match.group(1)
+        path_in_wpt = match.group(2)
+        file_path_in_wpt = self.wpt_manifest(wpt_path).file_path_for_test_url(
+            path_in_wpt)
+        if not file_path_in_wpt:
+            return None
+        return self._filesystem.join(wpt_path, file_path_in_wpt)
+
     def test_key(self, test_name):
         """Turns a test name into a pair of sublists: the natural sort key of the
         dirname, and the natural sort key of the basename.

third_party/blink/tools/blinkpy/web_tests/port/test.py

@@ -27,6 +27,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import base64
+import json
 import time
 
 from blinkpy.common import exit_codes
@@ -529,6 +530,105 @@ passes/slow.html [ Slow ]
     filesystem.clear_written_files()
 
 
+def add_manifest_to_mock_filesystem(port):
+    # Disable manifest update otherwise they'll be overwritten.
+    port.set_option_default('manifest_update', False)
+    filesystem = port.host.filesystem
+    filesystem.write_text_file(
+        WEB_TEST_DIR + '/external/wpt/MANIFEST.json',
+        json.dumps({
+            'items': {
+                'testharness': {
+                    'dom': {
+                        'ranges': {
+                            'Range-attributes.html': ['acbdef123', [None, {}]],
+                            'Range-attributes-slow.html':
+                            ['abcdef123', [None, {
+                                'timeout': 'long'
+                            }]],
+                        },
+                    },
+                    'console': {
+                        'console-is-a-namespace.any.js': [
+                            'abcdef1234',
+                            ['console/console-is-a-namespace.any.html', {}],
+                            [
+                                'console/console-is-a-namespace.any.worker.html',
+                                {
+                                    'timeout': 'long'
+                                }
+                            ],
+                        ],
+                    },
+                    'html': {
+                        'parse.html': [
+                            'abcdef123',
+                            ['html/parse.html?run_type=uri', {}],
+                            [
+                                'html/parse.html?run_type=write', {
+                                    'timeout': 'long'
+                                }
+                            ],
+                        ],
+                    },
+                },
+                'manual': {},
+                'reftest': {
+                    'html': {
+                        'dom': {
+                            'elements': {
+                                'global-attributes': {
+                                    'dir_auto-EN-L.html': [
+                                        'abcdef123',
+                                        [
+                                            None,
+                                            [[
+                                                '/html/dom/elements/global-attributes/dir_auto-EN-L-ref.html',
+                                                '=='
+                                            ]], {
+                                                'timeout': 'long'
+                                            }
+                                        ],
+                                    ]
+                                }
+                            }
+                        }
+                    }
+                },
+                'crashtest': {
+                    'portals': {
+                        'portals-no-frame-crash.html':
+                        ['abcdef123', [None, {}]],
+                    },
+                },
+            }
+        }))
+    filesystem.write_text_file(
+        WEB_TEST_DIR + '/external/wpt/dom/ranges/Range-attributes.html', '')
+    filesystem.write_text_file(
+        WEB_TEST_DIR + '/external/wpt/dom/ranges/Range-attributes-slow.html',
+        '')
+    filesystem.write_text_file(
+        WEB_TEST_DIR + '/external/wpt/console/console-is-a-namespace.any.js',
+        '')
+    filesystem.write_text_file(
+        WEB_TEST_DIR + '/external/wpt/common/blank.html', 'foo')
+
+    filesystem.write_text_file(
+        WEB_TEST_DIR + '/wpt_internal/MANIFEST.json',
+        json.dumps({
+            'items': {
+                'testharness': {
+                    'dom': {
+                        'bar.html': ['abcdef123', [None, {}]]
+                    }
+                }
+            }
+        }))
+    filesystem.write_text_file(WEB_TEST_DIR + '/wpt_internal/dom/bar.html',
+                               'baz')
+
+
 class TestPort(Port):
     port_name = 'test'
     default_port_name = 'test-mac-mac10.10'