wpt-import: Remove tests from expectations when WPT test harnesses are deleted

Several web platform tests do not have physical files checked into the
WPT repository. Instead they are generated and mapped to test harnesses
checked into the WPT repo. When those test harnesses are deleted then
we should delete all tests mapped to those files. Only affects the
behavior of the script when --clean-up-affected-tests-only is used.

Bug: 1050760, 1110003, 801357
Change-Id: Id1c30b096c73974ed0ded2d74284fbe43005ee0d
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2439417
Commit-Queue: Rakib Hasan <rmhasan@google.com>
Reviewed-by: Robert Ma <robertma@chromium.org>
Cr-Commit-Position: refs/heads/master@{#816865}

third_party/blink/tools/blinkpy/w3c/android_wpt_expectations_updater_unittest.py

@@ -2,6 +2,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import json
 import logging
 
 from blinkpy.common.host_mock import MockHost
@@ -10,6 +11,7 @@ from blinkpy.common.net.git_cl_mock import MockGitCL
 from blinkpy.common.net.results_fetcher import Build
 from blinkpy.common.net.web_test_results import WebTestResults
 from blinkpy.common.system.log_testing import LoggingTestCase
+from blinkpy.w3c.wpt_manifest import BASE_MANIFEST_NAME
 from blinkpy.web_tests.builder_list import BuilderList
 from blinkpy.web_tests.port.factory_mock import MockPortFactory
 from blinkpy.web_tests.port.android import (
@@ -76,6 +78,18 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
                 'is_try_builder': True,
             },
         })
+        host.filesystem.write_text_file(
+            host.port_factory.get().web_tests_dir() + '/external/' +
+            BASE_MANIFEST_NAME,
+            json.dumps({
+                'items': {
+                    'testharness': {
+                        'ghi.html': ['abcdef123', [None, {}]],
+                        'van.html': ['abcdef123', [None, {}]],
+                    },
+                },
+            }))
+
         # Write dummy expectations
         for path in PRODUCTS_TO_EXPECTATION_FILE_PATHS.values():
             host.filesystem.write_text_file(

third_party/blink/tools/blinkpy/w3c/test_importer.py

@@ -45,9 +45,10 @@ _log = logging.getLogger(__file__)
 
 
 class TestImporter(object):
-    def __init__(self, host, wpt_github=None):
+    def __init__(self, host, wpt_github=None, wpt_manifests=None):
         self.host = host
         self.wpt_github = wpt_github
+        self.port = host.port_factory.get()
 
         self.executive = host.executive
         self.fs = host.filesystem
@@ -72,7 +73,8 @@ class TestImporter(object):
 
         args = ['--clean-up-affected-tests-only',
                 '--clean-up-test-expectations']
-        self._expectations_updater = WPTExpectationsUpdater(self.host, args)
+        self._expectations_updater = WPTExpectationsUpdater(
+            self.host, args, wpt_manifests)
 
     def main(self, argv=None):
         # TODO(robertma): Test this method! Split it to make it easier to test
@@ -155,14 +157,15 @@ class TestImporter(object):
         # TODO(robertma): Implement `add --all` in Git (it is different from `commit --all`).
         self.chromium_git.run(['add', '--all', self.dest_path])
 
+        # Remove expectations for tests that were deleted and rename tests
+        # in expectations for renamed tests.
+        self._expectations_updater.cleanup_test_expectations_files()
+
         self._generate_manifest()
 
         # TODO(crbug.com/800570 robertma): Re-enable it once we fix the bug.
         # self._delete_orphaned_baselines()
 
-        # Remove expectations for tests that were deleted and rename tests
-        # in expectations for renamed tests.
-        self._expectations_updater.cleanup_test_expectations_files()
 
         if not self.chromium_git.has_working_directory_changes():
             _log.info('Done: no changes to import.')

third_party/blink/tools/blinkpy/w3c/test_importer_unittest.py

@@ -24,7 +24,13 @@ from blinkpy.w3c.wpt_manifest import BASE_MANIFEST_NAME
 from blinkpy.web_tests.port.android import PRODUCTS_TO_EXPECTATION_FILE_PATHS
 
 MOCK_WEB_TESTS = '/mock-checkout/' + RELATIVE_WEB_TESTS
-
+MANIFEST_INSTALL_CMD = [
+    'python',
+    '/mock-checkout/third_party/blink/tools/blinkpy/third_party/wpt/wpt/wpt',
+    'manifest',
+    '--no-download',
+    '--tests-root',
+    MOCK_WEB_TESTS + 'external/wpt']
 
 class TestImporterTest(LoggingTestCase):
 
@@ -34,11 +40,19 @@ class TestImporterTest(LoggingTestCase):
             host.filesystem.write_text_file(path, '')
         return host
 
+    @staticmethod
+    def _get_test_importer(host, wpt_github=None):
+        port = host.port_factory.get()
+        return TestImporter(
+            host,
+            wpt_github=wpt_github,
+            wpt_manifests=[port.wpt_manifest('external/wpt')])
+
     def test_update_expectations_for_cl_no_results(self):
         host = self.mock_host()
         host.filesystem.write_text_file(
             MOCK_WEB_TESTS + 'W3CImportExpectations', '')
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         importer.git_cl = MockGitCL(host, time_out=True)
         success = importer.update_expectations_for_cl()
         self.assertFalse(success)
@@ -52,7 +66,7 @@ class TestImporterTest(LoggingTestCase):
         host = self.mock_host()
         host.filesystem.write_text_file(
             MOCK_WEB_TESTS + 'W3CImportExpectations', '')
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         importer.git_cl = MockGitCL(
             host,
             status='closed',
@@ -70,7 +84,7 @@ class TestImporterTest(LoggingTestCase):
         host = self.mock_host()
         host.filesystem.write_text_file(
             MOCK_WEB_TESTS + 'W3CImportExpectations', '')
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         importer.git_cl = MockGitCL(
             host,
             status='lgtm',
@@ -88,7 +102,7 @@ class TestImporterTest(LoggingTestCase):
         host = self.mock_host()
         host.filesystem.write_text_file(
             MOCK_WEB_TESTS + 'W3CImportExpectations', '')
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         importer.git_cl = MockGitCL(
             host,
             status='lgtm',
@@ -107,7 +121,7 @@ class TestImporterTest(LoggingTestCase):
         host = self.mock_host()
         host.filesystem.write_text_file(
             MOCK_WEB_TESTS + 'W3CImportExpectations', '')
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         # Only the latest job for each builder is counted.
         importer.git_cl = MockGitCL(
             host,
@@ -136,7 +150,7 @@ class TestImporterTest(LoggingTestCase):
         host = self.mock_host()
         host.filesystem.write_text_file(
             MOCK_WEB_TESTS + 'W3CImportExpectations', '')
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         importer.git_cl = MockGitCL(
             host,
             status='lgtm',
@@ -165,7 +179,7 @@ class TestImporterTest(LoggingTestCase):
         host = self.mock_host()
         host.filesystem.write_text_file(
             MOCK_WEB_TESTS + 'W3CImportExpectations', '')
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         # Only the latest job for each builder is counted.
         importer.git_cl = MockGitCL(
             host,
@@ -196,7 +210,7 @@ class TestImporterTest(LoggingTestCase):
         host = self.mock_host()
         host.filesystem.write_text_file(
             MOCK_WEB_TESTS + 'W3CImportExpectations', '')
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         importer.git_cl = MockGitCL(
             host,
             status='closed',
@@ -219,7 +233,7 @@ class TestImporterTest(LoggingTestCase):
     def test_run_commit_queue_for_cl_timeout(self):
         # This simulates the case where we time out while waiting for try jobs.
         host = self.mock_host()
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         importer.git_cl = MockGitCL(host, time_out=True)
         success = importer.run_commit_queue_for_cl()
         self.assertFalse(success)
@@ -237,7 +251,7 @@ class TestImporterTest(LoggingTestCase):
         host = self.mock_host()
         host.filesystem.write_text_file(
             MOCK_WEB_TESTS + 'W3CImportExpectations', '')
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         # Define some error text that looks like a typical ScriptError.
         git_error_text = (
             'This is a git Script Error\n'
@@ -277,7 +291,7 @@ class TestImporterTest(LoggingTestCase):
     def test_apply_exportable_commits_locally(self):
         # TODO(robertma): Consider using MockLocalWPT.
         host = self.mock_host()
-        importer = TestImporter(
+        importer = self._get_test_importer(
             host, wpt_github=MockWPTGitHub(pull_requests=[]))
         importer.wpt_git = MockGit(cwd='/tmp/wpt', executive=host.executive)
         fake_commit = MockChromiumCommit(
@@ -296,6 +310,12 @@ class TestImporterTest(LoggingTestCase):
         # This assertion is implementation details of LocalWPT.apply_patch.
         # TODO(robertma): Move this to local_wpt_unittest.py.
         self.assertEqual(host.executive.full_calls, [
+            MockCall(MANIFEST_INSTALL_CMD,
+                     kwargs={
+                         'input': None,
+                         'cwd': None,
+                         'env': None
+                     }),
             MockCall(
                 ['git', 'apply', '-'], {
                     'input': ('Fake patch contents...\n'
@@ -322,7 +342,7 @@ class TestImporterTest(LoggingTestCase):
     def test_apply_exportable_commits_locally_returns_none_on_failure(self):
         host = self.mock_host()
         wpt_github = MockWPTGitHub(pull_requests=[])
-        importer = TestImporter(host, wpt_github=wpt_github)
+        importer = self._get_test_importer(host, wpt_github=wpt_github)
         commit = MockChromiumCommit(host, subject='My fake commit')
         importer.exportable_but_not_exported_commits = lambda _: [commit]
         # Failure to apply patch.
@@ -337,7 +357,7 @@ class TestImporterTest(LoggingTestCase):
         host.filesystem.write_text_file(
             MOCK_WEB_TESTS + 'external/wpt/foo/OWNERS',
             'someone@chromium.org\n')
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         importer.chromium_git.changed_files = lambda: [RELATIVE_WEB_TESTS + 'external/wpt/foo/x.html']
         self.assertEqual(importer.get_directory_owners(),
                          {('someone@chromium.org', ): ['external/wpt/foo']})
@@ -349,20 +369,20 @@ class TestImporterTest(LoggingTestCase):
         host.filesystem.write_text_file(
             MOCK_WEB_TESTS + 'external/wpt/foo/OWNERS',
             'someone@chromium.org\n')
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         self.assertEqual(importer.get_directory_owners(), {})
 
     # Tests for protected methods - pylint: disable=protected-access
 
     def test_commit_changes(self):
         host = self.mock_host()
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         importer._commit_changes('dummy message')
         self.assertEqual(importer.chromium_git.local_commits(),
                          [['dummy message']])
 
     def test_commit_message(self):
-        importer = TestImporter(self.mock_host())
+        importer = self._get_test_importer(self.mock_host())
         self.assertEqual(
             importer._commit_message('aaaa', '1111'), 'Import 1111\n\n'
             'Using wpt-import in Chromium aaaa.\n\n'
@@ -371,7 +391,7 @@ class TestImporterTest(LoggingTestCase):
     def test_cl_description_with_empty_environ(self):
         host = self.mock_host()
         host.executive = MockExecutive(output='Last commit message\n\n')
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         description = importer._cl_description(directory_owners={})
         self.assertEqual(
             description, 'Last commit message\n\n'
@@ -385,20 +405,22 @@ class TestImporterTest(LoggingTestCase):
             'No-Export: true\n'
             'Cq-Include-Trybots: luci.chromium.try:linux-wpt-identity-fyi-rel,'
             'linux-wpt-payments-fyi-rel')
+        print host.executive.calls
         self.assertEqual(host.executive.calls,
+                         [MANIFEST_INSTALL_CMD] +
                          [['git', 'log', '-1', '--format=%B']])
 
     def test_cl_description_moves_noexport_tag(self):
         host = self.mock_host()
         host.executive = MockExecutive(output='Summary\n\nNo-Export: true\n\n')
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         description = importer._cl_description(directory_owners={})
         self.assertIn('No-Export: true', description)
 
     def test_cl_description_with_directory_owners(self):
         host = self.mock_host()
         host.executive = MockExecutive(output='Last commit message\n\n')
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         description = importer._cl_description(
             directory_owners={
                 ('someone@chromium.org', ):
@@ -415,7 +437,7 @@ class TestImporterTest(LoggingTestCase):
 
     def test_tbr_reviewer_no_response_uses_backup(self):
         host = self.mock_host()
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         self.assertEqual(TBR_FALLBACK, importer.tbr_reviewer())
         self.assertLog([
             'ERROR: Exception while fetching current sheriff: '
@@ -426,7 +448,7 @@ class TestImporterTest(LoggingTestCase):
         host = self.mock_host()
         host.web.urls[ROTATIONS_URL] = json.dumps(
             {'updated_unix_timestamp': '1591108191'})
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         self.assertEqual(TBR_FALLBACK, importer.tbr_reviewer())
         self.assertLog([
             'ERROR: No email found for current sheriff. Retrieved content: %s\n'
@@ -440,7 +462,7 @@ class TestImporterTest(LoggingTestCase):
             'updated_unix_timestamp':
             '1591108191'
         })
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         self.assertEqual(TBR_FALLBACK, importer.tbr_reviewer())
         self.assertLog([
             'ERROR: No email found for current sheriff. Retrieved content: %s\n'
@@ -453,7 +475,7 @@ class TestImporterTest(LoggingTestCase):
 
         host = self.mock_host()
         host.web.get_binary = raise_exception
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         self.assertEqual(TBR_FALLBACK, importer.tbr_reviewer())
         self.assertLog(['ERROR: Cannot fetch %s\n' % ROTATIONS_URL])
 
@@ -464,14 +486,14 @@ class TestImporterTest(LoggingTestCase):
             'updated_unix_timestamp':
             '1591108191',
         })
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         self.assertEqual('current-sheriff@chromium.org',
                          importer.tbr_reviewer())
         self.assertLog([])
 
     def test_tbr_reviewer_skips_non_committer(self):
         host = self.mock_host()
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         importer._fetch_ecosystem_infra_sheriff_email = lambda: 'kyleju@google.com'
         self.assertEqual(TBR_FALLBACK, importer.tbr_reviewer())
         self.assertLog(
@@ -481,24 +503,17 @@ class TestImporterTest(LoggingTestCase):
         # This test doesn't test any aspect of the real manifest script, it just
         # asserts that TestImporter._generate_manifest would invoke the script.
         host = self.mock_host()
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         host.filesystem.write_text_file(
             MOCK_WEB_TESTS + 'external/wpt/MANIFEST.json', '{}')
         importer._generate_manifest()
-        self.assertEqual(host.executive.calls, [[
-            'python',
-            '/mock-checkout/third_party/blink/tools/blinkpy/third_party/wpt/wpt/wpt',
-            'manifest',
-            '--no-download',
-            '--tests-root',
-            MOCK_WEB_TESTS + 'external/wpt',
-        ]])
+        self.assertEqual(host.executive.calls, [MANIFEST_INSTALL_CMD] * 2)
         self.assertEqual(importer.chromium_git.added_paths,
                          {MOCK_WEB_TESTS + 'external/' + BASE_MANIFEST_NAME})
 
     def test_only_wpt_manifest_changed(self):
         host = self.mock_host()
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         importer.chromium_git.changed_files = lambda: [
             RELATIVE_WEB_TESTS + 'external/' + BASE_MANIFEST_NAME,
             RELATIVE_WEB_TESTS + 'external/wpt/foo/x.html']
@@ -513,7 +528,7 @@ class TestImporterTest(LoggingTestCase):
     @unittest.skip('Finding orphaned baselines is broken')
     def test_delete_orphaned_baselines_basic(self):
         host = self.mock_host()
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         dest_path = importer.dest_path
         host.filesystem.write_text_file(
             dest_path + '/MANIFEST.json',
@@ -542,7 +557,7 @@ class TestImporterTest(LoggingTestCase):
         # This test checks that baselines for existing tests shouldn't be
         # deleted, even if the test name isn't the same as the file name.
         host = self.mock_host()
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         dest_path = importer.dest_path
         host.filesystem.write_text_file(
             dest_path + '/MANIFEST.json',
@@ -585,7 +600,7 @@ class TestImporterTest(LoggingTestCase):
 
     def test_clear_out_dest_path(self):
         host = self.mock_host()
-        importer = TestImporter(host)
+        importer = self._get_test_importer(host)
         dest_path = importer.dest_path
         host.filesystem.write_text_file(dest_path + '/foo-test.html', '')
         host.filesystem.write_text_file(dest_path + '/foo-test-expected.txt',

third_party/blink/tools/blinkpy/w3c/wpt_expectations_updater.py

@@ -11,6 +11,7 @@ Specifically, this class fetches results from try bots for the current CL, then
 import argparse
 import copy
 import logging
+import re
 from collections import defaultdict, namedtuple
 
 from blinkpy.common.memoized import memoized
@@ -37,7 +38,7 @@ class WPTExpectationsUpdater(object):
     MARKER_COMMENT = '# ====== New tests from wpt-importer added here ======'
     UMBRELLA_BUG = 'crbug.com/626703'
 
-    def __init__(self, host, args=None):
+    def __init__(self, host, args=None, wpt_manifests=None):
         self.host = host
         self.port = self.host.port_factory.get()
         self.finder = PathFinder(self.host.filesystem)
@@ -45,6 +46,9 @@ class WPTExpectationsUpdater(object):
         self.git = self.host.git(self.finder.chromium_base())
         self.configs_with_no_results = []
         self.patchset = None
+        self.wpt_manifests = (
+            wpt_manifests or
+            [self.port.wpt_manifest(d) for d in self.port.WPT_DIRS])
 
         # Get options from command line arguments.
         parser = argparse.ArgumentParser(description=__doc__)
@@ -718,124 +722,126 @@ class WPTExpectationsUpdater(object):
         through this script. If that command line argument is not used then
         expectations for test files that no longer exist will be deleted.
         """
-        deleted_test_files = self._list_deleted_test_files()
-        renamed_test_files = self._list_renamed_test_files()
+        deleted_files = self._list_deleted_files()
+        renamed_files = self._list_renamed_files()
 
         for path in self._test_expectations.expectations_dict:
             _log.info(
                 'Updating %s for any removed or renamed tests.' %
                 self.host.filesystem.basename(path))
             self._clean_single_test_expectations_file(
-                path, deleted_test_files, renamed_test_files)
+                path, deleted_files, renamed_files)
         self._test_expectations.commit_changes()
 
+    def _list_deleted_files(self):
+        # TODO(robertma): Improve Git.changed_files so that we can use
+        # it here.
+        paths = self.git.run(
+            ['diff', 'origin/master', '-M100%', '--diff-filter=D',
+             '--name-only']).splitlines()
+        deleted_files = []
+        for p in paths:
+            rel_path = self._relative_to_web_test_dir(p)
+            if rel_path:
+                deleted_files.append(rel_path)
+        return deleted_files
+
+    def _list_renamed_files(self):
+        """Returns a dictionary mapping tests to their new name.
+
+        Regardless of the command line arguments used this test will only
+        return a dictionary for tests affected in the current CL.
+
+        Returns a dictionary mapping source name to destination name.
+        """
+        out = self.git.run(
+            ['diff', 'origin/master', '-M100%', '--diff-filter=R',
+             '--name-status'])
+        renamed_tests = {}
+        for line in out.splitlines():
+            _, source_path, dest_path = line.split()
+            source_test = self._relative_to_web_test_dir(source_path)
+            dest_test = self._relative_to_web_test_dir(dest_path)
+            if source_test and dest_test:
+                renamed_tests[source_test] = dest_test
+        return renamed_tests
+
     def _clean_single_test_expectations_file(
             self, path, deleted_files, renamed_files):
         """Cleans up a single test expectations file.
 
         Args:
             path: Path of expectations file that is being cleaned up.
-            deleted_files: List of test file paths relative to the web tests
+            deleted_files: List of file paths relative to the web tests
                 directory which were deleted.
-            renamed_files: Dictionary mapping test file paths to their new file
+            renamed_files: Dictionary mapping file paths to their new file
                 name after renaming.
         """
+        deleted_files = set(deleted_files)
         for line in self._test_expectations.get_updated_lines(path):
             # if a test is a glob type expectation or empty line or comment then
             # add it to the updated expectations file without modifications
             if not line.test or line.is_glob:
                 continue
-            root_test_file = self._get_root_file(line.test)
-
-            if root_test_file in renamed_files:
+            root_file = self._get_root_file(line.test)
+            if root_file in deleted_files:
+                self._test_expectations.remove_expectations(path, [line])
+            elif root_file in renamed_files:
                 self._test_expectations.remove_expectations(path, [line])
-                new_file_name = renamed_files[root_test_file]
-                if self.finder.is_webdriver_test_path(root_test_file):
-                    _, subtest_suffix = self.port.split_webdriver_test_name(
-                        line.test)
+                new_file_name = renamed_files[root_file]
+                if self.finder.is_webdriver_test_path(line.test):
+                    _, subtest_suffix = self.port.split_webdriver_test_name(line.test)
                     line.test = self.port.add_webdriver_subtest_suffix(
                         new_file_name, subtest_suffix)
-                elif '?' in line.test:
-                    line.test = (
-                        new_file_name + line.test[line.test.find('?'):])
+                elif self.port.is_wpt_test(line.test):
+                    # Based on logic in Base._wpt_test_urls_matching_paths
+                    line.test = line.test.replace(
+                        re.sub(r'\.js$', '.', root_file),
+                        re.sub(r'\.js$', '.', new_file_name))
                 else:
                     line.test = new_file_name
                 self._test_expectations.add_expectations(
                     path, [line], lineno=line.lineno)
-            elif root_test_file in deleted_files:
-                self._test_expectations.remove_expectations(
-                    path, [line])
+            elif not root_file or not self.port.test_isfile(root_file):
+                if not self.options.clean_up_affected_tests_only:
+                    self._test_expectations.remove_expectations(path, [line])
 
     @memoized
     def _get_root_file(self, test_name):
-        """Strips arguments from a web test name in order to get the file name.
+        """Finds the physical file in web tests directory for a test
 
-        It also removes the arguments for web driver tests. For instances for
-        the test test1/example.html?Hello this function will return
-        test1/example.html. For a webdriver test it would include arguments and
-        would have the following format, {test file}>>{argument}.
+        If a test is a WPT test then it will look in each of the WPT manifests
+        for the physical file. If test name cannot be found in any of the manifests
+        then the test no longer exists and the function will return None. If a file
+        is webdriver test then it will strip all subtest arguments and return the
+        file path. If a test is a legacy web test then it will return the test name.
 
         Args:
             test_name: Test name which may include test arguments.
 
         Returns:
-            Returns the test file which is the root of a test.
+            Returns the path of the physical file that backs
+            up a test. The path is relative to the web_tests directory.
         """
         if self.finder.is_webdriver_test_path(test_name):
             root_test_file, _ = (
                 self.port.split_webdriver_test_name(test_name))
-        elif '?' in test_name:
-            root_test_file = test_name[:test_name.find('?')]
-        else:
-            root_test_file = test_name
-        return root_test_file
-
-    def _list_deleted_test_files(self):
-        """Returns a list of web tests that have been deleted.
-
-        If --clean-up-affected-tests-only is true then only test files deleted
-        in the current CL may be removed from expectations. Otherwise, any test
-        file may be removed from expectations if it has been deleted.
-
-        Returns: A list of web test files that have been deleted.
-        """
-        if self.options.clean_up_affected_tests_only:
-            # TODO(robertma): Improve Git.changed_files so that we can use
-            # it here.
-            paths = set(self.git.run(
-                ['diff', 'origin/master', '-M100%', '--diff-filter=D',
-                 '--name-only']).splitlines())
-            deleted_tests = set()
-            for path in paths:
-                test = self._relative_to_web_test_dir(path)
-                if test:
-                    deleted_tests.add(test)
+            return root_test_file
+        elif self.port.is_wpt_test(test_name):
+            for wpt_manifest in self.wpt_manifests:
+                if test_name.startswith(wpt_manifest.wpt_dir):
+                    wpt_test = test_name[len(wpt_manifest.wpt_dir) + 1:]
+                    if wpt_manifest.is_test_url(wpt_test):
+                        return self.host.filesystem.join(
+                            wpt_manifest.wpt_dir,
+                            wpt_manifest.file_path_for_test_url(wpt_test))
+            # The test was not found in any of the wpt manifests, therefore
+            # the test does not exist. So we will return None in this case.
+            return None
         else:
-            # Remove expectations for all test which have files that
-            # were deleted. Paths are already relative to the web_tests
-            # directory
-            deleted_tests = self._deleted_test_files_in_expectations()
-        return deleted_tests
-
-    def _list_renamed_test_files(self):
-        """Returns a dictionary mapping tests to their new name.
-
-        Regardless of the command line arguments used this test will only
-        return a dictionary for tests affected in the current CL.
-
-        Returns a dictionary mapping source name to destination name.
-        """
-        out = self.git.run(
-            ['diff', 'origin/master', '-M100%', '--diff-filter=R',
-             '--name-status'])
-        renamed_tests = {}
-        for line in out.splitlines():
-            _, source_path, dest_path = line.split()
-            source_test = self._relative_to_web_test_dir(source_path)
-            dest_test = self._relative_to_web_test_dir(dest_path)
-            if source_test and dest_test:
-                renamed_tests[source_test] = dest_test
-        return renamed_tests
+            # Non WPT and non webdriver tests have no file parameters, and
+            # the physical file path is the actual name of the test.
+            return test_name
 
     def _relative_to_web_test_dir(self, path_relative_to_repo_root):
         """Returns a path that's relative to the web tests directory."""
@@ -846,25 +852,6 @@ class WPTExpectationsUpdater(object):
         return self.host.filesystem.relpath(
             abs_path, self.finder.web_tests_dir())
 
-    def _deleted_test_files_in_expectations(self):
-        """Returns a list of test files that were deleted.
-
-        Returns a list of test file names that are still in the expectations
-        files but no longer exists in the web tests directory.
-        """
-        deleted_files = set()
-        existing_files = {
-            self._get_root_file(p)
-            for p in self.port.tests()}
-        for path in self._test_expectations.expectations_dict:
-            for line in self._test_expectations.get_updated_lines(path):
-                if not line.test or line.is_glob:
-                    continue
-                root_test_file = self._get_root_file(line.test)
-                if root_test_file not in existing_files:
-                    deleted_files.add(root_test_file)
-        return deleted_files
-
     # TODO(robertma): Unit test this method.
     def download_text_baselines(self, test_results):
         """Fetches new baseline files for tests that should be rebaselined.

third_party/blink/tools/blinkpy/w3c/wpt_expectations_updater_unittest.py

@@ -18,7 +18,8 @@ from blinkpy.common.system.log_testing import LoggingTestCase
 
 from blinkpy.w3c.wpt_expectations_updater import (
     WPTExpectationsUpdater, SimpleTestResult, DesktopConfig)
-from blinkpy.w3c.wpt_manifest import BASE_MANIFEST_NAME
+from blinkpy.w3c.wpt_manifest import (
+    WPTManifest, BASE_MANIFEST_NAME, MANIFEST_NAME)
 
 from blinkpy.web_tests.builder_list import BuilderList
 from blinkpy.web_tests.port.android import PRODUCTS_TO_EXPECTATION_FILE_PATHS
@@ -84,6 +85,14 @@ class WPTExpectationsUpdaterTest(LoggingTestCase):
                     'testharness': {
                         'test/path.html': ['abcdef123', [None, {}]],
                         'test/zzzz.html': ['ghijkl456', [None, {}]],
+                        'fake/some_test.html': [
+                            'ghijkl456', ['fake/some_test.html?HelloWorld', {}]],
+                        'fake/file/deleted_path.html': [
+                            'ghijkl456', [None, {}]],
+                        'test/task.js': [
+                            'mnpqrs789',
+                            ['test/task.html', {}],
+                            ['test/task2.html', {}]],
                     },
                     'manual': {
                         'x-manual.html': ['abcdef123', [None, {}]],
@@ -801,6 +810,63 @@ class WPTExpectationsUpdaterTest(LoggingTestCase):
                     WPTExpectationsUpdater.MARKER_COMMENT + '\n' +
                     '[ linux ] external/wpt/fake/new.html?HelloWorld [ Failure ]\n'))
 
+    def test_clean_expectations_for_deleted_test_harness(self):
+        host = self.mock_host()
+        port = host.port_factory.get()
+        expectations_path = \
+            port.path_to_generic_test_expectations_file()
+        host.filesystem.write_text_file(
+            expectations_path,
+            '# tags: [ Win Linux ]\n' +
+            '# results: [ Pass Failure ]\n' +
+            WPTExpectationsUpdater.MARKER_COMMENT + '\n' +
+            '[ linux ] wpt_internal/test/task.html [ Failure ]\n' +
+            '[ win ] wpt_internal/test/task2.html [ Failure ]\n' +
+            '[ linux ] external/wpt/test/task.html [ Failure ]\n' +
+            'external/wpt/test/task2.html [ Pass ]\n')
+
+        def _git_command_return_val(cmd):
+            if '--diff-filter=D' in cmd:
+                return '\n'.join(['external/wpt/test/task.js',
+                                  'wpt_internal/test/task.js'])
+            return ''
+
+        wpt_manifest = port.wpt_manifest('external/wpt')
+        host.filesystem.maybe_make_directory(
+            port.web_tests_dir(), 'wpt_internal')
+        host.filesystem.copyfile(
+            host.filesystem.join(port.web_tests_dir(),
+                                 'external', 'wpt', MANIFEST_NAME),
+            host.filesystem.join(port.web_tests_dir(), 'wpt_internal',
+                                 MANIFEST_NAME))
+        wpt_internal_manifest = WPTManifest(host, host.filesystem.join(
+            port.web_tests_dir(), 'wpt_internal', MANIFEST_NAME))
+
+        updater = WPTExpectationsUpdater(
+            host,
+            ['--clean-up-affected-tests-only',
+             '--clean-up-test-expectations-only'],
+            [wpt_manifest, wpt_internal_manifest])
+        updater.git.run = _git_command_return_val
+        updater._relative_to_web_test_dir = lambda test_path: test_path
+        updater.cleanup_test_expectations_files()
+
+        test_expectations = {'external/wpt/fake/file/path.html': {
+            tuple([DesktopConfig(port_name='test-linux-trusty')]):
+            SimpleTestResult(actual='PASS', expected='', bug='crbug.com/123')}}
+        skip_path = host.port_factory.get().path_to_never_fix_tests_file()
+        skip_value_origin = host.filesystem.read_text_file(skip_path)
+
+        updater.write_to_test_expectations(test_expectations)
+        value = host.filesystem.read_text_file(expectations_path)
+        self.assertMultiLineEqual(
+            value, ('# tags: [ Win Linux ]\n' +
+                    '# results: [ Pass Failure ]\n\n' +
+                    WPTExpectationsUpdater.MARKER_COMMENT + '\n' +
+                    'crbug.com/123 [ Trusty ] external/wpt/fake/file/path.html [ Pass ]'))
+        skip_value = host.filesystem.read_text_file(skip_path)
+        self.assertMultiLineEqual(skip_value, skip_value_origin)
+
     def test_write_to_test_expectations_and_cleanup_expectations(self):
         host = self.mock_host()
         expectations_path = \
@@ -1225,16 +1291,21 @@ class WPTExpectationsUpdaterTest(LoggingTestCase):
 
     def test_cleanup_all_deleted_tests_in_expectations_files(self):
         host = MockHost()
-
+        port = host.port_factory.get()
         host.filesystem.files[MOCK_WEB_TESTS + 'TestExpectations'] = (
             '# results: [ Failure ]\n'
-            'some/test/a.html?hello%20world [ Failure ]\n'
+            'external/wpt/some/test/a.html?hello%20world [ Failure ]\n'
             'some/test/b.html [ Failure ]\n'
+            '# This line should be deleted\n'
+            'some/test/c.html [ Failure ]\n'
             '# line below should exist in new file\n'
             'some/test/d.html [ Failure ]\n')
         host.filesystem.files[MOCK_WEB_TESTS + 'VirtualTestSuites'] = '[]'
         host.filesystem.files[MOCK_WEB_TESTS + 'new/a.html'] = ''
         host.filesystem.files[MOCK_WEB_TESTS + 'new/b.html'] = ''
+        host.filesystem.files[
+            host.filesystem.join(
+                port.web_tests_dir(), 'some', 'test', 'd.html')] = ''
         # TODO(rmhasan): Remove creation of Android files within
         # tests.
         for path in PRODUCTS_TO_EXPECTATION_FILE_PATHS.values():
@@ -1248,7 +1319,6 @@ class WPTExpectationsUpdaterTest(LoggingTestCase):
             return ''
 
         updater.git.run = _git_command_return_val
-        updater.port.tests = lambda: ['some/test/d.html']
         updater._relative_to_web_test_dir = lambda test_path: test_path
         updater.cleanup_test_expectations_files()
         self.assertMultiLineEqual(
@@ -1297,8 +1367,8 @@ class WPTExpectationsUpdaterTest(LoggingTestCase):
             'external/wpt/webdriver/some/test/a*.html': 'old/a*.html',
             'external/wpt/webdriver/some/test/c.html': 'old/c.html',
         }
-        updater._list_deleted_test_files = lambda: deleted_files
-        updater._list_renamed_test_files = lambda: renamed_file_pairs
+        updater._list_deleted_files = lambda: deleted_files
+        updater._list_renamed_files = lambda: renamed_file_pairs
         updater.cleanup_test_expectations_files()
         self.assertMultiLineEqual(
             host.filesystem.read_text_file(MOCK_WEB_TESTS +

third_party/blink/tools/blinkpy/w3c/wpt_manifest.py

@@ -84,8 +84,11 @@ class WPTManifest(object):
         [[reference_url1, "=="], [reference_url2, "!="], ...]
     """
 
-    def __init__(self, json_content):
-        self.raw_dict = json.loads(json_content)
+    def __init__(self, host, manifest_path):
+        self.host = host
+        self.port = self.host.port_factory.get()
+        self.raw_dict = json.loads(
+            self.host.filesystem.read_text_file(manifest_path))
         # As a workaround to handle the change from a flat-list to a trie
         # structure in the v8 manifest, flatten the items back to the v7 format.
         #
@@ -93,9 +96,16 @@ class WPTManifest(object):
         self.raw_dict['items'] = self._flatten_items(
             self.raw_dict.get('items', {}))
 
+        self.wpt_manifest_path = manifest_path
         self.test_types = ('manual', 'reftest', 'testharness', 'crashtest')
         self.test_name_to_file = {}
 
+    @property
+    def wpt_dir(self):
+        return self.host.filesystem.dirname(
+            self.host.filesystem.relpath(
+                self.wpt_manifest_path, self.port.web_tests_dir()))
+
     def _items_for_file_path(self, path_in_wpt):
         """Finds manifest items for the given WPT path.
 
@@ -275,9 +285,8 @@ class WPTManifest(object):
     @staticmethod
     def generate_manifest(host, dest_path):
         """Generates MANIFEST.json on the specified directory."""
-        finder = PathFinder(host.filesystem)
-        wpt_exec_path = finder.path_from_blink_tools('blinkpy', 'third_party',
-                                                     'wpt', 'wpt', 'wpt')
+        wpt_exec_path = PathFinder(host.filesystem).path_from_blink_tools(
+            'blinkpy', 'third_party', 'wpt', 'wpt', 'wpt')
         cmd = [
             'python', wpt_exec_path, 'manifest', '--no-download',
             '--tests-root', dest_path

third_party/blink/tools/blinkpy/w3c/wpt_manifest_unittest.py

@@ -89,7 +89,11 @@ class WPTManifestUnitTest(unittest.TestCase):
     }
 }
         '''
-        manifest = WPTManifest(manifest_json)
+        host = MockHost()
+        host.filesystem.write_text_file(
+            WEB_TEST_DIR + '/external/wpt/MANIFEST.json', manifest_json)
+        manifest = WPTManifest(
+            host, WEB_TEST_DIR + '/external/wpt/MANIFEST.json')
         self.assertTrue(manifest.is_test_file('test.any.js'))
         self.assertEqual(manifest.all_url_items(),
                          {u'test.any.html': [u'test.any.html', {}]})
@@ -111,7 +115,11 @@ class WPTManifestUnitTest(unittest.TestCase):
     }
 }
         '''
-        manifest = WPTManifest(manifest_json)
+        host = MockHost()
+        host.filesystem.write_text_file(
+            WEB_TEST_DIR + '/external/wpt/MANIFEST.json', manifest_json)
+        manifest = WPTManifest(
+            host, WEB_TEST_DIR + '/external/wpt/MANIFEST.json')
         self.assertEqual(manifest.all_url_items(),
                          {u'test.any.html': [u'test.any.html', {}]})
 
@@ -132,7 +140,11 @@ class WPTManifestUnitTest(unittest.TestCase):
         }
     }
 }       '''
-        manifest = WPTManifest(manifest_json)
+        host = MockHost()
+        host.filesystem.write_text_file(
+            WEB_TEST_DIR + '/external/wpt/MANIFEST.json', manifest_json)
+        manifest = WPTManifest(
+            host, WEB_TEST_DIR + '/external/wpt/MANIFEST.json')
         self.assertEqual(
             manifest.all_url_items(), {
                 u'test.any.html': [u'test.any.html', {}],
@@ -168,7 +180,11 @@ class WPTManifestUnitTest(unittest.TestCase):
     }
 }
         '''
-        manifest = WPTManifest(manifest_json)
+        host = MockHost()
+        host.filesystem.write_text_file(
+            WEB_TEST_DIR + '/external/wpt/MANIFEST.json', manifest_json)
+        manifest = WPTManifest(
+            host, WEB_TEST_DIR + '/external/wpt/MANIFEST.json')
         self.assertEqual(
             manifest.all_url_items(), {
                 u'test.html': [u'test.html', {}],

third_party/blink/tools/blinkpy/web_tests/lint_test_expectations_unittest.py

@@ -144,7 +144,7 @@ class LintTest(LoggingTestCase):
         port = host.port_factory.get(options.platform, options=options)
         port.expectations_dict = lambda: {'foo': '-- syntax error1', 'bar': '-- syntax error2'}
 
-        host.port_factory.get = lambda platform, options=None: port
+        host.port_factory.get = lambda platform=None, options=None: port
         host.port_factory.all_port_names = lambda platform=None: [port.name()]
 
         failures, warnings = lint_test_expectations.lint(host, options)
@@ -166,7 +166,7 @@ class LintTest(LoggingTestCase):
         port = host.port_factory.get(options.platform, options=options)
         port.expectations_dict = lambda: {}
 
-        host.port_factory.get = lambda platform, options=None: port
+        host.port_factory.get = lambda platform=None, options=None: port
         host.port_factory.all_port_names = lambda platform=None: [port.name()]
         host.filesystem.write_text_file(WEB_TEST_DIR + '/LeakExpectations',
                                         '-- syntax error')
@@ -189,7 +189,7 @@ class LintTest(LoggingTestCase):
         port = host.port_factory.get(options.platform, options=options)
         port.expectations_dict = lambda: {'flag-specific': 'does/not/exist', 'noproblem': ''}
 
-        host.port_factory.get = lambda platform, options=None: port
+        host.port_factory.get = lambda platform=None, options=None: port
         host.port_factory.all_port_names = lambda platform=None: [port.name()]
 
         failures, warnings = lint_test_expectations.lint(host, options)
@@ -218,7 +218,7 @@ class LintTest(LoggingTestCase):
         port.expectations_dict = lambda: {
             'testexpectations': test_expectations}
 
-        host.port_factory.get = lambda platform, options=None: port
+        host.port_factory.get = lambda platform=None, options=None: port
         host.port_factory.all_port_names = lambda platform=None: [port.name()]
 
         failures, warnings = lint_test_expectations.lint(host, options)
@@ -267,7 +267,7 @@ class LintTest(LoggingTestCase):
             host.filesystem.join(port.web_tests_dir(), 'virtual', 'foo',
                                  'README.md'), 'foo')
 
-        host.port_factory.get = lambda platform, options=None: port
+        host.port_factory.get = lambda platform=None, options=None: port
         host.port_factory.all_port_names = lambda platform=None: [port.name()]
 
         failures, warnings = lint_test_expectations.lint(host, options)
@@ -300,7 +300,7 @@ class LintTest(LoggingTestCase):
                             'non-wpt/test.html [ Failure ]\n')
         for path in PRODUCTS_TO_EXPECTATION_FILE_PATHS.values():
             host.filesystem.write_text_file(path, raw_expectations)
-        host.port_factory.get = lambda platform, options=None: port
+        host.port_factory.get = lambda platform=None, options=None: port
         host.port_factory.all_port_names = lambda platform=None: [port.name()]
         port.test_exists = lambda _: True
         port.tests = lambda _: {'external/wpt/test.html', 'non-wpt/test.html'}
@@ -325,7 +325,7 @@ class LintTest(LoggingTestCase):
         host.filesystem.maybe_make_directory(
             host.filesystem.join(port.web_tests_dir(), 'test2'))
 
-        host.port_factory.get = lambda platform, options=None: port
+        host.port_factory.get = lambda platform=None, options=None: port
         host.port_factory.all_port_names = lambda platform=None: [port.name()]
 
         failures, warnings = lint_test_expectations.lint(host, options)
@@ -364,7 +364,7 @@ class LintTest(LoggingTestCase):
             'testexpectations': test_expectations
         }
         port.test_exists = lambda test: True
-        host.port_factory.get = lambda platform, options=None: port
+        host.port_factory.get = lambda platform=None, options=None: port
         host.port_factory.all_port_names = lambda platform=None: [port.name()]
 
         failures, warnings = lint_test_expectations.lint(host, options)
@@ -399,7 +399,7 @@ class LintTest(LoggingTestCase):
                              'virtual/foo/test1/* [ Pass ]\n')
         port.expectations_dict = lambda: {'NeverFixTests': test_expectations}
         port.test_exists = lambda test: True
-        host.port_factory.get = lambda platform, options=None: port
+        host.port_factory.get = lambda platform=None, options=None: port
         host.port_factory.all_port_names = lambda platform=None: [port.name()]
 
         failures, warnings = lint_test_expectations.lint(host, options)

third_party/blink/tools/blinkpy/web_tests/port/base.py

@@ -959,7 +959,7 @@ class Port(object):
                 'manifest_update', True):
             _log.debug('Generating MANIFEST.json for %s...', path)
             WPTManifest.ensure_manifest(self, path)
-        return WPTManifest(self._filesystem.read_text_file(manifest_path))
+        return WPTManifest(self.host, manifest_path)
 
     def is_wpt_crash_test(self, test_file):
         """Returns whether a WPT test is a crashtest.

third_party/blink/tools/blinkpy/web_tests/port/base_unittest.py

@@ -32,6 +32,7 @@ import unittest
 import mock
 
 from blinkpy.common.path_finder import RELATIVE_WEB_TESTS
+from blinkpy.common.host_mock import MockHost
 from blinkpy.common.system.executive_mock import MockExecutive
 from blinkpy.common.system.log_testing import LoggingTestCase
 from blinkpy.common.system.output_capture import OutputCapture
@@ -51,7 +52,7 @@ class PortTest(LoggingTestCase):
                   with_tests=False,
                   port_name=None,
                   **kwargs):
-        host = MockSystemHost()
+        host = MockHost()
         if executive:
             host.executive = executive
         if with_tests: