Skip to content

GitLab

T
tangled
Commit dfcecba4 authored by Rakib M. Hasan's avatar Rakib M. Hasan Committed by Commit Bot
Browse files
Options

wpt-import: Remove tests from expectations when WPT test harnesses are deleted 

Several web platform tests do not have physical files checked into the
WPT repository. Instead they are generated and mapped to test harnesses
checked into the WPT repo. When those test harnesses are deleted then
we should delete all tests mapped to those files. Only affects the
behavior of the script when --clean-up-affected-tests-only is used.

Bug: 1050760, 1110003, 801357
Change-Id: Id1c30b096c73974ed0ded2d74284fbe43005ee0d
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2439417
Commit-Queue: Rakib Hasan <rmhasan@google.com>
Reviewed-by: default avatarRobert Ma <robertma@chromium.org>
Cr-Commit-Position: refs/heads/master@{#816865}
parent 5d51d4e1
Hide whitespace changes
Inline Side-by-side
Showing with 280 additions and 165 deletions
third_party/blink/tools/blinkpy/w3c/android_wpt_expectations_updater_unittest.py
View file @ dfcecba4
......@@ -2,6 +2,7 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import json
import logging
from blinkpy.common.host_mock import MockHost
......@@ -10,6 +11,7 @@ from blinkpy.common.net.git_cl_mock import MockGitCL
from blinkpy.common.net.results_fetcher import Build
from blinkpy.common.net.web_test_results import WebTestResults
from blinkpy.common.system.log_testing import LoggingTestCase
from blinkpy.w3c.wpt_manifest import BASE_MANIFEST_NAME
from blinkpy.web_tests.builder_list import BuilderList
from blinkpy.web_tests.port.factory_mock import MockPortFactory
from blinkpy.web_tests.port.android import (
......@@ -76,6 +78,18 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
'is_try_builder': True,
},
})
host.filesystem.write_text_file(
host.port_factory.get().web_tests_dir() + '/external/' +
BASE_MANIFEST_NAME,
json.dumps({
'items': {
'testharness': {
'ghi.html': ['abcdef123', [None, {}]],
'van.html': ['abcdef123', [None, {}]],
},
},
}))
# Write dummy expectations
for path in PRODUCTS_TO_EXPECTATION_FILE_PATHS.values():
host.filesystem.write_text_file(
......
third_party/blink/tools/blinkpy/w3c/test_importer.py
View file @ dfcecba4
......@@ -45,9 +45,10 @@ _log = logging.getLogger(__file__)
class TestImporter(object):
def __init__(self, host, wpt_github=None):
def __init__(self, host, wpt_github=None, wpt_manifests=None):
self.host = host
self.wpt_github = wpt_github
self.port = host.port_factory.get()
self.executive = host.executive
self.fs = host.filesystem
......@@ -72,7 +73,8 @@ class TestImporter(object):
args = ['--clean-up-affected-tests-only',
'--clean-up-test-expectations']
self._expectations_updater = WPTExpectationsUpdater(self.host, args)
self._expectations_updater = WPTExpectationsUpdater(
self.host, args, wpt_manifests)
def main(self, argv=None):
# TODO(robertma): Test this method! Split it to make it easier to test
......@@ -155,14 +157,15 @@ class TestImporter(object):
# TODO(robertma): Implement `add --all` in Git (it is different from `commit --all`).
self.chromium_git.run(['add', '--all', self.dest_path])
# Remove expectations for tests that were deleted and rename tests
# in expectations for renamed tests.
self._expectations_updater.cleanup_test_expectations_files()
self._generate_manifest()
# TODO(crbug.com/800570 robertma): Re-enable it once we fix the bug.
# self._delete_orphaned_baselines()
# Remove expectations for tests that were deleted and rename tests
# in expectations for renamed tests.
self._expectations_updater.cleanup_test_expectations_files()
if not self.chromium_git.has_working_directory_changes():
_log.info('Done: no changes to import.')
......
third_party/blink/tools/blinkpy/w3c/test_importer_unittest.py
View file @ dfcecba4
......@@ -24,7 +24,13 @@ from blinkpy.w3c.wpt_manifest import BASE_MANIFEST_NAME
from blinkpy.web_tests.port.android import PRODUCTS_TO_EXPECTATION_FILE_PATHS
MOCK_WEB_TESTS = '/mock-checkout/' + RELATIVE_WEB_TESTS
MANIFEST_INSTALL_CMD = [
'python',
'/mock-checkout/third_party/blink/tools/blinkpy/third_party/wpt/wpt/wpt',
'manifest',
'--no-download',
'--tests-root',
MOCK_WEB_TESTS + 'external/wpt']
class TestImporterTest(LoggingTestCase):
......@@ -34,11 +40,19 @@ class TestImporterTest(LoggingTestCase):
host.filesystem.write_text_file(path, '')
return host
@staticmethod
def _get_test_importer(host, wpt_github=None):
port = host.port_factory.get()
return TestImporter(
host,
wpt_github=wpt_github,
wpt_manifests=[port.wpt_manifest('external/wpt')])
def test_update_expectations_for_cl_no_results(self):
host = self.mock_host()
host.filesystem.write_text_file(
MOCK_WEB_TESTS + 'W3CImportExpectations', '')
importer = TestImporter(host)
importer = self._get_test_importer(host)
importer.git_cl = MockGitCL(host, time_out=True)
success = importer.update_expectations_for_cl()
self.assertFalse(success)
......@@ -52,7 +66,7 @@ class TestImporterTest(LoggingTestCase):
host = self.mock_host()
host.filesystem.write_text_file(
MOCK_WEB_TESTS + 'W3CImportExpectations', '')
importer = TestImporter(host)
importer = self._get_test_importer(host)
importer.git_cl = MockGitCL(
host,
status='closed',
......@@ -70,7 +84,7 @@ class TestImporterTest(LoggingTestCase):
host = self.mock_host()
host.filesystem.write_text_file(
MOCK_WEB_TESTS + 'W3CImportExpectations', '')
importer = TestImporter(host)
importer = self._get_test_importer(host)
importer.git_cl = MockGitCL(
host,
status='lgtm',
......@@ -88,7 +102,7 @@ class TestImporterTest(LoggingTestCase):
host = self.mock_host()
host.filesystem.write_text_file(
MOCK_WEB_TESTS + 'W3CImportExpectations', '')
importer = TestImporter(host)
importer = self._get_test_importer(host)
importer.git_cl = MockGitCL(
host,
status='lgtm',
......@@ -107,7 +121,7 @@ class TestImporterTest(LoggingTestCase):
host = self.mock_host()
host.filesystem.write_text_file(
MOCK_WEB_TESTS + 'W3CImportExpectations', '')
importer = TestImporter(host)
importer = self._get_test_importer(host)
# Only the latest job for each builder is counted.
importer.git_cl = MockGitCL(
host,
......@@ -136,7 +150,7 @@ class TestImporterTest(LoggingTestCase):
host = self.mock_host()
host.filesystem.write_text_file(
MOCK_WEB_TESTS + 'W3CImportExpectations', '')
importer = TestImporter(host)
importer = self._get_test_importer(host)
importer.git_cl = MockGitCL(
host,
status='lgtm',
......@@ -165,7 +179,7 @@ class TestImporterTest(LoggingTestCase):
host = self.mock_host()
host.filesystem.write_text_file(
MOCK_WEB_TESTS + 'W3CImportExpectations', '')
importer = TestImporter(host)
importer = self._get_test_importer(host)
# Only the latest job for each builder is counted.
importer.git_cl = MockGitCL(
host,
......@@ -196,7 +210,7 @@ class TestImporterTest(LoggingTestCase):
host = self.mock_host()
host.filesystem.write_text_file(
MOCK_WEB_TESTS + 'W3CImportExpectations', '')
importer = TestImporter(host)
importer = self._get_test_importer(host)
importer.git_cl = MockGitCL(
host,
status='closed',
......@@ -219,7 +233,7 @@ class TestImporterTest(LoggingTestCase):
def test_run_commit_queue_for_cl_timeout(self):
# This simulates the case where we time out while waiting for try jobs.
host = self.mock_host()
importer = TestImporter(host)
importer = self._get_test_importer(host)
importer.git_cl = MockGitCL(host, time_out=True)
success = importer.run_commit_queue_for_cl()
self.assertFalse(success)
......@@ -237,7 +251,7 @@ class TestImporterTest(LoggingTestCase):
host = self.mock_host()
host.filesystem.write_text_file(
MOCK_WEB_TESTS + 'W3CImportExpectations', '')
importer = TestImporter(host)
importer = self._get_test_importer(host)
# Define some error text that looks like a typical ScriptError.
git_error_text = (
'This is a git Script Error\n'
......@@ -277,7 +291,7 @@ class TestImporterTest(LoggingTestCase):
def test_apply_exportable_commits_locally(self):
# TODO(robertma): Consider using MockLocalWPT.
host = self.mock_host()
importer = TestImporter(
importer = self._get_test_importer(
host, wpt_github=MockWPTGitHub(pull_requests=[]))
importer.wpt_git = MockGit(cwd='/tmp/wpt', executive=host.executive)
fake_commit = MockChromiumCommit(
......@@ -296,6 +310,12 @@ class TestImporterTest(LoggingTestCase):
# This assertion is implementation details of LocalWPT.apply_patch.
# TODO(robertma): Move this to local_wpt_unittest.py.
self.assertEqual(host.executive.full_calls, [
MockCall(MANIFEST_INSTALL_CMD,
kwargs={
'input': None,
'cwd': None,
'env': None
}),
MockCall(
['git', 'apply', '-'], {
'input': ('Fake patch contents...\n'
......@@ -322,7 +342,7 @@ class TestImporterTest(LoggingTestCase):
def test_apply_exportable_commits_locally_returns_none_on_failure(self):
host = self.mock_host()
wpt_github = MockWPTGitHub(pull_requests=[])
importer = TestImporter(host, wpt_github=wpt_github)
importer = self._get_test_importer(host, wpt_github=wpt_github)
commit = MockChromiumCommit(host, subject='My fake commit')
importer.exportable_but_not_exported_commits = lambda _: [commit]
# Failure to apply patch.
......@@ -337,7 +357,7 @@ class TestImporterTest(LoggingTestCase):
host.filesystem.write_text_file(
MOCK_WEB_TESTS + 'external/wpt/foo/OWNERS',
'someone@chromium.org\n')
importer = TestImporter(host)
importer = self._get_test_importer(host)
importer.chromium_git.changed_files = lambda: [RELATIVE_WEB_TESTS + 'external/wpt/foo/x.html']
self.assertEqual(importer.get_directory_owners(),
{('someone@chromium.org', ): ['external/wpt/foo']})
......@@ -349,20 +369,20 @@ class TestImporterTest(LoggingTestCase):
host.filesystem.write_text_file(
MOCK_WEB_TESTS + 'external/wpt/foo/OWNERS',
'someone@chromium.org\n')
importer = TestImporter(host)
importer = self._get_test_importer(host)
self.assertEqual(importer.get_directory_owners(), {})
# Tests for protected methods - pylint: disable=protected-access
def test_commit_changes(self):
host = self.mock_host()
importer = TestImporter(host)
importer = self._get_test_importer(host)
importer._commit_changes('dummy message')
self.assertEqual(importer.chromium_git.local_commits(),
[['dummy message']])
def test_commit_message(self):
importer = TestImporter(self.mock_host())
importer = self._get_test_importer(self.mock_host())
self.assertEqual(
importer._commit_message('aaaa', '1111'), 'Import 1111\n\n'
'Using wpt-import in Chromium aaaa.\n\n'
......@@ -371,7 +391,7 @@ class TestImporterTest(LoggingTestCase):
def test_cl_description_with_empty_environ(self):
host = self.mock_host()
host.executive = MockExecutive(output='Last commit message\n\n')
importer = TestImporter(host)
importer = self._get_test_importer(host)
description = importer._cl_description(directory_owners={})
self.assertEqual(
description, 'Last commit message\n\n'
......@@ -385,20 +405,22 @@ class TestImporterTest(LoggingTestCase):
'No-Export: true\n'
'Cq-Include-Trybots: luci.chromium.try:linux-wpt-identity-fyi-rel,'
'linux-wpt-payments-fyi-rel')
print host.executive.calls
self.assertEqual(host.executive.calls,
[MANIFEST_INSTALL_CMD] +
[['git', 'log', '-1', '--format=%B']])
def test_cl_description_moves_noexport_tag(self):
host = self.mock_host()
host.executive = MockExecutive(output='Summary\n\nNo-Export: true\n\n')
importer = TestImporter(host)
importer = self._get_test_importer(host)
description = importer._cl_description(directory_owners={})
self.assertIn('No-Export: true', description)
def test_cl_description_with_directory_owners(self):
host = self.mock_host()
host.executive = MockExecutive(output='Last commit message\n\n')
importer = TestImporter(host)
importer = self._get_test_importer(host)
description = importer._cl_description(
directory_owners={
('someone@chromium.org', ):
......@@ -415,7 +437,7 @@ class TestImporterTest(LoggingTestCase):
def test_tbr_reviewer_no_response_uses_backup(self):
host = self.mock_host()
importer = TestImporter(host)
importer = self._get_test_importer(host)
self.assertEqual(TBR_FALLBACK, importer.tbr_reviewer())
self.assertLog([
'ERROR: Exception while fetching current sheriff: '
......@@ -426,7 +448,7 @@ class TestImporterTest(LoggingTestCase):
host = self.mock_host()
host.web.urls[ROTATIONS_URL] = json.dumps(
{'updated_unix_timestamp': '1591108191'})
importer = TestImporter(host)
importer = self._get_test_importer(host)
self.assertEqual(TBR_FALLBACK, importer.tbr_reviewer())
self.assertLog([
'ERROR: No email found for current sheriff. Retrieved content: %s\n'
......@@ -440,7 +462,7 @@ class TestImporterTest(LoggingTestCase):
'updated_unix_timestamp':
'1591108191'
})
importer = TestImporter(host)
importer = self._get_test_importer(host)
self.assertEqual(TBR_FALLBACK, importer.tbr_reviewer())
self.assertLog([
'ERROR: No email found for current sheriff. Retrieved content: %s\n'
......@@ -453,7 +475,7 @@ class TestImporterTest(LoggingTestCase):
host = self.mock_host()
host.web.get_binary = raise_exception
importer = TestImporter(host)
importer = self._get_test_importer(host)
self.assertEqual(TBR_FALLBACK, importer.tbr_reviewer())
self.assertLog(['ERROR: Cannot fetch %s\n' % ROTATIONS_URL])
......@@ -464,14 +486,14 @@ class TestImporterTest(LoggingTestCase):
'updated_unix_timestamp':
'1591108191',
})
importer = TestImporter(host)
importer = self._get_test_importer(host)
self.assertEqual('current-sheriff@chromium.org',
importer.tbr_reviewer())
self.assertLog([])
def test_tbr_reviewer_skips_non_committer(self):
host = self.mock_host()
importer = TestImporter(host)
importer = self._get_test_importer(host)
importer._fetch_ecosystem_infra_sheriff_email = lambda: 'kyleju@google.com'
self.assertEqual(TBR_FALLBACK, importer.tbr_reviewer())
self.assertLog(
......@@ -481,24 +503,17 @@ class TestImporterTest(LoggingTestCase):
# This test doesn't test any aspect of the real manifest script, it just
# asserts that TestImporter._generate_manifest would invoke the script.
host = self.mock_host()
importer = TestImporter(host)
importer = self._get_test_importer(host)
host.filesystem.write_text_file(
MOCK_WEB_TESTS + 'external/wpt/MANIFEST.json', '{}')
importer._generate_manifest()
self.assertEqual(host.executive.calls, [[
'python',
'/mock-checkout/third_party/blink/tools/blinkpy/third_party/wpt/wpt/wpt',
'manifest',
'--no-download',
'--tests-root',
MOCK_WEB_TESTS + 'external/wpt',
]])
self.assertEqual(host.executive.calls, [MANIFEST_INSTALL_CMD] * 2)
self.assertEqual(importer.chromium_git.added_paths,
{MOCK_WEB_TESTS + 'external/' + BASE_MANIFEST_NAME})
def test_only_wpt_manifest_changed(self):
host = self.mock_host()
importer = TestImporter(host)
importer = self._get_test_importer(host)
importer.chromium_git.changed_files = lambda: [
RELATIVE_WEB_TESTS + 'external/' + BASE_MANIFEST_NAME,
RELATIVE_WEB_TESTS + 'external/wpt/foo/x.html']
......@@ -513,7 +528,7 @@ class TestImporterTest(LoggingTestCase):
@unittest.skip('Finding orphaned baselines is broken')
def test_delete_orphaned_baselines_basic(self):
host = self.mock_host()
importer = TestImporter(host)
importer = self._get_test_importer(host)
dest_path = importer.dest_path
host.filesystem.write_text_file(
dest_path + '/MANIFEST.json',
......@@ -542,7 +557,7 @@ class TestImporterTest(LoggingTestCase):
# This test checks that baselines for existing tests shouldn't be
# deleted, even if the test name isn't the same as the file name.
host = self.mock_host()
importer = TestImporter(host)
importer = self._get_test_importer(host)
dest_path = importer.dest_path
host.filesystem.write_text_file(
dest_path + '/MANIFEST.json',
......@@ -585,7 +600,7 @@ class TestImporterTest(LoggingTestCase):
def test_clear_out_dest_path(self):
host = self.mock_host()
importer = TestImporter(host)
importer = self._get_test_importer(host)
dest_path = importer.dest_path
host.filesystem.write_text_file(dest_path + '/foo-test.html', '')
host.filesystem.write_text_file(dest_path + '/foo-test-expected.txt',
......
third_party/blink/tools/blinkpy/w3c/wpt_expectations_updater.py
View file @ dfcecba4
......@@ -11,6 +11,7 @@ Specifically, this class fetches results from try bots for the current CL, then
import argparse
import copy
import logging
import re
from collections import defaultdict, namedtuple
from blinkpy.common.memoized import memoized
......@@ -37,7 +38,7 @@ class WPTExpectationsUpdater(object):
MARKER_COMMENT = '# ====== New tests from wpt-importer added here ======'
UMBRELLA_BUG = 'crbug.com/626703'
def __init__(self, host, args=None):
def __init__(self, host, args=None, wpt_manifests=None):
self.host = host
self.port = self.host.port_factory.get()
self.finder = PathFinder(self.host.filesystem)
......@@ -45,6 +46,9 @@ class WPTExpectationsUpdater(object):
self.git = self.host.git(self.finder.chromium_base())
self.configs_with_no_results = []
self.patchset = None
self.wpt_manifests = (
wpt_manifests or
[self.port.wpt_manifest(d) for d in self.port.WPT_DIRS])
# Get options from command line arguments.
parser = argparse.ArgumentParser(description=__doc__)
......@@ -718,124 +722,126 @@ class WPTExpectationsUpdater(object):
through this script. If that command line argument is not used then
expectations for test files that no longer exist will be deleted.
"""
deleted_test_files = self._list_deleted_test_files()
renamed_test_files = self._list_renamed_test_files()
deleted_files = self._list_deleted_files()
renamed_files = self._list_renamed_files()
for path in self._test_expectations.expectations_dict:
_log.info(
'Updating %s for any removed or renamed tests.' %
self.host.filesystem.basename(path))
self._clean_single_test_expectations_file(
path, deleted_test_files, renamed_test_files)
path, deleted_files, renamed_files)
self._test_expectations.commit_changes()
def _list_deleted_files(self):
# TODO(robertma): Improve Git.changed_files so that we can use
# it here.
paths = self.git.run(
['diff', 'origin/master', '-M100%', '--diff-filter=D',
'--name-only']).splitlines()
deleted_files = []
for p in paths:
rel_path = self._relative_to_web_test_dir(p)
if rel_path:
deleted_files.append(rel_path)
return deleted_files
def _list_renamed_files(self):
"""Returns a dictionary mapping tests to their new name.
Regardless of the command line arguments used this test will only
return a dictionary for tests affected in the current CL.
Returns a dictionary mapping source name to destination name.
"""
out = self.git.run(
['diff', 'origin/master', '-M100%', '--diff-filter=R',
'--name-status'])
renamed_tests = {}
for line in out.splitlines():
_, source_path, dest_path = line.split()
source_test = self._relative_to_web_test_dir(source_path)
dest_test = self._relative_to_web_test_dir(dest_path)
if source_test and dest_test:
renamed_tests[source_test] = dest_test
return renamed_tests
def _clean_single_test_expectations_file(
self, path, deleted_files, renamed_files):
"""Cleans up a single test expectations file.
Args:
path: Path of expectations file that is being cleaned up.
deleted_files: List of test file paths relative to the web tests
deleted_files: List of file paths relative to the web tests
directory which were deleted.
renamed_files: Dictionary mapping test file paths to their new file
renamed_files: Dictionary mapping file paths to their new file
name after renaming.
"""
deleted_files = set(deleted_files)
for line in self._test_expectations.get_updated_lines(path):
# if a test is a glob type expectation or empty line or comment then
# add it to the updated expectations file without modifications
if not line.test or line.is_glob:
continue
root_test_file = self._get_root_file(line.test)
if root_test_file in renamed_files:
root_file = self._get_root_file(line.test)
if root_file in deleted_files:
self._test_expectations.remove_expectations(path, [line])
elif root_file in renamed_files:
self._test_expectations.remove_expectations(path, [line])
new_file_name = renamed_files[root_test_file]
if self.finder.is_webdriver_test_path(root_test_file):
_, subtest_suffix = self.port.split_webdriver_test_name(
line.test)
new_file_name = renamed_files[root_file]
if self.finder.is_webdriver_test_path(line.test):
_, subtest_suffix = self.port.split_webdriver_test_name(line.test)
line.test = self.port.add_webdriver_subtest_suffix(
new_file_name, subtest_suffix)
elif '?' in line.test:
line.test = (
new_file_name + line.test[line.test.find('?'):])
elif self.port.is_wpt_test(line.test):
# Based on logic in Base._wpt_test_urls_matching_paths
line.test = line.test.replace(
re.sub(r'\.js$', '.', root_file),
re.sub(r'\.js$', '.', new_file_name))
else:
line.test = new_file_name
self._test_expectations.add_expectations(
path, [line], lineno=line.lineno)
elif root_test_file in deleted_files:
self._test_expectations.remove_expectations(
path, [line])
elif not root_file or not self.port.test_isfile(root_file):
if not self.options.clean_up_affected_tests_only:
self._test_expectations.remove_expectations(path, [line])
@memoized
def _get_root_file(self, test_name):
"""Strips arguments from a web test name in order to get the file name.
"""Finds the physical file in web tests directory for a test
It also removes the arguments for web driver tests. For instances for
the test test1/example.html?Hello this function will return
test1/example.html. For a webdriver test it would include arguments and
would have the following format, {test file}>>{argument}.
If a test is a WPT test then it will look in each of the WPT manifests
for the physical file. If test name cannot be found in any of the manifests
then the test no longer exists and the function will return None. If a file
is webdriver test then it will strip all subtest arguments and return the
file path. If a test is a legacy web test then it will return the test name.
Args:
test_name: Test name which may include test arguments.
Returns:
Returns the test file which is the root of a test.
Returns the path of the physical file that backs
up a test. The path is relative to the web_tests directory.
"""
if self.finder.is_webdriver_test_path(test_name):
root_test_file, _ = (
self.port.split_webdriver_test_name(test_name))
elif '?' in test_name:
root_test_file = test_name[:test_name.find('?')]
else:
root_test_file = test_name
return root_test_file
def _list_deleted_test_files(self):
"""Returns a list of web tests that have been deleted.
If --clean-up-affected-tests-only is true then only test files deleted
in the current CL may be removed from expectations. Otherwise, any test
file may be removed from expectations if it has been deleted.
Returns: A list of web test files that have been deleted.
"""
if self.options.clean_up_affected_tests_only:
# TODO(robertma): Improve Git.changed_files so that we can use
# it here.
paths = set(self.git.run(
['diff', 'origin/master', '-M100%', '--diff-filter=D',
'--name-only']).splitlines())
deleted_tests = set()
for path in paths:
test = self._relative_to_web_test_dir(path)
if test:
deleted_tests.add(test)
return root_test_file
elif self.port.is_wpt_test(test_name):
for wpt_manifest in self.wpt_manifests:
if test_name.startswith(wpt_manifest.wpt_dir):
wpt_test = test_name[len(wpt_manifest.wpt_dir) + 1:]
if wpt_manifest.is_test_url(wpt_test):
return self.host.filesystem.join(
wpt_manifest.wpt_dir,
wpt_manifest.file_path_for_test_url(wpt_test))
# The test was not found in any of the wpt manifests, therefore
# the test does not exist. So we will return None in this case.
return None
else:
# Remove expectations for all test which have files that
# were deleted. Paths are already relative to the web_tests
# directory
deleted_tests = self._deleted_test_files_in_expectations()
return deleted_tests
def _list_renamed_test_files(self):
"""Returns a dictionary mapping tests to their new name.
Regardless of the command line arguments used this test will only
return a dictionary for tests affected in the current CL.
Returns a dictionary mapping source name to destination name.
"""
out = self.git.run(
['diff', 'origin/master', '-M100%', '--diff-filter=R',
'--name-status'])
renamed_tests = {}
for line in out.splitlines():
_, source_path, dest_path = line.split()
source_test = self._relative_to_web_test_dir(source_path)
dest_test = self._relative_to_web_test_dir(dest_path)
if source_test and dest_test:
renamed_tests[source_test] = dest_test
return renamed_tests
# Non WPT and non webdriver tests have no file parameters, and
# the physical file path is the actual name of the test.
return test_name
def _relative_to_web_test_dir(self, path_relative_to_repo_root):
"""Returns a path that's relative to the web tests directory."""
......@@ -846,25 +852,6 @@ class WPTExpectationsUpdater(object):
return self.host.filesystem.relpath(
abs_path, self.finder.web_tests_dir())
def _deleted_test_files_in_expectations(self):
"""Returns a list of test files that were deleted.
Returns a list of test file names that are still in the expectations
files but no longer exists in the web tests directory.
"""
deleted_files = set()
existing_files = {
self._get_root_file(p)
for p in self.port.tests()}
for path in self._test_expectations.expectations_dict:
for line in self._test_expectations.get_updated_lines(path):
if not line.test or line.is_glob:
continue
root_test_file = self._get_root_file(line.test)
if root_test_file not in existing_files:
deleted_files.add(root_test_file)
return deleted_files
# TODO(robertma): Unit test this method.
def download_text_baselines(self, test_results):
"""Fetches new baseline files for tests that should be rebaselined.
......
third_party/blink/tools/blinkpy/w3c/wpt_expectations_updater_unittest.py
View file @ dfcecba4
......@@ -18,7 +18,8 @@ from blinkpy.common.system.log_testing import LoggingTestCase
from blinkpy.w3c.wpt_expectations_updater import (
WPTExpectationsUpdater, SimpleTestResult, DesktopConfig)
from blinkpy.w3c.wpt_manifest import BASE_MANIFEST_NAME
from blinkpy.w3c.wpt_manifest import (
WPTManifest, BASE_MANIFEST_NAME, MANIFEST_NAME)
from blinkpy.web_tests.builder_list import BuilderList
from blinkpy.web_tests.port.android import PRODUCTS_TO_EXPECTATION_FILE_PATHS
......@@ -84,6 +85,14 @@ class WPTExpectationsUpdaterTest(LoggingTestCase):
'testharness': {
'test/path.html': ['abcdef123', [None, {}]],
'test/zzzz.html': ['ghijkl456', [None, {}]],
'fake/some_test.html': [
'ghijkl456', ['fake/some_test.html?HelloWorld', {}]],
'fake/file/deleted_path.html': [
'ghijkl456', [None, {}]],
'test/task.js': [
'mnpqrs789',
['test/task.html', {}],
['test/task2.html', {}]],
},
'manual': {
'x-manual.html': ['abcdef123', [None, {}]],
......@@ -801,6 +810,63 @@ class WPTExpectationsUpdaterTest(LoggingTestCase):
WPTExpectationsUpdater.MARKER_COMMENT + '\n' +
'[ linux ] external/wpt/fake/new.html?HelloWorld [ Failure ]\n'))
def test_clean_expectations_for_deleted_test_harness(self):
host = self.mock_host()
port = host.port_factory.get()
expectations_path = \
port.path_to_generic_test_expectations_file()
host.filesystem.write_text_file(
expectations_path,
'# tags: [ Win Linux ]\n' +
'# results: [ Pass Failure ]\n' +
WPTExpectationsUpdater.MARKER_COMMENT + '\n' +
'[ linux ] wpt_internal/test/task.html [ Failure ]\n' +
'[ win ] wpt_internal/test/task2.html [ Failure ]\n' +
'[ linux ] external/wpt/test/task.html [ Failure ]\n' +
'external/wpt/test/task2.html [ Pass ]\n')
def _git_command_return_val(cmd):
if '--diff-filter=D' in cmd:
return '\n'.join(['external/wpt/test/task.js',
'wpt_internal/test/task.js'])
return ''
wpt_manifest = port.wpt_manifest('external/wpt')
host.filesystem.maybe_make_directory(
port.web_tests_dir(), 'wpt_internal')
host.filesystem.copyfile(
host.filesystem.join(port.web_tests_dir(),
'external', 'wpt', MANIFEST_NAME),
host.filesystem.join(port.web_tests_dir(), 'wpt_internal',
MANIFEST_NAME))
wpt_internal_manifest = WPTManifest(host, host.filesystem.join(
port.web_tests_dir(), 'wpt_internal', MANIFEST_NAME))
updater = WPTExpectationsUpdater(
host,
['--clean-up-affected-tests-only',
'--clean-up-test-expectations-only'],
[wpt_manifest, wpt_internal_manifest])
updater.git.run = _git_command_return_val
updater._relative_to_web_test_dir = lambda test_path: test_path
updater.cleanup_test_expectations_files()
test_expectations = {'external/wpt/fake/file/path.html': {
tuple([DesktopConfig(port_name='test-linux-trusty')]):
SimpleTestResult(actual='PASS', expected='', bug='crbug.com/123')}}
skip_path = host.port_factory.get().path_to_never_fix_tests_file()
skip_value_origin = host.filesystem.read_text_file(skip_path)
updater.write_to_test_expectations(test_expectations)
value = host.filesystem.read_text_file(expectations_path)
self.assertMultiLineEqual(
value, ('# tags: [ Win Linux ]\n' +
'# results: [ Pass Failure ]\n\n' +
WPTExpectationsUpdater.MARKER_COMMENT + '\n' +
'crbug.com/123 [ Trusty ] external/wpt/fake/file/path.html [ Pass ]'))
skip_value = host.filesystem.read_text_file(skip_path)
self.assertMultiLineEqual(skip_value, skip_value_origin)
def test_write_to_test_expectations_and_cleanup_expectations(self):
host = self.mock_host()
expectations_path = \
......@@ -1225,16 +1291,21 @@ class WPTExpectationsUpdaterTest(LoggingTestCase):
def test_cleanup_all_deleted_tests_in_expectations_files(self):
host = MockHost()
port = host.port_factory.get()
host.filesystem.files[MOCK_WEB_TESTS + 'TestExpectations'] = (
'# results: [ Failure ]\n'
'some/test/a.html?hello%20world [ Failure ]\n'
'external/wpt/some/test/a.html?hello%20world [ Failure ]\n'
'some/test/b.html [ Failure ]\n'
'# This line should be deleted\n'
'some/test/c.html [ Failure ]\n'
'# line below should exist in new file\n'
'some/test/d.html [ Failure ]\n')
host.filesystem.files[MOCK_WEB_TESTS + 'VirtualTestSuites'] = '[]'
host.filesystem.files[MOCK_WEB_TESTS + 'new/a.html'] = ''
host.filesystem.files[MOCK_WEB_TESTS + 'new/b.html'] = ''
host.filesystem.files[
host.filesystem.join(
port.web_tests_dir(), 'some', 'test', 'd.html')] = ''
# TODO(rmhasan): Remove creation of Android files within
# tests.
for path in PRODUCTS_TO_EXPECTATION_FILE_PATHS.values():
......@@ -1248,7 +1319,6 @@ class WPTExpectationsUpdaterTest(LoggingTestCase):
return ''
updater.git.run = _git_command_return_val
updater.port.tests = lambda: ['some/test/d.html']
updater._relative_to_web_test_dir = lambda test_path: test_path
updater.cleanup_test_expectations_files()
self.assertMultiLineEqual(
......@@ -1297,8 +1367,8 @@ class WPTExpectationsUpdaterTest(LoggingTestCase):
'external/wpt/webdriver/some/test/a*.html': 'old/a*.html',
'external/wpt/webdriver/some/test/c.html': 'old/c.html',
}
updater._list_deleted_test_files = lambda: deleted_files
updater._list_renamed_test_files = lambda: renamed_file_pairs
updater._list_deleted_files = lambda: deleted_files
updater._list_renamed_files = lambda: renamed_file_pairs
updater.cleanup_test_expectations_files()
self.assertMultiLineEqual(
host.filesystem.read_text_file(MOCK_WEB_TESTS +
......
third_party/blink/tools/blinkpy/w3c/wpt_manifest.py
View file @ dfcecba4
......@@ -84,8 +84,11 @@ class WPTManifest(object):
[[reference_url1, "=="], [reference_url2, "!="], ...]
"""
def __init__(self, json_content):
self.raw_dict = json.loads(json_content)
def __init__(self, host, manifest_path):
self.host = host
self.port = self.host.port_factory.get()
self.raw_dict = json.loads(
self.host.filesystem.read_text_file(manifest_path))
# As a workaround to handle the change from a flat-list to a trie
# structure in the v8 manifest, flatten the items back to the v7 format.
#
......@@ -93,9 +96,16 @@ class WPTManifest(object):
self.raw_dict['items'] = self._flatten_items(
self.raw_dict.get('items', {}))
self.wpt_manifest_path = manifest_path
self.test_types = ('manual', 'reftest', 'testharness', 'crashtest')
self.test_name_to_file = {}
@property
def wpt_dir(self):
return self.host.filesystem.dirname(
self.host.filesystem.relpath(
self.wpt_manifest_path, self.port.web_tests_dir()))
def _items_for_file_path(self, path_in_wpt):
"""Finds manifest items for the given WPT path.
......@@ -275,9 +285,8 @@ class WPTManifest(object):
@staticmethod
def generate_manifest(host, dest_path):
"""Generates MANIFEST.json on the specified directory."""
finder = PathFinder(host.filesystem)
wpt_exec_path = finder.path_from_blink_tools('blinkpy', 'third_party',
'wpt', 'wpt', 'wpt')
wpt_exec_path = PathFinder(host.filesystem).path_from_blink_tools(
'blinkpy', 'third_party', 'wpt', 'wpt', 'wpt')
cmd = [
'python', wpt_exec_path, 'manifest', '--no-download',
'--tests-root', dest_path
......
third_party/blink/tools/blinkpy/w3c/wpt_manifest_unittest.py
View file @ dfcecba4
......@@ -89,7 +89,11 @@ class WPTManifestUnitTest(unittest.TestCase):
}
}
'''
manifest = WPTManifest(manifest_json)
host = MockHost()
host.filesystem.write_text_file(
WEB_TEST_DIR + '/external/wpt/MANIFEST.json', manifest_json)
manifest = WPTManifest(
host, WEB_TEST_DIR + '/external/wpt/MANIFEST.json')
self.assertTrue(manifest.is_test_file('test.any.js'))
self.assertEqual(manifest.all_url_items(),
{u'test.any.html': [u'test.any.html', {}]})
......@@ -111,7 +115,11 @@ class WPTManifestUnitTest(unittest.TestCase):
}
}
'''
manifest = WPTManifest(manifest_json)
host = MockHost()
host.filesystem.write_text_file(
WEB_TEST_DIR + '/external/wpt/MANIFEST.json', manifest_json)
manifest = WPTManifest(
host, WEB_TEST_DIR + '/external/wpt/MANIFEST.json')
self.assertEqual(manifest.all_url_items(),
{u'test.any.html': [u'test.any.html', {}]})
......@@ -132,7 +140,11 @@ class WPTManifestUnitTest(unittest.TestCase):
}
}
} '''
manifest = WPTManifest(manifest_json)
host = MockHost()
host.filesystem.write_text_file(
WEB_TEST_DIR + '/external/wpt/MANIFEST.json', manifest_json)
manifest = WPTManifest(
host, WEB_TEST_DIR + '/external/wpt/MANIFEST.json')
self.assertEqual(
manifest.all_url_items(), {
u'test.any.html': [u'test.any.html', {}],
......@@ -168,7 +180,11 @@ class WPTManifestUnitTest(unittest.TestCase):
}
}
'''
manifest = WPTManifest(manifest_json)
host = MockHost()
host.filesystem.write_text_file(
WEB_TEST_DIR + '/external/wpt/MANIFEST.json', manifest_json)
manifest = WPTManifest(
host, WEB_TEST_DIR + '/external/wpt/MANIFEST.json')
self.assertEqual(
manifest.all_url_items(), {
u'test.html': [u'test.html', {}],
......
third_party/blink/tools/blinkpy/web_tests/lint_test_expectations_unittest.py
View file @ dfcecba4
......@@ -144,7 +144,7 @@ class LintTest(LoggingTestCase):
port = host.port_factory.get(options.platform, options=options)
port.expectations_dict = lambda: {'foo': '-- syntax error1', 'bar': '-- syntax error2'}
host.port_factory.get = lambda platform, options=None: port
host.port_factory.get = lambda platform=None, options=None: port
host.port_factory.all_port_names = lambda platform=None: [port.name()]
failures, warnings = lint_test_expectations.lint(host, options)
......@@ -166,7 +166,7 @@ class LintTest(LoggingTestCase):
port = host.port_factory.get(options.platform, options=options)
port.expectations_dict = lambda: {}
host.port_factory.get = lambda platform, options=None: port
host.port_factory.get = lambda platform=None, options=None: port
host.port_factory.all_port_names = lambda platform=None: [port.name()]
host.filesystem.write_text_file(WEB_TEST_DIR + '/LeakExpectations',
'-- syntax error')
......@@ -189,7 +189,7 @@ class LintTest(LoggingTestCase):
port = host.port_factory.get(options.platform, options=options)
port.expectations_dict = lambda: {'flag-specific': 'does/not/exist', 'noproblem': ''}
host.port_factory.get = lambda platform, options=None: port
host.port_factory.get = lambda platform=None, options=None: port
host.port_factory.all_port_names = lambda platform=None: [port.name()]
failures, warnings = lint_test_expectations.lint(host, options)
......@@ -218,7 +218,7 @@ class LintTest(LoggingTestCase):
port.expectations_dict = lambda: {
'testexpectations': test_expectations}
host.port_factory.get = lambda platform, options=None: port
host.port_factory.get = lambda platform=None, options=None: port
host.port_factory.all_port_names = lambda platform=None: [port.name()]
failures, warnings = lint_test_expectations.lint(host, options)
......@@ -267,7 +267,7 @@ class LintTest(LoggingTestCase):
host.filesystem.join(port.web_tests_dir(), 'virtual', 'foo',
'README.md'), 'foo')
host.port_factory.get = lambda platform, options=None: port
host.port_factory.get = lambda platform=None, options=None: port
host.port_factory.all_port_names = lambda platform=None: [port.name()]
failures, warnings = lint_test_expectations.lint(host, options)
......@@ -300,7 +300,7 @@ class LintTest(LoggingTestCase):
'non-wpt/test.html [ Failure ]\n')
for path in PRODUCTS_TO_EXPECTATION_FILE_PATHS.values():
host.filesystem.write_text_file(path, raw_expectations)
host.port_factory.get = lambda platform, options=None: port
host.port_factory.get = lambda platform=None, options=None: port
host.port_factory.all_port_names = lambda platform=None: [port.name()]
port.test_exists = lambda _: True
port.tests = lambda _: {'external/wpt/test.html', 'non-wpt/test.html'}
......@@ -325,7 +325,7 @@ class LintTest(LoggingTestCase):
host.filesystem.maybe_make_directory(
host.filesystem.join(port.web_tests_dir(), 'test2'))
host.port_factory.get = lambda platform, options=None: port
host.port_factory.get = lambda platform=None, options=None: port
host.port_factory.all_port_names = lambda platform=None: [port.name()]
failures, warnings = lint_test_expectations.lint(host, options)
......@@ -364,7 +364,7 @@ class LintTest(LoggingTestCase):
'testexpectations': test_expectations
}
port.test_exists = lambda test: True
host.port_factory.get = lambda platform, options=None: port
host.port_factory.get = lambda platform=None, options=None: port
host.port_factory.all_port_names = lambda platform=None: [port.name()]
failures, warnings = lint_test_expectations.lint(host, options)
......@@ -399,7 +399,7 @@ class LintTest(LoggingTestCase):
'virtual/foo/test1/* [ Pass ]\n')
port.expectations_dict = lambda: {'NeverFixTests': test_expectations}
port.test_exists = lambda test: True
host.port_factory.get = lambda platform, options=None: port
host.port_factory.get = lambda platform=None, options=None: port
host.port_factory.all_port_names = lambda platform=None: [port.name()]
failures, warnings = lint_test_expectations.lint(host, options)
......
third_party/blink/tools/blinkpy/web_tests/port/base.py
View file @ dfcecba4
......@@ -959,7 +959,7 @@ class Port(object):
'manifest_update', True):
_log.debug('Generating MANIFEST.json for %s...', path)
WPTManifest.ensure_manifest(self, path)
return WPTManifest(self._filesystem.read_text_file(manifest_path))
return WPTManifest(self.host, manifest_path)
def is_wpt_crash_test(self, test_file):
"""Returns whether a WPT test is a crashtest.
......
third_party/blink/tools/blinkpy/web_tests/port/base_unittest.py
View file @ dfcecba4
......@@ -32,6 +32,7 @@ import unittest
import mock
from blinkpy.common.path_finder import RELATIVE_WEB_TESTS
from blinkpy.common.host_mock import MockHost
from blinkpy.common.system.executive_mock import MockExecutive
from blinkpy.common.system.log_testing import LoggingTestCase
from blinkpy.common.system.output_capture import OutputCapture
......@@ -51,7 +52,7 @@ class PortTest(LoggingTestCase):
with_tests=False,
port_name=None,
**kwargs):
host = MockSystemHost()
host = MockHost()
if executive:
host.executive = executive
if with_tests:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment