Skip to content

GitLab

T
tangled
Commit e47aeeb7 authored by Rakib M. Hasan's avatar Rakib M. Hasan Committed by Commit Bot
Browse files
Options

weblayer,wpt: Update script so that it can mark tests as flaky 

Updates wpt_update_expectations.py so that it can mark tests as flaky
by adding pass expectations for tests with unexpected passes.

Bug: 1050754
Change-Id: I071c21d574763b9fa5d7d3f1637ad2236933941f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2303869
Commit-Queue: Rakib Hasan <rmhasan@google.com>
Reviewed-by: default avatarRobert Ma <robertma@chromium.org>
Cr-Commit-Position: refs/heads/master@{#790450}
parent afa8f831
Hide whitespace changes
Inline Side-by-side
Showing with 68 additions and 15 deletions
third_party/blink/tools/blinkpy/w3c/android_wpt_expectations_updater_unittest.py
View file @ e47aeeb7
......@@ -37,6 +37,7 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
'crbug.com/1050754 external/wpt/cat.html [ Failure ]\n'
'external/wpt/dog.html [ Crash Timeout ]\n'
'crbug.com/6789043 external/wpt/van.html [ Failure ]\n'
'external/wpt/unexpected_pass.html [ Failure ]\n'
'\n'
'# This comment will not be deleted\n'
'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n')
......@@ -105,6 +106,11 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
'actual': 'CRASH CRASH TIMEOUT',
'is_unexpected': True,
},
'unexpected_pass.html': {
'expected': 'FAIL',
'actual': 'PASS',
'is_unexpected': True
},
'dog.html': {
'expected': 'SKIP',
'actual': 'SKIP',
......@@ -116,7 +122,8 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
updater = AndroidWPTExpectationsUpdater(
host, ['-vvv', '--android-product', ANDROID_WEBVIEW,
'--clean-up-test-expectations',
'--clean-up-affected-tests-only'])
'--clean-up-affected-tests-only',
'--include-unexpected-pass'])
updater.git_cl = MockGitCL(host, {
Build('MOCK Android Pie', 123):
TryJobStatus('COMPLETED', 'FAILURE')})
......@@ -137,6 +144,7 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
'crbug.com/1050754 external/wpt/ghi.html [ Timeout ]\n'
'crbug.com/1111111 crbug.com/1050754'
' external/wpt/jkl.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/unexpected_pass.html [ Failure Pass ]\n'
'crbug.com/6789043 external/wpt/van.html [ Failure ]\n'
'external/wpt/www.html [ Crash Failure ]\n'
'\n'
......@@ -194,6 +202,11 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
'actual': 'CRASH CRASH TIMEOUT',
'is_unexpected': True,
},
'unexpected_pass.html': {
'expected': 'FAIL',
'actual': 'PASS',
'is_unexpected': True,
},
'new.html': {
'expected': 'PASS',
'actual': 'CRASH CRASH FAIL',
......@@ -205,7 +218,8 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
updater = AndroidWPTExpectationsUpdater(
host, ['-vvv', '--android-product', ANDROID_WEBLAYER,
'--clean-up-test-expectations',
'--clean-up-affected-tests-only'])
'--clean-up-affected-tests-only',
'--include-unexpected-pass'])
updater.git_cl = MockGitCL(host, {
Build('MOCK Android Weblayer - Pie', 123):
TryJobStatus('COMPLETED', 'FAILURE')})
......@@ -227,6 +241,7 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
'crbug.com/1111111 crbug.com/1050754'
' external/wpt/jkl.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/new.html [ Failure Crash ]\n'
'crbug.com/1050754 external/wpt/unexpected_pass.html [ Failure Pass ]\n'
'crbug.com/6789043 external/wpt/van.html [ Failure ]\n'
'external/wpt/www.html [ Crash Failure ]\n'
'\n'
......@@ -273,6 +288,11 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
'actual': 'SKIP',
'is_unexpected': True,
},
'unexpected_pass.html': {
'expected': 'FAIL',
'actual': 'PASS',
'is_unexpected': True
},
},
}, step_name=WEBLAYER_WPT_STEP + ' (with patch)'),
step_name=WEBLAYER_WPT_STEP + ' (with patch)')
......@@ -295,6 +315,11 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
'expected': 'SKIP',
'actual': 'SKIP',
},
'unexpected_pass.html': {
'expected': 'FAIL',
'actual': 'PASS',
'is_unexpected': True
},
},
}, step_name=WEBVIEW_WPT_STEP + ' (with patch)'),
step_name=WEBVIEW_WPT_STEP + ' (with patch)')
......@@ -318,6 +343,11 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
'actual': 'SKIP',
'is_unexpected': True,
},
'unexpected_pass.html': {
'expected': 'FAIL',
'actual': 'PASS',
'is_unexpected': True
},
},
}, step_name=CHROME_ANDROID_WPT_STEP + ' (with patch)'),
step_name=CHROME_ANDROID_WPT_STEP + ' (with patch)')
......@@ -325,6 +355,7 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
host, ['-vvv',
'--clean-up-test-expectations',
'--clean-up-affected-tests-only',
'--include-unexpected-pass',
'--android-product', ANDROID_WEBLAYER,
'--android-product', CHROME_ANDROID,
'--android-product', ANDROID_WEBVIEW])
......@@ -360,6 +391,7 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
'crbug.com/1050754 external/wpt/def.html [ Crash ]\n'
'external/wpt/dog.html [ Crash Timeout ]\n'
'crbug.com/1111111 external/wpt/jkl.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/unexpected_pass.html [ Failure Pass ]\n'
'crbug.com/6789043 external/wpt/wagon.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/weblayer_only.html [ Failure Crash ]\n'
'external/wpt/www.html [ Crash Failure ]\n'
......@@ -379,6 +411,7 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
'crbug.com/1050754 external/wpt/def.html [ Crash ]\n'
'external/wpt/dog.html [ Crash Timeout ]\n'
'crbug.com/1111111 external/wpt/jkl.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/unexpected_pass.html [ Failure Pass ]\n'
'crbug.com/6789043 external/wpt/wagon.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/webview_only.html [ Timeout ]\n'
'external/wpt/www.html [ Crash Failure ]\n'
......@@ -400,6 +433,7 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
'external/wpt/dog.html [ Crash Timeout ]\n'
'crbug.com/1111111 crbug.com/1050754'
' external/wpt/jkl.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/unexpected_pass.html [ Failure Pass ]\n'
'crbug.com/6789043 external/wpt/wagon.html [ Failure ]\n'
'external/wpt/www.html [ Crash Failure ]\n'
'\n'
......
third_party/blink/tools/blinkpy/w3c/wpt_expectations_updater.py
View file @ e47aeeb7
......@@ -44,7 +44,6 @@ class WPTExpectationsUpdater(object):
self.git_cl = GitCL(host)
self.git = self.host.git(self.finder.chromium_base())
self.configs_with_no_results = []
self.configs_with_all_pass = []
self.patchset = None
# Get options from command line arguments.
......@@ -83,6 +82,11 @@ class WPTExpectationsUpdater(object):
log_level = logging.DEBUG if self.options.verbose else logging.INFO
configure_logging(logging_level=log_level, include_time=True)
if not(self.options.android_product or
self.options.update_android_expectations_only):
assert not self.options.include_unexpected_pass, (
'Command line argument --include-unexpected-pass is not '
'supported in desktop mode.')
self.patchset = self.options.patchset
if (self.options.clean_up_test_expectations or
......@@ -132,6 +136,12 @@ class WPTExpectationsUpdater(object):
'--android-product', action='append', default=[],
help='Android products whose baselines will be updated.',
choices=PRODUCTS)
parser.add_argument(
'--include-unexpected-pass',
action='store_true',
help='Adds Pass to tests with failure expectations. '
'This command line argument can be used to mark tests '
'as flaky.')
def update_expectations(self):
"""Downloads text new baselines and adds test expectations lines.
......@@ -153,9 +163,8 @@ class WPTExpectationsUpdater(object):
# Here we build up a dict of failing test results for all platforms.
test_expectations = {}
for build, job_status in build_to_status.iteritems():
if job_status.result == 'SUCCESS':
self.configs_with_all_pass.extend(
self.get_builder_configs(build))
if (job_status.result == 'SUCCESS' and
not self.options.include_unexpected_pass):
continue
result_dicts = self.get_failing_results_dicts(build)
for result_dict in result_dicts:
......@@ -236,13 +245,13 @@ class WPTExpectationsUpdater(object):
self.configs_with_no_results.extend(self.get_builder_configs(build))
return []
failing_test_results = []
unexpected_test_results = []
for results_set in test_results_list:
results_dict = self.generate_failing_results_dict(
build, results_set)
if results_dict:
failing_test_results.append(results_dict)
return failing_test_results
unexpected_test_results.append(results_dict)
return unexpected_test_results
def _get_web_test_results(self, build):
"""Gets web tests results for a builder.
......@@ -291,9 +300,6 @@ class WPTExpectationsUpdater(object):
raise ScriptError('No configuration was found for builder and web test'
' step combination ')
config = configs[0]
if config in self.configs_with_all_pass:
return {}
for result in web_test_results.didnt_run_as_expected_results():
# TODO(rmhasan) If a test fails unexpectedly then it runs multiple
# times until, it passes or a retry limit is reached. Even though
......@@ -301,7 +307,11 @@ class WPTExpectationsUpdater(object):
# creating test expectations for. Maybe we should add a mode
# which creates expectations for tests that are flaky but still
# pass in a web test step.
if result.did_pass():
# Create flaky expectations for flaky tests on Android. In order to
# do this we should add 'Pass' to all tests with failing
# expectations that pass in the patchset's try job.
if result.did_pass() and not self.options.include_unexpected_pass:
continue
test_name = result.test_name()
......
third_party/blink/tools/blinkpy/w3c/wpt_expectations_updater_unittest.py
View file @ e47aeeb7
......@@ -157,6 +157,17 @@ class WPTExpectationsUpdaterTest(LoggingTestCase):
'crbug.com/626703 [ Mac10.10 ] external/wpt/test/path.html [ Timeout ]\n'
)
def test_cmd_arg_include_unexpected_pass_raieses_exception(self):
host = self.mock_host()
expectations_path = \
host.port_factory.get().path_to_generic_test_expectations_file()
host.filesystem.write_text_file(expectations_path,
WPTExpectationsUpdater.MARKER_COMMENT + '\n')
updater = WPTExpectationsUpdater(host, args=['--include-unexpected-pass'])
with self.assertRaises(AssertionError) as ctx:
updater.run()
self.assertIn('--include-unexpected-pass', str(ctx.exception))
def test_get_failing_results_dict_only_passing_results(self):
host = self.mock_host()
host.results_fetcher.set_results(
......@@ -1039,8 +1050,6 @@ class WPTExpectationsUpdaterTest(LoggingTestCase):
}
}
tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(two)
# external/wpt/test/zzzz.html is another possible candidate, but it
# is not listed in the results dict, so it shall not be rebaselined.
self.assertEqual(tests_to_rebaseline, ['external/wpt/test/path.html'])
def test_get_test_to_rebaseline_does_not_return_ref_tests(self):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment