Skip to content

GitLab

T
tangled
Commit 5bef0fc4 authored by Stephen Martinis's avatar Stephen Martinis Committed by Commit Bot
Browse files
Options

buildbot gen: Better sorting 

This CL modifies the sorting and duplicate checks in the *.pyl
files in //testing/buildbot. It adds more checks for subsections
of the files; for example, each entry inside of test_suite_exceptions.pyl
will now be checked, rather than just checking that the top level
list of tests is sorted.

A few duplicates have been removed; these won't cause any meaningful
behavior change, since python ignores duplicates in dictionaries.

Change-Id: I314338c904080b463ac6859011a538987a862530
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1986241
Commit-Queue: Stephen Martinis <martiniss@chromium.org>
Reviewed-by: default avatarGarrett Beaty <gbeaty@chromium.org>
Reviewed-by: default avatarKenneth Russell <kbr@chromium.org>
Cr-Commit-Position: refs/heads/master@{#728678}
parent 8b282fe6
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 269 additions and 109 deletions
testing/buildbot/generate_buildbot_json.py
View file @ 5bef0fc4
This diff is collapsed.
testing/buildbot/generate_buildbot_json_unittest.py
View file @ 5bef0fc4
......@@ -573,6 +573,17 @@ FOO_TEST_SUITE = """\
}
"""
FOO_TEST_SUITE_NOT_SORTED = """\
{
'basic_suites': {
'foo_tests': {
'foo_test': {},
'a_test': {},
},
},
}
"""
FOO_TEST_SUITE_WITH_ARGS = """\
{
'basic_suites': {
......@@ -913,6 +924,37 @@ EXCEPTIONS_UNSORTED = """\
}
"""
EXCEPTIONS_PER_TEST_UNSORTED = """\
{
'suite_d': {
'modifications': {
'Other Tester': {
'foo': 'baz',
},
'Fake Tester': {
'foo': 'baz',
},
},
},
}
"""
EXCEPTIONS_DUPS_REMOVE_FROM = """\
{
'suite_d': {
'remove_from': [
'Fake Tester',
'Fake Tester',
],
'modifications': {
'Fake Tester': {
'foo': 'baz',
},
},
},
}
"""
FOO_TEST_MODIFICATIONS = """\
{
'foo_test': {
......@@ -2454,9 +2496,9 @@ class UnitTest(unittest.TestCase):
fbb.check_input_file_consistency(verbose=True)
joined_lines = '\n'.join(fbb.printed_lines)
self.assertRegexpMatches(
joined_lines, '.*\+chromium\..*test.*')
joined_lines, '.*\+ chromium\..*test.*')
self.assertRegexpMatches(
joined_lines, '.*\-chromium\..*test.*')
joined_lines, '.*\- chromium\..*test.*')
fbb.printed_lines = []
self.assertFalse(fbb.printed_lines)
......@@ -2483,6 +2525,57 @@ class UnitTest(unittest.TestCase):
fbb.check_input_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
fbb = FakeBBGen(TEST_SUITE_SORTING_WATERFALL,
TEST_SUITE_SORTED,
LUCI_MILO_CFG,
exceptions=EXCEPTIONS_DUPS_REMOVE_FROM)
with self.assertRaises(generate_buildbot_json.BBGenErr):
fbb.check_input_file_consistency(verbose=True)
joined_lines = ' '.join(fbb.printed_lines)
self.assertRegexpMatches(
joined_lines, '.*\- Fake Tester.*')
fbb.printed_lines = []
self.assertFalse(fbb.printed_lines)
def test_test_suite_exceptions_no_dups_remove_from(self):
fbb = FakeBBGen(TEST_SUITE_SORTING_WATERFALL,
TEST_SUITE_SORTED,
LUCI_MILO_CFG,
exceptions=EXCEPTIONS_SORTED)
fbb.check_input_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
fbb = FakeBBGen(TEST_SUITE_SORTING_WATERFALL,
TEST_SUITE_SORTED,
LUCI_MILO_CFG,
exceptions=EXCEPTIONS_PER_TEST_UNSORTED)
real_dict_format = fbb.check_ast_dict_formatted
def fake_dict_format(*args, **kwargs):
kwargs['check_sorting'] = True
return real_dict_format(*args, **kwargs)
# Mock this out for now. A future CL will enable sorting by default. Still
# want to test the codepath, even if it's not used it production yet.
fbb.check_ast_dict_formatted = fake_dict_format
with self.assertRaises(generate_buildbot_json.BBGenErr):
fbb.check_input_file_consistency(verbose=True)
joined_lines = ' '.join(fbb.printed_lines)
self.assertRegexpMatches(
joined_lines, '.*\+ Fake Tester.*')
self.assertRegexpMatches(
joined_lines, '.*\- Fake Tester.*')
fbb.printed_lines = []
self.assertFalse(fbb.printed_lines)
def test_test_suite_exceptions_per_test_must_be_sorted(self):
fbb = FakeBBGen(TEST_SUITE_SORTING_WATERFALL,
TEST_SUITE_SORTED,
LUCI_MILO_CFG,
exceptions=EXCEPTIONS_SORTED)
fbb.check_input_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
fbb = FakeBBGen(TEST_SUITE_SORTING_WATERFALL,
TEST_SUITE_SORTED,
LUCI_MILO_CFG,
......@@ -2491,9 +2584,9 @@ class UnitTest(unittest.TestCase):
fbb.check_input_file_consistency(verbose=True)
joined_lines = ' '.join(fbb.printed_lines)
self.assertRegexpMatches(
joined_lines, '.*\+suite_.*')
joined_lines, '.*\+ suite_.*')
self.assertRegexpMatches(
joined_lines, '.*\-suite_.*')
joined_lines, '.*\- suite_.*')
fbb.printed_lines = []
self.assertFalse(fbb.printed_lines)
......@@ -2516,9 +2609,9 @@ class UnitTest(unittest.TestCase):
fbb.check_input_file_consistency(verbose=True)
joined_lines = ' '.join(fbb.printed_lines)
self.assertRegexpMatches(
joined_lines, '.*\+suite_.*')
joined_lines, '.*\+ suite_.*')
self.assertRegexpMatches(
joined_lines, '.*\-suite_.*')
joined_lines, '.*\- suite_.*')
fbb.printed_lines = []
self.assertFalse(fbb.printed_lines)
......@@ -3031,11 +3124,11 @@ class MixinTests(unittest.TestCase):
mixins=SWARMING_MIXINS_UNSORTED)
with self.assertRaises(generate_buildbot_json.BBGenErr):
fbb.check_input_file_consistency(verbose=True)
joined_lines = ' '.join(fbb.printed_lines)
joined_lines = '\n'.join(fbb.printed_lines)
self.assertRegexpMatches(
joined_lines, '.*\+._mixin.*')
joined_lines, '.*\+ ._mixin.*')
self.assertRegexpMatches(
joined_lines, '.*\-._mixin.*')
joined_lines, '.*\- ._mixin.*')
fbb.printed_lines = []
self.assertFalse(fbb.printed_lines)
......@@ -3153,9 +3246,32 @@ class MixinTests(unittest.TestCase):
generate_buildbot_json.BBGenErr,
'The following files have invalid keys: mixins.pyl'):
fbb.check_input_file_consistency(verbose=True)
joined_lines = ' '.join(fbb.printed_lines)
joined_lines = '\n'.join(fbb.printed_lines)
self.assertRegexpMatches(
joined_lines, 'Key .* is duplicated')
joined_lines, '.*\- builder_mixin')
fbb.printed_lines = []
self.assertFalse(fbb.printed_lines)
def test_no_duplicate_keys_basic_test_suite(self):
fbb = FakeBBGen(FOO_GTESTS_WATERFALL,
FOO_TEST_SUITE_NOT_SORTED,
LUCI_MILO_CFG)
real_dict_format = fbb.check_ast_dict_formatted
def fake_dict_format(*args, **kwargs):
kwargs['check_sorting'] = True
return real_dict_format(*args, **kwargs)
# Mock this out for now. A future CL will enable sorting by default. Still
# want to test the codepath, even if it's not used it production yet.
fbb.check_ast_dict_formatted = fake_dict_format
with self.assertRaisesRegexp(
generate_buildbot_json.BBGenErr,
'The following files have invalid keys: test_suites.pyl'):
fbb.check_input_file_consistency(verbose=True)
joined_lines = '\n'.join(fbb.printed_lines)
self.assertRegexpMatches(joined_lines, '.*\- a_test')
self.assertRegexpMatches(joined_lines, '.*\+ a_test')
fbb.printed_lines = []
self.assertFalse(fbb.printed_lines)
......
testing/buildbot/test_suite_exceptions.pyl
View file @ 5bef0fc4
......@@ -715,22 +715,6 @@
},
'use_xvfb': False,
},
'Linux ASan LSan Tests (1)': {
'args': [
'--use-gpu-in-tests',
'--no-xvfb',
],
'swarming': {
'dimension_sets': [
{
'gpu': '10de:1cb3',
'os': 'Ubuntu',
'pool': 'chromium.tests.gpu',
},
],
},
'use_xvfb': False,
},
'Linux CFI': {
'args': [
'--use-gpu-in-tests',
......@@ -1133,7 +1117,6 @@
'Mac10.13 Tests',
'Mac10.13 Tests (dbg)',
'Mac ASan 64 Tests (1)',
'Mac ASan 64 Tests (1)',
'ToTMacASan',
],
'modifications': {
......@@ -1165,7 +1148,6 @@
'Mac10.13 Tests',
'Mac10.13 Tests (dbg)',
'Mac ASan 64 Tests (1)',
'Mac ASan 64 Tests (1)',
'ToTMacASan',
],
},
......@@ -1186,7 +1168,6 @@
'Mac10.13 Tests',
'Mac10.13 Tests (dbg)',
'Mac ASan 64 Tests (1)',
'Mac ASan 64 Tests (1)',
'ToTMacASan',
],
},
......@@ -1206,7 +1187,6 @@
'Mac10.13 Tests',
'Mac10.13 Tests (dbg)',
'Mac ASan 64 Tests (1)',
'Mac ASan 64 Tests (1)',
'ToTMacASan',
],
},
......@@ -1232,7 +1212,6 @@
'Mac10.13 Tests',
'Mac10.13 Tests (dbg)',
'Mac ASan 64 Tests (1)',
'Mac ASan 64 Tests (1)',
'ToTMacASan',
],
'modifications': {
......
testing/buildbot/test_suites.pyl
View file @ 5bef0fc4
......@@ -3276,12 +3276,6 @@
'--test-launcher-timeout=120000'],
'test': 'content_browsertests',
},
'content_browsertests_stress': {
'args': ['--gtest_filter=WebRtc*MANUAL*:-UsingRealWebcam*',
'--run-manual', '--ui-test-action-max-timeout=110000',
'--test-launcher-timeout=120000'],
'test': 'content_browsertests',
},
'browser_tests_functional': {
'args': [
'--test-launcher-filter-file=../../testing/buildbot/filters/webrtc_functional.browser_tests.filter',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment