Skip to content

GitLab

T
tangled
Commit 264b87ff authored by Alexander Timin's avatar Alexander Timin Committed by Commit Bot
Browse files
Options

[base/test] Run tests in parallel for --gtest-repeat. 

Ensure that the test launcher can run a test multiple times in parallel
to enable us finding flaky tests faster.

Also add documentation for --gtest-repeat and --gtest-break-on-failure.

R=erikchen@chromium.org
BUG=976795

Change-Id: I767a618fda6ad8f7424b43b0bf1a06b5238f127a
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1667313
Commit-Queue: Alexander Timin <altimin@chromium.org>
Reviewed-by: default avatarIlia Samsonov <isamsonov@google.com>
Cr-Commit-Position: refs/heads/master@{#804465}
parent fdcf670f
Hide whitespace changes
Inline Side-by-side
Showing with 146 additions and 79 deletions
base/test/launcher/test_launcher.cc
View file @ 264b87ff
......@@ -928,11 +928,18 @@ bool TestLauncher::Run(CommandLine* command_line) {
// Indicate a test did not succeed.
bool test_failed = false;
int cycles = cycles_;
int iterations = cycles_;
if (cycles_ > 1 && !stop_on_failure_) {
// If we don't stop on failure, execute all the repeats in all iteration,
// which allows us to parallelize the execution.
iterations = 1;
repeats_per_iteration_ = cycles_;
}
// Set to false if any iteration fails.
bool run_result = true;
while ((cycles > 0 || cycles == -1) && !(stop_on_failure_ && test_failed)) {
while ((iterations > 0 || iterations == -1) &&
!(stop_on_failure_ && test_failed)) {
OnTestIterationStart();
RunTests();
......@@ -949,7 +956,7 @@ bool TestLauncher::Run(CommandLine* command_line) {
test_failed = test_success_count_ != test_finished_count_;
OnTestIterationFinished();
// Special value "-1" means "repeat indefinitely".
cycles = (cycles == -1) ? cycles : cycles - 1;
iterations = (iterations == -1) ? iterations : iterations - 1;
}
if (cycles_ != 1)
......@@ -1347,6 +1354,11 @@ bool TestLauncher::Init(CommandLine* command_line) {
}
retry_limit_ = retry_limit;
} else if (command_line->HasSwitch(kGTestRepeatFlag) ||
command_line->HasSwitch(kGTestBreakOnFailure)) {
// If we are repeating tests or waiting for the first test to fail, disable
// retries.
retry_limit_ = 0U;
} else if (!BotModeEnabled(command_line) &&
(command_line->HasSwitch(kGTestFilterFlag) ||
command_line->HasSwitch(kIsolatedScriptTestFilterFlag))) {
......@@ -1678,15 +1690,11 @@ void TestLauncher::CombinePositiveTestFilters(
}
}
void TestLauncher::RunTests() {
std::vector<std::string> TestLauncher::CollectTests() {
std::vector<std::string> test_names;
size_t test_found_count = 0;
for (const TestInfo& test_info : tests_) {
std::string test_name = test_info.GetFullName();
// Count tests in the binary, before we apply filter and sharding.
test_found_count++;
std::string prefix_stripped_name = test_info.GetPrefixStrippedName();
// Skip the test that doesn't match the filter (if given).
......@@ -1738,11 +1746,19 @@ void TestLauncher::RunTests() {
test_names.push_back(test_name);
}
// Save an early test summary in case the launcher crashes or gets killed.
results_tracker_.GeneratePlaceholderIteration();
MaybeSaveSummaryAsJSON({"EARLY_SUMMARY"});
return test_names;
}
void TestLauncher::RunTests() {
std::vector<std::string> original_test_names = CollectTests();
broken_threshold_ = std::max(static_cast<size_t>(20), test_found_count / 10);
std::vector<std::string> test_names;
for (int i = 0; i < repeats_per_iteration_; ++i) {
test_names.insert(test_names.end(), original_test_names.begin(),
original_test_names.end());
}
broken_threshold_ = std::max(static_cast<size_t>(20), tests_.size() / 10);
test_started_count_ = test_names.size();
......@@ -1754,8 +1770,17 @@ void TestLauncher::RunTests() {
fflush(stdout);
}
TestRunner test_runner(this, parallel_jobs_,
launcher_delegate_->GetBatchSize());
// Save an early test summary in case the launcher crashes or gets killed.
results_tracker_.GeneratePlaceholderIteration();
MaybeSaveSummaryAsJSON({"EARLY_SUMMARY"});
// If we are repeating the test, set batch size to 1 to ensure that batch size
// does not interfere with repeats (unittests are using filter for batches and
// can't run the same test twice in the same batch).
size_t batch_size =
repeats_per_iteration_ > 1 ? 1U : launcher_delegate_->GetBatchSize();
TestRunner test_runner(this, parallel_jobs_, batch_size);
test_runner.Run(test_names);
}
......
base/test/launcher/test_launcher.h
View file @ 264b87ff
......@@ -234,6 +234,8 @@ class TestLauncher {
bool was_timeout,
int leaked_items);
std::vector<std::string> CollectTests();
// Make sure we don't accidentally call the wrong methods e.g. on the worker
// pool thread. Should be the first member so that it's destroyed last: when
// destroying other members, especially the worker pool, we may check the code
......@@ -311,6 +313,11 @@ class TestLauncher {
// redirect stdio of subprocess
bool redirect_stdio_;
// Number of times all tests should be repeated during each iteration.
// 1 if gtest_repeat is not specified or gtest_break_on_failure is specified.
// Otherwise it matches gtest_repeat value.
int repeats_per_iteration_ = 1;
DISALLOW_COPY_AND_ASSIGN(TestLauncher);
};
......
base/test/launcher/test_launcher_unittest.cc
View file @ 264b87ff
......@@ -265,18 +265,37 @@ TEST_F(TestLauncherTest, FilterIncludePreTest) {
}
// Test TestLauncher "gtest_repeat" switch.
TEST_F(TestLauncherTest, RunningMultipleIterations) {
TEST_F(TestLauncherTest, RepeatTest) {
AddMockedTests("Test", {"firstTest"});
SetUpExpectCalls();
// Unless --gtest-break-on-failure is specified,
command_line->AppendSwitchASCII("gtest_repeat", "2");
using ::testing::_;
EXPECT_CALL(test_launcher, LaunchChildGTestProcess(_, _, _, _))
.Times(2)
.WillRepeatedly(OnTestResult(&test_launcher, "Test.firstTest",
TestResult::TEST_SUCCESS));
.WillRepeatedly(::testing::DoAll(OnTestResult(
&test_launcher, "Test.firstTest", TestResult::TEST_SUCCESS)));
EXPECT_TRUE(test_launcher.Run(command_line.get()));
}
// Test TestLauncher --gtest_repeat and --gtest_break_on_failure.
TEST_F(TestLauncherTest, RunningMultipleIterationsUntilFailure) {
AddMockedTests("Test", {"firstTest"});
SetUpExpectCalls();
// Unless --gtest-break-on-failure is specified,
command_line->AppendSwitchASCII("gtest_repeat", "4");
command_line->AppendSwitch("gtest_break_on_failure");
using ::testing::_;
EXPECT_CALL(test_launcher, LaunchChildGTestProcess(_, _, _, _))
.WillOnce(::testing::DoAll(OnTestResult(&test_launcher, "Test.firstTest",
TestResult::TEST_SUCCESS)))
.WillOnce(::testing::DoAll(OnTestResult(&test_launcher, "Test.firstTest",
TestResult::TEST_SUCCESS)))
.WillOnce(::testing::DoAll(OnTestResult(&test_launcher, "Test.firstTest",
TestResult::TEST_FAILURE)));
EXPECT_FALSE(test_launcher.Run(command_line.get()));
}
// Test TestLauncher will retry failed test, and stop on success.
TEST_F(TestLauncherTest, SuccessOnRetryTests) {
AddMockedTests("Test", {"firstTest"});
......@@ -516,6 +535,8 @@ TEST_F(TestLauncherTest, JsonSummary) {
FilePath path = dir.GetPath().AppendASCII("SaveSummaryResult.json");
command_line->AppendSwitchPath("test-launcher-summary-output", path);
command_line->AppendSwitchASCII("gtest_repeat", "2");
// Force the repeats to run sequentially.
command_line->AppendSwitch("gtest_break_on_failure");
// Setup results to be returned by the test launcher delegate.
TestResult first_result =
......
base/test/launcher/unit_test_launcher.cc
View file @ 264b87ff
......@@ -49,68 +49,82 @@ const size_t kDefaultTestBatchLimit = 10;
#if !defined(OS_ANDROID)
void PrintUsage() {
fprintf(stdout,
"Runs tests using the gtest framework, each batch of tests being\n"
"run in their own process. Supported command-line flags:\n"
"\n"
" Common flags:\n"
" --gtest_filter=...\n"
" Runs a subset of tests (see --gtest_help for more info).\n"
"\n"
" --help\n"
" Shows this message.\n"
"\n"
" --gtest_help\n"
" Shows the gtest help message.\n"
"\n"
" --test-launcher-jobs=N\n"
" Sets the number of parallel test jobs to N.\n"
"\n"
" --single-process-tests\n"
" Runs the tests and the launcher in the same process. Useful\n"
" for debugging a specific test in a debugger.\n"
"\n"
" Other flags:\n"
" --test-launcher-filter-file=PATH\n"
" Like --gtest_filter, but read the test filter from PATH.\n"
" Supports multiple filter paths separated by ';'.\n"
" One pattern per line; lines starting with '-' are exclusions.\n"
" See also //testing/buildbot/filters/README.md file.\n"
"\n"
" --test-launcher-batch-limit=N\n"
" Sets the limit of test batch to run in a single process to N.\n"
"\n"
" --test-launcher-debug-launcher\n"
" Disables autodetection of debuggers and similar tools,\n"
" making it possible to use them to debug launcher itself.\n"
"\n"
" --test-launcher-retry-limit=N\n"
" Sets the limit of test retries on failures to N.\n"
"\n"
" --test-launcher-summary-output=PATH\n"
" Saves a JSON machine-readable summary of the run.\n"
"\n"
" --test-launcher-print-test-stdio=auto|always|never\n"
" Controls when full test output is printed.\n"
" auto means to print it when the test failed.\n"
"\n"
" --test-launcher-test-part-results-limit=N\n"
" Sets the limit of failed EXPECT/ASSERT entries in the xml and\n"
" JSON outputs per test to N (default N=10). Negative value \n"
" will disable this limit.\n"
"\n"
" --test-launcher-total-shards=N\n"
" Sets the total number of shards to N.\n"
"\n"
" --test-launcher-shard-index=N\n"
" Sets the shard index to run to N (from 0 to TOTAL - 1).\n"
"\n"
" --dont-use-job-objects\n"
" Avoids using job objects in Windows.\n"
"\n"
" --test-launcher-print-temp-leaks\n"
" Prints information about leaked files and/or directories in\n"
" child process's temporary directories (Windows and macOS).\n");
fprintf(
stdout,
"Runs tests using the gtest framework, each batch of tests being\n"
"run in their own process. Supported command-line flags:\n"
"\n"
" Common flags:\n"
" --gtest_filter=...\n"
" Runs a subset of tests (see --gtest_help for more info).\n"
"\n"
" --help\n"
" Shows this message.\n"
"\n"
" --gtest_help\n"
" Shows the gtest help message.\n"
"\n"
" --test-launcher-jobs=N\n"
" Sets the number of parallel test jobs to N.\n"
"\n"
" --single-process-tests\n"
" Runs the tests and the launcher in the same process. Useful\n"
" for debugging a specific test in a debugger.\n"
"\n"
" Other flags:\n"
" --test-launcher-filter-file=PATH\n"
" Like --gtest_filter, but read the test filter from PATH.\n"
" Supports multiple filter paths separated by ';'.\n"
" One pattern per line; lines starting with '-' are exclusions.\n"
" See also //testing/buildbot/filters/README.md file.\n"
"\n"
" --test-launcher-batch-limit=N\n"
" Sets the limit of test batch to run in a single process to N.\n"
"\n"
" --test-launcher-debug-launcher\n"
" Disables autodetection of debuggers and similar tools,\n"
" making it possible to use them to debug launcher itself.\n"
"\n"
" --test-launcher-retry-limit=N\n"
" Sets the limit of test retries on failures to N.\n"
" --gtest-repeat=N\n"
" Forces the launcher to run every test N times. -1 is a special"
" value, causing the infinite amount of iterations."
" Repeated tests are run in parallel, unless the number of"
" iterations is infinite or --gtest-break-on-failure is specified"
" (see below)."
" Consider using --test_launcher-jobs flag to speed up the"
" parallel execution."
"\n"
" --gtest-break-on-failure\n"
" Stop running repeated tests as soon as one repeat of the test fails."
" This flag forces sequential repeats and prevents parallelised"
" execution."
"\n"
" --test-launcher-summary-output=PATH\n"
" Saves a JSON machine-readable summary of the run.\n"
"\n"
" --test-launcher-print-test-stdio=auto|always|never\n"
" Controls when full test output is printed.\n"
" auto means to print it when the test failed.\n"
"\n"
" --test-launcher-test-part-results-limit=N\n"
" Sets the limit of failed EXPECT/ASSERT entries in the xml and\n"
" JSON outputs per test to N (default N=10). Negative value \n"
" will disable this limit.\n"
"\n"
" --test-launcher-total-shards=N\n"
" Sets the total number of shards to N.\n"
"\n"
" --test-launcher-shard-index=N\n"
" Sets the shard index to run to N (from 0 to TOTAL - 1).\n"
"\n"
" --dont-use-job-objects\n"
" Avoids using job objects in Windows.\n"
"\n"
" --test-launcher-print-temp-leaks\n"
" Prints information about leaked files and/or directories in\n"
" child process's temporary directories (Windows and macOS).\n");
fflush(stdout);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment