Add --isolated-script-test-* to gtest launcher

As part of the effort to make all test harness supported by Findit,
this CL adds four new flags that are unified across different test harness:
1. --isolated-script-test-filter=test1::test2::test3
2. --isolated-script-test-repeat=30
3. --isolated-script-test-launcher-retry-limit=0
4. --isolated-script-test-also-run-disabled-tests

For test filter, both ":" and "::" are supported.

Bug: 836417
Change-Id: Ie931812ade9899a2b2ae1a97bdac215ed9e0ea73
Reviewed-on: https://chromium-review.googlesource.com/c/1272655
Commit-Queue: Shuotao Gao <stgao@chromium.org>
Reviewed-by: Joe Downing <joedow@chromium.org>
Reviewed-by: Lei Zhang <thestig@chromium.org>
Reviewed-by: Chan Li <chanli@chromium.org>
Cr-Commit-Position: refs/heads/master@{#600063}

base/test/launcher/test_launcher.cc

@@ -252,6 +252,7 @@ CommandLine PrepareCommandLineForGTest(const CommandLine& command_line,
 
   // Handled by the launcher process.
   switches.erase(kGTestRepeatFlag);
+  switches.erase(kIsolatedScriptTestRepeatFlag);
   switches.erase(kGTestShuffleFlag);
   switches.erase(kGTestRandomSeedFlag);
 
@@ -568,6 +569,20 @@ void DoLaunchChildTestProcess(
                output_file_contents));
 }
 
+std::vector<std::string> ExtractTestsFromFilter(const std::string& filter,
+                                                bool double_colon_supported) {
+  std::vector<std::string> tests;
+  if (double_colon_supported) {
+    tests =
+        SplitString(filter, "::", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
+  }
+  if (tests.size() <= 1) {
+    tests =
+        SplitString(filter, ":", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
+  }
+  return tests;
+}
+
 }  // namespace
 
 const char kGTestBreakOnFailure[] = "gtest_break_on_failure";
@@ -580,6 +595,10 @@ const char kGTestRunDisabledTestsFlag[] = "gtest_also_run_disabled_tests";
 const char kGTestOutputFlag[] = "gtest_output";
 const char kGTestShuffleFlag[] = "gtest_shuffle";
 const char kGTestRandomSeedFlag[] = "gtest_random_seed";
+const char kIsolatedScriptRunDisabledTestsFlag[] =
+    "isolated-script-test-also-run-disabled-tests";
+const char kIsolatedScriptTestFilterFlag[] = "isolated-script-test-filter";
+const char kIsolatedScriptTestRepeatFlag[] = "isolated-script-test-repeat";
 
 TestLauncherDelegate::~TestLauncherDelegate() = default;
 
@@ -946,6 +965,13 @@ bool TestLauncher::Init() {
     LOG(ERROR) << "Invalid value for " << kGTestRepeatFlag;
     return false;
   }
+  if (command_line->HasSwitch(kIsolatedScriptTestRepeatFlag) &&
+      !StringToInt(
+          command_line->GetSwitchValueASCII(kIsolatedScriptTestRepeatFlag),
+          &cycles_)) {
+    LOG(ERROR) << "Invalid value for " << kIsolatedScriptTestRepeatFlag;
+    return false;
+  }
 
   if (command_line->HasSwitch(switches::kTestLauncherRetryLimit)) {
     int retry_limit = -1;
@@ -957,7 +983,22 @@ bool TestLauncher::Init() {
     }
 
     retry_limit_ = retry_limit;
-  } else if (!command_line->HasSwitch(kGTestFilterFlag) || BotModeEnabled()) {
+  } else if (command_line->HasSwitch(
+                 switches::kIsolatedScriptTestLauncherRetryLimit)) {
+    int retry_limit = -1;
+    if (!StringToInt(command_line->GetSwitchValueASCII(
+                         switches::kIsolatedScriptTestLauncherRetryLimit),
+                     &retry_limit) ||
+        retry_limit < 0) {
+      LOG(ERROR) << "Invalid value for "
+                 << switches::kIsolatedScriptTestLauncherRetryLimit;
+      return false;
+    }
+
+    retry_limit_ = retry_limit;
+  } else if (BotModeEnabled() ||
+             !(command_line->HasSwitch(kGTestFilterFlag) ||
+               command_line->HasSwitch(kIsolatedScriptTestFilterFlag))) {
     // Retry failures 3 times by default if we are running all of the tests or
     // in bot mode.
     retry_limit_ = 3;
@@ -1022,21 +1063,22 @@ bool TestLauncher::Init() {
 
   // Split --gtest_filter at '-', if there is one, to separate into
   // positive filter and negative filter portions.
-  std::string filter = command_line->GetSwitchValueASCII(kGTestFilterFlag);
+  bool double_colon_supported = !command_line->HasSwitch(kGTestFilterFlag);
+  std::string filter = command_line->GetSwitchValueASCII(
+      double_colon_supported ? kIsolatedScriptTestFilterFlag
+                             : kGTestFilterFlag);
   size_t dash_pos = filter.find('-');
   if (dash_pos == std::string::npos) {
     positive_gtest_filter =
-        SplitString(filter, ":", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
+        ExtractTestsFromFilter(filter, double_colon_supported);
   } else {
     // Everything up to the dash.
-    positive_gtest_filter =
-        SplitString(filter.substr(0, dash_pos), ":", base::TRIM_WHITESPACE,
-                    base::SPLIT_WANT_ALL);
+    positive_gtest_filter = ExtractTestsFromFilter(filter.substr(0, dash_pos),
+                                                   double_colon_supported);
 
     // Everything after the dash.
-    for (std::string pattern :
-         SplitString(filter.substr(dash_pos + 1), ":", base::TRIM_WHITESPACE,
-                     base::SPLIT_WANT_ALL)) {
+    for (std::string pattern : ExtractTestsFromFilter(
+             filter.substr(dash_pos + 1), double_colon_supported)) {
       negative_test_filter_.push_back(pattern);
     }
   }
@@ -1168,7 +1210,8 @@ void TestLauncher::RunTests() {
       results_tracker_.AddDisabledTest(test_name);
 
       // Skip disabled tests unless explicitly requested.
-      if (!command_line->HasSwitch(kGTestRunDisabledTestsFlag))
+      if (!command_line->HasSwitch(kGTestRunDisabledTestsFlag) &&
+          !command_line->HasSwitch(kIsolatedScriptRunDisabledTestsFlag))
         continue;
     }
 
@@ -1366,7 +1409,9 @@ size_t NumParallelJobs() {
     }
     return jobs;
   }
-  if (command_line->HasSwitch(kGTestFilterFlag) && !BotModeEnabled()) {
+  if (!BotModeEnabled() &&
+      (command_line->HasSwitch(kGTestFilterFlag) ||
+       command_line->HasSwitch(kIsolatedScriptTestFilterFlag))) {
     // Do not run jobs in parallel by default if we are running a subset of
     // the tests and if bot mode is off.
     return 1U;

base/test/launcher/test_launcher.h

@@ -40,6 +40,9 @@ extern const char kGTestRunDisabledTestsFlag[];
 extern const char kGTestOutputFlag[];
 extern const char kGTestShuffleFlag[];
 extern const char kGTestRandomSeedFlag[];
+extern const char kIsolatedScriptRunDisabledTestsFlag[];
+extern const char kIsolatedScriptTestFilterFlag[];
+extern const char kIsolatedScriptTestRepeatFlag[];
 
 // Interface for use with LaunchTests that abstracts away exact details
 // which tests and how are run.

base/test/test_switches.cc

@@ -40,8 +40,12 @@ const char switches::kTestLauncherListTests[] = "test-launcher-list-tests";
 // Path to test results file in our custom test launcher format.
 const char switches::kTestLauncherOutput[] = "test-launcher-output";
 
+// These two flags has the same effect, but don't use them at the same time.
+// And isolated-script-test-launcher-retry-limit is preferred in the future.
 // Maximum number of times to retry a test after failure.
 const char switches::kTestLauncherRetryLimit[] = "test-launcher-retry-limit";
+const char switches::kIsolatedScriptTestLauncherRetryLimit[] =
+    "isolated-script-test-launcher-retry-limit";
 
 // Path to test results file with all the info from the test launcher.
 const char switches::kTestLauncherSummaryOutput[] =

base/test/test_switches.h

@@ -19,6 +19,7 @@ extern const char kTestLauncherJobs[];
 extern const char kTestLauncherListTests[];
 extern const char kTestLauncherOutput[];
 extern const char kTestLauncherRetryLimit[];
+extern const char kIsolatedScriptTestLauncherRetryLimit[];
 extern const char kTestLauncherSummaryOutput[];
 extern const char kTestLauncherPrintTestStdio[];
 extern const char kTestLauncherPrintWritablePath[];

remoting/test/app_remoting_test_driver.cc

@@ -164,6 +164,8 @@ int main(int argc, char** argv) {
   // We do not want to retry failures as a failed test should signify an error
   // to be investigated.
   command_line->AppendSwitchASCII(switches::kTestLauncherRetryLimit, "0");
+  command_line->AppendSwitchASCII(
+      switches::kIsolatedScriptTestLauncherRetryLimit, "0");
 
   // We do not want to run the tests in parallel and we do not want to retry
   // failures.  The reason for running in a single process is that some tests

remoting/test/chromoting_test_driver.cc

@@ -165,6 +165,8 @@ int main(int argc, char* argv[]) {
 
   // Do not retry if tests fails.
   command_line->AppendSwitchASCII(switches::kTestLauncherRetryLimit, "0");
+  command_line->AppendSwitchASCII(
+      switches::kIsolatedScriptTestLauncherRetryLimit, "0");
 
   // Different tests may require access to the same host if run in parallel.
   // To avoid shared resource contention, tests will be run one at a time.