Extend TestLauncher unit tests code coverage.

Moved unit test delegate to general test launcher tests
to share results file parsing.

Added test for unit test delegate and test launcher to
launch 3 tests, and validate their results.

Windows versions prior to Windows 8 required a new flag
'dont-use-job-objects' to avoid unsupported nested job objects.

This test is more comprehensive, and also tests
the gtest xml results parsing of the delegate.

Bug: 936244
Change-Id: I28afe983f8960c366f9b61a22c05103b32fafaa2
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1732885
Commit-Queue: Ilia Samsonov <isamsonov@google.com>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Erik Chen <erikchen@chromium.org>
Cr-Commit-Position: refs/heads/master@{#685211}

base/BUILD.gn

@@ -2732,7 +2732,6 @@ test("base_unittests") {
     "template_util_unittest.cc",
     "test/gmock_callback_support_unittest.cc",
     "test/launcher/test_launcher_unittest.cc",
-    "test/launcher/unit_test_launcher_unittest.cc",
     "test/metrics/histogram_enum_reader_unittest.cc",
     "test/metrics/histogram_tester_unittest.cc",
     "test/metrics/user_action_tester_unittest.cc",
@@ -2910,9 +2909,12 @@ test("base_unittests") {
       add_unwind_tables_in_apk = true
       sources += [ "trace_event/cfi_backtrace_android_unittest.cc" ]
     }
+
+    # Android does not use test_launcher to run gtests.
     sources -= [
       "process/process_unittest.cc",
       "process/process_util_unittest.cc",
+      "test/launcher/test_launcher_unittest.cc",
     ]
     deps += [
       ":base_java",
@@ -2922,6 +2924,7 @@ test("base_unittests") {
   }
 
   if (is_ios) {
+    # ios does not use test_launcher to run gtests.
     sources -= [
       "files/file_path_watcher_unittest.cc",
       "memory/discardable_shared_memory_unittest.cc",
@@ -2932,7 +2935,6 @@ test("base_unittests") {
       "sync_socket_unittest.cc",
       "synchronization/waitable_event_watcher_unittest.cc",
       "test/launcher/test_launcher_unittest.cc",
-      "test/launcher/unit_test_launcher_unittest.cc",
     ]
 
     # Pull in specific Mac files for iOS (which have been filtered out by file

base/test/launcher/test_launcher_unittest.cc

@@ -11,14 +11,21 @@
 #include "base/files/scoped_temp_dir.h"
 #include "base/json/json_reader.h"
 #include "base/json/json_writer.h"
+#include "base/process/launch.h"
+#include "base/strings/string_split.h"
 #include "base/test/launcher/test_launcher.h"
 #include "base/test/launcher/unit_test_launcher.h"
 #include "base/test/scoped_task_environment.h"
 #include "base/test/test_timeouts.h"
 #include "base/threading/thread_task_runner_handle.h"
+#include "build/build_config.h"
 #include "testing/gmock/include/gmock/gmock.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
+#if defined(OS_WIN)
+#include "base/win/windows_version.h"
+#endif
+
 namespace base {
 namespace {
 
@@ -101,7 +108,6 @@ class TestLauncherTest : public testing::Test {
         scoped_task_environment(
             base::test::ScopedTaskEnvironment::MainThreadType::IO) {}
 
-
   // Adds tests to be returned by the delegate.
   void AddMockedTests(std::string test_case_name,
                       const std::vector<std::string>& test_names) {
@@ -137,20 +143,11 @@ class TestLauncherTest : public testing::Test {
         .WillRepeatedly(testing::Return(batch_size));
   }
 
-  void ReadSummary(FilePath path) {
-    File resultFile(path, File::FLAG_OPEN | File::FLAG_READ);
-    const int size = 2048;
-    std::string json;
-    ASSERT_TRUE(ReadFileToStringWithMaxSize(path, &json, size));
-    root = JSONReader::Read(json);
-  }
-
   std::unique_ptr<CommandLine> command_line;
   MockTestLauncher test_launcher;
   MockTestLauncherDelegate delegate;
   base::test::ScopedTaskEnvironment scoped_task_environment;
   ScopedTempDir dir;
-  Optional<Value> root;
 
  private:
   std::vector<TestIdentifier> tests_;
@@ -469,6 +466,24 @@ void ValidateTestResult(Value* root, TestResult& result) {
   }
 }
 
+// Validate vital information of actual test results.
+void ValidateTestResult(Value* root,
+                        const std::string& test_name,
+                        const std::string& status,
+                        size_t result_part_count) {
+  Value* val = root->FindListKey(test_name);
+  ASSERT_TRUE(val) << test_name;
+  ASSERT_EQ(1u, val->GetList().size());
+  val = &val->GetList().at(0);
+  ASSERT_TRUE(val->is_dict()) << test_name;
+
+  ValidateKeyValue(val, "status", status);
+
+  Value* value = val->FindListKey("result_parts");
+  ASSERT_TRUE(value) << test_name;
+  EXPECT_EQ(result_part_count, value->GetList().size()) << test_name;
+}
+
 void ValidateStringList(Optional<Value>& root,
                         const std::string& key,
                         std::vector<const char*> values) {
@@ -492,6 +507,16 @@ void ValidateTestLocation(Value* root,
   EXPECT_EQ(line, val->FindIntKey("line").value_or(0));
 }
 
+Optional<Value> ReadSummary(FilePath path) {
+  Optional<Value> result;
+  File resultFile(path, File::FLAG_OPEN | File::FLAG_READ);
+  const int size = 2e7;
+  std::string json;
+  CHECK(ReadFileToStringWithMaxSize(path, &json, size));
+  result = JSONReader::Read(json);
+  return result;
+}
+
 // Unit tests to validate TestLauncher outputs the correct JSON file.
 TEST_F(TestLauncherTest, JsonSummary) {
   AddMockedTests("DISABLED_TestDisabled", {"firstTest"});
@@ -523,7 +548,8 @@ TEST_F(TestLauncherTest, JsonSummary) {
   EXPECT_TRUE(test_launcher.Run(command_line.get()));
 
   // Validate the resulting JSON file is the expected output.
-  ReadSummary(path);
+  Optional<Value> root = ReadSummary(path);
+  ASSERT_TRUE(root);
   ValidateStringList(root, "all_tests",
                      {"Test.firstTest", "Test.firstTestDisabled",
                       "Test.secondTest", "TestDisabled.firstTest"});
@@ -571,7 +597,8 @@ TEST_F(TestLauncherTest, JsonSummaryWithDisabledTests) {
   EXPECT_TRUE(test_launcher.Run(command_line.get()));
 
   // Validate the resulting JSON file is the expected output.
-  ReadSummary(path);
+  Optional<Value> root = ReadSummary(path);
+  ASSERT_TRUE(root);
   Value* val = root->FindDictKey("test_locations");
   ASSERT_TRUE(val);
   EXPECT_EQ(1u, val->DictSize());
@@ -590,6 +617,117 @@ TEST_F(TestLauncherTest, JsonSummaryWithDisabledTests) {
   ValidateTestResult(iteration_val, test_result);
 }
 
+// Unit tests to validate UnitTestLauncherDelegate implementation.
+class UnitTestLauncherDelegateTester : public testing::Test {
+ protected:
+  DefaultUnitTestPlatformDelegate defaultPlatform;
+  ScopedTempDir dir;
+
+ private:
+  base::test::ScopedTaskEnvironment scoped_task_environment;
+};
+
+// Validate delegate produces correct command line.
+TEST_F(UnitTestLauncherDelegateTester, GetCommandLine) {
+  UnitTestLauncherDelegate launcher_delegate(&defaultPlatform, 10u, true);
+  TestLauncherDelegate* delegate_ptr = &launcher_delegate;
+
+  std::vector<std::string> test_names(5, "Tests");
+  base::FilePath temp_dir;
+  base::FilePath result_file;
+  CreateNewTempDirectory(FilePath::StringType(), &temp_dir);
+
+  CommandLine cmd_line =
+      delegate_ptr->GetCommandLine(test_names, temp_dir, &result_file);
+  EXPECT_TRUE(cmd_line.HasSwitch("single-process-tests"));
+  EXPECT_EQ(cmd_line.GetSwitchValuePath("test-launcher-output"), result_file);
+
+  const int size = 2048;
+  std::string content;
+  ASSERT_TRUE(ReadFileToStringWithMaxSize(
+      cmd_line.GetSwitchValuePath("gtest_flagfile"), &content, size));
+  EXPECT_EQ(content.find("--gtest_filter="), 0u);
+  base::ReplaceSubstringsAfterOffset(&content, 0, "--gtest_filter=", "");
+  std::vector<std::string> gtest_filter_tests =
+      SplitString(content, ":", TRIM_WHITESPACE, SPLIT_WANT_ALL);
+  ASSERT_EQ(gtest_filter_tests.size(), test_names.size());
+  for (unsigned i = 0; i < test_names.size(); i++) {
+    EXPECT_EQ(gtest_filter_tests.at(i), test_names.at(i));
+  }
+}
+
+// Validate delegate sets batch size correctly.
+TEST_F(UnitTestLauncherDelegateTester, BatchSize) {
+  UnitTestLauncherDelegate launcher_delegate(&defaultPlatform, 15u, true);
+  TestLauncherDelegate* delegate_ptr = &launcher_delegate;
+  EXPECT_EQ(delegate_ptr->GetBatchSize(), 15u);
+}
+
+// The following 3 tests are disabled as they are meant to only run from
+// |RunMockTests| to validate tests launcher output for known results.
+
+// Basic Test to pass
+TEST(MockUnitTests, DISABLED_PassTest) {
+  ASSERT_TRUE(true);
+}
+// Basic Test to fail
+TEST(MockUnitTests, DISABLED_FailTest) {
+  ASSERT_TRUE(false);
+}
+// Basic Test to crash
+TEST(MockUnitTests, DISABLED_CrashTest) {
+  CHECK(false);
+}
+
+// Using UnitTestLauncherDelegate and TestLauncher (with minor mocks)
+// to launch 3 basic tests, and validate the resulting json file.
+TEST_F(UnitTestLauncherDelegateTester, RunMockTests) {
+  CommandLine command_line(CommandLine::ForCurrentProcess()->GetProgram());
+  command_line.AppendSwitchASCII("gtest_filter", "MockUnitTests.DISABLED_*");
+
+  ASSERT_TRUE(dir.CreateUniqueTempDir());
+  FilePath path = dir.GetPath().AppendASCII("SaveSummaryResult.json");
+  command_line.AppendSwitchPath("test-launcher-summary-output", path);
+  command_line.AppendSwitch("gtest_also_run_disabled_tests");
+#if defined(OS_WIN)
+  // In Windows versions prior to Windows 8, nested job objects are
+  // not allowed and cause this test to fail.
+  if (win::GetVersion() < win::Version::WIN8) {
+    command_line.AppendSwitch(kDontUseJobObjectFlag);
+  }
+#endif  // defined(OS_WIN)
+
+  std::string output;
+  GetAppOutputAndError(command_line, &output);
+
+  // Validate the resulting JSON file is the expected output.
+  Optional<Value> root = ReadSummary(path);
+  ASSERT_TRUE(root);
+
+  Value* val = root->FindDictKey("test_locations");
+  ASSERT_TRUE(val);
+  EXPECT_EQ(3u, val->DictSize());
+  // If path or test location changes, the following expectation
+  // will need to change accordingly.
+  std::string file_name = "../../base/test/launcher/test_launcher_unittest.cc";
+  ValidateTestLocation(val, "MockUnitTests.DISABLED_PassTest", file_name, 670);
+  ValidateTestLocation(val, "MockUnitTests.DISABLED_FailTest", file_name, 674);
+  ValidateTestLocation(val, "MockUnitTests.DISABLED_CrashTest", file_name, 678);
+
+  val = root->FindListKey("per_iteration_data");
+  ASSERT_TRUE(val);
+  ASSERT_EQ(1u, val->GetList().size());
+
+  Value* iteration_val = &(val->GetList().at(0));
+  ASSERT_TRUE(iteration_val);
+  ASSERT_TRUE(iteration_val->is_dict());
+  EXPECT_EQ(3u, iteration_val->DictSize());
+  // We expect the result to be stripped of disabled prefix.
+  ValidateTestResult(iteration_val, "MockUnitTests.PassTest", "SUCCESS", 0u);
+  ValidateTestResult(iteration_val, "MockUnitTests.FailTest", "FAILURE", 1u);
+  ValidateTestResult(iteration_val, "MockUnitTests.CrashTest", "CRASH", 1u);
+}
+
 }  // namespace
 
 }  // namespace base

base/test/launcher/unit_test_launcher.cc

@@ -106,7 +106,10 @@ void PrintUsage() {
           "    Sets the total number of shards to N.\n"
           "\n"
           "  --test-launcher-shard-index=N\n"
-          "    Sets the shard index to run to N (from 0 to TOTAL - 1).\n");
+          "    Sets the shard index to run to N (from 0 to TOTAL - 1).\n"
+          "\n"
+          "  --dont-use-job-objects\n"
+          "    Avoids using job objects in Windows.\n");
   fflush(stdout);
 }
 
@@ -184,10 +187,13 @@ int LaunchUnitTestsInternal(RunTestSuiteCallback run_test_suite,
 #if defined(OS_POSIX)
   FileDescriptorWatcher file_descriptor_watcher(message_loop.task_runner());
 #endif
+  use_job_objects =
+      use_job_objects &&
+      !CommandLine::ForCurrentProcess()->HasSwitch(kDontUseJobObjectFlag);
 
   DefaultUnitTestPlatformDelegate platform_delegate;
-  UnitTestLauncherDelegate delegate(
-      &platform_delegate, batch_limit, use_job_objects);
+  UnitTestLauncherDelegate delegate(&platform_delegate, batch_limit,
+                                    use_job_objects);
   TestLauncher launcher(&delegate, parallel_jobs);
   bool success = launcher.Run();
 
@@ -343,6 +349,9 @@ std::vector<TestResult> UnitTestProcessTestResults(
 
 }  // namespace
 
+// Flag to avoid using job objects
+const char kDontUseJobObjectFlag[] = "dont-use-job-objects";
+
 int LaunchUnitTests(int argc,
                     char** argv,
                     RunTestSuiteCallback run_test_suite) {
@@ -450,8 +459,7 @@ UnitTestLauncherDelegate::UnitTestLauncherDelegate(
     bool use_job_objects)
     : platform_delegate_(platform_delegate),
       batch_limit_(batch_limit),
-      use_job_objects_(use_job_objects) {
-}
+      use_job_objects_(use_job_objects) {}
 
 UnitTestLauncherDelegate::~UnitTestLauncherDelegate() {
   DCHECK(thread_checker_.CalledOnValidThread());

base/test/launcher/unit_test_launcher.h

@@ -19,6 +19,8 @@
 
 namespace base {
 
+extern const char kDontUseJobObjectFlag[];
+
 // Callback that runs a test suite and returns exit code.
 using RunTestSuiteCallback = OnceCallback<int(void)>;
 

base/test/launcher/unit_test_launcher_unittest.cc

-// Copyright 2019 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#include <stddef.h>
-
-#include "base/command_line.h"
-#include "base/files/file_util.h"
-#include "base/run_loop.h"
-#include "base/strings/string_split.h"
-#include "base/strings/string_util.h"
-#include "base/test/gtest_util.h"
-#include "base/test/launcher/test_launcher.h"
-#include "base/test/launcher/unit_test_launcher.h"
-#include "base/test/scoped_task_environment.h"
-#include "base/test/test_switches.h"
-#include "testing/gmock/include/gmock/gmock.h"
-#include "testing/gtest/include/gtest/gtest.h"
-
-namespace base {
-namespace {
-// Mock TestLauncher to validate LaunchChildGTestProcess
-// is called correctly inside the test launcher delegate.
-class MockTestLauncher : public TestLauncher {
- public:
-  MockTestLauncher(TestLauncherDelegate* launcher_delegate,
-                   size_t parallel_jobs)
-      : TestLauncher(launcher_delegate, parallel_jobs) {}
-
-  MOCK_METHOD3(LaunchChildGTestProcess,
-               void(scoped_refptr<TaskRunner> task_runner,
-                    const std::vector<std::string>& test_names,
-                    const FilePath& temp_dir));
-};
-
-// Unit tests to validate UnitTestLauncherDelegateTester implementations.
-class UnitTestLauncherDelegateTester : public testing::Test {
- protected:
-  DefaultUnitTestPlatformDelegate defaultPlatform;
-};
-
-// Validate delegate produces correct command line.
-TEST_F(UnitTestLauncherDelegateTester, GetCommandLine) {
-  UnitTestLauncherDelegate launcher_delegate(&defaultPlatform, 10u, true);
-  TestLauncherDelegate* delegate_ptr = &launcher_delegate;
-
-  std::vector<std::string> test_names(5, "Tests");
-  base::FilePath temp_dir;
-  base::FilePath result_file;
-  CreateNewTempDirectory(FilePath::StringType(), &temp_dir);
-
-  CommandLine cmd_line =
-      delegate_ptr->GetCommandLine(test_names, temp_dir, &result_file);
-  EXPECT_TRUE(cmd_line.HasSwitch("single-process-tests"));
-  EXPECT_EQ(cmd_line.GetSwitchValuePath("test-launcher-output"), result_file);
-
-  const int size = 2048;
-  std::string content;
-  ASSERT_TRUE(ReadFileToStringWithMaxSize(
-      cmd_line.GetSwitchValuePath("gtest_flagfile"), &content, size));
-  EXPECT_EQ(content.find("--gtest_filter="), 0u);
-  base::ReplaceSubstringsAfterOffset(&content, 0, "--gtest_filter=", "");
-  std::vector<std::string> gtest_filter_tests =
-      SplitString(content, ":", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
-  ASSERT_EQ(gtest_filter_tests.size(), test_names.size());
-  for (unsigned i = 0; i < test_names.size(); i++) {
-    EXPECT_EQ(gtest_filter_tests.at(i), test_names.at(i));
-  }
-}
-
-// Validate delegate sets batch size correctly.
-TEST_F(UnitTestLauncherDelegateTester, BatchSize) {
-  UnitTestLauncherDelegate launcher_delegate(&defaultPlatform, 15u, true);
-  TestLauncherDelegate* delegate_ptr = &launcher_delegate;
-  EXPECT_EQ(delegate_ptr->GetBatchSize(), 15u);
-}
-
-}  // namespace
-
-}  // namespace base