Added code coverage for browser tests launcher.

Added tests similar to TestLauncher unittests to run
3 simple content tests and validate the TestLauncher json result file.

Common methods for validating resulting json file were
moved to test_launcher_test_utils.h/cc.

Bug: 936244
Change-Id: I8dee29f80f82ca5981f7343e9ffec4cacc95eeaa
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1743099Reviewed-by: Scott Violet <sky@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Erik Chen <erikchen@chromium.org>
Commit-Queue: Ilia Samsonov <isamsonov@google.com>
Cr-Commit-Position: refs/heads/master@{#688321}

base/test/BUILD.gn

@@ -170,6 +170,8 @@ static_library("test_support") {
     sources += [
       "launcher/test_launcher.cc",
       "launcher/test_launcher.h",
+      "launcher/test_launcher_test_utils.cc",
+      "launcher/test_launcher_test_utils.h",
       "launcher/test_launcher_tracer.cc",
       "launcher/test_launcher_tracer.h",
       "launcher/test_results_tracker.cc",

base/test/launcher/test_launcher_test_utils.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/test/launcher/test_launcher_test_utils.h"
+
+#include "base/files/file_util.h"
+#include "base/json/json_reader.h"
+#include "base/json/json_writer.h"
+#include "base/optional.h"
+#include "base/test/launcher/test_result.h"
+
+namespace base {
+
+namespace test_launcher_utils {
+
+namespace {
+
+// Helper function to return |Value::FindStringKey| by value instead of
+// pointer to string, or empty string if nullptr.
+std::string FindStringKeyOrEmpty(const Value& dict_value,
+                                 const std::string& key) {
+  const std::string* value = dict_value.FindStringKey(key);
+  return value ? *value : std::string();
+}
+
+}  // namespace
+
+bool ValidateKeyValue(const Value& dict_value,
+                      const std::string& key,
+                      const std::string& expected_value) {
+  std::string actual_value = FindStringKeyOrEmpty(dict_value, key);
+  bool result = !actual_value.compare(expected_value);
+  if (!result)
+    ADD_FAILURE() << key << " expected value: " << expected_value
+                  << ", actual: " << actual_value;
+  return result;
+}
+
+bool ValidateKeyValue(const Value& dict_value,
+                      const std::string& key,
+                      int64_t expected_value) {
+  int actual_value = dict_value.FindIntKey(key).value_or(0);
+  bool result = (actual_value == expected_value);
+  if (!result)
+    ADD_FAILURE() << key << " expected value: " << expected_value
+                  << ", actual: " << actual_value;
+  return result;
+}
+
+bool ValidateTestResult(const Value* iteration_data,
+                        const std::string& test_name,
+                        const std::string& status,
+                        size_t result_part_count) {
+  const Value* results = iteration_data->FindListKey(test_name);
+  if (!results) {
+    ADD_FAILURE() << "Cannot find result";
+    return false;
+  }
+  if (1u != results->GetList().size()) {
+    ADD_FAILURE() << "Expected one result";
+    return false;
+  }
+
+  const Value& val = results->GetList().at(0);
+  if (!val.is_dict()) {
+    ADD_FAILURE() << "Value must be of type DICTIONARY";
+    return false;
+  }
+
+  if (!ValidateKeyValue(val, "status", status))
+    return false;
+
+  const Value* value = val.FindListKey("result_parts");
+  if (!value) {
+    ADD_FAILURE() << "Result must contain 'result_parts' key";
+    return false;
+  }
+
+  if (result_part_count != value->GetList().size()) {
+    ADD_FAILURE() << "result_parts count expected: " << result_part_count
+                  << ", actual:" << value->GetList().size();
+    return false;
+  }
+  return true;
+}
+
+bool ValidateTestLocation(const Value* test_locations,
+                          const std::string& test_name,
+                          const std::string& file,
+                          int line) {
+  const Value* val = test_locations->FindDictKey(test_name);
+  if (!val) {
+    ADD_FAILURE() << "|test_locations| missing location for " << test_name;
+    return false;
+  }
+
+  bool result = ValidateKeyValue(*val, "file", file);
+  result &= ValidateKeyValue(*val, "line", line);
+  return result;
+}
+
+Optional<Value> ReadSummary(const FilePath& path) {
+  Optional<Value> result;
+  File resultFile(path, File::FLAG_OPEN | File::FLAG_READ);
+  const int size = 2e7;
+  std::string json;
+  CHECK(ReadFileToStringWithMaxSize(path, &json, size));
+  result = JSONReader::Read(json);
+  return result;
+}
+
+}  // namespace test_launcher_utils
+
+}  // namespace base
\ No newline at end of file

base/test/launcher/test_launcher_test_utils.h

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef BASE_TEST_LAUNCHER_TEST_LAUNCHER_TEST_UTILS_H_
+#define BASE_TEST_LAUNCHER_TEST_LAUNCHER_TEST_UTILS_H_
+
+#include <stddef.h>
+
+#include <string>
+#include <vector>
+
+#include "base/optional.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace base {
+
+class Value;
+class FilePath;
+
+namespace test_launcher_utils {
+
+// Validate |dict_value| value in |key| is equal to |expected_value|
+bool ValidateKeyValue(const Value& dict_value,
+                      const std::string& key,
+                      const std::string& expected_value);
+
+// Validate |dict_value| value in |key| is equal to |expected_value|
+bool ValidateKeyValue(const Value& dict_value,
+                      const std::string& key,
+                      int64_t expected_value);
+
+// Validate |iteration_data| contains one test result under |test_name|
+// with |status|, and |result_part_count| number of result parts.
+bool ValidateTestResult(const Value* iteration_data,
+                        const std::string& test_name,
+                        const std::string& status,
+                        size_t result_part_count);
+
+// Validate test_locations contains the correct file name and line number.
+bool ValidateTestLocation(const Value* test_locations,
+                          const std::string& test_name,
+                          const std::string& file,
+                          int line);
+
+// Read json output file of test launcher.
+Optional<Value> ReadSummary(const FilePath& path);
+
+}  // namespace test_launcher_utils
+
+}  // namespace base
+
+#endif  // BASE_TEST_LAUNCHER_TEST_LAUNCHER_TEST_UTILS_H_
\ No newline at end of file

content/test/content_browser_test_test.cc

@@ -14,6 +14,7 @@
 #include "base/single_thread_task_runner.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/test/launcher/test_launcher.h"
+#include "base/test/launcher/test_launcher_test_utils.h"
 #include "base/test/scoped_feature_list.h"
 #include "base/test/test_timeouts.h"
 #include "base/threading/thread_restrictions.h"
@@ -143,6 +144,78 @@ IN_PROC_BROWSER_TEST_F(ContentBrowserTest, BrowserCrashCallStack) {
   }
 }
 
+// The following 3 tests are disabled as they are meant to only run from
+// |RunMockTests| to validate tests launcher output for known results.
+using MockContentBrowserTest = ContentBrowserTest;
+
+// Basic Test to pass
+IN_PROC_BROWSER_TEST_F(MockContentBrowserTest, DISABLED_PassTest) {
+  ASSERT_TRUE(true);
+}
+// Basic Test to fail
+IN_PROC_BROWSER_TEST_F(MockContentBrowserTest, DISABLED_FailTest) {
+  ASSERT_TRUE(false);
+}
+// Basic Test to crash
+IN_PROC_BROWSER_TEST_F(MockContentBrowserTest, DISABLED_CrashTest) {
+  IMMEDIATE_CRASH();
+}
+
+// Using TestLauncher to launch 3 simple browser tests
+// and validate the resulting json file.
+IN_PROC_BROWSER_TEST_F(ContentBrowserTest, RunMockTests) {
+  base::ScopedAllowBlockingForTesting allow_blocking;
+  base::ScopedTempDir temp_dir;
+
+  base::CommandLine command_line(
+      base::CommandLine::ForCurrentProcess()->GetProgram());
+  command_line.AppendSwitchASCII("gtest_filter",
+                                 "MockContentBrowserTest.DISABLED_*");
+  ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
+  base::FilePath path =
+      temp_dir.GetPath().AppendASCII("SaveSummaryResult.json");
+  command_line.AppendSwitchPath("test-launcher-summary-output", path);
+  command_line.AppendSwitch("gtest_also_run_disabled_tests");
+  command_line.AppendSwitch("--test-launcher-retry-limit=0");
+
+  std::string output;
+  base::GetAppOutputAndError(command_line, &output);
+
+  // Validate the resulting JSON file is the expected output.
+  base::Optional<base::Value> root =
+      base::test_launcher_utils::ReadSummary(path);
+  ASSERT_TRUE(root);
+
+  base::Value* val = root->FindDictKey("test_locations");
+  ASSERT_TRUE(val);
+  EXPECT_EQ(3u, val->DictSize());
+  // If path or test location changes, the following expectation
+  // will need to change accordingly.
+  std::string file_name = "../../content/test/content_browser_test_test.cc";
+  EXPECT_TRUE(base::test_launcher_utils::ValidateTestLocation(
+      val, "MockContentBrowserTest.DISABLED_PassTest", file_name, 152));
+  EXPECT_TRUE(base::test_launcher_utils::ValidateTestLocation(
+      val, "MockContentBrowserTest.DISABLED_FailTest", file_name, 156));
+  EXPECT_TRUE(base::test_launcher_utils::ValidateTestLocation(
+      val, "MockContentBrowserTest.DISABLED_CrashTest", file_name, 160));
+
+  val = root->FindListKey("per_iteration_data");
+  ASSERT_TRUE(val);
+  ASSERT_EQ(1u, val->GetList().size());
+
+  base::Value* iteration_val = &(val->GetList().at(0));
+  ASSERT_TRUE(iteration_val);
+  ASSERT_TRUE(iteration_val->is_dict());
+  EXPECT_EQ(3u, iteration_val->DictSize());
+  // We expect the result to be stripped of disabled prefix.
+  EXPECT_TRUE(base::test_launcher_utils::ValidateTestResult(
+      iteration_val, "MockContentBrowserTest.PassTest", "SUCCESS", 0u));
+  EXPECT_TRUE(base::test_launcher_utils::ValidateTestResult(
+      iteration_val, "MockContentBrowserTest.FailTest", "FAILURE", 1u));
+  EXPECT_TRUE(base::test_launcher_utils::ValidateTestResult(
+      iteration_val, "MockContentBrowserTest.CrashTest", "CRASH", 0u));
+}
+
 #endif
 
 class ContentBrowserTestSanityTest : public ContentBrowserTest {