Commit d69f6005 authored by Ilia Samsonov's avatar Ilia Samsonov Committed by Commit Bot

Added code coverage for browser tests launcher.

Added tests similar to TestLauncher unittests to run
3 simple content tests and validate the TestLauncher json result file.

Common methods for validating resulting json file were
moved to test_launcher_test_utils.h/cc.

Bug: 936244
Change-Id: I8dee29f80f82ca5981f7343e9ffec4cacc95eeaa
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1743099Reviewed-by: default avatarScott Violet <sky@chromium.org>
Reviewed-by: default avatarDaniel Cheng <dcheng@chromium.org>
Reviewed-by: default avatarErik Chen <erikchen@chromium.org>
Commit-Queue: Ilia Samsonov <isamsonov@google.com>
Cr-Commit-Position: refs/heads/master@{#688321}
parent 7e51f3dd
......@@ -170,6 +170,8 @@ static_library("test_support") {
sources += [
"launcher/test_launcher.cc",
"launcher/test_launcher.h",
"launcher/test_launcher_test_utils.cc",
"launcher/test_launcher_test_utils.h",
"launcher/test_launcher_tracer.cc",
"launcher/test_launcher_tracer.h",
"launcher/test_results_tracker.cc",
......
// Copyright 2019 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "base/test/launcher/test_launcher_test_utils.h"
#include "base/files/file_util.h"
#include "base/json/json_reader.h"
#include "base/json/json_writer.h"
#include "base/optional.h"
#include "base/test/launcher/test_result.h"
namespace base {
namespace test_launcher_utils {
namespace {
// Helper function to return |Value::FindStringKey| by value instead of
// pointer to string, or empty string if nullptr.
std::string FindStringKeyOrEmpty(const Value& dict_value,
const std::string& key) {
const std::string* value = dict_value.FindStringKey(key);
return value ? *value : std::string();
}
} // namespace
bool ValidateKeyValue(const Value& dict_value,
const std::string& key,
const std::string& expected_value) {
std::string actual_value = FindStringKeyOrEmpty(dict_value, key);
bool result = !actual_value.compare(expected_value);
if (!result)
ADD_FAILURE() << key << " expected value: " << expected_value
<< ", actual: " << actual_value;
return result;
}
bool ValidateKeyValue(const Value& dict_value,
const std::string& key,
int64_t expected_value) {
int actual_value = dict_value.FindIntKey(key).value_or(0);
bool result = (actual_value == expected_value);
if (!result)
ADD_FAILURE() << key << " expected value: " << expected_value
<< ", actual: " << actual_value;
return result;
}
bool ValidateTestResult(const Value* iteration_data,
const std::string& test_name,
const std::string& status,
size_t result_part_count) {
const Value* results = iteration_data->FindListKey(test_name);
if (!results) {
ADD_FAILURE() << "Cannot find result";
return false;
}
if (1u != results->GetList().size()) {
ADD_FAILURE() << "Expected one result";
return false;
}
const Value& val = results->GetList().at(0);
if (!val.is_dict()) {
ADD_FAILURE() << "Value must be of type DICTIONARY";
return false;
}
if (!ValidateKeyValue(val, "status", status))
return false;
const Value* value = val.FindListKey("result_parts");
if (!value) {
ADD_FAILURE() << "Result must contain 'result_parts' key";
return false;
}
if (result_part_count != value->GetList().size()) {
ADD_FAILURE() << "result_parts count expected: " << result_part_count
<< ", actual:" << value->GetList().size();
return false;
}
return true;
}
bool ValidateTestLocation(const Value* test_locations,
const std::string& test_name,
const std::string& file,
int line) {
const Value* val = test_locations->FindDictKey(test_name);
if (!val) {
ADD_FAILURE() << "|test_locations| missing location for " << test_name;
return false;
}
bool result = ValidateKeyValue(*val, "file", file);
result &= ValidateKeyValue(*val, "line", line);
return result;
}
Optional<Value> ReadSummary(const FilePath& path) {
Optional<Value> result;
File resultFile(path, File::FLAG_OPEN | File::FLAG_READ);
const int size = 2e7;
std::string json;
CHECK(ReadFileToStringWithMaxSize(path, &json, size));
result = JSONReader::Read(json);
return result;
}
} // namespace test_launcher_utils
} // namespace base
\ No newline at end of file
// Copyright 2019 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef BASE_TEST_LAUNCHER_TEST_LAUNCHER_TEST_UTILS_H_
#define BASE_TEST_LAUNCHER_TEST_LAUNCHER_TEST_UTILS_H_
#include <stddef.h>
#include <string>
#include <vector>
#include "base/optional.h"
#include "testing/gtest/include/gtest/gtest.h"
namespace base {
class Value;
class FilePath;
namespace test_launcher_utils {
// Validate |dict_value| value in |key| is equal to |expected_value|
bool ValidateKeyValue(const Value& dict_value,
const std::string& key,
const std::string& expected_value);
// Validate |dict_value| value in |key| is equal to |expected_value|
bool ValidateKeyValue(const Value& dict_value,
const std::string& key,
int64_t expected_value);
// Validate |iteration_data| contains one test result under |test_name|
// with |status|, and |result_part_count| number of result parts.
bool ValidateTestResult(const Value* iteration_data,
const std::string& test_name,
const std::string& status,
size_t result_part_count);
// Validate test_locations contains the correct file name and line number.
bool ValidateTestLocation(const Value* test_locations,
const std::string& test_name,
const std::string& file,
int line);
// Read json output file of test launcher.
Optional<Value> ReadSummary(const FilePath& path);
} // namespace test_launcher_utils
} // namespace base
#endif // BASE_TEST_LAUNCHER_TEST_LAUNCHER_TEST_UTILS_H_
\ No newline at end of file
......@@ -9,11 +9,10 @@
#include "base/command_line.h"
#include "base/files/file_util.h"
#include "base/files/scoped_temp_dir.h"
#include "base/json/json_reader.h"
#include "base/json/json_writer.h"
#include "base/process/launch.h"
#include "base/strings/string_split.h"
#include "base/test/launcher/test_launcher.h"
#include "base/test/launcher/test_launcher_test_utils.h"
#include "base/test/launcher/unit_test_launcher.h"
#include "base/test/scoped_task_environment.h"
#include "base/test/test_timeouts.h"
......@@ -422,98 +421,87 @@ TEST_F(TestLauncherTest, RedirectStdio) {
EXPECT_TRUE(test_launcher.Run(command_line.get()));
}
void ValidateKeyValue(Value* dict_value,
const std::string& key,
const std::string& expected_value) {
const std::string* value = dict_value->FindStringKey(key);
ASSERT_TRUE(value);
EXPECT_EQ(expected_value, *value);
}
// Validate |iteration_data| contains one test result matching |result|.
bool ValidateTestResultObject(const Value* iteration_data,
TestResult& test_result) {
const Value* results = iteration_data->FindListKey(test_result.full_name);
if (!results) {
ADD_FAILURE() << "Results not found";
return false;
}
if (1u != results->GetList().size()) {
ADD_FAILURE() << "Expected one result, actual: "
<< results->GetList().size();
return false;
}
const Value& val = results->GetList().at(0);
if (!val.is_dict()) {
ADD_FAILURE() << "Unexpected type";
return false;
}
// Validate a json child node for a particular test result.
void ValidateTestResult(Value* root, TestResult& result) {
Value* val = root->FindListKey(result.full_name);
ASSERT_TRUE(val);
ASSERT_EQ(1u, val->GetList().size());
val = &val->GetList().at(0);
ASSERT_TRUE(val->is_dict());
using test_launcher_utils::ValidateKeyValue;
bool result = ValidateKeyValue(val, "elapsed_time_ms",
test_result.elapsed_time.InMilliseconds());
EXPECT_EQ(result.elapsed_time.InMilliseconds(),
val->FindIntKey("elapsed_time_ms").value_or(0));
EXPECT_EQ(true, val->FindBoolKey("losless_snippet").value_or(false));
ValidateKeyValue(val, "output_snippet", result.output_snippet);
if (!val.FindBoolKey("losless_snippet").value_or(false)) {
ADD_FAILURE() << "losless_snippet expected to be true";
result = false;
}
result &= ValidateKeyValue(val, "output_snippet", test_result.output_snippet);
std::string base64_output_snippet;
Base64Encode(result.output_snippet, &base64_output_snippet);
Base64Encode(test_result.output_snippet, &base64_output_snippet);
result &=
ValidateKeyValue(val, "output_snippet_base64", base64_output_snippet);
ValidateKeyValue(val, "status", result.StatusAsString());
Value* value = val->FindListKey("result_parts");
ASSERT_TRUE(value);
EXPECT_EQ(result.test_result_parts.size(), value->GetList().size());
for (unsigned i = 0; i < result.test_result_parts.size(); i++) {
TestResultPart result_part = result.test_result_parts.at(0);
Value* part_dict = &(value->GetList().at(i));
ASSERT_TRUE(part_dict);
ASSERT_TRUE(part_dict->is_dict());
ValidateKeyValue(part_dict, "type", result_part.TypeAsString());
ValidateKeyValue(part_dict, "file", result_part.file_name);
EXPECT_EQ(result_part.line_number,
part_dict->FindIntKey("line").value_or(0));
ValidateKeyValue(part_dict, "summary", result_part.summary);
ValidateKeyValue(part_dict, "message", result_part.message);
}
}
// Validate vital information of actual test results.
void ValidateTestResult(Value* root,
const std::string& test_name,
const std::string& status,
size_t result_part_count) {
Value* val = root->FindListKey(test_name);
ASSERT_TRUE(val) << test_name;
ASSERT_EQ(1u, val->GetList().size());
val = &val->GetList().at(0);
ASSERT_TRUE(val->is_dict()) << test_name;
result &= ValidateKeyValue(val, "status", test_result.StatusAsString());
const Value* value = val.FindListKey("result_parts");
if (test_result.test_result_parts.size() != value->GetList().size()) {
ADD_FAILURE() << "test_result_parts count is not valid";
return false;
}
ValidateKeyValue(val, "status", status);
for (unsigned i = 0; i < test_result.test_result_parts.size(); i++) {
TestResultPart result_part = test_result.test_result_parts.at(i);
const Value& part_dict = value->GetList().at(i);
Value* value = val->FindListKey("result_parts");
ASSERT_TRUE(value) << test_name;
EXPECT_EQ(result_part_count, value->GetList().size()) << test_name;
result &= ValidateKeyValue(part_dict, "type", result_part.TypeAsString());
result &= ValidateKeyValue(part_dict, "file", result_part.file_name);
result &= ValidateKeyValue(part_dict, "line", result_part.line_number);
result &= ValidateKeyValue(part_dict, "summary", result_part.summary);
result &= ValidateKeyValue(part_dict, "message", result_part.message);
}
return result;
}
void ValidateStringList(Optional<Value>& root,
// Validate |root| dictionary value contains a list with |values|
// at |key| value.
bool ValidateStringList(const Optional<Value>& root,
const std::string& key,
std::vector<const char*> values) {
Value* val = root->FindListKey(key);
ASSERT_TRUE(val);
ASSERT_EQ(values.size(), val->GetList().size());
for (unsigned i = 0; i < values.size(); i++) {
ASSERT_TRUE(val->GetList().at(i).is_string());
EXPECT_EQ(values.at(i), val->GetList().at(i).GetString());
const Value* val = root->FindListKey(key);
if (!val) {
ADD_FAILURE() << "|root| has no list_value in key: " << key;
return false;
}
}
void ValidateTestLocation(Value* root,
const std::string& key,
const std::string& file,
int line) {
Value* val = root->FindDictKey(key);
ASSERT_TRUE(val);
EXPECT_EQ(2u, val->DictSize());
ValidateKeyValue(val, "file", file);
EXPECT_EQ(line, val->FindIntKey("line").value_or(0));
}
if (values.size() != val->GetList().size()) {
ADD_FAILURE() << "expected size: " << values.size()
<< ", actual size:" << val->GetList().size();
return false;
}
Optional<Value> ReadSummary(FilePath path) {
Optional<Value> result;
File resultFile(path, File::FLAG_OPEN | File::FLAG_READ);
const int size = 2e7;
std::string json;
CHECK(ReadFileToStringWithMaxSize(path, &json, size));
result = JSONReader::Read(json);
return result;
for (unsigned i = 0; i < values.size(); i++) {
if (!val->GetList().at(i).is_string() &&
val->GetList().at(i).GetString().compare(values.at(i))) {
ADD_FAILURE() << "Expected list values do not match actual list";
return false;
}
}
return true;
}
// Unit tests to validate TestLauncher outputs the correct JSON file.
......@@ -547,30 +535,34 @@ TEST_F(TestLauncherTest, JsonSummary) {
EXPECT_TRUE(test_launcher.Run(command_line.get()));
// Validate the resulting JSON file is the expected output.
Optional<Value> root = ReadSummary(path);
Optional<Value> root = test_launcher_utils::ReadSummary(path);
ASSERT_TRUE(root);
EXPECT_TRUE(
ValidateStringList(root, "all_tests",
{"Test.firstTest", "Test.firstTestDisabled",
"Test.secondTest", "TestDisabled.firstTest"});
"Test.secondTest", "TestDisabled.firstTest"}));
EXPECT_TRUE(
ValidateStringList(root, "disabled_tests",
{"Test.firstTestDisabled", "TestDisabled.firstTest"});
{"Test.firstTestDisabled", "TestDisabled.firstTest"}));
Value* val = root->FindDictKey("test_locations");
const Value* val = root->FindDictKey("test_locations");
ASSERT_TRUE(val);
EXPECT_EQ(2u, val->DictSize());
ValidateTestLocation(val, "Test.firstTest", "File", 100);
ValidateTestLocation(val, "Test.secondTest", "File", 100);
ASSERT_TRUE(test_launcher_utils::ValidateTestLocation(val, "Test.firstTest",
"File", 100));
ASSERT_TRUE(test_launcher_utils::ValidateTestLocation(val, "Test.secondTest",
"File", 100));
val = root->FindListKey("per_iteration_data");
ASSERT_TRUE(val);
ASSERT_EQ(2u, val->GetList().size());
for (size_t i = 0; i < val->GetList().size(); i++) {
Value* iteration_val = &(val->GetList().at(i));
const Value* iteration_val = &(val->GetList().at(i));
ASSERT_TRUE(iteration_val);
ASSERT_TRUE(iteration_val->is_dict());
EXPECT_EQ(2u, iteration_val->DictSize());
ValidateTestResult(iteration_val, first_result);
ValidateTestResult(iteration_val, second_result);
EXPECT_TRUE(ValidateTestResultObject(iteration_val, first_result));
EXPECT_TRUE(ValidateTestResultObject(iteration_val, second_result));
}
}
......@@ -596,12 +588,13 @@ TEST_F(TestLauncherTest, JsonSummaryWithDisabledTests) {
EXPECT_TRUE(test_launcher.Run(command_line.get()));
// Validate the resulting JSON file is the expected output.
Optional<Value> root = ReadSummary(path);
Optional<Value> root = test_launcher_utils::ReadSummary(path);
ASSERT_TRUE(root);
Value* val = root->FindDictKey("test_locations");
ASSERT_TRUE(val);
EXPECT_EQ(1u, val->DictSize());
ValidateTestLocation(val, "Test.DISABLED_Test", "File", 100);
EXPECT_TRUE(test_launcher_utils::ValidateTestLocation(
val, "Test.DISABLED_Test", "File", 100));
val = root->FindListKey("per_iteration_data");
ASSERT_TRUE(val);
......@@ -613,7 +606,7 @@ TEST_F(TestLauncherTest, JsonSummaryWithDisabledTests) {
EXPECT_EQ(1u, iteration_val->DictSize());
// We expect the result to be stripped of disabled prefix.
test_result.full_name = "Test.Test";
ValidateTestResult(iteration_val, test_result);
EXPECT_TRUE(ValidateTestResultObject(iteration_val, test_result));
}
// Unit tests to validate UnitTestLauncherDelegate implementation.
......@@ -678,8 +671,8 @@ TEST(MockUnitTests, DISABLED_CrashTest) {
IMMEDIATE_CRASH();
}
// Using UnitTestLauncherDelegate and TestLauncher (with minor mocks)
// to launch 3 basic tests, and validate the resulting json file.
// Using TestLauncher to launch 3 simple unitests
// and validate the resulting json file.
TEST_F(UnitTestLauncherDelegateTester, RunMockTests) {
CommandLine command_line(CommandLine::ForCurrentProcess()->GetProgram());
command_line.AppendSwitchASCII("gtest_filter", "MockUnitTests.DISABLED_*");
......@@ -701,7 +694,7 @@ TEST_F(UnitTestLauncherDelegateTester, RunMockTests) {
GetAppOutputAndError(command_line, &output);
// Validate the resulting JSON file is the expected output.
Optional<Value> root = ReadSummary(path);
Optional<Value> root = test_launcher_utils::ReadSummary(path);
ASSERT_TRUE(root);
Value* val = root->FindDictKey("test_locations");
......@@ -710,9 +703,12 @@ TEST_F(UnitTestLauncherDelegateTester, RunMockTests) {
// If path or test location changes, the following expectation
// will need to change accordingly.
std::string file_name = "../../base/test/launcher/test_launcher_unittest.cc";
ValidateTestLocation(val, "MockUnitTests.DISABLED_PassTest", file_name, 669);
ValidateTestLocation(val, "MockUnitTests.DISABLED_FailTest", file_name, 673);
ValidateTestLocation(val, "MockUnitTests.DISABLED_CrashTest", file_name, 677);
EXPECT_TRUE(test_launcher_utils::ValidateTestLocation(
val, "MockUnitTests.DISABLED_PassTest", file_name, 662));
EXPECT_TRUE(test_launcher_utils::ValidateTestLocation(
val, "MockUnitTests.DISABLED_FailTest", file_name, 666));
EXPECT_TRUE(test_launcher_utils::ValidateTestLocation(
val, "MockUnitTests.DISABLED_CrashTest", file_name, 670));
val = root->FindListKey("per_iteration_data");
ASSERT_TRUE(val);
......@@ -723,9 +719,12 @@ TEST_F(UnitTestLauncherDelegateTester, RunMockTests) {
ASSERT_TRUE(iteration_val->is_dict());
EXPECT_EQ(3u, iteration_val->DictSize());
// We expect the result to be stripped of disabled prefix.
ValidateTestResult(iteration_val, "MockUnitTests.PassTest", "SUCCESS", 0u);
ValidateTestResult(iteration_val, "MockUnitTests.FailTest", "FAILURE", 1u);
ValidateTestResult(iteration_val, "MockUnitTests.CrashTest", "CRASH", 0u);
EXPECT_TRUE(test_launcher_utils::ValidateTestResult(
iteration_val, "MockUnitTests.PassTest", "SUCCESS", 0u));
EXPECT_TRUE(test_launcher_utils::ValidateTestResult(
iteration_val, "MockUnitTests.FailTest", "FAILURE", 1u));
EXPECT_TRUE(test_launcher_utils::ValidateTestResult(
iteration_val, "MockUnitTests.CrashTest", "CRASH", 0u));
}
} // namespace
......
......@@ -14,6 +14,7 @@
#include "base/single_thread_task_runner.h"
#include "base/strings/utf_string_conversions.h"
#include "base/test/launcher/test_launcher.h"
#include "base/test/launcher/test_launcher_test_utils.h"
#include "base/test/scoped_feature_list.h"
#include "base/test/test_timeouts.h"
#include "base/threading/thread_restrictions.h"
......@@ -143,6 +144,78 @@ IN_PROC_BROWSER_TEST_F(ContentBrowserTest, BrowserCrashCallStack) {
}
}
// The following 3 tests are disabled as they are meant to only run from
// |RunMockTests| to validate tests launcher output for known results.
using MockContentBrowserTest = ContentBrowserTest;
// Basic Test to pass
IN_PROC_BROWSER_TEST_F(MockContentBrowserTest, DISABLED_PassTest) {
ASSERT_TRUE(true);
}
// Basic Test to fail
IN_PROC_BROWSER_TEST_F(MockContentBrowserTest, DISABLED_FailTest) {
ASSERT_TRUE(false);
}
// Basic Test to crash
IN_PROC_BROWSER_TEST_F(MockContentBrowserTest, DISABLED_CrashTest) {
IMMEDIATE_CRASH();
}
// Using TestLauncher to launch 3 simple browser tests
// and validate the resulting json file.
IN_PROC_BROWSER_TEST_F(ContentBrowserTest, RunMockTests) {
base::ScopedAllowBlockingForTesting allow_blocking;
base::ScopedTempDir temp_dir;
base::CommandLine command_line(
base::CommandLine::ForCurrentProcess()->GetProgram());
command_line.AppendSwitchASCII("gtest_filter",
"MockContentBrowserTest.DISABLED_*");
ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
base::FilePath path =
temp_dir.GetPath().AppendASCII("SaveSummaryResult.json");
command_line.AppendSwitchPath("test-launcher-summary-output", path);
command_line.AppendSwitch("gtest_also_run_disabled_tests");
command_line.AppendSwitch("--test-launcher-retry-limit=0");
std::string output;
base::GetAppOutputAndError(command_line, &output);
// Validate the resulting JSON file is the expected output.
base::Optional<base::Value> root =
base::test_launcher_utils::ReadSummary(path);
ASSERT_TRUE(root);
base::Value* val = root->FindDictKey("test_locations");
ASSERT_TRUE(val);
EXPECT_EQ(3u, val->DictSize());
// If path or test location changes, the following expectation
// will need to change accordingly.
std::string file_name = "../../content/test/content_browser_test_test.cc";
EXPECT_TRUE(base::test_launcher_utils::ValidateTestLocation(
val, "MockContentBrowserTest.DISABLED_PassTest", file_name, 152));
EXPECT_TRUE(base::test_launcher_utils::ValidateTestLocation(
val, "MockContentBrowserTest.DISABLED_FailTest", file_name, 156));
EXPECT_TRUE(base::test_launcher_utils::ValidateTestLocation(
val, "MockContentBrowserTest.DISABLED_CrashTest", file_name, 160));
val = root->FindListKey("per_iteration_data");
ASSERT_TRUE(val);
ASSERT_EQ(1u, val->GetList().size());
base::Value* iteration_val = &(val->GetList().at(0));
ASSERT_TRUE(iteration_val);
ASSERT_TRUE(iteration_val->is_dict());
EXPECT_EQ(3u, iteration_val->DictSize());
// We expect the result to be stripped of disabled prefix.
EXPECT_TRUE(base::test_launcher_utils::ValidateTestResult(
iteration_val, "MockContentBrowserTest.PassTest", "SUCCESS", 0u));
EXPECT_TRUE(base::test_launcher_utils::ValidateTestResult(
iteration_val, "MockContentBrowserTest.FailTest", "FAILURE", 1u));
EXPECT_TRUE(base::test_launcher_utils::ValidateTestResult(
iteration_val, "MockContentBrowserTest.CrashTest", "CRASH", 0u));
}
#endif
class ContentBrowserTestSanityTest : public ContentBrowserTest {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment