Skip to content

GitLab

T
tangled
Commit 7a0fc27a authored by Kevin Babbitt's avatar Kevin Babbitt Committed by Commit Bot
Browse files
Options

Create accessibility dump validation helper class 

Move the helper functions for loading an accessibility dump expectation
file and validating a dump into their own test helper class. This is
purely code movement - no changes apart from fixing up dereferences
where necessary.

I originally wanted to add these methods to AccessibilityTreeFormatter
but that failed a presubmit check for LOG(INFO) spam.
AccessibilityTreeFormatter isn't test-only code - it's used from the
chrome://accessibility page - so I opted for creating a new class for
the test helper methods instead.

R=nektar@chromium.org

Bug: 974397
Change-Id: Ib9d5105747b1418c215c5d11020c92dedade099f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1669947
Commit-Queue: Kevin Babbitt <kbabbitt@microsoft.com>
Reviewed-by: default avatarNektarios Paisios <nektar@chromium.org>
Cr-Commit-Position: refs/heads/master@{#671739}
parent d48e784d
Hide whitespace changes
Inline Side-by-side
Showing with 234 additions and 175 deletions
content/browser/accessibility/dump_accessibility_browsertest_base.cc
View file @ 7a0fc27a
......@@ -24,6 +24,7 @@
#include "content/browser/accessibility/browser_accessibility.h"
#include "content/browser/accessibility/browser_accessibility_manager.h"
#include "content/browser/accessibility/browser_accessibility_state_impl.h"
#include "content/browser/accessibility/dump_accessibility_test_helper.h"
#include "content/browser/renderer_host/render_widget_host_view_child_frame.h"
#include "content/browser/web_contents/web_contents_impl.h"
#include "content/public/browser/web_contents.h"
......@@ -45,11 +46,6 @@ namespace content {
namespace {
const char kCommentToken = '#';
const char kMarkSkipFile[] = "#<skip";
const char kMarkEndOfFile[] = "<-- End-of-file -->";
const char kSignalDiff[] = "*";
// Searches recursively and returns true if an accessibility node is found
// that represents a fully loaded web document with the given url.
bool AccessibilityTreeContainsLoadedDocWithUrl(BrowserAccessibility* node,
......@@ -135,148 +131,6 @@ DumpAccessibilityTestBase::DumpUnfilteredAccessibilityTreeAsString() {
return ax_tree_dump;
}
base::Optional<base::FilePath>
DumpAccessibilityTestBase::GetExpectationFilePath(
const base::FilePath& test_file_path) {
base::ScopedAllowBlockingForTesting allow_blocking;
base::FilePath expected_file_path;
// Try to get version specific expected file.
base::FilePath::StringType expected_file_suffix =
formatter_->GetVersionSpecificExpectedFileSuffix();
if (expected_file_suffix != FILE_PATH_LITERAL("")) {
expected_file_path = base::FilePath(
test_file_path.RemoveExtension().value() + expected_file_suffix);
if (base::PathExists(expected_file_path))
return expected_file_path;
}
// If a version specific file does not exist, get the generic one.
expected_file_suffix = formatter_->GetExpectedFileSuffix();
expected_file_path = base::FilePath(test_file_path.RemoveExtension().value() +
expected_file_suffix);
if (base::PathExists(expected_file_path))
return expected_file_path;
// If no expected file could be found, display error.
LOG(INFO) << "File not found: " << expected_file_path.LossyDisplayName();
LOG(INFO) << "To run this test, create "
<< expected_file_path.LossyDisplayName()
<< " (it can be empty) and then run this test "
<< "with the switch: --"
<< switches::kGenerateAccessibilityTestExpectations;
return base::nullopt;
}
base::Optional<std::vector<std::string>>
DumpAccessibilityTestBase::LoadExpectationFile(
const base::FilePath& expected_file) {
base::ScopedAllowBlockingForTesting allow_blocking;
std::string expected_contents_raw;
base::ReadFileToString(expected_file, &expected_contents_raw);
// Tolerate Windows-style line endings (\r\n) in the expected file:
// normalize by deleting all \r from the file (if any) to leave only \n.
std::string expected_contents;
base::RemoveChars(expected_contents_raw, "\r", &expected_contents);
if (!expected_contents.compare(0, strlen(kMarkSkipFile), kMarkSkipFile)) {
return base::nullopt;
}
std::vector<std::string> expected_lines =
base::SplitString(expected_contents, "\n", base::KEEP_WHITESPACE,
base::SPLIT_WANT_NONEMPTY);
// Marking the end of the file with a line of text ensures that
// file length differences are found.
expected_lines.push_back(kMarkEndOfFile);
return expected_lines;
}
bool DumpAccessibilityTestBase::ValidateAgainstExpectation(
const base::FilePath& test_file_path,
const base::FilePath& expected_file,
const std::vector<std::string>& actual_lines,
const std::vector<std::string>& expected_lines) {
// Output the test path to help anyone who encounters a failure and needs
// to know where to look.
LOG(INFO) << "Testing: "
<< test_file_path.NormalizePathSeparatorsTo('/').LossyDisplayName();
LOG(INFO) << "Expected output: "
<< expected_file.NormalizePathSeparatorsTo('/').LossyDisplayName();
// Perform a diff (or write the initial baseline).
std::vector<int> diff_lines = DiffLines(expected_lines, actual_lines);
bool is_different = diff_lines.size() > 0;
if (is_different) {
std::string diff;
// Mark the expected lines which did not match actual output with a *.
diff += "* Line Expected\n";
diff += "- ---- --------\n";
for (int line = 0, diff_index = 0;
line < static_cast<int>(expected_lines.size()); ++line) {
bool is_diff = false;
if (diff_index < static_cast<int>(diff_lines.size()) &&
diff_lines[diff_index] == line) {
is_diff = true;
++diff_index;
}
diff += base::StringPrintf("%1s %4d %s\n", is_diff ? kSignalDiff : "",
line + 1, expected_lines[line].c_str());
}
diff += "\nActual\n";
diff += "------\n";
diff += base::JoinString(actual_lines, "\n");
diff += "\n";
diff += kMarkEndOfFile;
LOG(ERROR) << "Diff:\n" << diff;
} else {
LOG(INFO) << "Test output matches expectations.";
}
if (base::CommandLine::ForCurrentProcess()->HasSwitch(
switches::kGenerateAccessibilityTestExpectations)) {
base::ScopedAllowBlockingForTesting allow_blocking;
std::string actual_contents_for_output =
base::JoinString(actual_lines, "\n") + "\n";
CHECK(base::WriteFile(expected_file, actual_contents_for_output.c_str(),
actual_contents_for_output.size()) ==
static_cast<int>(actual_contents_for_output.size()));
LOG(INFO) << "Wrote expectations to: " << expected_file.LossyDisplayName();
}
return !is_different;
}
std::vector<int> DumpAccessibilityTestBase::DiffLines(
const std::vector<std::string>& expected_lines,
const std::vector<std::string>& actual_lines) {
int actual_lines_count = actual_lines.size();
int expected_lines_count = expected_lines.size();
std::vector<int> diff_lines;
int i = 0, j = 0;
while (i < actual_lines_count && j < expected_lines_count) {
if (expected_lines[j].size() == 0 ||
expected_lines[j][0] == kCommentToken) {
// Skip comment lines and blank lines in expected output.
++j;
continue;
}
if (actual_lines[i] != expected_lines[j])
diff_lines.push_back(j);
++i;
++j;
}
// Actual file has been fully checked.
return diff_lines;
}
void DumpAccessibilityTestBase::ParseHtmlForExtraDirectives(
const std::string& test_html,
std::vector<std::string>* wait_for,
......@@ -352,6 +206,7 @@ void DumpAccessibilityTestBase::RunTestForPlatform(
const base::FilePath file_path,
const char* file_dir) {
formatter_ = formatter_factory_();
DumpAccessibilityTestHelper test_helper(formatter_.get());
// Disable the "hot tracked" state (set when the mouse is hovering over
// an object) because it makes test output change based on the mouse position.
......@@ -372,7 +227,7 @@ void DumpAccessibilityTestBase::RunTestForPlatform(
// WAIT-FOR directive in the source file that's looking for something not
// supported on the current platform.
base::Optional<base::FilePath> expected_file =
GetExpectationFilePath(file_path);
test_helper.GetExpectationFilePath(file_path);
if (!expected_file) {
LOG(INFO) << "No expectation file present, ignoring test on this "
"platform.";
......@@ -380,7 +235,7 @@ void DumpAccessibilityTestBase::RunTestForPlatform(
}
base::Optional<std::vector<std::string>> expected_lines =
LoadExpectationFile(*expected_file);
test_helper.LoadExpectationFile(*expected_file);
if (!expected_lines) {
LOG(INFO) << "Skipping this test on this platform.";
return;
......@@ -522,7 +377,7 @@ void DumpAccessibilityTestBase::RunTestForPlatform(
std::vector<std::string> actual_lines = Dump(run_until);
// Validate against the expectation file.
bool matches_expectation = ValidateAgainstExpectation(
bool matches_expectation = test_helper.ValidateAgainstExpectation(
file_path, *expected_file, actual_lines, *expected_lines);
EXPECT_TRUE(matches_expectation);
if (!matches_expectation)
......
content/browser/accessibility/dump_accessibility_browsertest_base.h
View file @ 7a0fc27a
......@@ -71,31 +71,6 @@ class DumpAccessibilityTestBase : public ContentBrowserTest,
// and return it as a string.
base::string16 DumpUnfilteredAccessibilityTreeAsString();
// Returns a path to an expectation file for the current platform. If no
// suitable expectation file can be found, logs an error message and returns
// nullopt.
base::Optional<base::FilePath> GetExpectationFilePath(
const base::FilePath& test_file_path);
// Loads the given expectation file. Returns nullopt if the file contains a
// skip marker.
base::Optional<std::vector<std::string>> LoadExpectationFile(
const base::FilePath& expected_file);
// Compares the given actual dump against the given expectation and generates
// a new expectation file if switches::kGenerateAccessibilityTestExpectations
// has been set. Returns true if the result matches the expectation.
bool ValidateAgainstExpectation(
const base::FilePath& test_file_path,
const base::FilePath& expected_file,
const std::vector<std::string>& actual_lines,
const std::vector<std::string>& expected_lines);
// Utility helper that does a comment-aware equality check.
// Returns array of lines from expected file which are different.
std::vector<int> DiffLines(const std::vector<std::string>& expected_lines,
const std::vector<std::string>& actual_lines);
// Parse the test html file and parse special directives, usually
// beginning with an '@' and inside an HTML comment, that control how the
// test is run and how the results are interpreted.
......
content/browser/accessibility/dump_accessibility_test_helper.cc 0 → 100644
View file @ 7a0fc27a
// Copyright 2019 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "content/browser/accessibility/dump_accessibility_test_helper.h"
#include "base/command_line.h"
#include "base/files/file_util.h"
#include "base/strings/string_split.h"
#include "base/strings/string_util.h"
#include "base/strings/stringprintf.h"
#include "base/threading/thread_restrictions.h"
#include "content/browser/accessibility/accessibility_tree_formatter.h"
#include "content/public/common/content_switches.h"
namespace content {
namespace {
const char kCommentToken = '#';
const char kMarkSkipFile[] = "#<skip";
const char kMarkEndOfFile[] = "<-- End-of-file -->";
const char kSignalDiff[] = "*";
} // namespace
DumpAccessibilityTestHelper::DumpAccessibilityTestHelper(
AccessibilityTreeFormatter* formatter)
: formatter_(formatter) {}
base::Optional<base::FilePath>
DumpAccessibilityTestHelper::GetExpectationFilePath(
const base::FilePath& test_file_path) {
base::ScopedAllowBlockingForTesting allow_blocking;
base::FilePath expected_file_path;
// Try to get version specific expected file.
base::FilePath::StringType expected_file_suffix =
formatter_->GetVersionSpecificExpectedFileSuffix();
if (expected_file_suffix != FILE_PATH_LITERAL("")) {
expected_file_path = base::FilePath(
test_file_path.RemoveExtension().value() + expected_file_suffix);
if (base::PathExists(expected_file_path))
return expected_file_path;
}
// If a version specific file does not exist, get the generic one.
expected_file_suffix = formatter_->GetExpectedFileSuffix();
expected_file_path = base::FilePath(test_file_path.RemoveExtension().value() +
expected_file_suffix);
if (base::PathExists(expected_file_path))
return expected_file_path;
// If no expected file could be found, display error.
LOG(INFO) << "File not found: " << expected_file_path.LossyDisplayName();
LOG(INFO) << "To run this test, create "
<< expected_file_path.LossyDisplayName()
<< " (it can be empty) and then run this test "
<< "with the switch: --"
<< switches::kGenerateAccessibilityTestExpectations;
return base::nullopt;
}
base::Optional<std::vector<std::string>>
DumpAccessibilityTestHelper::LoadExpectationFile(
const base::FilePath& expected_file) {
base::ScopedAllowBlockingForTesting allow_blocking;
std::string expected_contents_raw;
base::ReadFileToString(expected_file, &expected_contents_raw);
// Tolerate Windows-style line endings (\r\n) in the expected file:
// normalize by deleting all \r from the file (if any) to leave only \n.
std::string expected_contents;
base::RemoveChars(expected_contents_raw, "\r", &expected_contents);
if (!expected_contents.compare(0, strlen(kMarkSkipFile), kMarkSkipFile)) {
return base::nullopt;
}
std::vector<std::string> expected_lines =
base::SplitString(expected_contents, "\n", base::KEEP_WHITESPACE,
base::SPLIT_WANT_NONEMPTY);
// Marking the end of the file with a line of text ensures that
// file length differences are found.
expected_lines.push_back(kMarkEndOfFile);
return expected_lines;
}
bool DumpAccessibilityTestHelper::ValidateAgainstExpectation(
const base::FilePath& test_file_path,
const base::FilePath& expected_file,
const std::vector<std::string>& actual_lines,
const std::vector<std::string>& expected_lines) {
// Output the test path to help anyone who encounters a failure and needs
// to know where to look.
LOG(INFO) << "Testing: "
<< test_file_path.NormalizePathSeparatorsTo('/').LossyDisplayName();
LOG(INFO) << "Expected output: "
<< expected_file.NormalizePathSeparatorsTo('/').LossyDisplayName();
// Perform a diff (or write the initial baseline).
std::vector<int> diff_lines = DiffLines(expected_lines, actual_lines);
bool is_different = diff_lines.size() > 0;
if (is_different) {
std::string diff;
// Mark the expected lines which did not match actual output with a *.
diff += "* Line Expected\n";
diff += "- ---- --------\n";
for (int line = 0, diff_index = 0;
line < static_cast<int>(expected_lines.size()); ++line) {
bool is_diff = false;
if (diff_index < static_cast<int>(diff_lines.size()) &&
diff_lines[diff_index] == line) {
is_diff = true;
++diff_index;
}
diff += base::StringPrintf("%1s %4d %s\n", is_diff ? kSignalDiff : "",
line + 1, expected_lines[line].c_str());
}
diff += "\nActual\n";
diff += "------\n";
diff += base::JoinString(actual_lines, "\n");
diff += "\n";
diff += kMarkEndOfFile;
LOG(ERROR) << "Diff:\n" << diff;
} else {
LOG(INFO) << "Test output matches expectations.";
}
if (base::CommandLine::ForCurrentProcess()->HasSwitch(
switches::kGenerateAccessibilityTestExpectations)) {
base::ScopedAllowBlockingForTesting allow_blocking;
std::string actual_contents_for_output =
base::JoinString(actual_lines, "\n") + "\n";
CHECK(base::WriteFile(expected_file, actual_contents_for_output.c_str(),
actual_contents_for_output.size()) ==
static_cast<int>(actual_contents_for_output.size()));
LOG(INFO) << "Wrote expectations to: " << expected_file.LossyDisplayName();
}
return !is_different;
}
std::vector<int> DumpAccessibilityTestHelper::DiffLines(
const std::vector<std::string>& expected_lines,
const std::vector<std::string>& actual_lines) {
int actual_lines_count = actual_lines.size();
int expected_lines_count = expected_lines.size();
std::vector<int> diff_lines;
int i = 0, j = 0;
while (i < actual_lines_count && j < expected_lines_count) {
if (expected_lines[j].size() == 0 ||
expected_lines[j][0] == kCommentToken) {
// Skip comment lines and blank lines in expected output.
++j;
continue;
}
if (actual_lines[i] != expected_lines[j])
diff_lines.push_back(j);
++i;
++j;
}
// Actual file has been fully checked.
return diff_lines;
}
} // namespace content
content/browser/accessibility/dump_accessibility_test_helper.h 0 → 100644
View file @ 7a0fc27a
// Copyright 2019 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef CONTENT_BROWSER_ACCESSIBILITY_DUMP_ACCESSIBILITY_TEST_HELPER_H_
#define CONTENT_BROWSER_ACCESSIBILITY_DUMP_ACCESSIBILITY_TEST_HELPER_H_
#include "base/optional.h"
namespace base {
class FilePath;
}
namespace content {
class AccessibilityTreeFormatter;
// A helper class for writing accessibility tree dump tests.
class DumpAccessibilityTestHelper {
public:
DumpAccessibilityTestHelper(AccessibilityTreeFormatter* formatter);
~DumpAccessibilityTestHelper() = default;
// Returns a path to an expectation file for the current platform. If no
// suitable expectation file can be found, logs an error message and returns
// nullopt.
base::Optional<base::FilePath> GetExpectationFilePath(
const base::FilePath& test_file_path);
// Loads the given expectation file. Returns nullopt if the file contains a
// skip marker.
static base::Optional<std::vector<std::string>> LoadExpectationFile(
const base::FilePath& expected_file);
// Compares the given actual dump against the given expectation and generates
// a new expectation file if switches::kGenerateAccessibilityTestExpectations
// has been set. Returns true if the result matches the expectation.
static bool ValidateAgainstExpectation(
const base::FilePath& test_file_path,
const base::FilePath& expected_file,
const std::vector<std::string>& actual_lines,
const std::vector<std::string>& expected_lines);
private:
// Utility helper that does a comment-aware equality check.
// Returns array of lines from expected file which are different.
static std::vector<int> DiffLines(
const std::vector<std::string>& expected_lines,
const std::vector<std::string>& actual_lines);
AccessibilityTreeFormatter* formatter_;
};
} // namespace content
#endif // CONTENT_BROWSER_ACCESSIBILITY_DUMP_ACCESSIBILITY_TEST_HELPER_H_
content/test/BUILD.gn
View file @ 7a0fc27a
......@@ -38,6 +38,8 @@ jumbo_static_library("test_support") {
"../browser/accessibility/accessibility_event_recorder_uia_win.cc",
"../browser/accessibility/accessibility_event_recorder_uia_win.h",
"../browser/accessibility/accessibility_event_recorder_win.cc",
"../browser/accessibility/dump_accessibility_test_helper.cc",
"../browser/accessibility/dump_accessibility_test_helper.h",
"../browser/background_fetch/background_fetch_test_base.cc",
"../browser/background_fetch/background_fetch_test_base.h",
"../browser/background_fetch/background_fetch_test_browser_context.cc",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment