Commit d1e85573 authored by Hiroki Sato's avatar Hiroki Sato Committed by Chromium LUCI CQ

arc-a11y: Extract DrawerLayout handling logic to a class

This is a preparation of auto complete handling.
This change adds a new interface AXTreeSourceArc::Hook, which hooks
serialization process of AXTreeSourceArc and can modify the default
output of the serialization for special Android widgets.

See go/arc-autocomplete-a11y for details.

AX-Relnotes: n/a.
Bug: b:150827488
Test: unit tests
Change-Id: Ifa9a6cf4801e97ebca58ebe9636be66910c3b3a4
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2569199
Commit-Queue: Hiroki Sato <hirokisato@chromium.org>
Reviewed-by: default avatarSara Kato <sarakato@chromium.org>
Reviewed-by: default avatarDavid Tseng <dtseng@chromium.org>
Cr-Commit-Position: refs/heads/master@{#836979}
parent a59cfd35
......@@ -575,6 +575,8 @@ source_set("chromeos") {
"arc/accessibility/arc_accessibility_util.h",
"arc/accessibility/ax_tree_source_arc.cc",
"arc/accessibility/ax_tree_source_arc.h",
"arc/accessibility/drawer_layout_handler.cc",
"arc/accessibility/drawer_layout_handler.h",
"arc/accessibility/geometry_util.cc",
"arc/accessibility/geometry_util.h",
"arc/adbd/arc_adbd_monitor_bridge.cc",
......@@ -3337,6 +3339,7 @@ source_set("unit_tests") {
"arc/accessibility/arc_accessibility_helper_bridge_unittest.cc",
"arc/accessibility/arc_accessibility_util_unittest.cc",
"arc/accessibility/ax_tree_source_arc_unittest.cc",
"arc/accessibility/drawer_layout_handler_unittest.cc",
"arc/adbd/arc_adbd_monitor_bridge_unittest.cc",
"arc/app_shortcuts/arc_app_shortcuts_menu_builder_unittest.cc",
"arc/app_shortcuts/arc_app_shortcuts_request_unittest.cc",
......
......@@ -124,11 +124,6 @@ void AccessibilityNodeInfoDataWrapper::PopulateAXRole(
class_name);
}
if (role_) {
out_data->role = *role_;
return;
}
if (GetProperty(AXBooleanProperty::EDITABLE)) {
out_data->role = ax::mojom::Role::kTextField;
return;
......@@ -326,7 +321,7 @@ void AccessibilityNodeInfoDataWrapper::Serialize(
bool is_node_tree_root = tree_source_->IsRootOfNodeTree(GetId());
// String properties that doesn't belong to any of existing chrome
// automation string properties are pushed into description.
// TODO (sahok): Refactor this to make clear the functionality(b/158633575).
// TODO(sahok): Refactor this to make clear the functionality(b/158633575).
std::vector<std::string> descriptions;
// String properties.
......@@ -539,8 +534,6 @@ std::string AccessibilityNodeInfoDataWrapper::ComputeAXName(
}
if (!hint_text.empty())
names.push_back(hint_text);
if (cached_name_ && !(*cached_name_).empty())
names.push_back(*cached_name_);
// If a node is accessibility focusable, but has no name, the name should be
// computed from its descendants.
......@@ -748,8 +741,7 @@ bool AccessibilityNodeInfoDataWrapper::HasImportantPropertyInternal() const {
AXStringProperty::CONTENT_DESCRIPTION) ||
HasNonEmptyStringProperty(node_ptr_, AXStringProperty::TEXT) ||
HasNonEmptyStringProperty(node_ptr_, AXStringProperty::PANE_TITLE) ||
HasNonEmptyStringProperty(node_ptr_, AXStringProperty::HINT_TEXT) ||
cached_name_.has_value()) {
HasNonEmptyStringProperty(node_ptr_, AXStringProperty::HINT_TEXT)) {
return true;
}
......
......@@ -45,8 +45,6 @@ class AccessibilityNodeInfoDataWrapper : public AccessibilityInfoDataWrapper {
mojom::AccessibilityNodeInfoData* node() { return node_ptr_; }
void set_role(ax::mojom::Role role) { role_ = role; }
void set_cached_name(const std::string& name) { cached_name_ = name; }
void set_container_live_status(mojom::AccessibilityLiveRegionType status) {
container_live_status_ = status;
}
......@@ -85,8 +83,6 @@ class AccessibilityNodeInfoDataWrapper : public AccessibilityInfoDataWrapper {
mojom::AccessibilityNodeInfoData* node_ptr_ = nullptr;
base::Optional<ax::mojom::Role> role_;
base::Optional<std::string> cached_name_;
mojom::AccessibilityLiveRegionType container_live_status_ =
mojom::AccessibilityLiveRegionType::NONE;
......
......@@ -10,12 +10,10 @@
#include "ash/public/cpp/external_arc/message_center/arc_notification_surface.h"
#include "ash/public/cpp/external_arc/message_center/arc_notification_surface_manager.h"
#include "base/strings/string_util.h"
#include "base/strings/stringprintf.h"
#include "chrome/browser/chromeos/arc/accessibility/accessibility_node_info_data_wrapper.h"
#include "chrome/browser/chromeos/arc/accessibility/accessibility_window_info_data_wrapper.h"
#include "chrome/browser/chromeos/arc/accessibility/arc_accessibility_util.h"
#include "chrome/browser/chromeos/arc/accessibility/geometry_util.h"
#include "chrome/browser/chromeos/arc/accessibility/drawer_layout_handler.h"
#include "components/exo/input_method_surface.h"
#include "components/exo/wm_helper.h"
#include "extensions/browser/api/automation_internal/automation_event_router.h"
......@@ -33,24 +31,9 @@ using AXEventType = mojom::AccessibilityEventType;
using AXIntProperty = mojom::AccessibilityIntProperty;
using AXIntListProperty = mojom::AccessibilityIntListProperty;
using AXNodeInfoData = mojom::AccessibilityNodeInfoData;
using AXStringProperty = mojom::AccessibilityStringProperty;
using AXWindowInfoData = mojom::AccessibilityWindowInfoData;
using AXWindowIntListProperty = mojom::AccessibilityWindowIntListProperty;
namespace {
bool IsDrawerLayout(AXNodeInfoData* node) {
if (!node || !node->string_properties)
return false;
auto it = node->string_properties->find(AXStringProperty::CLASS_NAME);
if (it == node->string_properties->end())
return false;
return it->second == "androidx.drawerlayout.widget.DrawerLayout" ||
it->second == "android.support.v4.widget.DrawerLayout";
}
} // namespace
AXTreeSourceArc::AXTreeSourceArc(Delegate* delegate)
: current_tree_serializer_(new AXTreeArcSerializer(this)),
is_notification_(false),
......@@ -152,6 +135,10 @@ void AXTreeSourceArc::SerializeNode(AccessibilityInfoDataWrapper* info_data,
return;
info_data->Serialize(out_data);
const auto& itr = hooks_.find(info_data->GetId());
if (itr != hooks_.end())
itr->second->PostSerializeNode(out_data);
}
aura::Window* AXTreeSourceArc::GetWindow() const {
......@@ -249,15 +236,9 @@ void AXTreeSourceArc::NotifyAccessibilityEventInternal(
return;
}
if (event_data.event_type == AXEventType::WINDOW_STATE_CHANGED &&
event_data.event_text) {
AccessibilityInfoDataWrapper* source_node = GetFromId(event_data.source_id);
if (IsValid(source_node))
UpdateAXNameCache(source_node, *event_data.event_text);
}
ApplyCachedProperties();
ProcessHooksOnEvent(event_data);
// Bundle the event and send it to automation.
ExtensionMsg_AccessibilityEventBundleParams event_bundle;
event_bundle.tree_id = ax_tree_id();
......@@ -509,51 +490,16 @@ bool AXTreeSourceArc::UpdateAndroidFocusedId(const AXEventData& event_data) {
return true;
}
void AXTreeSourceArc::UpdateAXNameCache(
AccessibilityInfoDataWrapper* source_node,
const std::vector<std::string>& event_text) {
if (IsDrawerLayout(source_node->GetNode())) {
// When drawer menu opened, make the menu title announced.
// When focus is changed, ChromeVox computes the diff in ancestry between
// the previously focused and new focused node.
// As the DrawerLayout is LCA of them, set the new title to be the first
// visible child node (which is usually drawer menu).
std::vector<AccessibilityInfoDataWrapper*> children;
source_node->GetChildren(&children);
for (auto* child : children) {
if (child->IsNode() && child->IsVisibleToUser() &&
GetBooleanProperty(child->GetNode(), AXBooleanProperty::IMPORTANCE)) {
cached_roles_[child->GetId()] = ax::mojom::Role::kMenu;
if (!event_text.empty())
cached_names_[child->GetId()] = base::JoinString(event_text, " ");
return;
}
}
}
}
void AXTreeSourceArc::ProcessHooksOnEvent(const AXEventData& event_data) {
base::EraseIf(hooks_, [this](const auto& it) {
return this->GetFromId(it.first) == nullptr;
});
void AXTreeSourceArc::ApplyCachedProperties() {
for (auto it = cached_names_.begin(); it != cached_names_.end();) {
AccessibilityInfoDataWrapper* node = GetFromId(it->first);
if (node) {
static_cast<AccessibilityNodeInfoDataWrapper*>(node)->set_cached_name(
it->second);
it++;
} else {
it = cached_names_.erase(it);
}
}
for (auto it = cached_roles_.begin(); it != cached_roles_.end();) {
AccessibilityInfoDataWrapper* node = GetFromId(it->first);
if (node) {
static_cast<AccessibilityNodeInfoDataWrapper*>(node)->set_role(
it->second);
it++;
} else {
it = cached_roles_.erase(it);
}
}
// Add new hook implementations if necessary.
auto drawer_layout_hook =
DrawerLayoutHandler::CreateIfNecessary(this, event_data);
if (drawer_layout_hook.has_value())
hooks_.insert(std::move(*drawer_layout_hook));
}
void AXTreeSourceArc::HandleLiveRegions(std::vector<ui::AXEvent>* events) {
......
......@@ -11,6 +11,7 @@
#include <string>
#include <vector>
#include "base/containers/flat_map.h"
#include "chrome/browser/chromeos/arc/accessibility/accessibility_info_data_wrapper.h"
#include "components/arc/mojom/accessibility_helper.mojom-forward.h"
#include "extensions/browser/api/automation_internal/automation_event_router.h"
......@@ -45,6 +46,17 @@ class AXTreeSourceArc : public ui::AXTreeSource<AccessibilityInfoDataWrapper*,
virtual bool UseFullFocusMode() const = 0;
};
// The interface to hook the event handling and the node serialization.
class Hook {
public:
Hook() = default;
virtual ~Hook() = default;
// Called after the default serialization of the attaching node.
// Implementations can modify the serialization of given |out_data|.
virtual void PostSerializeNode(ui::AXNodeData* out_data) const = 0;
};
explicit AXTreeSourceArc(Delegate* delegate);
~AXTreeSourceArc() override;
......@@ -132,10 +144,7 @@ class AXTreeSourceArc : public ui::AXTreeSource<AccessibilityInfoDataWrapper*,
// event to chrome automation. Otherwise, this returns true.
bool UpdateAndroidFocusedId(const mojom::AccessibilityEventData& event_data);
void UpdateAXNameCache(AccessibilityInfoDataWrapper* source_node,
const std::vector<std::string>& event_text);
void ApplyCachedProperties();
void ProcessHooksOnEvent(const mojom::AccessibilityEventData& event_data);
// Compare previous live region and current live region, and add event to the
// given vector if there is any difference.
......@@ -174,9 +183,6 @@ class AXTreeSourceArc : public ui::AXTreeSource<AccessibilityInfoDataWrapper*,
base::Optional<std::string> notification_key_;
std::map<int32_t, std::string> cached_names_;
std::map<int32_t, ax::mojom::Role> cached_roles_;
// Cache of mapping from the *Android* window id to the last focused node id.
std::map<int32_t, int32_t> window_id_to_last_focus_node_id_;
......@@ -187,6 +193,9 @@ class AXTreeSourceArc : public ui::AXTreeSource<AccessibilityInfoDataWrapper*,
// Mapping from Chrome node ID to the previous computed name for live region.
std::map<int32_t, std::string> previous_live_region_name_;
// Mapping from Chrome node ID to the attached hook implementations.
base::flat_map<int32_t, std::unique_ptr<Hook>> hooks_;
// A delegate that handles accessibility actions on behalf of this tree. The
// delegate is valid during the lifetime of this tree.
const Delegate* const delegate_;
......
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "chrome/browser/chromeos/arc/accessibility/drawer_layout_handler.h"
#include "base/strings/string_util.h"
#include "chrome/browser/chromeos/arc/accessibility/accessibility_info_data_wrapper.h"
#include "chrome/browser/chromeos/arc/accessibility/arc_accessibility_util.h"
#include "chrome/browser/chromeos/arc/accessibility/ax_tree_source_arc.h"
#include "components/arc/mojom/accessibility_helper.mojom-forward.h"
#include "ui/accessibility/ax_enums.mojom.h"
#include "ui/accessibility/ax_node_data.h"
namespace {
constexpr char kDrawerLayoutClassNameAndroidX[] =
"androidx.drawerlayout.widget.DrawerLayout";
constexpr char kDrawerLayoutClassNameLegacy[] =
"android.support.v4.widget.DrawerLayout";
bool IsDrawerLayout(arc::mojom::AccessibilityNodeInfoData* node) {
if (!node || !node->string_properties)
return false;
auto it = node->string_properties->find(
arc::mojom::AccessibilityStringProperty::CLASS_NAME);
if (it == node->string_properties->end())
return false;
return it->second == kDrawerLayoutClassNameAndroidX ||
it->second == kDrawerLayoutClassNameLegacy;
}
} // namespace
namespace arc {
// static
base::Optional<std::pair<int32_t, std::unique_ptr<DrawerLayoutHandler>>>
DrawerLayoutHandler::CreateIfNecessary(
AXTreeSourceArc* tree_source,
const mojom::AccessibilityEventData& event_data) {
if (event_data.event_type !=
mojom::AccessibilityEventType::WINDOW_STATE_CHANGED) {
return base::nullopt;
}
AccessibilityInfoDataWrapper* source_node =
tree_source->GetFromId(event_data.source_id);
if (!source_node || !IsDrawerLayout(source_node->GetNode()))
return base::nullopt;
// Find a node with accessibility importance. That is a menu node opened now.
// Extract the accessibility name of the drawer menu from the event text.
std::vector<AccessibilityInfoDataWrapper*> children;
source_node->GetChildren(&children);
for (auto* child : children) {
if (!child->IsNode() || !child->IsVisibleToUser() ||
!GetBooleanProperty(child->GetNode(),
mojom::AccessibilityBooleanProperty::IMPORTANCE)) {
continue;
}
return std::make_pair(
child->GetId(),
std::make_unique<DrawerLayoutHandler>(base::JoinString(
event_data.event_text.value_or<std::vector<std::string>>({}),
" ")));
}
return base::nullopt;
}
void DrawerLayoutHandler::PostSerializeNode(ui::AXNodeData* out_data) const {
out_data->role = ax::mojom::Role::kMenu;
if (!name_.empty())
out_data->SetName(name_);
}
} // namespace arc
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef CHROME_BROWSER_CHROMEOS_ARC_ACCESSIBILITY_DRAWER_LAYOUT_HANDLER_H_
#define CHROME_BROWSER_CHROMEOS_ARC_ACCESSIBILITY_DRAWER_LAYOUT_HANDLER_H_
#include <memory>
#include <string>
#include <utility>
#include "base/optional.h"
#include "chrome/browser/chromeos/arc/accessibility/ax_tree_source_arc.h"
namespace ui {
struct AXNodeData;
}
namespace arc {
namespace mojom {
class AccessibilityEventData;
}
class DrawerLayoutHandler : public AXTreeSourceArc::Hook {
public:
static base::Optional<
std::pair<int32_t, std::unique_ptr<DrawerLayoutHandler>>>
CreateIfNecessary(AXTreeSourceArc* tree_source,
const mojom::AccessibilityEventData& event_data);
explicit DrawerLayoutHandler(const std::string& name) : name_(name) {}
// AXTreeSourceArc::Hook overrides:
void PostSerializeNode(ui::AXNodeData* out_data) const override;
private:
const std::string name_;
};
} // namespace arc
#endif // CHROME_BROWSER_CHROMEOS_ARC_ACCESSIBILITY_DRAWER_LAYOUT_HANDLER_H_
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "chrome/browser/chromeos/arc/accessibility/drawer_layout_handler.h"
#include <map>
#include <memory>
#include <utility>
#include "chrome/browser/chromeos/arc/accessibility/accessibility_info_data_wrapper.h"
#include "chrome/browser/chromeos/arc/accessibility/accessibility_node_info_data_wrapper.h"
#include "chrome/browser/chromeos/arc/accessibility/accessibility_window_info_data_wrapper.h"
#include "chrome/browser/chromeos/arc/accessibility/arc_accessibility_util.h"
#include "chrome/browser/chromeos/arc/accessibility/ax_tree_source_arc.h"
#include "components/arc/mojom/accessibility_helper.mojom.h"
#include "testing/gtest/include/gtest/gtest.h"
#include "ui/accessibility/ax_enums.mojom.h"
#include "ui/accessibility/ax_role_properties.h"
namespace arc {
using AXBooleanProperty = mojom::AccessibilityBooleanProperty;
using AXEventData = mojom::AccessibilityEventData;
using AXEventType = mojom::AccessibilityEventType;
using AXIntListProperty = mojom::AccessibilityIntListProperty;
using AXNodeInfoData = mojom::AccessibilityNodeInfoData;
using AXStringProperty = mojom::AccessibilityStringProperty;
using AXWindowInfoData = mojom::AccessibilityWindowInfoData;
namespace {
void SetProperty(AXNodeInfoData* node, AXBooleanProperty prop, bool value) {
arc::SetProperty(node->boolean_properties, prop, value);
}
void SetProperty(AXNodeInfoData* node,
AXIntListProperty prop,
const std::vector<int>& value) {
arc::SetProperty(node->int_list_properties, prop, value);
}
void SetProperty(AXNodeInfoData* node,
AXStringProperty prop,
const std::string& value) {
arc::SetProperty(node->string_properties, prop, value);
}
} // namespace
class DrawerLayoutHandlerTest : public testing::Test,
public AXTreeSourceArc::Delegate {
public:
class TestAXTreeSourceArc : public AXTreeSourceArc {
public:
explicit TestAXTreeSourceArc(AXTreeSourceArc::Delegate* delegate)
: AXTreeSourceArc(delegate) {}
// AXTreeSourceArc overrides.
AccessibilityInfoDataWrapper* GetFromId(int32_t id) const override {
auto itr = wrapper_map_.find(id);
if (itr == wrapper_map_.end())
return nullptr;
return itr->second.get();
}
void SetId(std::unique_ptr<AccessibilityInfoDataWrapper>&& wrapper) {
wrapper_map_[wrapper->GetId()] = std::move(wrapper);
}
private:
std::map<int32_t, std::unique_ptr<AccessibilityInfoDataWrapper>>
wrapper_map_;
};
DrawerLayoutHandlerTest() : tree_source_(new TestAXTreeSourceArc(this)) {}
void SetNodeIdToTree(mojom::AccessibilityNodeInfoData* wrapper) {
tree_source_->SetId(std::make_unique<AccessibilityNodeInfoDataWrapper>(
tree_source(), wrapper));
}
void SetWindowIdToTree(mojom::AccessibilityWindowInfoData* wrapper) {
tree_source_->SetId(std::make_unique<AccessibilityWindowInfoDataWrapper>(
tree_source(), wrapper));
}
// AXTreeSourceArc::Delegate overrides.
bool UseFullFocusMode() const override { return true; }
void OnAction(const ui::AXActionData& data) const override {}
AXTreeSourceArc* tree_source() { return tree_source_.get(); }
mojom::AccessibilityEventDataPtr CreateEventWithDrawer() {
auto event = AXEventData::New();
event->source_id = 10;
event->task_id = 1;
event->event_type = AXEventType::WINDOW_STATE_CHANGED;
event->window_data = std::vector<mojom::AccessibilityWindowInfoDataPtr>();
event->window_data->push_back(AXWindowInfoData::New());
AXWindowInfoData* root_window = event->window_data->back().get();
root_window->window_id = 100;
root_window->root_node_id = 10;
SetWindowIdToTree(root_window);
event->node_data.push_back(AXNodeInfoData::New());
AXNodeInfoData* root = event->node_data.back().get();
root->id = 10;
SetNodeIdToTree(root);
SetProperty(root, AXIntListProperty::CHILD_NODE_IDS,
std::vector<int>({1, 2}));
SetProperty(root, AXBooleanProperty::IMPORTANCE, true);
SetProperty(root, AXStringProperty::CLASS_NAME,
"android.support.v4.widget.DrawerLayout");
event->node_data.push_back(AXNodeInfoData::New());
AXNodeInfoData* node1 = event->node_data.back().get();
node1->id = 1;
SetNodeIdToTree(node1);
SetProperty(node1, AXBooleanProperty::VISIBLE_TO_USER, true);
event->node_data.push_back(AXNodeInfoData::New());
AXNodeInfoData* node2 = event->node_data.back().get();
node2->id = 2;
SetNodeIdToTree(node2);
SetProperty(node2, AXIntListProperty::CHILD_NODE_IDS,
std::vector<int>({3}));
SetProperty(node2, AXBooleanProperty::IMPORTANCE, true);
SetProperty(node2, AXBooleanProperty::VISIBLE_TO_USER, true);
event->node_data.push_back(AXNodeInfoData::New());
AXNodeInfoData* node3 = event->node_data.back().get();
node3->id = 3;
SetNodeIdToTree(node3);
SetProperty(node3, AXBooleanProperty::IMPORTANCE, true);
SetProperty(node3, AXBooleanProperty::VISIBLE_TO_USER, true);
SetProperty(node3, AXStringProperty::TEXT, "sample string.");
return event;
}
private:
const std::unique_ptr<TestAXTreeSourceArc> tree_source_;
};
TEST_F(DrawerLayoutHandlerTest, CreateAndSerialize) {
auto event_data = CreateEventWithDrawer();
event_data->event_text = std::vector<std::string>({"Test", "Navigation"});
auto create_result =
DrawerLayoutHandler::CreateIfNecessary(tree_source(), *event_data);
ASSERT_TRUE(create_result.has_value());
ASSERT_EQ(2, create_result.value().first);
ui::AXNodeData data;
create_result.value().second->PostSerializeNode(&data);
ASSERT_EQ(ax::mojom::Role::kMenu, data.role);
ASSERT_EQ("Test Navigation",
data.GetStringAttribute(ax::mojom::StringAttribute::kName));
}
TEST_F(DrawerLayoutHandlerTest, CreateAndSerializeWithoutText) {
auto event_data = CreateEventWithDrawer();
auto create_result =
DrawerLayoutHandler::CreateIfNecessary(tree_source(), *event_data);
ASSERT_TRUE(create_result.has_value());
ASSERT_EQ(2, create_result.value().first);
ui::AXNodeData data;
data.SetName("node description");
create_result.value().second->PostSerializeNode(&data);
// Modifier doesn't override the name by an empty string.
ASSERT_EQ(ax::mojom::Role::kMenu, data.role);
ASSERT_EQ("node description",
data.GetStringAttribute(ax::mojom::StringAttribute::kName));
}
TEST_F(DrawerLayoutHandlerTest, NoCreation) {
auto event_data = CreateEventWithDrawer();
event_data->event_type = AXEventType::WINDOW_CONTENT_CHANGED;
auto create_result =
DrawerLayoutHandler::CreateIfNecessary(tree_source(), *event_data);
ASSERT_FALSE(create_result.has_value());
}
} // namespace arc
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment