arc-a11y: Handle VIEW_SELECTED event appropriately

When user press tab in ListView, VIEW_SELECTED event is dispatched.
This event is not only dispatched from the newly selected node but also
from List itself and child nodes of it. Previously filtering out these
event were took place in Android AccessibilityService side.
This CL migrates the event handling to Chrome side.

Also, this type of event was converted to chrome selection event, but
focus event is more appropriate. This CL also fixes it.

Bug: b/148837372
Bug: b/146916101
Bug: b/143336586
Bug: b/139645142
Bug: b/152374820
Test: unit_tests --gtest_filter="AXTreeSourceArcTest.*"
Test: manual with ag/10901596. tab navigation works in PlayStore hamburger menu and talkback test appp list menu.
Change-Id: Ia8d2f3bb71dcb5a9b9b796dc89e4d8d2a733a940
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2129381
Commit-Queue: Hiroki Sato <hirokisato@chromium.org>
Reviewed-by: David Tseng <dtseng@chromium.org>
Reviewed-by: Sara Kato <sarakato@chromium.org>
Cr-Commit-Position: refs/heads/master@{#756140}

chrome/browser/chromeos/arc/accessibility/arc_accessibility_util.cc

@@ -3,6 +3,7 @@
 // found in the LICENSE file.
 
 #include "chrome/browser/chromeos/arc/accessibility/arc_accessibility_util.h"
+#include "chrome/browser/chromeos/arc/accessibility/accessibility_info_data_wrapper.h"
 
 #include "components/arc/mojom/accessibility_helper.mojom.h"
 #include "ui/accessibility/ax_enums.mojom.h"
@@ -15,9 +16,9 @@ using AXIntListProperty = mojom::AccessibilityIntListProperty;
 using AXNodeInfoData = mojom::AccessibilityNodeInfoData;
 using AXStringProperty = mojom::AccessibilityStringProperty;
 
-ax::mojom::Event ToAXEvent(
-    mojom::AccessibilityEventType arc_event_type,
-    mojom::AccessibilityNodeInfoData* focused_node_info_data) {
+ax::mojom::Event ToAXEvent(mojom::AccessibilityEventType arc_event_type,
+                           AccessibilityInfoDataWrapper* source_node,
+                           AccessibilityInfoDataWrapper* focused_node) {
   switch (arc_event_type) {
     case mojom::AccessibilityEventType::VIEW_FOCUSED:
     case mojom::AccessibilityEventType::VIEW_ACCESSIBILITY_FOCUSED:
@@ -30,7 +31,7 @@ ax::mojom::Event ToAXEvent(
     case mojom::AccessibilityEventType::VIEW_TEXT_SELECTION_CHANGED:
       return ax::mojom::Event::kTextSelectionChanged;
     case mojom::AccessibilityEventType::WINDOW_STATE_CHANGED: {
-      if (focused_node_info_data)
+      if (focused_node)
         return ax::mojom::Event::kFocus;
       else
         return ax::mojom::Event::kLayoutComplete;
@@ -50,16 +51,14 @@ ax::mojom::Event ToAXEvent(
     case mojom::AccessibilityEventType::VIEW_SCROLLED:
       return ax::mojom::Event::kScrollPositionChanged;
     case mojom::AccessibilityEventType::VIEW_SELECTED: {
-      // In Android, VIEW_SELECTED event is fired in the two cases below:
-      // 1. Changing a value in ProgressBar or TimePicker.
-      //    (this usage is NOT documented)
-      // 2. Selecting an item in the context of an AdapterView.
-      //    (officially documented in Android Developer doc below)
-      //    https://developer.android.com/reference/android/view/accessibility/AccessibilityEvent#TYPE_VIEW_SELECTED
-      if (focused_node_info_data && focused_node_info_data->range_info)
+      // VIEW_SELECTED event is not selection event in Chrome.
+      // See the comment on AXTreeSourceArc::NotifyAccessibilityEvent.
+      if (source_node && source_node->IsNode() &&
+          source_node->GetNode()->range_info) {
         return ax::mojom::Event::kValueChanged;
-      else
-        return ax::mojom::Event::kSelection;
+      } else {
+        return ax::mojom::Event::kFocus;
+      }
     }
     case mojom::AccessibilityEventType::VIEW_HOVER_EXIT:
     case mojom::AccessibilityEventType::TOUCH_EXPLORATION_GESTURE_START:

chrome/browser/chromeos/arc/accessibility/arc_accessibility_util.h

@@ -14,9 +14,11 @@
 #include "ui/accessibility/ax_enum_util.h"
 
 namespace arc {
+class AccessibilityInfoDataWrapper;
 
 ax::mojom::Event ToAXEvent(mojom::AccessibilityEventType arc_event_type,
-                           mojom::AccessibilityNodeInfoData* node_info_data);
+                           AccessibilityInfoDataWrapper* source_node,
+                           AccessibilityInfoDataWrapper* focused_node);
 
 base::Optional<mojom::AccessibilityActionType> ConvertToAndroidAction(
     ax::mojom::Action action);

chrome/browser/chromeos/arc/accessibility/ax_tree_source_arc.cc

@@ -24,6 +24,7 @@ namespace arc {
 
 using AXBooleanProperty = mojom::AccessibilityBooleanProperty;
 using AXEventData = mojom::AccessibilityEventData;
+using AXEventIntProperty = mojom::AccessibilityEventIntProperty;
 using AXEventType = mojom::AccessibilityEventType;
 using AXIntProperty = mojom::AccessibilityIntProperty;
 using AXIntListProperty = mojom::AccessibilityIntListProperty;
@@ -143,6 +144,26 @@ void AXTreeSourceArc::NotifyAccessibilityEvent(AXEventData* event_data) {
       android_focused_id_ = IsValid(adjusted_node) ? adjusted_node->GetId()
                                                    : event_data->source_id;
     }
+  } else if (event_data->event_type == AXEventType::VIEW_SELECTED) {
+    // In Android, VIEW_SELECTED event is dispatched in the two cases below:
+    // 1. Changing a value in ProgressBar or TimePicker.
+    // 2. Selecting an item in the context of an AdapterView.
+    AccessibilityInfoDataWrapper* source_node =
+        GetFromId(event_data->source_id);
+    if (!source_node || !source_node->IsNode())
+      return;
+    AXNodeInfoData* node_info = source_node->GetNode();
+    DCHECK(node_info);
+
+    bool is_range_change = !node_info->range_info.is_null();
+    if (!is_range_change) {
+      AccessibilityInfoDataWrapper* selected_node =
+          GetSelectedNodeInfoFromAdapterView(event_data);
+      if (!selected_node)
+        return;
+
+      android_focused_id_ = selected_node->GetId();
+    }
   } else if (event_data->event_type == AXEventType::WINDOW_STATE_CHANGED) {
     // When accessibility window changed, a11y event of WINDOW_CONTENT_CHANGED
     // is fired from Android multiple times.
@@ -196,8 +217,8 @@ void AXTreeSourceArc::NotifyAccessibilityEvent(AXEventData* event_data) {
 
   event_bundle.events.emplace_back();
   ui::AXEvent& event = event_bundle.events.back();
-  event.event_type = ToAXEvent(
-      event_data->event_type, focused_node ? focused_node->GetNode() : nullptr);
+  event.event_type = ToAXEvent(event_data->event_type,
+                               GetFromId(event_data->source_id), focused_node);
   event.id = event_data->source_id;
 
   if (HasProperty(event_data->int_properties,
@@ -452,6 +473,57 @@ AccessibilityInfoDataWrapper* AXTreeSourceArc::FindFirstFocusableNode(
   return nullptr;
 }
 
+AccessibilityInfoDataWrapper*
+AXTreeSourceArc::GetSelectedNodeInfoFromAdapterView(
+    AXEventData* event_data) const {
+  AccessibilityInfoDataWrapper* source_node = GetFromId(event_data->source_id);
+  if (!source_node || !source_node->IsNode())
+    return nullptr;
+
+  AXNodeInfoData* node_info = source_node->GetNode();
+  if (!node_info)
+    return nullptr;
+
+  AccessibilityInfoDataWrapper* selected_node = source_node;
+  if (!node_info->collection_item_info) {
+    // The event source is not an item of AdapterView. If the event source is
+    // AdapterView, select the child. Otherwise, this is an unrelated event.
+    int item_count, from_index, current_item_index;
+    if (!GetProperty(event_data->int_properties, AXEventIntProperty::ITEM_COUNT,
+                     &item_count) ||
+        !GetProperty(event_data->int_properties, AXEventIntProperty::FROM_INDEX,
+                     &from_index) ||
+        !GetProperty(event_data->int_properties,
+                     AXEventIntProperty::CURRENT_ITEM_INDEX,
+                     &current_item_index)) {
+      return nullptr;
+    }
+
+    int index = current_item_index - from_index;
+    if (index < 0)
+      return nullptr;
+
+    std::vector<AccessibilityInfoDataWrapper*> children;
+    source_node->GetChildren(&children);
+    if (index >= static_cast<int>(children.size()))
+      return nullptr;
+
+    selected_node = children[index];
+  }
+
+  // Sometimes a collection item is wrapped by a non-focusable node.
+  // Find a node with focusable property.
+  while (selected_node && !GetBooleanProperty(selected_node->GetNode(),
+                                              AXBooleanProperty::FOCUSABLE)) {
+    std::vector<AccessibilityInfoDataWrapper*> children;
+    selected_node->GetChildren(&children);
+    if (children.size() != 1)
+      break;
+    selected_node = children[0];
+  }
+  return selected_node;
+}
+
 void AXTreeSourceArc::UpdateAXNameCache(
     AccessibilityInfoDataWrapper* focused_node,
     const std::vector<std::string>& event_text) {

chrome/browser/chromeos/arc/accessibility/ax_tree_source_arc.h

@@ -126,6 +126,9 @@ class AXTreeSourceArc : public ui::AXTreeSource<AccessibilityInfoDataWrapper*,
   AccessibilityInfoDataWrapper* FindFirstFocusableNode(
       AccessibilityInfoDataWrapper* info_data) const;
 
+  AccessibilityInfoDataWrapper* GetSelectedNodeInfoFromAdapterView(
+      mojom::AccessibilityEventData* event_data) const;
+
   void UpdateAXNameCache(AccessibilityInfoDataWrapper* focused_node,
                          const std::vector<std::string>& event_text);
 

chrome/browser/chromeos/arc/accessibility/ax_tree_source_arc_unittest.cc

@@ -22,6 +22,7 @@ using AXBooleanProperty = mojom::AccessibilityBooleanProperty;
 using AXCollectionInfoData = mojom::AccessibilityCollectionInfoData;
 using AXCollectionItemInfoData = mojom::AccessibilityCollectionItemInfoData;
 using AXEventData = mojom::AccessibilityEventData;
+using AXEventIntProperty = mojom::AccessibilityEventIntProperty;
 using AXEventType = mojom::AccessibilityEventType;
 using AXIntListProperty = mojom::AccessibilityIntListProperty;
 using AXIntProperty = mojom::AccessibilityIntProperty;
@@ -98,6 +99,15 @@ void SetProperty(AXWindowInfoData* window,
   prop_map.insert(std::make_pair(prop, value));
 }
 
+void SetProperty(AXEventData* event, AXEventIntProperty prop, int32_t value) {
+  if (!event->int_properties) {
+    event->int_properties = base::flat_map<AXEventIntProperty, int>();
+  }
+  auto& prop_map = event->int_properties.value();
+  base::EraseIf(prop_map, [prop](auto it) { return it.first == prop; });
+  prop_map.insert(std::make_pair(prop, value));
+}
+
 class MockAutomationEventRouter
     : public extensions::AutomationEventRouterInterface {
  public:
@@ -965,6 +975,7 @@ TEST_F(AXTreeSourceArcTest, GetTreeDataAppliesFocus) {
 TEST_F(AXTreeSourceArcTest, OnViewSelectedEvent) {
   auto event = AXEventData::New();
   event->task_id = 1;
+  event->event_type = AXEventType::VIEW_SELECTED;
 
   event->window_data = std::vector<mojom::AccessibilityWindowInfoDataPtr>();
   event->window_data->emplace_back(AXWindowInfoData::New());
@@ -973,32 +984,83 @@ TEST_F(AXTreeSourceArcTest, OnViewSelectedEvent) {
   root_window->root_node_id = 10;
 
   event->node_data.emplace_back(AXNodeInfoData::New());
-  event->source_id = 1;  // button->id
   AXNodeInfoData* root = event->node_data.back().get();
   root->id = 10;
   SetProperty(root, AXIntListProperty::CHILD_NODE_IDS, std::vector<int>({1}));
 
-  // Add child node.
   event->node_data.emplace_back(AXNodeInfoData::New());
-  AXNodeInfoData* button = event->node_data.back().get();
-  button->id = 1;
-  SetProperty(button, AXBooleanProperty::FOCUSABLE, true);
-  SetProperty(button, AXBooleanProperty::IMPORTANCE, true);
+  AXNodeInfoData* list = event->node_data.back().get();
+  list->id = 1;
+  SetProperty(list, AXBooleanProperty::FOCUSABLE, true);
+  SetProperty(list, AXBooleanProperty::IMPORTANCE, true);
+  SetProperty(list, AXIntListProperty::CHILD_NODE_IDS,
+              std::vector<int>({2, 3, 4}));
+
+  // Slider.
+  event->node_data.emplace_back(AXNodeInfoData::New());
+  AXNodeInfoData* slider = event->node_data.back().get();
+  slider->id = 2;
+  SetProperty(slider, AXBooleanProperty::FOCUSABLE, true);
+  SetProperty(slider, AXBooleanProperty::IMPORTANCE, true);
+  slider->range_info = AXRangeInfoData::New();
 
-  // Ensure that button has a focus.
-  event->event_type = AXEventType::VIEW_FOCUSED;
+  // Simple list item.
+  event->node_data.emplace_back(AXNodeInfoData::New());
+  AXNodeInfoData* simple_item = event->node_data.back().get();
+  simple_item->id = 3;
+  SetProperty(simple_item, AXBooleanProperty::FOCUSABLE, true);
+  SetProperty(simple_item, AXBooleanProperty::IMPORTANCE, true);
+  simple_item->collection_item_info = AXCollectionItemInfoData::New();
+
+  // This node is not focusable.
+  event->node_data.emplace_back(AXNodeInfoData::New());
+  AXNodeInfoData* wrap_node = event->node_data.back().get();
+  wrap_node->id = 4;
+  SetProperty(wrap_node, AXBooleanProperty::IMPORTANCE, true);
+  SetProperty(wrap_node, AXIntListProperty::CHILD_NODE_IDS,
+              std::vector<int>({5}));
+  wrap_node->collection_item_info = AXCollectionItemInfoData::New();
+
+  // A list item expected to get the focus.
+  event->node_data.emplace_back(AXNodeInfoData::New());
+  AXNodeInfoData* item = event->node_data.back().get();
+  item->id = 5;
+  SetProperty(item, AXBooleanProperty::FOCUSABLE, true);
+  SetProperty(item, AXBooleanProperty::IMPORTANCE, true);
+
+  // A selected event from Slider is kValueChanged.
+  event->source_id = slider->id;
   CallNotifyAccessibilityEvent(event.get());
+  EXPECT_EQ(1, GetDispatchedEventCount(ax::mojom::Event::kValueChanged));
 
-  // Without range_info, kSelection event should be emitted. Usually this event
-  // is fired from AdapterView.
-  event->event_type = AXEventType::VIEW_SELECTED;
+  // A selected event from a collection. In Android, these event properties are
+  // populated by AdapterView.
+  event->source_id = list->id;
+  SetProperty(event.get(), AXEventIntProperty::ITEM_COUNT, 3);
+  SetProperty(event.get(), AXEventIntProperty::FROM_INDEX, 0);
+  SetProperty(event.get(), AXEventIntProperty::CURRENT_ITEM_INDEX, 2);
   CallNotifyAccessibilityEvent(event.get());
-  EXPECT_EQ(1, GetDispatchedEventCount(ax::mojom::Event::kSelection));
+  EXPECT_EQ(1, GetDispatchedEventCount(ax::mojom::Event::kFocus));
+
+  ui::AXTreeData data;
+  EXPECT_TRUE(CallGetTreeData(&data));
+  EXPECT_EQ(item->id, data.focus_id);
 
-  // Set range_info, the event should be kValueChanged.
-  button->range_info = AXRangeInfoData::New();
+  // A selected event from a collection item.
+  event->source_id = simple_item->id;
+  event->int_properties->clear();
   CallNotifyAccessibilityEvent(event.get());
-  EXPECT_EQ(1, GetDispatchedEventCount(ax::mojom::Event::kValueChanged));
+  EXPECT_EQ(2, GetDispatchedEventCount(ax::mojom::Event::kFocus));
+
+  EXPECT_TRUE(CallGetTreeData(&data));
+  EXPECT_EQ(simple_item->id, data.focus_id);
+
+  // A selected event from non collection node is dropped.
+  event->source_id = item->id;
+  event->int_properties->clear();
+  CallNotifyAccessibilityEvent(event.get());
+  EXPECT_EQ(2,
+            GetDispatchedEventCount(ax::mojom::Event::kFocus));  // not changed
 }
 
 TEST_F(AXTreeSourceArcTest, OnWindowStateChangedEvent) {