Extend AXLiveRegionTracker to generate live region change text.

When a live region changes, optionally provide a way to generate the
text associated with that change. On some platforms we might expose
that text directly so that AT doesn't need to try to compute it.

The computation is somewhat subtle because it involves worrying about
deleted nodes and handling aria-atomic, aria-relevant, and aria-busy.

Bug: 1042893
Change-Id: Ia79f0c976bef73ea25a8821dbb9fc7f22fde0e3c
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2006131
Commit-Queue: Dominic Mazzoni <dmazzoni@chromium.org>
Reviewed-by: Kevin Babbitt <kbabbitt@microsoft.com>
Cr-Commit-Position: refs/heads/master@{#738876}

ui/accessibility/BUILD.gn

@@ -253,6 +253,7 @@ test("accessibility_unittests") {
     "ax_event_generator_unittest.cc",
     "ax_generated_tree_unittest.cc",
     "ax_language_detection_unittest.cc",
+    "ax_live_region_tracker_unittest.cc",
     "ax_node_data_unittest.cc",
     "ax_node_position_unittest.cc",
     "ax_range_unittest.cc",

ui/accessibility/ax_event_generator.cc

@@ -265,7 +265,8 @@ void AXEventGenerator::OnStringAttributeChanged(AXTree* tree,
 
       if (node->data().HasStringAttribute(
               ax::mojom::StringAttribute::kContainerLiveStatus)) {
-        FireLiveRegionEvents(node);
+        FireLiveRegionEvents(node,
+                             AXLiveRegionTracker::ChangeType::kTextChanged);
       }
       break;
     case ax::mojom::StringAttribute::kPlaceholder:
@@ -480,7 +481,7 @@ void AXEventGenerator::OnTreeDataChanged(AXTree* tree,
 
 void AXEventGenerator::OnNodeWillBeDeleted(AXTree* tree, AXNode* node) {
   if (live_region_tracker_->GetLiveRoot(node))
-    FireLiveRegionEvents(node);
+    FireLiveRegionEvents(node, AXLiveRegionTracker::ChangeType::kNodeRemoved);
   live_region_tracker_->OnNodeWillBeDeleted(node);
 
   DCHECK_EQ(tree_, tree);
@@ -529,19 +530,25 @@ void AXEventGenerator::OnAtomicUpdateFinished(
       continue;
     }
 
-    if (IsAlert(change.node->data().role))
+    if (IsAlert(change.node->data().role)) {
       AddEvent(change.node, Event::ALERT);
-    else if (IsActiveLiveRegion(change))
+    } else if (IsActiveLiveRegion(change)) {
       AddEvent(change.node, Event::LIVE_REGION_CREATED);
-    else if (IsContainedInLiveRegion(change))
-      FireLiveRegionEvents(change.node);
+    } else if (IsContainedInLiveRegion(change)) {
+      FireLiveRegionEvents(change.node,
+                           AXLiveRegionTracker::ChangeType::kNodeAdded);
+    }
   }
 
   FireActiveDescendantEvents();
+  if (should_compute_live_region_change_description_)
+    ComputeLiveRegionChangeDescription();
   live_region_tracker_->OnAtomicUpdateFinished();
 }
 
-void AXEventGenerator::FireLiveRegionEvents(AXNode* node) {
+void AXEventGenerator::FireLiveRegionEvents(
+    AXNode* node,
+    AXLiveRegionTracker::ChangeType type) {
   AXNode* live_root = live_region_tracker_->GetLiveRootIfNotBusy(node);
 
   // Note that |live_root| might be nullptr if a live region was just added,
@@ -554,12 +561,44 @@ void AXEventGenerator::FireLiveRegionEvents(AXNode* node) {
            .GetStringAttribute(ax::mojom::StringAttribute::kName)
            .empty()) {
     AddEvent(node, Event::LIVE_REGION_NODE_CHANGED);
+
+    if (should_compute_live_region_change_description_)
+      live_region_tracker_->ComputeTextForChangedNode(node, type);
   }
 
   // Fire LIVE_REGION_CHANGED on the root of the live region.
   AddEvent(live_root, Event::LIVE_REGION_CHANGED);
 }
 
+void AXEventGenerator::ComputeLiveRegionChangeDescription() {
+  std::map<AXNode*, std::string> live_region_change_description;
+  live_region_tracker_->ComputeLiveRegionChangeDescription(
+      &live_region_change_description);
+
+  for (auto& key_value : live_region_change_description) {
+    AXNode* live_root = key_value.first;
+    std::string text = key_value.second;
+
+    std::map<AXNode*, std::set<EventParams>>::iterator tree_events_iter =
+        tree_events_.find(live_root);
+    if (tree_events_iter == tree_events_.end())
+      continue;
+    std::set<EventParams>& live_root_events = tree_events_iter->second;
+    for (std::set<EventParams>::iterator event_params_iter =
+             live_root_events.begin();
+         event_params_iter != live_root_events.end(); ++event_params_iter) {
+      if (event_params_iter->event != Event::LIVE_REGION_CHANGED)
+        continue;
+
+      EventParams params = *event_params_iter;
+      live_root_events.erase(event_params_iter);
+      params.live_region_change_description = text;
+      live_root_events.insert(params);
+      break;
+    }
+  }
+}
+
 void AXEventGenerator::FireActiveDescendantEvents() {
   for (AXNode* node : active_descendant_changed_) {
     AXNode* descendant = tree_->GetFromId(node->data().GetIntAttribute(

ui/accessibility/ax_event_generator.h

@@ -13,6 +13,7 @@
 #include <vector>
 
 #include "ui/accessibility/ax_export.h"
+#include "ui/accessibility/ax_live_region_tracker.h"
 #include "ui/accessibility/ax_tree.h"
 #include "ui/accessibility/ax_tree_observer.h"
 
@@ -96,6 +97,10 @@ class AX_EXPORT AXEventGenerator : public AXTreeObserver {
     Event event;
     ax::mojom::EventFrom event_from;
 
+    // Only for LIVE_REGION_CHANGED events, if explicitly enabled
+    // with set_compute_live_region_change_description(true).
+    std::string live_region_change_description;
+
     bool operator==(const EventParams& rhs);
     bool operator<(const EventParams& rhs) const;
   };
@@ -167,6 +172,12 @@ class AX_EXPORT AXEventGenerator : public AXTreeObserver {
   // same order they were added.
   void AddEvent(ui::AXNode* node, Event event);
 
+  // Set whether or not the text of live region changed events should
+  // be computed. (It's only needed on some platforms.)
+  void set_should_compute_live_region_change_description(bool should_compute) {
+    should_compute_live_region_change_description_ = should_compute;
+  }
+
  protected:
   // AXTreeObserver overrides.
   void OnNodeDataChanged(AXTree* tree,
@@ -217,10 +228,11 @@ class AX_EXPORT AXEventGenerator : public AXTreeObserver {
                               const std::vector<Change>& changes) override;
 
  private:
-  void FireLiveRegionEvents(AXNode* node);
+  void FireLiveRegionEvents(AXNode* node, AXLiveRegionTracker::ChangeType type);
   void FireActiveDescendantEvents();
   void FireRelationSourceEvents(AXTree* tree, AXNode* target_node);
   bool ShouldFireLoadEvents(AXNode* node);
+  void ComputeLiveRegionChangeDescription();
   static void GetRestrictionStates(ax::mojom::Restriction restriction,
                                    bool* is_enabled,
                                    bool* is_readonly);
@@ -237,7 +249,12 @@ class AX_EXPORT AXEventGenerator : public AXTreeObserver {
   // OnAtomicUpdateFinished. List of nodes whose active descendant changed.
   std::vector<AXNode*> active_descendant_changed_;
 
-  // Helper that tracks live regions.
+  // Whether or not we should compute the text of LIVE_REGION_CHANGED events.
+  // Only needed on some platforms.
+  bool should_compute_live_region_change_description_ = false;
+
+  // Helper class that optionally tracks live regions and computes the
+  // text that should be spoken when a live region changes.
   std::unique_ptr<AXLiveRegionTracker> live_region_tracker_;
 };
 

ui/accessibility/ax_live_region_tracker.cc

@@ -5,10 +5,31 @@
 #include "ui/accessibility/ax_live_region_tracker.h"
 
 #include "base/stl_util.h"
+#include "base/strings/string_util.h"
 #include "ui/accessibility/ax_node.h"
 #include "ui/accessibility/ax_role_properties.h"
 
 namespace ui {
+namespace {
+
+void GetTextFromSubtreeHelper(AXNode* node, std::vector<std::string>* strings) {
+  if (node->data().role == ax::mojom::Role::kInlineTextBox)
+    return;
+
+  std::string name =
+      node->GetStringAttribute(ax::mojom::StringAttribute::kName);
+  if (!name.empty())
+    strings->push_back(name);
+  for (AXNode* child : node->children())
+    GetTextFromSubtreeHelper(child, strings);
+}
+
+std::string GetTextFromSubtree(AXNode* node) {
+  std::vector<std::string> strings;
+  GetTextFromSubtreeHelper(node, &strings);
+  return base::JoinString(strings, " ");
+}
+}  // namespace
 
 AXLiveRegionTracker::AXLiveRegionTracker(AXTree* tree) : tree_(tree) {
   InitializeLiveRegionNodeToRoot(tree_->root(), nullptr);
@@ -76,4 +97,106 @@ bool AXLiveRegionTracker::IsLiveRegionRoot(const AXNode* node) {
              ax::mojom::StringAttribute::kLiveStatus) != "off";
 }
 
+void AXLiveRegionTracker::ComputeTextForChangedNode(AXNode* node,
+                                                    ChangeType type) {
+  AXNode* live_root = GetLiveRoot(node);
+  if (!live_root)
+    return;
+
+  LiveRegionChange change;
+  change.node_id = node->id();
+  change.data = node->data();
+  change.type = type;
+  live_region_changes_[live_root].push_back(change);
+}
+
+void AXLiveRegionTracker::ComputeLiveRegionChangeDescription(
+    std::map<AXNode*, std::string>* live_region_change_description) {
+  for (auto iter : live_region_changes_) {
+    AXNode* live_root = iter.first;
+    std::set<int32_t> already_handled_node_ids;
+    std::vector<std::string> additions_vector;
+    std::vector<std::string> removals_vector;
+
+    for (LiveRegionChange change : iter.second) {
+      int32_t node_id = change.node_id;
+      ui::AXNodeData data = change.data;
+      bool is_deleted =
+          deleted_node_ids_.find(node_id) != deleted_node_ids_.end();
+      if (!is_deleted) {
+        AXNode* node = tree_->GetFromId(node_id);
+        DCHECK(node);
+        if (!node)
+          continue;
+
+        while (node->data().GetBoolAttribute(
+                   ax::mojom::BoolAttribute::kContainerLiveAtomic) &&
+               !node->data().GetBoolAttribute(
+                   ax::mojom::BoolAttribute::kLiveAtomic)) {
+          node = node->parent();
+        }
+        if (node) {
+          node_id = node->id();
+          data = node->data();
+        }
+      }
+
+      if (already_handled_node_ids.find(node_id) !=
+          already_handled_node_ids.end())
+        continue;
+      already_handled_node_ids.insert(node_id);
+
+      bool explore_subtree =
+          data.GetBoolAttribute(ax::mojom::BoolAttribute::kLiveAtomic);
+      std::string relevant;
+      if (data.GetStringAttribute(
+              ax::mojom::StringAttribute::kContainerLiveRelevant, &relevant)) {
+        relevant = base::ToLowerASCII(relevant);
+      } else {
+        relevant = "additions text";
+      }
+
+      bool all = relevant.find("all") != std::string::npos;
+      bool additions = all || relevant.find("additions") != std::string::npos;
+      bool removals = all || relevant.find("removals") != std::string::npos;
+      bool text = all || relevant.find("text") != std::string::npos;
+
+      std::string live_text;
+      if (explore_subtree && !is_deleted) {
+        AXNode* node = tree_->GetFromId(node_id);
+        DCHECK(node);
+        if (node)
+          live_text = GetTextFromSubtree(node);
+      } else {
+        live_text = data.GetStringAttribute(ax::mojom::StringAttribute::kName);
+      }
+      if (live_text.empty())
+        continue;
+
+      if (change.type == ChangeType::kNodeAdded && additions)
+        additions_vector.push_back(live_text);
+      else if (change.type == ChangeType::kTextChanged && text)
+        additions_vector.push_back(live_text);
+      else if (change.type == ChangeType::kNodeRemoved && removals)
+        removals_vector.push_back(live_text);
+    }
+
+    if (additions_vector.empty() && removals_vector.empty())
+      continue;
+
+    std::string additions_text = base::JoinString(additions_vector, " ");
+    std::string removals_text = base::JoinString(removals_vector, " ");
+
+    std::string final_text = additions_text;
+    if (!additions_text.empty() && !removals_text.empty())
+      final_text += ", ";
+    if (!removals_text.empty())
+      final_text += removals_text + " removed";  // TODO: i18n
+
+    (*live_region_change_description)[live_root] = final_text;
+  }
+
+  live_region_changes_.clear();
+}
+
 }  // namespace ui

ui/accessibility/ax_live_region_tracker.h

@@ -17,6 +17,8 @@ namespace ui {
 // an AXTree.
 class AXLiveRegionTracker {
  public:
+  enum class ChangeType : int32_t { kNodeAdded, kNodeRemoved, kTextChanged };
+
   explicit AXLiveRegionTracker(AXTree* tree);
 
   ~AXLiveRegionTracker();
@@ -29,15 +31,31 @@ class AXLiveRegionTracker {
 
   static bool IsLiveRegionRoot(const AXNode* node);
 
+  void ComputeTextForChangedNode(AXNode* node, ChangeType type);
+  void ComputeLiveRegionChangeDescription(
+      std::map<AXNode*, std::string>* live_region_change_description);
+
  private:
+  struct LiveRegionChange {
+    int32_t node_id;
+    ui::AXNodeData data;
+    ChangeType type;
+  };
+
   void InitializeLiveRegionNodeToRoot(AXNode* node, AXNode* current_root);
 
   AXTree* tree_;  // Not owned.
 
   // Map from live region node to its live region root ID.
   std::map<AXNode*, int32_t> live_region_node_to_root_id_;
+
+  // Set of node IDs that are no longer valid. Cleared after each call to
+  // OnAtomicUpdateFinished.
   std::set<int32_t> deleted_node_ids_;
 
+  // Map from live region root to a list of changes.
+  std::map<AXNode*, std::vector<LiveRegionChange>> live_region_changes_;
+
   DISALLOW_COPY_AND_ASSIGN(AXLiveRegionTracker);
 };