Implement scopes_route/names_route processing in flutter ax bridge

When a node with scopes_route flag is added, perform a depth first
search for a child with names_route flag.  If found, that node
should receive focus and be spoken.  Subsequent updates for a
the node should not speak or refocus.  When a scopes node is
removed, refocus a names node under the last scopes node unless
there is none. Refocused nodes after a removed node should not
be spoken, only focused.  If there is no scopes node found, focus
on the first focusable node in the tree.

This is used by flutter alert dialogs and drawers that require
the user's attention if the screen reader is enabled.

Bug: None
Test: Local display assistant and unittest
Change-Id: I69b0205d47269ec75265ca1af56400cc28faea56
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2436768Reviewed-by: Daniel Nicoara <dnicoara@chromium.org>
Commit-Queue: Randy Rossi <rmrossi@chromium.org>
Cr-Commit-Position: refs/heads/master@{#812086}

chromecast/browser/accessibility/flutter/ax_tree_source_flutter.cc

@@ -217,7 +217,7 @@ void AXTreeSourceFlutter::NotifyAccessibilityEvent(
     }
   }
 
-  // Do we need to put focus back on the root after a child tree
+  // Do we need to put focus somewhere after a child tree
   // has been removed?
   bool need_focus_clear = false;
   for (std::string id : child_trees_) {
@@ -281,6 +281,9 @@ void AXTreeSourceFlutter::NotifyAccessibilityEvent(
     // Clear reparented children.
     reparented_children_.clear();
 
+    // Handle routes added/removed from the tree.
+    HandleRoutes(&event_bundle.events);
+
     event_bundle.updates.emplace_back();
     current_tree_serializer_->SerializeChanges(GetFromId(event.id),
                                                &event_bundle.updates.back());
@@ -293,12 +296,12 @@ void AXTreeSourceFlutter::NotifyAccessibilityEvent(
     HandleNativeTTS();
   }
 
-  // Place focus back on the root if a child tree has disappeared
+  // Need to refocus
   if (need_focus_clear) {
     event_bundle.events.emplace_back();
     ui::AXEvent& focus_event = event_bundle.events.back();
     focus_event.event_type = ax::mojom::Event::kFocus;
-    focus_event.id = root_id_;
+    focus_event.id = focused_id_;
     focus_event.event_from = ax::mojom::EventFrom::kNone;
   }
 
@@ -622,6 +625,135 @@ void AXTreeSourceFlutter::HandleLiveRegions(std::vector<ui::AXEvent>* events) {
   std::swap(live_region_name_cache_, new_live_region_map);
 }
 
+// Handle created/deleted nodes with scopes routes flag set.
+void AXTreeSourceFlutter::HandleRoutes(std::vector<ui::AXEvent>* events) {
+  bool focused_new = false;
+  for (const auto& it : tree_map_) {
+    FlutterSemanticsNode* node = it.second.get();
+    if (!node->HasScopesRoute())
+      continue;
+
+    // Do we know about this node already? If so, skip.
+    if (std::find(scopes_route_cache_.begin(), scopes_route_cache_.end(),
+                  node->GetId()) != scopes_route_cache_.end()) {
+      continue;
+    }
+
+    scopes_route_cache_.push_back(node->GetId());
+
+    // Find a node in the sub-tree with names route flag set.
+    FlutterSemanticsNode* sub_node = FindRoutesNode(node);
+    if (sub_node) {
+      ui::AXNodeData data;
+      SerializeNode(sub_node, &data);
+      std::string name;
+      data.GetStringAttribute(ax::mojom::StringAttribute::kName, &name);
+      if (name.length() > 0) {
+        focused_new = true;
+
+        // Focus the node.
+        focused_id_ = sub_node->GetId();
+        events->emplace_back();
+        ui::AXEvent& focus_event = events->back();
+        focus_event.event_type = ax::mojom::Event::kFocus;
+        focus_event.id = focused_id_;
+        focus_event.event_from = ax::mojom::EventFrom::kNone;
+
+        // Speak it.
+        std::unique_ptr<content::TtsUtterance> utterance =
+            content::TtsUtterance::Create(browser_context_);
+        utterance->SetText(name);
+        auto* tts_controller = content::TtsController::GetInstance();
+        tts_controller->Stop();
+        tts_controller->SpeakOrEnqueue(std::move(utterance));
+      }
+    }
+  }
+
+  // Detect any removed nodes with scopes_route flag.
+  bool need_refocus = false;
+  for (std::vector<int32_t>::iterator it = scopes_route_cache_.begin();
+       it != scopes_route_cache_.end();) {
+    int32_t id = *it;
+    if (GetFromId(id) == nullptr) {
+      // This was removed.
+      it = scopes_route_cache_.erase(it++);
+      need_refocus = true;
+    } else {
+      it++;
+    }
+  }
+
+  // After a deletion, use the last scopes route node to refocus on a child
+  // with names route set (unless we already focused on a new node from above).
+  if (need_refocus && !focused_new) {
+    // Select the last in-depth node with scopesRoute for refocus
+    FlutterSemanticsNode* refocused_routes_node = nullptr;
+    if (scopes_route_cache_.size() > 0)
+      refocused_routes_node =
+          FindRoutesNode(GetFromId(scopes_route_cache_.back()));
+    if (refocused_routes_node) {
+      focused_id_ = refocused_routes_node->GetId();
+    } else {
+      focused_id_ = FindFirstFocusableNodeId();
+    }
+    events->emplace_back();
+    ui::AXEvent& focus_event = events->back();
+    focus_event.event_type = ax::mojom::Event::kFocus;
+    focus_event.id = focused_id_;
+    focus_event.event_from = ax::mojom::EventFrom::kNone;
+  }
+}
+
+// Perform depth first search for a subtree node under 'parent'
+// with names route flag set.
+FlutterSemanticsNode* AXTreeSourceFlutter::FindRoutesNode(
+    FlutterSemanticsNode* parent) {
+  if (parent == nullptr)
+    return nullptr;
+
+  std::stack<FlutterSemanticsNode*> stack;
+  stack.push(parent);
+  while (!stack.empty()) {
+    FlutterSemanticsNode* node = stack.top();
+    stack.pop();
+    DCHECK(node);
+
+    if (node->HasNamesRoute()) {
+      return node;
+    }
+
+    std::vector<FlutterSemanticsNode*> children;
+    node->GetChildren(&children);
+    for (FlutterSemanticsNode* child : children)
+      stack.push(child);
+  }
+  return nullptr;
+}
+
+// Find the first focusable node.
+int32_t AXTreeSourceFlutter::FindFirstFocusableNodeId() {
+  std::stack<FlutterSemanticsNode*> stack;
+  stack.push(GetFromId(root_id_));
+  while (!stack.empty()) {
+    FlutterSemanticsNode* node = stack.top();
+    stack.pop();
+    DCHECK(node);
+
+    if (node->CanBeAccessibilityFocused()) {
+      return node->GetId();
+    }
+
+    std::vector<FlutterSemanticsNode*> children;
+    node->GetChildren(&children);
+    for (FlutterSemanticsNode* child : children)
+      stack.push(child);
+  }
+
+  // Fallback to root if none found.
+  return root_id_;
+}
+
 void AXTreeSourceFlutter::UpdateTree() {
   // Update the tree with last known flutter nodes.
   // TODO: A more efficient update would be to isolate just the parent node

chromecast/browser/accessibility/flutter/ax_tree_source_flutter.h

@@ -153,9 +153,20 @@ class AXTreeSourceFlutter : public ui::AXTreeSource<FlutterSemanticsNode*,
   // Detects live region changes and generates events for them.
   void HandleLiveRegions(std::vector<ui::AXEvent>* events);
 
+  // Detects added or deleted routes that trigger TTS from edge
+  // transitions (i.e. alert dialogs).
+  void HandleRoutes(std::vector<ui::AXEvent>* events);
+
   // Detects rapidly changing nodes and use native TTS instead.
   void HandleNativeTTS();
 
+  // Depth first search for a node under 'parent' with names route flag.
+  FlutterSemanticsNode* FindRoutesNode(FlutterSemanticsNode* parent);
+
+  // Find the first focusable node from the root.  If none found, return
+  // the root node id.
+  int32_t FindFirstFocusableNodeId();
+
   std::unique_ptr<AXTreeFlutterSerializer> current_tree_serializer_;
   int32_t root_id_;
   int32_t window_id_;
@@ -182,6 +193,9 @@ class AXTreeSourceFlutter : public ui::AXTreeSource<FlutterSemanticsNode*,
   // Cache from node id to computed name for live region.
   std::map<int32_t, std::string> live_region_name_cache_;
 
+  // Cache for nodes with scopes route flags.
+  std::vector<int32_t> scopes_route_cache_;
+
   // Cache form node id to tts string for native tts components.
   std::map<int32_t, std::string> native_tts_name_cache_;
 

chromecast/browser/accessibility/flutter/ax_tree_source_flutter_unittest.cc

@@ -7,6 +7,10 @@
 #include <string>
 
 #include "chromecast/browser/accessibility/flutter/flutter_semantics_node_wrapper.h"
+#include "content/public/browser/content_browser_client.h"
+#include "content/public/browser/tts_controller.h"
+#include "content/public/browser/tts_platform.h"
+#include "content/public/common/content_client.h"
 #include "extensions/browser/api/automation_internal/automation_event_router_interface.h"
 #include "testing/gmock/include/gmock/gmock.h"
 #include "testing/gtest/include/gtest/gtest.h"
@@ -17,9 +21,71 @@ using ::testing::StrictMock;
 using ::gallium::castos::ActionProperties;
 using ::gallium::castos::BooleanProperties;
 using ::gallium::castos::OnAccessibilityEventRequest;
+using ::gallium::castos::OnAccessibilityEventRequest_EventType_CONTENT_CHANGED;
 using ::gallium::castos::OnAccessibilityEventRequest_EventType_FOCUSED;
 using ::gallium::castos::Rect;
 
+namespace content {
+
+class MockTtsPlatformImpl : public TtsPlatform {
+ public:
+  MockTtsPlatformImpl() = default;
+  virtual ~MockTtsPlatformImpl() = default;
+
+  void set_voices(const std::vector<VoiceData>& voices) { voices_ = voices; }
+
+  void set_run_speak_callback(bool value) { run_speak_callback_ = value; }
+  void set_is_speaking(bool value) { is_speaking_ = value; }
+
+  // TtsPlatform:
+  bool PlatformImplAvailable() override { return true; }
+  void Speak(int utterance_id,
+             const std::string& utterance,
+             const std::string& lang,
+             const VoiceData& voice,
+             const UtteranceContinuousParameters& params,
+             base::OnceCallback<void(bool)> on_speak_finished) override {
+    if (run_speak_callback_)
+      std::move(on_speak_finished).Run(true);
+    last_spoken_utterance_ = utterance;
+  }
+  bool IsSpeaking() override { return is_speaking_; }
+  bool StopSpeaking() override { return true; }
+  void Pause() override {}
+  void Resume() override {}
+  void GetVoices(std::vector<VoiceData>* out_voices) override {
+    *out_voices = voices_;
+  }
+  bool LoadBuiltInTtsEngine(BrowserContext* browser_context) override {
+    return false;
+  }
+  void WillSpeakUtteranceWithVoice(TtsUtterance* utterance,
+                                   const VoiceData& voice_data) override {}
+  void SetError(const std::string& error) override {}
+  std::string GetError() override { return std::string(); }
+  void ClearError() override {}
+  const std::string& GetLastSpokenUtterance() { return last_spoken_utterance_; }
+  void ClearLastSpokenUtterance() { last_spoken_utterance_ = ""; }
+
+ private:
+  std::vector<VoiceData> voices_;
+  bool run_speak_callback_ = true;
+  bool is_speaking_ = false;
+  std::string last_spoken_utterance_;
+};
+
+class MockContentBrowserClient : public ContentBrowserClient {
+ public:
+  ~MockContentBrowserClient() override {}
+};
+
+class MockContentClient : public ContentClient {
+ public:
+  ~MockContentClient() override {}
+};
+
+}  // namespace content
+
 namespace chromecast {
 namespace accessibility {
 
@@ -654,5 +720,129 @@ TEST_F(AXTreeSourceFlutterTest, NoClickable) {
   ASSERT_FALSE(data->GetBoolAttribute(ax::mojom::BoolAttribute::kClickable));
 }
 
+// Tests a new node with scopes route will focus and speak
+// a child with names route set.
+TEST_F(AXTreeSourceFlutterTest, ScopesRoute) {
+  // Install a mock tts platform
+  auto* tts_controller = content::TtsController::GetInstance();
+  content::MockTtsPlatformImpl mock_tts_platform;
+  tts_controller->SetTtsPlatform(&mock_tts_platform);
+
+  // Setup some mocks required for tts platform impl
+  content::MockContentBrowserClient mock_content_browser_client;
+  content::MockContentClient client;
+  content::SetContentClient(&client);
+  content::SetBrowserClientForTesting(&mock_content_browser_client);
+
+  // Add node with scopes route and child with names route. Focus
+  // should move to node with names route.
+  //
+  OnAccessibilityEventRequest event;
+
+  event.set_source_id(0);
+  event.set_window_id(1);
+  event.set_event_type(OnAccessibilityEventRequest_EventType_CONTENT_CHANGED);
+
+  SemanticsNode* root = event.add_node_data();
+  root->set_node_id(0);
+
+  SemanticsNode* child1;
+  SemanticsNode* child2;
+  SemanticsNode* child3;
+  SemanticsNode* child4;
+  SemanticsNode* child5;
+  SemanticsNode* child6;
+
+  child1 = AddChild(&event, root, 1, 0, 0, 1, 1, false);
+  child2 = AddChild(&event, child1, 2, 0, 0, 1, 1, false);
+  child3 = AddChild(&event, child2, 3, 0, 0, 1, 1, false);
+  std::string child_3_label = "Speak This";
+  child3->set_label(child_3_label);
+
+  child4 = AddChild(&event, root, 4, 0, 0, 1, 1, false);
+  child5 = AddChild(&event, child4, 5, 0, 0, 1, 1, false);
+  child6 = AddChild(&event, child5, 6, 0, 0, 1, 1, false);
+  std::string child_6_label = "Speak That";
+  child6->set_label(child_6_label);
+
+  BooleanProperties* boolean_properties;
+
+  // Set scopes on child2, names on child3
+  boolean_properties = child2->mutable_boolean_properties();
+  boolean_properties->set_scopes_route(true);
+  boolean_properties = child3->mutable_boolean_properties();
+  boolean_properties->set_names_route(true);
+
+  // Set scopes on child5, names on child6
+  boolean_properties = child5->mutable_boolean_properties();
+  boolean_properties->set_scopes_route(true);
+  boolean_properties = child6->mutable_boolean_properties();
+  boolean_properties->set_names_route(true);
+
+  CallNotifyAccessibilityEvent(&event);
+
+  // focus should have moved to child 6 since that is the first
+  // node that will be found with scopes
+  ui::AXTreeData tree_data;
+  CallGetTreeData(&tree_data);
+  ASSERT_EQ(6, tree_data.focus_id);
+
+  // Child 3 should have been spoken
+  ASSERT_TRUE(mock_tts_platform.GetLastSpokenUtterance() == child_6_label);
+
+  mock_tts_platform.ClearLastSpokenUtterance();
+
+  // Same tree should not speak the same scopes_route/names_route
+  CallNotifyAccessibilityEvent(&event);
+
+  ASSERT_TRUE(mock_tts_platform.GetLastSpokenUtterance() == "");
+
+  // Now setup another tree but with 5&6 removed. This should
+  // make the tree source focus (but not speak) 3
+
+  OnAccessibilityEventRequest event2;
+  event2.set_source_id(0);
+  event2.set_window_id(1);
+  event2.set_event_type(OnAccessibilityEventRequest_EventType_CONTENT_CHANGED);
+
+  root = event2.add_node_data();
+  root->set_node_id(0);
+
+  child1 = AddChild(&event2, root, 1, 0, 0, 1, 1, false);
+  child2 = AddChild(&event2, child1, 2, 0, 0, 1, 1, false);
+  child3 = AddChild(&event2, child2, 3, 0, 0, 1, 1, false);
+  child3->set_label(child_3_label);
+
+  // Set scopes on child2, names on child3
+  boolean_properties = child2->mutable_boolean_properties();
+  boolean_properties->set_scopes_route(true);
+  boolean_properties = child3->mutable_boolean_properties();
+  boolean_properties->set_names_route(true);
+
+  CallNotifyAccessibilityEvent(&event2);
+
+  // Focus moves to 3
+  CallGetTreeData(&tree_data);
+  ASSERT_EQ(3, tree_data.focus_id);
+
+  // Nothings spoken
+  ASSERT_TRUE(mock_tts_platform.GetLastSpokenUtterance() == "");
+
+  // Finally, remove 2&3
+  //
+  OnAccessibilityEventRequest event3;
+  event3.set_source_id(0);
+  event3.set_window_id(1);
+  event3.set_event_type(OnAccessibilityEventRequest_EventType_CONTENT_CHANGED);
+
+  root = event3.add_node_data();
+  root->set_node_id(0);
+
+  CallNotifyAccessibilityEvent(&event3);
+
+  CallGetTreeData(&tree_data);
+  ASSERT_EQ(0, tree_data.focus_id);
+}
+
 }  // namespace accessibility
 }  // namespace chromecast

chromecast/browser/accessibility/flutter/flutter_semantics_node.h

@@ -29,6 +29,8 @@ class FlutterSemanticsNode {
   virtual bool IsVisibleToUser() const = 0;
   virtual bool IsFocused() const = 0;
   virtual bool IsLiveRegion() const = 0;
+  virtual bool HasScopesRoute() const = 0;
+  virtual bool HasNamesRoute() const = 0;
   virtual bool IsRapidChangingSlider() const = 0;
   virtual bool CanBeAccessibilityFocused() const = 0;
   virtual void PopulateAXRole(ui::AXNodeData* out_data) const = 0;

chromecast/browser/accessibility/flutter/flutter_semantics_node_wrapper.cc

@@ -65,6 +65,16 @@ bool FlutterSemanticsNodeWrapper::IsLiveRegion() const {
   return boolean_properties.is_live_region();
 }
 
+bool FlutterSemanticsNodeWrapper::HasScopesRoute() const {
+  const BooleanProperties& boolean_properties = node_ptr_->boolean_properties();
+  return boolean_properties.scopes_route();
+}
+
+bool FlutterSemanticsNodeWrapper::HasNamesRoute() const {
+  const BooleanProperties& boolean_properties = node_ptr_->boolean_properties();
+  return boolean_properties.names_route();
+}
+
 bool FlutterSemanticsNodeWrapper::CanBeAccessibilityFocused() const {
   // In Chrome, this means:
   // a node with a non-generic role and:
@@ -516,7 +526,7 @@ bool FlutterSemanticsNodeWrapper::IsScrollable() const {
 
 bool FlutterSemanticsNodeWrapper::IsFocusable() const {
   const BooleanProperties& boolean_properties = node_ptr_->boolean_properties();
-  if (boolean_properties.scopes_route())
+  if (boolean_properties.scopes_route() && !boolean_properties.names_route())
     return false;
 
   bool focusable_flags =

chromecast/browser/accessibility/flutter/flutter_semantics_node_wrapper.h

@@ -37,6 +37,8 @@ class FlutterSemanticsNodeWrapper : public FlutterSemanticsNode {
   bool IsVisibleToUser() const override;
   bool IsFocused() const override;
   bool IsLiveRegion() const override;
+  bool HasScopesRoute() const override;
+  bool HasNamesRoute() const override;
   bool IsRapidChangingSlider() const override;
   bool CanBeAccessibilityFocused() const override;
   void PopulateAXRole(ui::AXNodeData* out_data) const override;