Upstream chromecast gallium accessibility bridge

Patch set 1 is baseline of unmodified downstream
files.  Downstream refs will change to point to
this ax bridge once this lands.

Test: Unittest
Bug: None
Change-Id: I5157c0a9caa1f662998e1f3222b519008369d621
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2386102Reviewed-by: Daniel Nicoara <dnicoara@chromium.org>
Reviewed-by: Avi Drissman <avi@chromium.org>
Reviewed-by: Mitsuru Oshima (Slow: gardener) <oshima@chromium.org>
Reviewed-by: Devlin <rdevlin.cronin@chromium.org>
Commit-Queue: Randy Rossi <rmrossi@chromium.org>
Cr-Commit-Position: refs/heads/master@{#805757}

chromecast/BUILD.gn

@@ -94,6 +94,12 @@ cast_test_group("cast_tests") {
     "//ui/base:ui_base_unittests",
   ]
 
+  if (enable_chromecast_extensions) {
+    tests += [
+      "//chromecast/browser/accessibility/flutter:cast_accessibility_unittests",
+    ]
+  }
+
   if (!is_cast_audio_only) {
     tests += [ "//gpu:gpu_unittests" ]
 

chromecast/browser/accessibility/flutter/BUILD.gn

+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//chromecast/build/tests/cast_test.gni")
+import("//chromecast/chromecast.gni")
+
+cast_source_set("accessibility") {
+  sources = [
+    "ax_tree_source_flutter.cc",
+    "ax_tree_source_flutter.h",
+    "flutter_accessibility_helper_bridge.cc",
+    "flutter_accessibility_helper_bridge.h",
+    "flutter_semantics_node.h",
+    "flutter_semantics_node_wrapper.cc",
+    "flutter_semantics_node_wrapper.h",
+  ]
+  public_deps = [ "//ui/accessibility" ]
+  deps = [
+    "//base",
+    "//chromecast/base",
+    "//chromecast/browser",
+    "//chromecast/browser/accessibility/proto:gallium_accessibility_proto",
+    "//chromecast/common",
+    "//components/exo",
+    "//content/public/browser",
+    "//extensions/browser/api",
+    "//skia",
+    "//third_party/grpc:grpcpp",
+    "//ui/views",
+  ]
+}
+
+test("cast_accessibility_unittests") {
+  sources = [ "ax_tree_source_flutter_unittest.cc" ]
+
+  deps = [
+    ":accessibility",
+    "//base",
+    "//base/test:run_all_unittests",
+    "//chromecast/browser",
+    "//extensions/browser/api",
+    "//skia",
+    "//testing/gmock",
+    "//testing/gtest",
+    "//third_party/protobuf:protobuf_lite",
+    "//ui/views",
+  ]
+}

chromecast/browser/accessibility/flutter/DEPS

+include_rules = [
+  "+chromecast/browser",
+  "+chromecast/common/extensions_api",
+  "+components/exo",
+  "+content/browser",
+  "+content/public/browser",
+  "+extensions/browser",
+  '+third_party/grpc',
+  "+ui/accessibility",
+  "+ui/aura",
+  "+ui/gfx",
+  "+ui/views",
+]

chromecast/browser/accessibility/flutter/ax_tree_source_flutter.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chromecast/browser/accessibility/flutter/ax_tree_source_flutter.h"
+
+#include <stack>
+
+#include "base/check_op.h"
+#include "base/strings/string_number_conversions.h"
+#include "chromecast/browser/accessibility/flutter/flutter_semantics_node_wrapper.h"
+#include "chromecast/browser/ui/aura/accessibility/automation_manager_aura.h"
+#include "content/public/browser/tts_controller.h"
+#include "content/public/browser/tts_utterance.h"
+#include "extensions/browser/api/automation_internal/automation_event_router.h"
+#include "extensions/browser/api/automation_internal/automation_event_router_interface.h"
+#include "ui/accessibility/aura/aura_window_properties.h"
+#include "ui/aura/window.h"
+#include "ui/views/view.h"
+#include "ui/views/widget/widget.h"
+#include "ui/views/widget/widget_delegate.h"
+
+namespace {
+ax::mojom::Event ToAXEvent(
+    gallium::castos::OnAccessibilityEventRequest_EventType flutter_event_type) {
+  switch (flutter_event_type) {
+    case gallium::castos::OnAccessibilityEventRequest_EventType_FOCUSED:
+      return ax::mojom::Event::kFocus;
+    case gallium::castos::OnAccessibilityEventRequest_EventType_CLICKED:
+    case gallium::castos::OnAccessibilityEventRequest_EventType_LONG_CLICKED:
+      return ax::mojom::Event::kClicked;
+    case gallium::castos::OnAccessibilityEventRequest_EventType_TEXT_CHANGED:
+      return ax::mojom::Event::kTextChanged;
+    case gallium::castos::
+        OnAccessibilityEventRequest_EventType_TEXT_SELECTION_CHANGED:
+      return ax::mojom::Event::kTextSelectionChanged;
+    case gallium::castos::OnAccessibilityEventRequest_EventType_HOVER_ENTER:
+      return ax::mojom::Event::kHover;
+    case gallium::castos::OnAccessibilityEventRequest_EventType_SCROLLED:
+      return ax::mojom::Event::kScrollPositionChanged;
+    case gallium::castos::OnAccessibilityEventRequest_EventType_CONTENT_CHANGED:
+      return ax::mojom::Event::kChildrenChanged;
+    case gallium::castos::
+        OnAccessibilityEventRequest_EventType_WINDOW_STATE_CHANGED:
+      return ax::mojom::Event::kLayoutComplete;
+    default:
+      return ax::mojom::Event::kNone;
+  }
+}
+}  // namespace
+
+namespace chromecast {
+namespace accessibility {
+
+AXTreeSourceFlutter::AXTreeWebContentsObserver::AXTreeWebContentsObserver(
+    content::WebContents* web_contents,
+    AXTreeSourceFlutter* ax_tree_source)
+    : WebContentsObserver(web_contents), ax_tree_source_(ax_tree_source) {}
+
+void AXTreeSourceFlutter::AXTreeWebContentsObserver::RenderFrameHostChanged(
+    content::RenderFrameHost* old_host,
+    content::RenderFrameHost* new_host) {
+  ax_tree_source_->UpdateTree();
+}
+
+constexpr int kInvalidId = -1;
+
+AXTreeSourceFlutter::AXTreeSourceFlutter(
+    Delegate* delegate,
+    content::BrowserContext* browser_context,
+    extensions::AutomationEventRouterInterface* event_router)
+    : current_tree_serializer_(std::make_unique<AXTreeFlutterSerializer>(this)),
+      root_id_(kInvalidId),
+      window_id_(kInvalidId),
+      focused_id_(kInvalidId),
+      delegate_(delegate),
+      browser_context_(browser_context),
+      event_router_(event_router
+                        ? event_router
+                        : extensions::AutomationEventRouter::GetInstance()) {
+  DCHECK(delegate_);
+}
+
+AXTreeSourceFlutter::~AXTreeSourceFlutter() {
+  Reset();
+}
+
+void AXTreeSourceFlutter::NotifyAccessibilityEvent(
+    const gallium::castos::OnAccessibilityEventRequest* event_data) {
+  DCHECK(event_data);
+
+  if (event_data->node_data_size() > 0) {
+    // Remember most recent tree in case we need to update parents
+    // of child trees with new ax tree ids (i.e. due to embedded webview
+    // navigation)
+    last_event_data_ = *event_data;
+  }
+
+  // First find out if we know what to do with this event type from flutter.
+  ax::mojom::Event translated_event = ToAXEvent(event_data->event_type());
+  if (translated_event == ax::mojom::Event::kNone) {
+    LOG(INFO) << "Ignoring unknown flutter ax event "
+              << event_data->event_type() << ". No mapping available.";
+    return;
+  }
+
+  // b/150992421 - We sometimes get nodes that have been reparented.
+  // Any node that is reparented must first be deleted from its old
+  // parent before appearing under a new one. The tree serializer
+  // should be handling this case for us but isn't so the tree
+  // update fails. As a workaround until this is fixed upstream, we
+  // will identify these nodes ourselves and provide a separate update
+  // that will delete them prior to the full update.
+  reparented_children_.clear();
+  std::vector<int32_t> parents_with_deleted_children;
+  for (int i = 0; i < event_data->node_data_size(); ++i) {
+    const SemanticsNode& node = event_data->node_data(i);
+    for (int j = 0; j < node.child_node_ids_size(); ++j) {
+      int child_id = node.child_node_ids(j);
+      auto it = parent_map_.find(child_id);
+      if (it != parent_map_.end()) {
+        if (node.node_id() != it->second) {
+          // Remember this child and who its parent was.
+          reparented_children_.push_back(child_id);
+          parents_with_deleted_children.push_back(it->second);
+        }
+      }
+    }
+  }
+
+  if (event_data->node_data_size() > 0) {
+    // Unless there are new nodes, don't clear previous maps so we
+    // can detect reparenting above.
+    tree_map_.clear();
+    parent_map_.clear();
+    cached_computed_bounds_.clear();
+  }
+
+  window_id_ = event_data->window_id();
+
+  // The following loops perform caching to prepare for AXTreeSerializer.
+  // First, we need to cache parent links, which are implied by a node's child
+  // ids. Next, we cache the nodes by id. During this process, we can detect
+  // the root node based upon the parent links we cached above. Finally, we
+  // cache each node's computed bounds, based on its descendants.
+  std::map<int32_t, int32_t> all_parent_map;
+  std::map<int32_t, std::vector<int32_t>> all_children_map;
+  for (int i = 0; i < event_data->node_data_size(); ++i) {
+    const SemanticsNode& node = event_data->node_data(i);
+    for (int j = 0; j < node.child_node_ids_size(); ++j) {
+      all_children_map[node.node_id()].push_back(node.child_node_ids(j));
+      all_parent_map[node.child_node_ids(j)] = node.node_id();
+    }
+  }
+
+  // Now copy just the relevant subtree containing the source_id into the
+  // |parent_map_|.
+  root_id_ = event_data->source_id();
+  // Walk up to the root from the source_id.
+  for (auto it = all_parent_map.find(root_id_); it != all_parent_map.end();
+       it = all_parent_map.find(root_id_)) {
+    root_id_ = it->second;
+  }
+
+  // Walk back down through children map to populate parent_map_.
+  std::stack<int32_t> stack;
+  stack.push(root_id_);
+  while (!stack.empty()) {
+    int32_t parent = stack.top();
+    stack.pop();
+    for (int32_t child_id : all_children_map[parent]) {
+      parent_map_[child_id] = parent;
+      stack.push(child_id);
+    }
+  }
+
+  std::vector<std::string> new_child_trees;
+  for (int i = 0; i < event_data->node_data_size(); ++i) {
+    int32_t id = event_data->node_data(i).node_id();
+    // Only map nodes in the parent_map and the root.
+    // This avoids adding other subtrees that are not interesting.
+    if (parent_map_.find(id) == parent_map_.end() && id != root_id_)
+      continue;
+    const SemanticsNode& node = event_data->node_data(i);
+    tree_map_[id] = std::make_unique<FlutterSemanticsNodeWrapper>(this, &node);
+    if (tree_map_[id]->IsFocused()) {
+      focused_id_ = id;
+    }
+    // Place focus on the node that holds a child tree. When the
+    // child tree is removed, focus will be placed back on the root.
+    if (node.has_plugin_id()) {
+      std::string ax_tree_id = node.plugin_id();
+      new_child_trees.push_back(ax_tree_id);
+
+      if (std::find(child_trees_.begin(), child_trees_.end(), ax_tree_id) ==
+          child_trees_.end()) {
+        if (ax_tree_id.rfind("T:", 0) == 0) {
+          int web_contents_id;
+          base::StringToInt(ax_tree_id.substr(2), &web_contents_id);
+          std::vector<CastWebContents*> all_contents =
+              CastWebContents::GetAll();
+          for (CastWebContents* contents : all_contents) {
+            if (contents->id() == web_contents_id &&
+                child_tree_observers_.find(web_contents_id) ==
+                    child_tree_observers_.end()) {
+              child_tree_observers_[contents->id()] = std::make_unique<
+                  AXTreeSourceFlutter::AXTreeWebContentsObserver>(
+                  contents->web_contents(), this);
+              contents->AddObserver(this);
+              break;
+            }
+          }
+        }
+      }
+
+      focused_id_ = node.node_id();
+    }
+  }
+
+  // Do we need to put focus back on the root after a child tree
+  // has been removed?
+  bool need_focus_clear = false;
+  for (std::string id : child_trees_) {
+    // Is this old child tree still known?
+    if (std::find(new_child_trees.begin(), new_child_trees.end(), id) ==
+        new_child_trees.end()) {
+      // No, clear focus.
+      need_focus_clear = true;
+      focused_id_ = root_id_;
+      break;
+    }
+  }
+  child_trees_ = new_child_trees;
+
+  // Assuming |nodeData| is in pre-order, compute cached bounds in post-order to
+  // avoid an O(n^2) amount of work as the computed bounds uses descendant
+  // bounds.
+  for (int i = 0; i < event_data->node_data_size(); ++i) {
+    int32_t id = event_data->node_data(i).node_id();
+    if (parent_map_.find(id) == parent_map_.end() && id != root_id_)
+      continue;
+    cached_computed_bounds_[id] = ComputeEnclosingBounds(tree_map_[id].get());
+  }
+
+  // If focus was not set from above, set it now on the root node.
+  if (focused_id_ < 0 && root_id_ >= 0) {
+    focused_id_ = root_id_;
+  }
+
+  ExtensionMsg_AccessibilityEventBundleParams event_bundle;
+  event_bundle.tree_id = ax_tree_id();
+
+  event_bundle.events.emplace_back();
+  ui::AXEvent& event = event_bundle.events.back();
+  event.event_type = translated_event;
+  event.id = event_data->source_id();
+
+  if (event_data->event_type() ==
+      gallium::castos::OnAccessibilityEventRequest_EventType_CONTENT_CHANGED) {
+    current_tree_serializer_->InvalidateSubtree(GetFromId(event.id));
+  }
+
+  if (event_data->event_type() !=
+          gallium::castos::OnAccessibilityEventRequest_EventType_HOVER_ENTER &&
+      event_data->event_type() !=
+          gallium::castos::OnAccessibilityEventRequest_EventType_HOVER_EXIT) {
+    // For every parent whose child has been moved, serialize an update.
+    // This update will filter all the children that have moved.
+    for (int32_t nid : parents_with_deleted_children) {
+      event_bundle.updates.emplace_back();
+      current_tree_serializer_->SerializeChanges(GetFromId(nid),
+                                                 &event_bundle.updates.back());
+    }
+
+    // If there were any children that were reparented, invalidate the entire
+    // tree so the new parents get the children.
+    if (reparented_children_.size() > 0) {
+      current_tree_serializer_->InvalidateSubtree(GetFromId(root_id_));
+    }
+
+    // Clear reparented children.
+    reparented_children_.clear();
+
+    event_bundle.updates.emplace_back();
+    current_tree_serializer_->SerializeChanges(GetFromId(event.id),
+                                               &event_bundle.updates.back());
+
+    HandleLiveRegions(&event_bundle.events);
+
+    // b/162311902: For nodes that have scroll extents, rapidly changing the
+    // value will result in queueing up the values and speak out one by one.
+    // Here we handle the tts natively.
+    HandleNativeTTS();
+  }
+
+  // Place focus back on the root if a child tree has disappeared
+  if (need_focus_clear) {
+    event_bundle.events.emplace_back();
+    ui::AXEvent& focus_event = event_bundle.events.back();
+    focus_event.event_type = ax::mojom::Event::kFocus;
+    focus_event.id = root_id_;
+    focus_event.event_from = ax::mojom::EventFrom::kNone;
+  }
+
+  if (event_router_)
+    event_router_->DispatchAccessibilityEvents(event_bundle);
+}
+
+void AXTreeSourceFlutter::NotifyActionResult(const ui::AXActionData& data,
+                                             bool result) {
+  if (!event_router_)
+    return;
+
+  event_router_->DispatchActionResult(data, result, browser_context_);
+}
+
+bool AXTreeSourceFlutter::GetTreeData(ui::AXTreeData* data) const {
+  DCHECK(data);
+  data->tree_id = ax_tree_id();
+  if (focused_id_ >= 0) {
+    data->focus_id = focused_id_;
+  } else if (root_id_ >= 0) {
+    data->focus_id = root_id_;
+  }
+  return true;
+}
+
+FlutterSemanticsNode* AXTreeSourceFlutter::GetRoot() const {
+  return GetFromId(root_id_);
+}
+
+FlutterSemanticsNode* AXTreeSourceFlutter::GetFromId(int32_t id) const {
+  auto it = tree_map_.find(id);
+  if (it == tree_map_.end())
+    return nullptr;
+  return it->second.get();
+}
+
+int32_t AXTreeSourceFlutter::GetId(FlutterSemanticsNode* info_data) const {
+  if (!info_data)
+    return kInvalidId;
+  return info_data->GetId();
+}
+
+void AXTreeSourceFlutter::GetChildren(
+    FlutterSemanticsNode* info_data,
+    std::vector<FlutterSemanticsNode*>* out_children) const {
+  if (!info_data)
+    return;
+
+  info_data->GetChildren(out_children);
+  if (out_children->empty())
+    return;
+
+  // Filter out any reparented children so the update doesn't see them.
+  auto it = out_children->begin();
+  while (it != out_children->end()) {
+    if (std::find(reparented_children_.begin(), reparented_children_.end(),
+                  (*it)->GetId()) != reparented_children_.end()) {
+      it = out_children->erase(it);
+    } else {
+      ++it;
+    }
+  }
+
+  std::map<int32_t, size_t> id_to_index;
+  for (size_t i = 0; i < out_children->size(); i++)
+    id_to_index[(*out_children)[i]->GetId()] = i;
+
+  // Sort children based on their enclosing bounding rectangles, based on their
+  // descendants.
+  std::sort(
+      out_children->begin(), out_children->end(),
+      [this, id_to_index](auto left, auto right) {
+        auto left_bounds = ComputeEnclosingBounds(left);
+        auto right_bounds = ComputeEnclosingBounds(right);
+
+        if (left_bounds.IsEmpty() || right_bounds.IsEmpty()) {
+          return id_to_index.at(left->GetId()) < id_to_index.at(right->GetId());
+        }
+
+        // Left to right sort (non-overlapping).
+        if (!left_bounds.Intersects(right_bounds)) {
+          return left_bounds.x() < right_bounds.x();
+        }
+
+        // Overlapping
+        // Left to right.
+        int left_difference = left_bounds.x() - right_bounds.x();
+        if (left_difference != 0) {
+          return left_difference < 0;
+        }
+
+        // Top to bottom.
+        int top_difference = left_bounds.y() - right_bounds.y();
+        if (top_difference != 0) {
+          return top_difference < 0;
+        }
+
+        // Larger to smaller.
+        int height_difference = left_bounds.height() - right_bounds.height();
+        if (height_difference != 0) {
+          return height_difference > 0;
+        }
+
+        int width_difference = left_bounds.width() - right_bounds.width();
+        if (width_difference != 0) {
+          return width_difference > 0;
+        }
+
+        // The rects are equal.
+        return id_to_index.at(left->GetId()) < id_to_index.at(right->GetId());
+      });
+}
+
+FlutterSemanticsNode* AXTreeSourceFlutter::GetParent(
+    FlutterSemanticsNode* info_data) const {
+  if (!info_data)
+    return nullptr;
+  auto it = parent_map_.find(info_data->GetId());
+  if (it != parent_map_.end())
+    return GetFromId(it->second);
+  return nullptr;
+}
+
+bool AXTreeSourceFlutter::IsValid(FlutterSemanticsNode* info_data) const {
+  return info_data;
+}
+
+bool AXTreeSourceFlutter::IsIgnored(FlutterSemanticsNode* info_data) const {
+  return false;
+}
+
+bool AXTreeSourceFlutter::IsEqual(FlutterSemanticsNode* info_data1,
+                                  FlutterSemanticsNode* info_data2) const {
+  if (!info_data1 || !info_data2)
+    return false;
+  return info_data1->GetId() == info_data2->GetId();
+}
+
+FlutterSemanticsNode* AXTreeSourceFlutter::GetNull() const {
+  return nullptr;
+}
+
+void AXTreeSourceFlutter::SerializeNode(FlutterSemanticsNode* info_data,
+                                        ui::AXNodeData* out_data) const {
+  if (!info_data)
+    return;
+
+  int32_t id = info_data->GetId();
+  out_data->id = id;
+  if (id == root_id_) {
+    out_data->role = ax::mojom::Role::kRootWebArea;
+  } else {
+    info_data->PopulateAXRole(out_data);
+  }
+
+  info_data->Serialize(out_data);
+}
+
+const gfx::Rect AXTreeSourceFlutter::GetBounds(
+    FlutterSemanticsNode* info_data) const {
+  DCHECK(info_data);
+  DCHECK_NE(root_id_, kInvalidId);
+
+  gfx::Rect node_bounds = info_data->GetBounds();
+
+  // TODO(rmrossi): If embedded flutter is ever not full screen, we will have
+  // to pass in the embedded object tag's screen coordinates to this function
+  // and set the offset of the root node here separately from other nodes.
+  // The bounds of the root node are supposed to be relative to its container
+  // but since we are full screen, we leave them alone.  See
+  // ax_tree_source_arc.cc for an example.
+  if (info_data->GetId() == root_id_) {
+    gfx::Rect root_bounds = GetFromId(root_id_)->GetBounds();
+    // No offset applied since we are full screen.  See TODO above.
+    return root_bounds;
+  }
+  // Bounds of non-root node is relative to its tree's root.
+  gfx::Rect root_bounds = GetFromId(root_id_)->GetBounds();
+  node_bounds.Offset(-1 * root_bounds.x(), -1 * root_bounds.y());
+  return node_bounds;
+}
+
+gfx::Rect AXTreeSourceFlutter::ComputeEnclosingBounds(
+    FlutterSemanticsNode* info_data) const {
+  gfx::Rect computed_bounds;
+  // Exit early if the node or window is invisible.
+  if (!info_data->IsVisibleToUser())
+    return computed_bounds;
+
+  ComputeEnclosingBoundsInternal(info_data, &computed_bounds);
+  return computed_bounds;
+}
+
+void AXTreeSourceFlutter::ComputeEnclosingBoundsInternal(
+    FlutterSemanticsNode* info_data,
+    gfx::Rect* computed_bounds) const {
+  auto cached_bounds = cached_computed_bounds_.find(info_data->GetId());
+  if (cached_bounds != cached_computed_bounds_.end()) {
+    computed_bounds->Union(cached_bounds->second);
+    return;
+  }
+
+  if (!info_data->IsVisibleToUser())
+    return;
+
+  if (info_data->CanBeAccessibilityFocused()) {
+    // Only consider nodes that can possibly be accessibility focused.
+    computed_bounds->Union(info_data->GetBounds());
+    return;
+  }
+
+  std::vector<FlutterSemanticsNode*> children;
+  info_data->GetChildren(&children);
+  if (children.empty())
+    return;
+
+  for (FlutterSemanticsNode* child : children)
+    ComputeEnclosingBoundsInternal(child, computed_bounds);
+}
+
+void AXTreeSourceFlutter::PerformAction(const ui::AXActionData& data) {
+  delegate_->OnAction(data);
+}
+
+void AXTreeSourceFlutter::Reset() {
+  tree_map_.clear();
+  parent_map_.clear();
+  cached_computed_bounds_.clear();
+  current_tree_serializer_ = std::make_unique<AXTreeFlutterSerializer>(this);
+  root_id_ = kInvalidId;
+  focused_id_ = kInvalidId;
+  if (!event_router_)
+    return;
+  event_router_->DispatchTreeDestroyedEvent(ax_tree_id(), browser_context_);
+}
+
+void AXTreeSourceFlutter::HandleNativeTTS() {
+  std::map<int32_t, std::string> new_native_tts_name_cache;
+
+  // Cache current native tts name cache.
+  for (const auto& it : tree_map_) {
+    FlutterSemanticsNode* node_info = it.second.get();
+    if (!node_info->IsRapidChangingSlider())
+      continue;
+
+    std::vector<std::string> names;
+    if (node_info->HasLabelHint()) {
+      names.push_back(node_info->GetLabelHint());
+    }
+    if (node_info->HasValue()) {
+      names.push_back(node_info->GetValue());
+    }
+
+    new_native_tts_name_cache[node_info->GetId()] =
+        base::JoinString(names, " ");
+  }
+
+  // Compare to the previous one, and send out TTS if needed.
+  for (const auto& it : new_native_tts_name_cache) {
+    auto prev_it = native_tts_name_cache_.find(it.first);
+    if (prev_it != native_tts_name_cache_.end() &&
+        prev_it->second != it.second) {
+      // Send to TTS controller.
+      std::unique_ptr<content::TtsUtterance> utterance =
+          content::TtsUtterance::Create(browser_context_);
+      utterance->SetText(it.second);
+
+      auto* tts_controller = content::TtsController::GetInstance();
+      tts_controller->Stop();
+      tts_controller->SpeakOrEnqueue(std::move(utterance));
+    }
+  }
+
+  std::swap(native_tts_name_cache_, new_native_tts_name_cache);
+}
+
+void AXTreeSourceFlutter::HandleLiveRegions(std::vector<ui::AXEvent>* events) {
+  std::map<int32_t, std::string> new_live_region_map;
+
+  // Cache current live region's name.
+  for (const auto& it : tree_map_) {
+    FlutterSemanticsNode* node_info = it.second.get();
+    if (!node_info->IsLiveRegion())
+      continue;
+
+    std::stack<FlutterSemanticsNode*> stack;
+    stack.push(node_info);
+    while (!stack.empty()) {
+      FlutterSemanticsNode* node = stack.top();
+      stack.pop();
+      DCHECK(node);
+
+      ui::AXNodeData data;
+      SerializeNode(node, &data);
+      std::string name;
+      data.GetStringAttribute(ax::mojom::StringAttribute::kName, &name);
+      new_live_region_map[node->GetId()] = name;
+
+      std::vector<FlutterSemanticsNode*> children;
+      node->GetChildren(&children);
+      for (FlutterSemanticsNode* child : children)
+        stack.push(child);
+    }
+  }
+
+  // Compare to the previous one, and add an event if needed.
+  for (const auto& it : new_live_region_map) {
+    auto prev_it = live_region_name_cache_.find(it.first);
+    if (prev_it == live_region_name_cache_.end())
+      continue;
+
+    if (prev_it->second != it.second) {
+      events->emplace_back();
+      ui::AXEvent& event = events->back();
+      event.event_type = ax::mojom::Event::kLiveRegionChanged;
+      event.id = it.first;
+    }
+  }
+
+  std::swap(live_region_name_cache_, new_live_region_map);
+}
+
+void AXTreeSourceFlutter::UpdateTree() {
+  // Update the tree with last known flutter nodes.
+  // TODO: A more efficient update would be to isolate just the parent node
+  // whose child has changed. Consider giving the node to the observer
+  // class on creation and passing through here.
+  NotifyAccessibilityEvent(&last_event_data_);
+}
+
+void AXTreeSourceFlutter::OnPageStopped(CastWebContents* cast_web_contents,
+                                        int error_code) {
+  // Webview is gone. Stop observing.
+  cast_web_contents->RemoveObserver(this);
+  child_tree_observers_.erase(cast_web_contents->id());
+}
+
+}  // namespace accessibility
+}  // namespace chromecast

chromecast/browser/accessibility/flutter/ax_tree_source_flutter.h

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_AX_TREE_SOURCE_FLUTTER_H_
+#define CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_AX_TREE_SOURCE_FLUTTER_H_
+
+#include <stdint.h>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "base/containers/flat_map.h"
+#include "chromecast/browser/accessibility/proto/cast_server_accessibility.pb.h"
+#include "chromecast/browser/cast_web_contents.h"
+#include "content/public/browser/web_contents.h"
+#include "content/public/browser/web_contents_observer.h"
+#include "ui/accessibility/ax_action_handler.h"
+#include "ui/accessibility/ax_node.h"
+#include "ui/accessibility/ax_node_data.h"
+#include "ui/accessibility/ax_tree_data.h"
+#include "ui/accessibility/ax_tree_serializer.h"
+#include "ui/accessibility/ax_tree_source.h"
+
+namespace aura {
+class Window;
+}  // namespace aura
+
+namespace content {
+class BrowserContext;
+}  // namespace content
+
+namespace extensions {
+class AutomationEventRouterInterface;
+}  // namespace extensions
+
+namespace ui {
+struct AXEvent;
+}  // namespace ui
+
+namespace chromecast {
+namespace accessibility {
+
+class FlutterSemanticsNode;
+
+// This class translates accessibility trees found in the gallium accessibility
+// OnAccessibilityEventRequest proto into a tree update Chrome's accessibility
+// API can work with.
+class AXTreeSourceFlutter : public ui::AXTreeSource<FlutterSemanticsNode*,
+                                                    ui::AXNodeData,
+                                                    ui::AXTreeData>,
+                            public CastWebContents::Observer,
+                            public ui::AXActionHandler {
+ public:
+  class Delegate {
+   public:
+    virtual ~Delegate() {}
+    virtual void OnAction(const ui::AXActionData& data) = 0;
+  };
+
+  AXTreeSourceFlutter(
+      Delegate* delegate,
+      content::BrowserContext* browser_context,
+      extensions::AutomationEventRouterInterface* event_router = nullptr);
+  AXTreeSourceFlutter(const AXTreeSourceFlutter&) = delete;
+  ~AXTreeSourceFlutter() override;
+  AXTreeSourceFlutter& operator=(const AXTreeSourceFlutter&) = delete;
+
+  // AXTreeSource implementation.
+  bool GetTreeData(ui::AXTreeData* data) const override;
+
+  // AXTreeSource implementation used by FlutterAccessibilityInfoData
+  // subclasses.
+  FlutterSemanticsNode* GetRoot() const override;
+  FlutterSemanticsNode* GetFromId(int32_t id) const override;
+  void SerializeNode(FlutterSemanticsNode* node,
+                     ui::AXNodeData* out_data) const override;
+  FlutterSemanticsNode* GetParent(FlutterSemanticsNode* node) const override;
+
+  // Notifies automation of an accessibility event.
+  void NotifyAccessibilityEvent(
+      const ::gallium::castos::OnAccessibilityEventRequest* event_data);
+
+  // Notifies automation of a result to an action.
+  void NotifyActionResult(const ui::AXActionData& data, bool result);
+
+  // Attaches tree to an aura window and gives it system focus.
+  void Focus(aura::Window* window);
+
+  // Gets the window id of this tree.
+  int32_t window_id() const { return window_id_; }
+
+  // Returns bounds of a node which can be passed to AXNodeData.location. Bounds
+  // are returned in the following coordinates depending on whether it's root or
+  // not.
+  // - Root node is relative to its container.
+  // - Non-root node is relative to the root node of this tree.
+  const gfx::Rect GetBounds(FlutterSemanticsNode* node) const;
+
+  void UpdateTree();
+
+  // CastWebContents::Observer
+  void OnPageStopped(CastWebContents* cast_web_contents,
+                     int error_code) override;
+
+ private:
+  class AXTreeWebContentsObserver : public content::WebContentsObserver {
+   public:
+    AXTreeWebContentsObserver(
+        content::WebContents* web_contents,
+        chromecast::accessibility::AXTreeSourceFlutter* ax_tree_source);
+
+    void RenderFrameHostChanged(content::RenderFrameHost* old_host,
+                                content::RenderFrameHost* new_host) override;
+
+   private:
+    chromecast::accessibility::AXTreeSourceFlutter* ax_tree_source_;
+
+    DISALLOW_COPY_AND_ASSIGN(AXTreeWebContentsObserver);
+  };
+
+  using AXTreeFlutterSerializer = ui::
+      AXTreeSerializer<FlutterSemanticsNode*, ui::AXNodeData, ui::AXTreeData>;
+
+  friend class AXTreeSourceFlutterTest;
+
+  // AXTreeSource overrides.
+  int32_t GetId(FlutterSemanticsNode* node) const override;
+  void GetChildren(
+      FlutterSemanticsNode* node,
+      std::vector<FlutterSemanticsNode*>* out_children) const override;
+  bool IsValid(FlutterSemanticsNode* node) const override;
+  bool IsIgnored(FlutterSemanticsNode* node) const override;
+  bool IsEqual(FlutterSemanticsNode* node1,
+               FlutterSemanticsNode* node2) const override;
+  FlutterSemanticsNode* GetNull() const override;
+
+  // Computes the smallest rect that encloses all of the descendants of |node|.
+  gfx::Rect ComputeEnclosingBounds(FlutterSemanticsNode* node) const;
+
+  // Helper to recursively compute bounds for |node|. Returns true if non-empty
+  // bounds were encountered.
+  void ComputeEnclosingBoundsInternal(FlutterSemanticsNode* node,
+                                      gfx::Rect* computed_bounds) const;
+
+  // AXHostDelegate implementation.
+  void PerformAction(const ui::AXActionData& data) override;
+
+  // Resets tree state.
+  void Reset();
+
+  // Detects live region changes and generates events for them.
+  void HandleLiveRegions(std::vector<ui::AXEvent>* events);
+
+  // Detects rapidly changing nodes and use native TTS instead.
+  void HandleNativeTTS();
+
+  std::unique_ptr<AXTreeFlutterSerializer> current_tree_serializer_;
+  int32_t root_id_;
+  int32_t window_id_;
+  int32_t focused_id_;
+
+  // A delegate that handles accessibility actions on behalf of this tree. The
+  // delegate is valid during the lifetime of this tree.
+  Delegate* delegate_;
+  content::BrowserContext* const browser_context_;
+  extensions::AutomationEventRouterInterface* const event_router_;
+
+  // Maps a node id to its tree data.
+  base::flat_map<int32_t /* node_id */, std::unique_ptr<FlutterSemanticsNode>>
+      tree_map_;
+
+  // Maps a node id to its parent.
+  base::flat_map<int32_t /* node_id */, int32_t /* parent_node_id */>
+      parent_map_;
+
+  // Mapping from ArcAccessibilityInfoData ID to its cached computed bounds.
+  // This simplifies bounds calculations.
+  base::flat_map<int32_t, gfx::Rect> cached_computed_bounds_;
+
+  // Cache from node id to computed name for live region.
+  std::map<int32_t, std::string> live_region_name_cache_;
+
+  // Cache form node id to tts string for native tts components.
+  std::map<int32_t, std::string> native_tts_name_cache_;
+
+  std::vector<int32_t> reparented_children_;
+  std::vector<std::string> child_trees_;
+
+  // Maps web contents id to the web contents observer
+  base::flat_map<int32_t, std::unique_ptr<AXTreeWebContentsObserver>>
+      child_tree_observers_;
+
+  // Copy of most recent tree data
+  gallium::castos::OnAccessibilityEventRequest last_event_data_;
+};
+
+}  // namespace accessibility
+}  // namespace chromecast
+
+#endif  // CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_AX_TREE_SOURCE_FLUTTER_H_

chromecast/browser/accessibility/flutter/ax_tree_source_flutter_unittest.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chromecast/browser/accessibility/flutter/ax_tree_source_flutter.h"
+
+#include <string>
+
+#include "chromecast/browser/accessibility/flutter/flutter_semantics_node_wrapper.h"
+#include "extensions/browser/api/automation_internal/automation_event_router_interface.h"
+#include "testing/gmock/include/gmock/gmock.h"
+#include "testing/gtest/include/gtest/gtest.h"
+#include "ui/accessibility/ax_action_data.h"
+
+using ::testing::StrictMock;
+
+using ::gallium::castos::ActionProperties;
+using ::gallium::castos::BooleanProperties;
+using ::gallium::castos::OnAccessibilityEventRequest;
+using ::gallium::castos::OnAccessibilityEventRequest_EventType_FOCUSED;
+using ::gallium::castos::Rect;
+
+namespace chromecast {
+namespace accessibility {
+
+class MockAutomationEventRouter
+    : public extensions::AutomationEventRouterInterface {
+ public:
+  MockAutomationEventRouter() {}
+  ~MockAutomationEventRouter() override = default;
+  void DispatchAccessibilityEvents(
+      const ExtensionMsg_AccessibilityEventBundleParams& events) override {
+    for (const auto& event : events.events)
+      event_count_[event.event_type]++;
+  }
+  MOCK_METHOD(void,
+              DispatchAccessibilityLocationChange,
+              (const ExtensionMsg_AccessibilityLocationChangeParams&),
+              (override));
+  MOCK_METHOD(void,
+              DispatchTreeDestroyedEvent,
+              (ui::AXTreeID, content::BrowserContext*),
+              (override));
+  MOCK_METHOD(void,
+              DispatchActionResult,
+              (const ui::AXActionData&, bool, content::BrowserContext*),
+              (override));
+  std::map<ax::mojom::Event, int> event_count_;
+};
+
+class AXTreeSourceFlutterTest : public testing::Test,
+                                public AXTreeSourceFlutter::Delegate {
+ public:
+  AXTreeSourceFlutterTest()
+      : tree_(
+            std::make_unique<AXTreeSourceFlutter>(this,
+                                                  nullptr /* browser_context */,
+                                                  &router_)) {}
+  AXTreeSourceFlutterTest(const AXTreeSourceFlutterTest&) = delete;
+  ~AXTreeSourceFlutterTest() override {
+    EXPECT_CALL(router_, DispatchTreeDestroyedEvent).Times(1);
+  }
+  AXTreeSourceFlutterTest& operator=(const AXTreeSourceFlutterTest&) = delete;
+
+ protected:
+  void CallNotifyAccessibilityEvent(OnAccessibilityEventRequest* event_data) {
+    tree_->NotifyAccessibilityEvent(event_data);
+  }
+
+  void CallGetChildren(SemanticsNode* node,
+                       std::vector<FlutterSemanticsNode*>* out_children) const {
+    FlutterSemanticsNodeWrapper node_data(tree_.get(), node);
+    tree_->GetChildren(&node_data, out_children);
+  }
+
+  void CallSerializeNode(SemanticsNode* node,
+                         std::unique_ptr<ui::AXNodeData>* out_data) const {
+    ASSERT_TRUE(out_data);
+    FlutterSemanticsNodeWrapper node_data(tree_.get(), node);
+    *out_data = std::make_unique<ui::AXNodeData>();
+    tree_->SerializeNode(&node_data, out_data->get());
+  }
+
+  FlutterSemanticsNode* CallGetFromId(int32_t id) const {
+    return tree_->GetFromId(id);
+  }
+
+  bool CallGetTreeData(ui::AXTreeData* data) {
+    return tree_->GetTreeData(data);
+  }
+
+  int GetDispatchedEventCount(ax::mojom::Event type) {
+    return router_.event_count_[type];
+  }
+
+  void SetRect(Rect* rect, int left, int top, int right, int bottom) {
+    rect->set_left(left);
+    rect->set_top(top);
+    rect->set_right(right);
+    rect->set_bottom(bottom);
+  }
+
+  SemanticsNode* AddChild(OnAccessibilityEventRequest* event,
+                          SemanticsNode* parent,
+                          int id,
+                          int x,
+                          int y,
+                          int w,
+                          int h,
+                          bool click) {
+    BooleanProperties* boolean_properties;
+    ActionProperties* action_properties;
+
+    SemanticsNode* new_node = event->add_node_data();
+    new_node->set_node_id(id);
+    Rect* bounds = new_node->mutable_bounds_in_screen();
+    SetRect(bounds, x, y, x + w, y + h);
+    boolean_properties = new_node->mutable_boolean_properties();
+    boolean_properties->set_is_button(click);
+    if (click) {
+      new_node->set_label("Button");
+    }
+    action_properties = new_node->mutable_action_properties();
+    action_properties->set_tap(click);
+
+    parent->add_child_node_ids(id);
+    return new_node;
+  }
+
+ private:
+  void OnAction(const ui::AXActionData& data) override {}
+
+  MockAutomationEventRouter router_;
+  std::unique_ptr<AXTreeSourceFlutter> tree_;
+};
+
+TEST_F(AXTreeSourceFlutterTest, ReorderChildrenByLayout) {
+  BooleanProperties* boolean_properties;
+  ActionProperties* action_properties;
+
+  OnAccessibilityEventRequest event;
+  event.set_source_id(0);
+  event.set_window_id(1);
+  event.set_event_type(OnAccessibilityEventRequest_EventType_FOCUSED);
+
+  //     0
+  //    / \
+  //   1   2
+  SemanticsNode* root = event.add_node_data();
+  root->set_node_id(0);
+  root->add_child_node_ids(1);
+  root->add_child_node_ids(2);
+
+  // Child button 1.
+  SemanticsNode* button1 = event.add_node_data();
+  button1->set_node_id(1);
+  button1->set_label("Button");
+  boolean_properties = button1->mutable_boolean_properties();
+  boolean_properties->set_is_button(true);
+  boolean_properties->set_is_hidden(false);
+  action_properties = button1->mutable_action_properties();
+  action_properties->set_tap(true);
+
+  // Another child button.
+  SemanticsNode* button2 = event.add_node_data();
+  button2->set_node_id(2);
+  button2->set_label("Button");
+  boolean_properties = button2->mutable_boolean_properties();
+  boolean_properties->set_is_button(true);
+  boolean_properties->set_is_hidden(false);
+  action_properties = button2->mutable_action_properties();
+  action_properties->set_tap(true);
+
+  // Non-overlapping: 2
+  //                   1
+  // Expect left(2) to right(1)
+  Rect* bounds1 = button1->mutable_bounds_in_screen();
+  Rect* bounds2 = button2->mutable_bounds_in_screen();
+  SetRect(bounds1, 100, 100, 200, 200);
+  SetRect(bounds2, 0, 0, 50, 50);
+
+  // Trigger an update which refreshes the computed bounds used for reordering.
+  CallNotifyAccessibilityEvent(&event);
+  EXPECT_EQ(1, GetDispatchedEventCount(ax::mojom::Event::kFocus));
+  std::vector<FlutterSemanticsNode*> order;
+  order.clear();
+  CallGetChildren(root, &order);
+  ASSERT_EQ(2U, order.size());
+  EXPECT_EQ(2, order[0]->GetId());
+  EXPECT_EQ(1, order[1]->GetId());
+
+  // Non-overlapping:  2
+  //                  1
+  // Expect left(1) to right(2)
+  SetRect(bounds1, 0, 50, 50, 50);
+  SetRect(bounds2, 100, 0, 200, 200);
+  CallNotifyAccessibilityEvent(&event);
+  EXPECT_EQ(2, GetDispatchedEventCount(ax::mojom::Event::kFocus));
+  order.clear();
+  CallGetChildren(root, &order);
+  ASSERT_EQ(2U, order.size());
+  EXPECT_EQ(1, order[0]->GetId());
+  EXPECT_EQ(2, order[1]->GetId());
+
+  // Non-overlapping: 1
+  //                   2
+  // Expect left(1) to right(2)
+  SetRect(bounds1, 0, 0, 50, 50);
+  SetRect(bounds2, 100, 100, 200, 200);
+  CallNotifyAccessibilityEvent(&event);
+  EXPECT_EQ(3, GetDispatchedEventCount(ax::mojom::Event::kFocus));
+  order.clear();
+  CallGetChildren(root, &order);
+  ASSERT_EQ(2U, order.size());
+  EXPECT_EQ(1, order[0]->GetId());
+  EXPECT_EQ(2, order[1]->GetId());
+
+  // Non-overlapping:  1
+  //                  2
+  // Expect left(2) to right(1)
+  SetRect(bounds1, 100, 0, 200, 50);
+  SetRect(bounds2, 0, 100, 50, 200);
+  CallNotifyAccessibilityEvent(&event);
+  EXPECT_EQ(4, GetDispatchedEventCount(ax::mojom::Event::kFocus));
+  order.clear();
+  CallGetChildren(root, &order);
+  ASSERT_EQ(2U, order.size());
+  EXPECT_EQ(2, order[0]->GetId());
+  EXPECT_EQ(1, order[1]->GetId());
+
+  // Overlapping: Same y, 2_x < 1_x
+  // Expect left(2) to right(1)
+  SetRect(bounds1, 101, 100, 200, 200);
+  SetRect(bounds2, 100, 100, 200, 200);
+  CallNotifyAccessibilityEvent(&event);
+  EXPECT_EQ(5, GetDispatchedEventCount(ax::mojom::Event::kFocus));
+  order.clear();
+  CallGetChildren(root, &order);
+  ASSERT_EQ(2U, order.size());
+  EXPECT_EQ(2, order[0]->GetId());
+  EXPECT_EQ(1, order[1]->GetId());
+
+  // Overlapping: Same y, 1_x < 2_x
+  // Expect left(1) to right(2)
+  SetRect(bounds1, 100, 100, 200, 200);
+  SetRect(bounds2, 101, 100, 200, 200);
+  CallNotifyAccessibilityEvent(&event);
+  EXPECT_EQ(6, GetDispatchedEventCount(ax::mojom::Event::kFocus));
+  order.clear();
+  CallGetChildren(root, &order);
+  ASSERT_EQ(2U, order.size());
+  EXPECT_EQ(1, order[0]->GetId());
+  EXPECT_EQ(2, order[1]->GetId());
+
+  // Overlapping, 1st_x == 2nd_x
+  // Expect top(2) to bottom(1)
+  SetRect(bounds1, 100, 100, 200, 200);
+  SetRect(bounds2, 100, 99, 200, 199);
+  CallNotifyAccessibilityEvent(&event);
+  EXPECT_EQ(7, GetDispatchedEventCount(ax::mojom::Event::kFocus));
+  order.clear();
+  CallGetChildren(root, &order);
+  ASSERT_EQ(2U, order.size());
+  EXPECT_EQ(2, order[0]->GetId());
+  EXPECT_EQ(1, order[1]->GetId());
+
+  // Overlapping, 1st_x > 2nd_x
+  // Expect left(1) to right(2)
+  SetRect(bounds1, 100, 100, 200, 200);
+  SetRect(bounds2, 99, 100, 200, 199);
+  CallNotifyAccessibilityEvent(&event);
+  EXPECT_EQ(8, GetDispatchedEventCount(ax::mojom::Event::kFocus));
+  order.clear();
+  CallGetChildren(root, &order);
+  ASSERT_EQ(2U, order.size());
+  EXPECT_EQ(2, order[0]->GetId());
+  EXPECT_EQ(1, order[1]->GetId());
+
+  // Same top,left,x
+  // Expect larger(2) to smaller(1)
+  SetRect(bounds1, 100, 100, 200, 110);
+  SetRect(bounds2, 100, 100, 200, 200);
+  CallNotifyAccessibilityEvent(&event);
+  EXPECT_EQ(9, GetDispatchedEventCount(ax::mojom::Event::kFocus));
+  order.clear();
+  CallGetChildren(root, &order);
+  ASSERT_EQ(2U, order.size());
+  EXPECT_EQ(2, order[0]->GetId());
+  EXPECT_EQ(1, order[1]->GetId());
+
+  // Same top,left,y
+  // Expect larger(2) to smaller(1)
+  SetRect(bounds1, 100, 100, 110, 200);
+  SetRect(bounds2, 100, 100, 200, 200);
+  CallNotifyAccessibilityEvent(&event);
+  EXPECT_EQ(10, GetDispatchedEventCount(ax::mojom::Event::kFocus));
+  order.clear();
+  CallGetChildren(root, &order);
+  ASSERT_EQ(2U, order.size());
+  EXPECT_EQ(2, order[0]->GetId());
+  EXPECT_EQ(1, order[1]->GetId());
+
+  // Same top,left,x
+  // Expect larger(1) to smaller(2)
+  SetRect(bounds1, 100, 100, 200, 200);
+  SetRect(bounds2, 100, 100, 200, 110);
+  CallNotifyAccessibilityEvent(&event);
+  EXPECT_EQ(11, GetDispatchedEventCount(ax::mojom::Event::kFocus));
+  order.clear();
+  CallGetChildren(root, &order);
+  ASSERT_EQ(2U, order.size());
+  EXPECT_EQ(1, order[0]->GetId());
+  EXPECT_EQ(2, order[1]->GetId());
+
+  // Same top,left,y
+  // Expect larger(1) to smaller(2)
+  SetRect(bounds1, 100, 100, 200, 200);
+  SetRect(bounds2, 100, 100, 110, 200);
+  CallNotifyAccessibilityEvent(&event);
+  EXPECT_EQ(12, GetDispatchedEventCount(ax::mojom::Event::kFocus));
+  order.clear();
+  CallGetChildren(root, &order);
+  ASSERT_EQ(2U, order.size());
+  EXPECT_EQ(1, order[0]->GetId());
+  EXPECT_EQ(2, order[1]->GetId());
+}
+
+// This tree was taken from real tree data. When the enclosing
+// bounds have been computed for children and they don't overlap,
+// we should prefer left to right ordering as it aligns
+// better with the way the UI is laid out. Preferring top
+// to bottom usually results in a strange ordering which
+// may make sense if you knew what the enclosing bounds were
+// but does not behave with what you would expect visually.
+TEST_F(AXTreeSourceFlutterTest, ReorderChildrenByLayoutPreferLeftToRight) {
+  OnAccessibilityEventRequest event;
+
+  event.set_source_id(0);
+  event.set_window_id(1);
+  event.set_event_type(OnAccessibilityEventRequest_EventType_FOCUSED);
+
+  SemanticsNode* root = event.add_node_data();
+  root->set_node_id(0);
+  Rect* bounds = root->mutable_bounds_in_screen();
+  SetRect(bounds, 0, 0, 1280, 800);
+
+  SemanticsNode* child;
+  AddChild(&event, root, 17, 0, 0, 422, 800, true);
+
+  child = AddChild(&event, root, 29, 1022, 0, 257, 800, false);
+  child = AddChild(&event, child, 30, 1022, 55, 257, 690, true);
+  AddChild(&event, child, 31, 1232, 165, 47, 150, false);
+  AddChild(&event, child, 32, 1232, 165, 47, 150, false);
+
+  child = AddChild(&event, root, 18, 422, 0, 600, 800, false);
+  child = AddChild(&event, child, 19, 422, 55, 570, 690, false);
+  AddChild(&event, child, 20, 507, 95, 445, 40, false);
+  AddChild(&event, child, 21, 463, 186, 488, 70, true);
+  AddChild(&event, child, 22, 463, 283, 488, 70, true);
+  AddChild(&event, child, 23, 463, 381, 488, 70, true);
+  AddChild(&event, child, 24, 463, 478, 488, 70, true);
+
+  // Enclosing bounds for 29 end up being smaller y
+  // than enclosing bounds for 18 but 18 is left of 29
+  // so order should be 17,18,29
+
+  CallNotifyAccessibilityEvent(&event);
+  EXPECT_EQ(1, GetDispatchedEventCount(ax::mojom::Event::kFocus));
+  std::vector<FlutterSemanticsNode*> ordered;
+  CallGetChildren(root, &ordered);
+  ASSERT_EQ(3U, ordered.size());
+  EXPECT_EQ(17, ordered[0]->GetId());
+  EXPECT_EQ(18, ordered[1]->GetId());
+  EXPECT_EQ(29, ordered[2]->GetId());
+}
+
+TEST_F(AXTreeSourceFlutterTest, AccessibleNameComputation) {
+  ActionProperties* action_properties;
+
+  OnAccessibilityEventRequest event;
+  event.set_source_id(0);
+  event.set_window_id(1);
+  event.set_event_type(OnAccessibilityEventRequest_EventType_FOCUSED);
+
+  //     0
+  //    / \
+  //   1   2
+  SemanticsNode* root = event.add_node_data();
+  root->set_node_id(0);
+  root->add_child_node_ids(1);
+  root->add_child_node_ids(2);
+
+  // Child.
+  SemanticsNode* child1 = event.add_node_data();
+  child1->set_node_id(1);
+
+  // Another child.
+  SemanticsNode* child2 = event.add_node_data();
+  child2->set_node_id(2);
+
+  // Populate the tree source with the data.
+  CallNotifyAccessibilityEvent(&event);
+  EXPECT_EQ(1, GetDispatchedEventCount(ax::mojom::Event::kFocus));
+
+  // No attributes.
+  std::unique_ptr<ui::AXNodeData> data;
+  CallSerializeNode(root, &data);
+  std::string name;
+  ASSERT_FALSE(
+      data->GetStringAttribute(ax::mojom::StringAttribute::kName, &name));
+  EXPECT_EQ("", name);
+
+  // Label (empty).
+  root->set_label("");
+  CallSerializeNode(root, &data);
+  ASSERT_TRUE(
+      data->GetStringAttribute(ax::mojom::StringAttribute::kName, &name));
+  EXPECT_EQ("", name);
+
+  // Label (non-empty).
+  root->set_label("label text");
+  CallSerializeNode(root, &data);
+  ASSERT_TRUE(
+      data->GetStringAttribute(ax::mojom::StringAttribute::kName, &name));
+  EXPECT_EQ("label text", name);
+
+  // Hint (empty), Label (non-empty).
+  root->set_hint("");
+  CallSerializeNode(root, &data);
+  ASSERT_TRUE(
+      data->GetStringAttribute(ax::mojom::StringAttribute::kName, &name));
+  EXPECT_EQ("label text", name);
+
+  // Hint (non-empty), Label (empty).
+  root->set_hint("hint");
+  root->clear_label();
+  CallSerializeNode(root, &data);
+  ASSERT_TRUE(
+      data->GetStringAttribute(ax::mojom::StringAttribute::kName, &name));
+  EXPECT_EQ("hint", name);
+
+  // Name from contents.
+
+  // Root node has no name, but has descendants with name.
+  root->clear_hint();
+  root->clear_label();
+  // Name from contents only happens if a node is clickable.
+  action_properties = root->mutable_action_properties();
+  action_properties->set_tap(true);
+  child1->set_label("child1 label text");
+  child2->set_label("child2 label text");
+  CallSerializeNode(root, &data);
+  ASSERT_TRUE(
+      data->GetStringAttribute(ax::mojom::StringAttribute::kName, &name));
+  ASSERT_EQ("child1 label text child2 label text", name);
+
+  // If the node has a name, it should override the contents.
+  root->set_label("root label text");
+  CallSerializeNode(root, &data);
+  ASSERT_TRUE(
+      data->GetStringAttribute(ax::mojom::StringAttribute::kName, &name));
+  ASSERT_EQ("root label text", name);
+
+  // Clearing both clickable and name from root, the name should not be
+  // populated.
+  root->clear_label();
+  action_properties->clear_tap();
+  CallSerializeNode(root, &data);
+  ASSERT_FALSE(
+      data->GetStringAttribute(ax::mojom::StringAttribute::kName, &name));
+}
+
+// Flutter's 'hidden' attribute should not translate to the node being
+// ignored or in any way not a11y focusable.
+TEST_F(AXTreeSourceFlutterTest, NeverHidden) {
+  OnAccessibilityEventRequest event;
+  event.set_source_id(0);
+  event.set_window_id(1);
+  event.set_event_type(OnAccessibilityEventRequest_EventType_FOCUSED);
+
+  BooleanProperties* boolean_properties;
+  SemanticsNode* root = event.add_node_data();
+  root->add_child_node_ids(1);
+  SemanticsNode* child1 = event.add_node_data();
+  child1->set_node_id(1);
+  child1->set_label("some label text");
+  boolean_properties = child1->mutable_boolean_properties();
+  boolean_properties->set_is_hidden(true);
+
+  CallNotifyAccessibilityEvent(&event);
+  EXPECT_EQ(1, GetDispatchedEventCount(ax::mojom::Event::kFocus));
+
+  std::unique_ptr<ui::AXNodeData> data;
+  CallSerializeNode(child1, &data);
+
+  ASSERT_TRUE(data->role == ax::mojom::Role::kStaticText);
+  ASSERT_FALSE(data->HasState(ax::mojom::State::kInvisible));
+}
+
+TEST_F(AXTreeSourceFlutterTest, GetTreeDataAppliesFocus) {
+  OnAccessibilityEventRequest event;
+  event.set_source_id(5);
+  event.set_window_id(1);
+  event.set_event_type(OnAccessibilityEventRequest_EventType_FOCUSED);
+
+  SemanticsNode* root = event.add_node_data();
+  root->set_node_id(5);
+
+  CallNotifyAccessibilityEvent(&event);
+  EXPECT_EQ(1, GetDispatchedEventCount(ax::mojom::Event::kFocus));
+  ui::AXTreeData data;
+
+  // If no node claimed focus, the root node should get it.
+  EXPECT_TRUE(CallGetTreeData(&data));
+  EXPECT_EQ(5, data.focus_id);
+
+  // Add a child node with focus.
+  root->add_child_node_ids(6);
+  SemanticsNode* child = event.add_node_data();
+  child->set_node_id(6);
+  child->mutable_boolean_properties()->set_is_focused(true);
+
+  CallNotifyAccessibilityEvent(&event);
+  EXPECT_EQ(2, GetDispatchedEventCount(ax::mojom::Event::kFocus));
+
+  EXPECT_TRUE(CallGetTreeData(&data));
+  EXPECT_EQ(6, data.focus_id);
+}
+
+TEST_F(AXTreeSourceFlutterTest, LiveRegion) {
+  OnAccessibilityEventRequest event;
+  event.set_source_id(1);
+  event.set_window_id(1);
+  event.set_event_type(OnAccessibilityEventRequest_EventType_FOCUSED);
+
+  SemanticsNode* root = event.add_node_data();
+  root->set_node_id(10);
+  root->add_child_node_ids(1);
+  root->add_child_node_ids(2);
+  BooleanProperties* boolean_properties = root->mutable_boolean_properties();
+  boolean_properties->set_is_live_region(true);
+
+  // Add child nodes.
+  SemanticsNode* node1 = event.add_node_data();
+  node1->set_node_id(1);
+  node1->set_label("text 1");
+
+  SemanticsNode* node2 = event.add_node_data();
+  node2->set_node_id(2);
+  node2->set_label("text 2");
+
+  CallNotifyAccessibilityEvent(&event);
+  EXPECT_EQ(1, GetDispatchedEventCount(ax::mojom::Event::kFocus));
+
+  std::unique_ptr<ui::AXNodeData> data;
+  CallSerializeNode(root, &data);
+  std::string status;
+  ASSERT_TRUE(data->GetStringAttribute(
+      ax::mojom::StringAttribute::kContainerLiveStatus, &status));
+  ASSERT_EQ(status, "polite");
+
+  EXPECT_EQ(0, GetDispatchedEventCount(ax::mojom::Event::kLiveRegionChanged));
+
+  // Modify text of node1.
+  node1->set_label("modified text 1");
+  CallNotifyAccessibilityEvent(&event);
+
+  EXPECT_EQ(1, GetDispatchedEventCount(ax::mojom::Event::kLiveRegionChanged));
+}
+
+TEST_F(AXTreeSourceFlutterTest, ResetFocus) {
+  //
+  // tree 1: no child tree
+  //
+  OnAccessibilityEventRequest event;
+
+  event.set_source_id(0);
+  event.set_window_id(1);
+  event.set_event_type(OnAccessibilityEventRequest_EventType_FOCUSED);
+
+  SemanticsNode* root = event.add_node_data();
+  root->set_node_id(0);
+  Rect* bounds = root->mutable_bounds_in_screen();
+  SetRect(bounds, 0, 0, 1280, 800);
+
+  SemanticsNode* child;
+  AddChild(&event, root, 1, 0, 0, 800, 600, false);
+
+  child = AddChild(&event, root, 2, 0, 0, 400, 600, false);
+  child = AddChild(&event, root, 3, 400, 0, 400, 600, false);
+
+  CallNotifyAccessibilityEvent(&event);
+
+  // initial focus on root
+  ui::AXTreeData tree_data;
+  CallGetTreeData(&tree_data);
+  ASSERT_EQ(0, tree_data.focus_id);
+
+  //
+  // tree 2: add a node with a child tree id
+  //
+  OnAccessibilityEventRequest event2;
+
+  event2.set_source_id(0);
+  event2.set_window_id(1);
+  event2.set_event_type(OnAccessibilityEventRequest_EventType_FOCUSED);
+
+  SemanticsNode* root2 = event2.add_node_data();
+  root2->set_node_id(0);
+  Rect* bounds2 = root2->mutable_bounds_in_screen();
+  SetRect(bounds2, 0, 0, 1280, 800);
+
+  AddChild(&event2, root2, 1, 0, 0, 800, 600, false);
+
+  child = AddChild(&event2, root2, 2, 0, 0, 400, 600, false);
+  child = AddChild(&event2, root2, 3, 400, 0, 400, 600, false);
+  child = AddChild(&event2, root2, 4, 0, 0, 200, 200, false);
+  child->set_plugin_id("1234");
+
+  // focus should move to node with child tree
+  CallNotifyAccessibilityEvent(&event2);
+  CallGetTreeData(&tree_data);
+  ASSERT_EQ(4, tree_data.focus_id);
+
+  //
+  // tree 2: back to initial tree
+  //
+  CallNotifyAccessibilityEvent(&event);
+  CallGetTreeData(&tree_data);
+
+  // focus back to root
+  ASSERT_EQ(0, tree_data.focus_id);
+}
+
+}  // namespace accessibility
+}  // namespace chromecast

chromecast/browser/accessibility/flutter/flutter_accessibility_helper_bridge.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chromecast/browser/accessibility/flutter/flutter_accessibility_helper_bridge.h"
+
+#include <utility>
+
+#include "base/task/post_task.h"
+#include "chromecast/browser/accessibility/accessibility_manager.h"
+#include "chromecast/browser/accessibility/proto/gallium_server_accessibility.grpc.pb.h"
+#include "chromecast/browser/cast_browser_process.h"
+#include "components/exo/fullscreen_shell_surface.h"
+#include "components/exo/shell_surface_util.h"
+#include "components/exo/surface.h"
+#include "content/public/browser/browser_context.h"
+#include "content/public/browser/browser_task_traits.h"
+#include "content/public/browser/browser_thread.h"
+#include "ui/accessibility/aura/aura_window_properties.h"
+#include "ui/accessibility/ax_action_data.h"
+#include "ui/aura/window.h"
+#include "ui/aura/window_tree_host.h"
+#include "ui/views/view.h"
+#include "ui/views/widget/widget.h"
+
+namespace chromecast {
+namespace gallium {
+namespace accessibility {
+
+using ::gallium::castos::
+    OnAccessibilityActionRequest_AccessibilityActionType_CUSTOM_ACTION;
+using ::gallium::castos::
+    // NOLINTNEXTLINE(whitespace/line_length)
+    OnAccessibilityActionRequest_AccessibilityActionType_DID_GAIN_ACCESSIBILITY_FOCUS;
+using ::gallium::castos::
+    // NOLINTNEXTLINE(whitespace/line_length)
+    OnAccessibilityActionRequest_AccessibilityActionType_DID_LOSE_ACCESSIBILITY_FOCUS;
+using ::gallium::castos::
+    OnAccessibilityActionRequest_AccessibilityActionType_SCROLL_DOWN;
+using ::gallium::castos::
+    OnAccessibilityActionRequest_AccessibilityActionType_SCROLL_LEFT;
+using ::gallium::castos::
+    OnAccessibilityActionRequest_AccessibilityActionType_SCROLL_RIGHT;
+using ::gallium::castos::
+    OnAccessibilityActionRequest_AccessibilityActionType_SCROLL_UP;
+using ::gallium::castos::
+    OnAccessibilityActionRequest_AccessibilityActionType_SET_SELECTION;
+using ::gallium::castos::
+    OnAccessibilityActionRequest_AccessibilityActionType_SHOW_ON_SCREEN;
+using ::gallium::castos::
+    OnAccessibilityActionRequest_AccessibilityActionType_TAP;
+
+FlutterAccessibilityHelperBridge::FlutterAccessibilityHelperBridge(
+    Delegate* bridge_delegate,
+    content::BrowserContext* browser_context)
+    : tree_source_(
+          std::make_unique<AXTreeSourceFlutter>(this, browser_context)),
+      bridge_delegate_(bridge_delegate) {}
+
+FlutterAccessibilityHelperBridge::~FlutterAccessibilityHelperBridge() = default;
+
+void FlutterAccessibilityHelperBridge::AccessibilityStateChanged(bool value) {
+  if (value) {
+    aura::Window* window = chromecast::shell::CastBrowserProcess::GetInstance()
+                               ->accessibility_manager()
+                               ->window_tree_host()
+                               ->window();
+
+    // Find the full screen shell surface for the exo::Surface representing
+    // the ui.  We must ensure our tree id is the child ax tree id for
+    // that view so when the root window is serialized, the flutter ax tree
+    // will be parented by that view.
+    bool found = false;
+    if (window) {
+      for (aura::Window* child : window->children()) {
+        exo::Surface* surface = exo::GetShellMainSurface(child);
+        if (surface) {
+          views::Widget* widget =
+              views::Widget::GetWidgetForNativeWindow(child);
+          if (widget) {
+            exo::FullscreenShellSurface* full_screen_shell_surface =
+                static_cast<exo::FullscreenShellSurface*>(
+                    widget->widget_delegate());
+            full_screen_shell_surface->SetChildAxTreeId(
+                tree_source_->ax_tree_id());
+            full_screen_shell_surface->GetContentsView()
+                ->NotifyAccessibilityEvent(ax::mojom::Event::kChildrenChanged,
+                                           false);
+            child->Focus();
+            found = true;
+          }
+          break;
+        }
+      }
+    }
+
+    if (!found) {
+      LOG(ERROR) << "Could not find full screen shell surface for ax tree.";
+    }
+  }
+}
+
+void FlutterAccessibilityHelperBridge::OnAccessibilityEventRequestInternal(
+    std::unique_ptr<::gallium::castos::OnAccessibilityEventRequest>
+        event_data) {
+  // Tell the tree source to serialize these changes.
+  tree_source_->NotifyAccessibilityEvent(event_data.get());
+}
+
+bool FlutterAccessibilityHelperBridge::OnAccessibilityEventRequest(
+    const ::gallium::castos::OnAccessibilityEventRequest* event_data) {
+  std::unique_ptr<::gallium::castos::OnAccessibilityEventRequest> event =
+      std::make_unique<::gallium::castos::OnAccessibilityEventRequest>(
+          *event_data);
+
+  base::PostTask(FROM_HERE, {content::BrowserThread::UI},
+                 base::BindOnce(&FlutterAccessibilityHelperBridge::
+                                    OnAccessibilityEventRequestInternal,
+                                base::Unretained(this), std::move(event)));
+  return true;
+}
+
+void FlutterAccessibilityHelperBridge::OnAction(const ui::AXActionData& data) {
+  // Called by tree source to dispatch ax action to flutter. Translate this
+  // to gallium accessibility proto and forward to the delegate for
+  // dispatching.
+  ::gallium::castos::OnAccessibilityActionRequest request;
+  request.set_node_id(data.target_node_id);
+
+  switch (data.action) {
+    case ax::mojom::Action::kDoDefault:
+      request.set_action_type(
+          OnAccessibilityActionRequest_AccessibilityActionType_TAP);
+      break;
+    case ax::mojom::Action::kScrollToMakeVisible:
+      request.set_action_type(
+          OnAccessibilityActionRequest_AccessibilityActionType_SHOW_ON_SCREEN);
+      break;
+    case ax::mojom::Action::kScrollBackward:
+      request.set_action_type(
+          OnAccessibilityActionRequest_AccessibilityActionType_SCROLL_LEFT);
+      break;
+    case ax::mojom::Action::kScrollForward:
+      request.set_action_type(
+          OnAccessibilityActionRequest_AccessibilityActionType_SCROLL_RIGHT);
+      break;
+    case ax::mojom::Action::kScrollUp:
+      request.set_action_type(
+          OnAccessibilityActionRequest_AccessibilityActionType_SCROLL_UP);
+      break;
+    case ax::mojom::Action::kScrollDown:
+      request.set_action_type(
+          OnAccessibilityActionRequest_AccessibilityActionType_SCROLL_DOWN);
+      break;
+    case ax::mojom::Action::kScrollLeft:
+      request.set_action_type(
+          OnAccessibilityActionRequest_AccessibilityActionType_SCROLL_LEFT);
+      break;
+    case ax::mojom::Action::kScrollRight:
+      request.set_action_type(
+          OnAccessibilityActionRequest_AccessibilityActionType_SCROLL_RIGHT);
+      break;
+    case ax::mojom::Action::kCustomAction:
+      request.set_action_type(
+          OnAccessibilityActionRequest_AccessibilityActionType_CUSTOM_ACTION);
+      request.set_custom_action_id(data.custom_action_id);
+      break;
+    case ax::mojom::Action::kSetAccessibilityFocus:
+      request.set_action_type(
+          // NOLINTNEXTLINE(whitespace/line_length)
+          OnAccessibilityActionRequest_AccessibilityActionType_DID_GAIN_ACCESSIBILITY_FOCUS);
+      break;
+    case ax::mojom::Action::kClearAccessibilityFocus:
+      request.set_action_type(
+          // NOLINTNEXTLINE(whitespace/line_length)
+          OnAccessibilityActionRequest_AccessibilityActionType_DID_LOSE_ACCESSIBILITY_FOCUS);
+      break;
+    case ax::mojom::Action::kGetTextLocation:
+      request.set_action_type(
+          OnAccessibilityActionRequest_AccessibilityActionType_SET_SELECTION);
+      request.set_start_index(data.start_index);
+      request.set_end_index(data.end_index);
+      break;
+    default:
+      LOG(WARNING) << "Cast ax action " << data.action
+                   << " not mapped to flutter action - dropped.";
+
+      return;
+  }
+
+  bridge_delegate_->SendAccessibilityAction(request);
+}
+
+}  // namespace accessibility
+}  // namespace gallium
+}  // namespace chromecast

chromecast/browser/accessibility/flutter/flutter_accessibility_helper_bridge.h

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_FLUTTER_ACCESSIBILITY_HELPER_BRIDGE_H_
+#define CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_FLUTTER_ACCESSIBILITY_HELPER_BRIDGE_H_
+
+#include <memory>
+
+#include "chromecast/browser/accessibility/flutter/ax_tree_source_flutter.h"
+
+using chromecast::accessibility::AXTreeSourceFlutter;
+
+namespace content {
+class BrowserContext;
+}  // namespace content
+
+namespace gallium {
+namespace castos {
+class OnAccessibilityEventRequest;
+class OnAccessibilityActionRequest;
+}  // namespace castos
+}  // namespace gallium
+
+namespace chromecast {
+namespace gallium {
+namespace accessibility {
+
+// FlutterAccessibilityHelperBridge receives Flutter accessibility
+// events from gallium, translates them to chrome tree updates and dispatches
+// them to chromecast accessibility services.
+class FlutterAccessibilityHelperBridge : public AXTreeSourceFlutter::Delegate {
+ public:
+  class Delegate {
+   public:
+    virtual void SendAccessibilityAction(
+        ::gallium::castos::OnAccessibilityActionRequest request) = 0;
+
+   protected:
+    virtual ~Delegate() = default;
+  };
+
+  FlutterAccessibilityHelperBridge(Delegate* bridge_delegate,
+                                   content::BrowserContext* browser_context);
+  FlutterAccessibilityHelperBridge(const FlutterAccessibilityHelperBridge&) =
+      delete;
+  ~FlutterAccessibilityHelperBridge() override;
+  FlutterAccessibilityHelperBridge& operator=(
+      const FlutterAccessibilityHelperBridge&) = delete;
+
+  // Receive an accessibility event from flutter.
+  bool OnAccessibilityEventRequest(
+      const ::gallium::castos::OnAccessibilityEventRequest* event_data);
+
+  // AXTreeSourceArc::Delegate implementation:
+  // Dispatch a chrome accessibility action to flutter.
+  void OnAction(const ui::AXActionData& data) override;
+
+  void AccessibilityStateChanged(bool value);
+
+ private:
+  void OnAccessibilityEventRequestInternal(
+      std::unique_ptr<::gallium::castos::OnAccessibilityEventRequest>
+          event_data);
+
+  std::unique_ptr<AXTreeSourceFlutter> tree_source_;
+  Delegate* bridge_delegate_;
+};
+
+}  // namespace accessibility
+}  // namespace gallium
+}  // namespace chromecast
+
+#endif  // CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_FLUTTER_ACCESSIBILITY_HELPER_BRIDGE_H_

chromecast/browser/accessibility/flutter/flutter_semantics_node.h

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_FLUTTER_SEMANTICS_NODE_H_
+#define CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_FLUTTER_SEMANTICS_NODE_H_
+
+#include <ui/gfx/geometry/rect.h>
+
+#include <string>
+#include <vector>
+
+namespace ui {
+struct AXNodeData;
+}  // namespace ui
+
+namespace chromecast {
+namespace accessibility {
+
+// FlutterSemanticsNode represents a single flutter semantics object from
+// flutter. This class is used by AXTreeSourceFlutter to encapsulate
+// flutter information which maps to a single AXNodeData.
+class FlutterSemanticsNode {
+ public:
+  virtual ~FlutterSemanticsNode() = default;
+
+  virtual int32_t GetId() const = 0;
+  virtual const gfx::Rect GetBounds() const = 0;
+  virtual bool IsVisibleToUser() const = 0;
+  virtual bool IsFocused() const = 0;
+  virtual bool IsLiveRegion() const = 0;
+  virtual bool IsRapidChangingSlider() const = 0;
+  virtual bool CanBeAccessibilityFocused() const = 0;
+  virtual void PopulateAXRole(ui::AXNodeData* out_data) const = 0;
+  virtual void PopulateAXState(ui::AXNodeData* out_data) const = 0;
+  virtual void Serialize(ui::AXNodeData* out_data) const = 0;
+  virtual void GetChildren(
+      std::vector<FlutterSemanticsNode*>* children) const = 0;
+  virtual void ComputeNameFromContents(
+      std::vector<std::string>* names) const = 0;
+  virtual bool HasLabelHint() const = 0;
+  virtual std::string GetLabelHint() const = 0;
+  virtual bool HasValue() const = 0;
+  virtual std::string GetValue() const = 0;
+};
+
+}  // namespace accessibility
+}  // namespace chromecast
+
+#endif  // CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_FLUTTER_SEMANTICS_NODE_H_

chromecast/browser/accessibility/flutter/flutter_semantics_node_wrapper.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chromecast/browser/accessibility/flutter/flutter_semantics_node_wrapper.h"
+
+#include "base/check.h"
+#include "base/strings/string_util.h"
+#include "chromecast/browser/accessibility/flutter/ax_tree_source_flutter.h"
+#include "chromecast/browser/cast_web_contents.h"
+#include "content/public/browser/render_frame_host.h"
+#include "content/public/browser/web_contents.h"
+#include "ui/accessibility/ax_tree_id_registry.h"
+
+using gallium::castos::ActionProperties;
+using gallium::castos::BooleanProperties;
+
+namespace chromecast {
+namespace accessibility {
+
+FlutterSemanticsNodeWrapper::FlutterSemanticsNodeWrapper(
+    AXTreeSourceFlutter* tree_source,
+    const SemanticsNode* node)
+    : tree_source_(tree_source), node_ptr_(node) {
+  DCHECK(tree_source_);
+  DCHECK(node_ptr_);
+}
+
+int32_t FlutterSemanticsNodeWrapper::GetId() const {
+  return node_ptr_->node_id();
+}
+
+const gfx::Rect FlutterSemanticsNodeWrapper::GetBounds() const {
+  if (node_ptr_->has_bounds_in_screen()) {
+    return gfx::Rect(node_ptr_->bounds_in_screen().left(),
+                     node_ptr_->bounds_in_screen().top(),
+                     node_ptr_->bounds_in_screen().right() -
+                         node_ptr_->bounds_in_screen().left(),
+                     node_ptr_->bounds_in_screen().bottom() -
+                         node_ptr_->bounds_in_screen().top());
+  }
+  return gfx::Rect(0, 0, 0, 0);
+}
+
+bool FlutterSemanticsNodeWrapper::IsVisibleToUser() const {
+  if (node_ptr_->has_boolean_properties()) {
+    const BooleanProperties& boolean_properties =
+        node_ptr_->boolean_properties();
+    return !boolean_properties.is_hidden();
+  }
+  return true;
+}
+
+bool FlutterSemanticsNodeWrapper::IsFocused() const {
+  if (node_ptr_->has_boolean_properties()) {
+    const BooleanProperties& boolean_properties =
+        node_ptr_->boolean_properties();
+    return boolean_properties.is_focused();
+  }
+  return false;
+}
+
+bool FlutterSemanticsNodeWrapper::IsLiveRegion() const {
+  const BooleanProperties& boolean_properties = node_ptr_->boolean_properties();
+  return boolean_properties.is_live_region();
+}
+
+bool FlutterSemanticsNodeWrapper::CanBeAccessibilityFocused() const {
+  // In Chrome, this means:
+  // a node with a non-generic role and:
+  // actionable nodes or top level scrollables with a name
+  ui::AXNodeData data;
+  PopulateAXRole(&data);
+  bool non_generic_role = data.role != ax::mojom::Role::kGenericContainer &&
+                          data.role != ax::mojom::Role::kGroup;
+  bool actionable = node_ptr_->action_properties().tap() ||
+                    node_ptr_->action_properties().long_press();
+  bool top_level_scrollable =
+      HasLabelHint() && (node_ptr_->action_properties().scroll_up() ||
+                         node_ptr_->action_properties().scroll_down() ||
+                         node_ptr_->action_properties().scroll_left() ||
+                         node_ptr_->action_properties().scroll_right());
+  return non_generic_role && (actionable || top_level_scrollable);
+}
+
+void FlutterSemanticsNodeWrapper::PopulateAXRole(
+    ui::AXNodeData* out_data) const {
+  const BooleanProperties& boolean_properties = node_ptr_->boolean_properties();
+  const ActionProperties& action_properties = node_ptr_->action_properties();
+
+  if (boolean_properties.is_text_field()) {
+    out_data->role = ax::mojom::Role::kTextField;
+    return;
+  }
+
+  if (boolean_properties.is_header()) {
+    out_data->role = ax::mojom::Role::kHeading;
+    return;
+  }
+
+  // b/148808637: Flutter allows buttons to be containers but ChromeVox
+  // expects buttons to be atomic.  If we allow this node to be marked a
+  // button and it contains other clickable nodes, none of those nodes will
+  // be focusable by the user. This means no button that is also a
+  // container of other actionable items will ever be read as 'button'
+  // by the screen reader.  However, buttons that have no actionable
+  // descendants will still say 'Button' as expected.
+  if (boolean_properties.is_button() && !AnyChildIsActionable() &&
+      HasLabelHint() && GetLabelHint().length() > 0) {
+    out_data->role = ax::mojom::Role::kButton;
+    return;
+  }
+
+  if (boolean_properties.is_image() &&
+      node_ptr_->child_node_ids().size() == 0) {
+    out_data->role = ax::mojom::Role::kImage;
+    return;
+  }
+
+  if (action_properties.increase() || action_properties.decrease()) {
+    out_data->role = ax::mojom::Role::kSlider;
+    return;
+  }
+
+  bool has_checked_state = boolean_properties.has_checked_state();
+  bool has_toggled_state = boolean_properties.has_toggled_state();
+
+  if (has_checked_state) {
+    if (boolean_properties.is_in_mutually_exclusive_group()) {
+      out_data->role = ax::mojom::Role::kRadioButton;
+    } else {
+      out_data->role = ax::mojom::Role::kCheckBox;
+    }
+    return;
+  }
+  if (has_toggled_state) {
+    out_data->role = ax::mojom::Role::kSwitch;
+    return;
+  }
+
+  // b/149934151 : Flutter sends us nodes with labels that
+  // have children. Don't mark these as static text or
+  // no children will ever be focused properly via swipe
+  // navigation. Only nodes that have labels with no children
+  // should get the static text role. Use kHeader role
+  // instead as it is allowed to be a container.
+  if (HasLabelHint() && GetLabelHint().length() > 0) {
+    if (node_ptr_->child_node_ids().size() == 0) {
+      if (HasTapOrPress()) {
+        out_data->role = ax::mojom::Role::kButton;
+      } else {
+        out_data->role = ax::mojom::Role::kStaticText;
+      }
+    } else {
+      if (IsListItem()) {
+        out_data->role = ax::mojom::Role::kListBoxOption;
+      } else {
+        out_data->role = ax::mojom::Role::kHeader;
+      }
+    }
+    return;
+  }
+
+  if (node_ptr_->scroll_children() > 0) {
+    out_data->role = ax::mojom::Role::kList;
+    return;
+  }
+
+  out_data->role = ax::mojom::Role::kGenericContainer;
+}
+
+FlutterSemanticsNodeWrapper* FlutterSemanticsNodeWrapper::IsListItem() const {
+  // To consider it a list item, the node has to have a ancestor that has scroll
+  // children.
+  std::vector<FlutterSemanticsNodeWrapper*> ancestors;
+  FlutterSemanticsNodeWrapper* node = static_cast<FlutterSemanticsNodeWrapper*>(
+      tree_source_->GetFromId(GetId()));
+
+  while (node) {
+    FlutterSemanticsNodeWrapper* parent =
+        static_cast<FlutterSemanticsNodeWrapper*>(
+            tree_source_->GetParent(node));
+    if (parent)
+      ancestors.push_back(parent);
+    node = parent;
+  }
+
+  // |ancestors| is with order from closest ancestor to root. Find the closest
+  // ancestor that has scroll children and in between there is no actionable
+  // nodes.
+  for (FlutterSemanticsNodeWrapper* ancestor : ancestors) {
+    if (!ancestor->IsActionable()) {
+      if (ancestor->node()->scroll_children() > 0) {
+        return ancestor;
+      }
+    } else {
+      break;
+    }
+  }
+
+  return nullptr;
+}
+
+void FlutterSemanticsNodeWrapper::GetActionableChildren(
+    std::vector<FlutterSemanticsNodeWrapper*>* out_children) const {
+  std::vector<FlutterSemanticsNode*> children;
+  GetChildren(&children);
+
+  for (FlutterSemanticsNode* child : children) {
+    FlutterSemanticsNodeWrapper* child_wrapper =
+        static_cast<FlutterSemanticsNodeWrapper*>(child);
+
+    if (child_wrapper->HasTapOrPress() ||
+        child_wrapper->AnyChildIsActionable()) {
+      out_children->push_back(child_wrapper);
+    }
+  }
+}
+
+bool FlutterSemanticsNodeWrapper::IsDescendant(
+    FlutterSemanticsNodeWrapper* ancestor) const {
+  FlutterSemanticsNodeWrapper* parent =
+      static_cast<FlutterSemanticsNodeWrapper*>(
+          tree_source_->GetParent(tree_source_->GetFromId(GetId())));
+
+  while (parent) {
+    if (parent == ancestor)
+      return true;
+    parent = static_cast<FlutterSemanticsNodeWrapper*>(
+        tree_source_->GetParent(parent));
+  }
+
+  return false;
+}
+
+bool FlutterSemanticsNodeWrapper::IsRapidChangingSlider() const {
+  return (node_ptr_->scroll_extent_min() || node_ptr_->scroll_extent_min() ||
+          node_ptr_->action_properties().increase() ||
+          node_ptr_->action_properties().decrease());
+}
+
+void FlutterSemanticsNodeWrapper::PopulateAXState(
+    ui::AXNodeData* out_data) const {
+  const BooleanProperties& boolean_properties = node_ptr_->boolean_properties();
+
+  if (boolean_properties.is_obscured()) {
+    out_data->AddState(ax::mojom::State::kProtected);
+  }
+
+  if (boolean_properties.is_text_field()) {
+    out_data->AddState(ax::mojom::State::kEditable);
+  }
+
+  if (IsFocusable()) {
+    out_data->AddState(ax::mojom::State::kFocusable);
+  }
+
+  if (boolean_properties.has_checked_state()) {
+    out_data->SetCheckedState(boolean_properties.is_checked()
+                                  ? ax::mojom::CheckedState::kTrue
+                                  : ax::mojom::CheckedState::kFalse);
+  }
+
+  if (boolean_properties.has_toggled_state()) {
+    out_data->SetCheckedState(boolean_properties.is_toggled()
+                                  ? ax::mojom::CheckedState::kTrue
+                                  : ax::mojom::CheckedState::kFalse);
+  }
+
+  // Put this back after b/148875421 is resolved. Flutter is sending
+  // many elements with a disabled flag which causes the reader to
+  // speak 'Disabled' even though it is not.
+  // if (!boolean_properties.is_enabled()) {
+  // out_data->SetRestriction(ax::mojom::Restriction::kDisabled);
+  //}
+}
+
+void FlutterSemanticsNodeWrapper::Serialize(ui::AXNodeData* out_data) const {
+  PopulateAXState(out_data);
+
+  const BooleanProperties& boolean_properties = node_ptr_->boolean_properties();
+  const ActionProperties& action_properties = node_ptr_->action_properties();
+
+  // b/162311902: For nodes that have scroll extents and can be changed rapidly,
+  // set the name as empty so that ChromeVox will skip speaking them.
+  if (IsRapidChangingSlider()) {
+    out_data->SetName("");
+  } else if (HasLabelHint()) {
+    out_data->SetName(GetLabelHint());
+  } else if (IsActionable()) {
+    // Compute the name by joining all nodes with names.
+    std::vector<std::string> names;
+    ComputeNameFromContents(&names);
+    if (!names.empty())
+      out_data->SetName(base::JoinString(names, " "));
+  }
+
+  if (HasValue()) {
+    out_data->SetValue(GetValue());
+  }
+
+  out_data->AddBoolAttribute(ax::mojom::BoolAttribute::kClickable,
+                             IsActionable());
+  out_data->AddBoolAttribute(ax::mojom::BoolAttribute::kScrollable,
+                             IsScrollable());
+
+  if (boolean_properties.is_selected()) {
+    out_data->AddBoolAttribute(ax::mojom::BoolAttribute::kSelected, true);
+  }
+
+  if (node_ptr_->has_hint()) {
+    out_data->AddStringAttribute(ax::mojom::StringAttribute::kPlaceholder,
+                                 node_ptr_->hint());
+  }
+
+  // Set bounds
+  if (tree_source_->GetRoot()->GetId() != -1) {
+    // TODO(rmrossi) Pass in nullptr for now for active window. Root node will
+    // get bounds relative to 0,0 anyway since we are full screen.  This may
+    // change if flutter is ever not full screen in which case we will have
+    // to pass in the bounds of whatever container it resides in.
+    const gfx::Rect& local_bounds =
+        tree_source_->GetBounds(tree_source_->GetFromId(GetId()));
+    out_data->relative_bounds.bounds.SetRect(local_bounds.x(), local_bounds.y(),
+                                             local_bounds.width(),
+                                             local_bounds.height());
+  }
+
+  if (node_ptr_->has_text_selection_base()) {
+    out_data->AddIntAttribute(ax::mojom::IntAttribute::kTextSelStart,
+                              node_ptr_->text_selection_base());
+  }
+
+  if (node_ptr_->has_text_selection_extent()) {
+    out_data->AddIntAttribute(ax::mojom::IntAttribute::kTextSelEnd,
+                              node_ptr_->text_selection_extent());
+  }
+
+  if (action_properties.has_scroll_left() ||
+      action_properties.has_scroll_up()) {
+    out_data->AddAction(ax::mojom::Action::kScrollBackward);
+  }
+
+  if (action_properties.has_scroll_right() ||
+      action_properties.has_scroll_down()) {
+    out_data->AddAction(ax::mojom::Action::kScrollForward);
+  }
+
+  if (action_properties.set_selection()) {
+    out_data->AddAction(ax::mojom::Action::kSetSelection);
+  }
+
+  if (action_properties.increase()) {
+    out_data->AddAction(ax::mojom::Action::kIncrement);
+  }
+
+  if (action_properties.decrease()) {
+    out_data->AddAction(ax::mojom::Action::kDecrement);
+  }
+
+  if (IsScrollable()) {
+    const float position = node_ptr_->scroll_position();
+    const float min = node_ptr_->scroll_extent_min();
+    const float max = node_ptr_->scroll_extent_max();
+    if (node_ptr_->action_properties().scroll_up() ||
+        node_ptr_->action_properties().scroll_down()) {
+      out_data->AddIntAttribute(ax::mojom::IntAttribute::kScrollY, position);
+      out_data->AddIntAttribute(ax::mojom::IntAttribute::kScrollYMin, min);
+      out_data->AddIntAttribute(ax::mojom::IntAttribute::kScrollYMax, max);
+    } else if (node_ptr_->action_properties().scroll_left() ||
+               node_ptr_->action_properties().scroll_right()) {
+      out_data->AddIntAttribute(ax::mojom::IntAttribute::kScrollX, position);
+      out_data->AddIntAttribute(ax::mojom::IntAttribute::kScrollXMin, min);
+      out_data->AddIntAttribute(ax::mojom::IntAttribute::kScrollXMax, max);
+    }
+  }
+
+  if (node_ptr_->custom_actions_size() > 0) {
+    std::vector<int32_t> ids;
+    std::vector<std::string> labels;
+    for (int i = 0; i < node_ptr_->custom_actions_size(); i++) {
+      ids.push_back(node_ptr_->custom_actions(i).id());
+      labels.push_back(node_ptr_->custom_actions(i).label());
+    }
+    out_data->AddAction(ax::mojom::Action::kCustomAction);
+    out_data->AddIntListAttribute(ax::mojom::IntListAttribute::kCustomActionIds,
+                                  ids);
+    out_data->AddStringListAttribute(
+        ax::mojom::StringListAttribute::kCustomActionDescriptions, labels);
+  }
+
+  if (node_ptr_->has_plugin_id()) {
+    std::string ax_tree_id = node_ptr_->plugin_id();
+    if (ax_tree_id.rfind("T:", 0) == 0) {
+      // This is a cast web contents id. Find the matching
+      // CastWebContents and find the ax tree id from there.
+      int web_contents_id;
+      base::StringToInt(ax_tree_id.substr(2), &web_contents_id);
+      std::vector<CastWebContents*> all_contents = CastWebContents::GetAll();
+      // There will likely only ever be one active at any time.
+      for (CastWebContents* contents : all_contents) {
+        if (contents->id() == web_contents_id) {
+          content::WebContents* web_contents = contents->web_contents();
+          out_data->AddStringAttribute(
+              ax::mojom::StringAttribute::kChildTreeId,
+              web_contents->GetMainFrame()->GetAXTreeID().ToString());
+          break;
+        }
+      }
+    } else {
+      // Use the value as a tree id.
+      out_data->AddStringAttribute(ax::mojom::StringAttribute::kChildTreeId,
+                                   ax_tree_id);
+    }
+  }
+
+  if (boolean_properties.is_live_region()) {
+    out_data->AddStringAttribute(
+        ax::mojom::StringAttribute::kContainerLiveStatus, "polite");
+    out_data->AddStringAttribute(
+        ax::mojom::StringAttribute::kContainerLiveRelevant, "text");
+  }
+
+  if (out_data->role == ax::mojom::Role::kListBoxOption) {
+    // Find the ancestor whose role is kList.
+    FlutterSemanticsNodeWrapper* ancestor = IsListItem();
+
+    std::vector<FlutterSemanticsNodeWrapper*> ancestor_actionable_children;
+    ancestor->GetActionableChildren(&ancestor_actionable_children);
+
+    // kSetSize should be the number of actionable children that the scrollable
+    // ancestor has.
+    out_data->AddIntAttribute(ax::mojom::IntAttribute::kSetSize,
+                              ancestor_actionable_children.size());
+
+    // Find which children tree that this node is in.
+    for (size_t i = 0; i < ancestor_actionable_children.size(); ++i) {
+      if (IsDescendant(ancestor_actionable_children[i])) {
+        out_data->AddIntAttribute(ax::mojom::IntAttribute::kPosInSet, i + 1);
+        break;
+      }
+    }
+  }
+
+  if (out_data->role == ax::mojom::Role::kList) {
+    // Find the size of actionable children.
+    std::vector<FlutterSemanticsNodeWrapper*> actionable_children;
+    GetActionableChildren(&actionable_children);
+
+    out_data->AddIntAttribute(ax::mojom::IntAttribute::kSetSize,
+                              actionable_children.size());
+  }
+}
+
+void FlutterSemanticsNodeWrapper::GetChildren(
+    std::vector<FlutterSemanticsNode*>* children) const {
+  for (auto id : node_ptr_->child_node_ids()) {
+    FlutterSemanticsNode* node = tree_source_->GetFromId(id);
+    if (node) {
+      children->push_back(tree_source_->GetFromId(id));
+    } else {
+      LOG(ERROR) << "Node id present for which there is no child node given";
+    }
+  }
+}
+
+bool FlutterSemanticsNodeWrapper::AnyChildIsActionable() const {
+  // b/156940104 - If this node is host to a child tree, we
+  // can assume at least one of it's child nodes is actionable.
+  // Otherwise, none of it's descendants will be traversed by
+  // the reader.
+  if (node_ptr_->has_plugin_id()) {
+    return true;
+  }
+  for (auto child_id : node_ptr_->child_node_ids()) {
+    FlutterSemanticsNodeWrapper* child =
+        static_cast<FlutterSemanticsNodeWrapper*>(
+            tree_source_->GetFromId(child_id));
+    if (child->HasTapOrPress()) {
+      return true;
+    }
+    if (child->AnyChildIsActionable()) {
+      return true;
+    }
+  }
+  return false;
+}
+
+bool FlutterSemanticsNodeWrapper::HasTapOrPress() const {
+  return node_ptr_->boolean_properties().is_button() ||
+         node_ptr_->action_properties().tap() ||
+         node_ptr_->action_properties().long_press();
+}
+
+bool FlutterSemanticsNodeWrapper::IsActionable() const {
+  // When flutter tells us a generic container is actionable, do not allow this
+  // unless all children of this node are NOT actionable.  Otherwise, the
+  // tree walker will consider this node a leaf and it will not navigate to
+  // any actionable children.
+  bool actionable = HasTapOrPress();
+
+  ui::AXNodeData data;
+  PopulateAXRole(&data);
+  if (actionable && data.role == ax::mojom::Role::kGenericContainer &&
+      AnyChildIsActionable()) {
+    actionable = false;
+  }
+  // If this node is actionable but is also the host for a child tree,
+  // don't make it actionable or else chromevox won't traverse into
+  // any child.
+  if (node_ptr_->has_plugin_id()) {
+    actionable = false;
+  }
+  return actionable;
+}
+
+bool FlutterSemanticsNodeWrapper::IsScrollable() const {
+  return node_ptr_->action_properties().scroll_up() ||
+         node_ptr_->action_properties().scroll_down() ||
+         node_ptr_->action_properties().scroll_left() ||
+         node_ptr_->action_properties().scroll_right();
+}
+
+bool FlutterSemanticsNodeWrapper::IsFocusable() const {
+  const BooleanProperties& boolean_properties = node_ptr_->boolean_properties();
+  if (boolean_properties.scopes_route())
+    return false;
+
+  bool focusable_flags =
+      boolean_properties.has_checked_state() ||
+      boolean_properties.is_checked() || boolean_properties.is_selected() ||
+      HasTapOrPress() || boolean_properties.is_text_field() ||
+      boolean_properties.is_focused() ||
+      boolean_properties.has_enabled_state() ||
+      boolean_properties.is_enabled() ||
+      boolean_properties.is_in_mutually_exclusive_group() ||
+      boolean_properties.is_header() || boolean_properties.is_obscured() ||
+      boolean_properties.names_route() ||
+      (boolean_properties.is_image() &&
+       node_ptr_->child_node_ids().size() == 0) ||
+      boolean_properties.is_live_region() ||
+      boolean_properties.has_toggled_state() || boolean_properties.is_toggled();
+
+  // b/149934151 : If a node has a label but also has children, don't
+  // mark it focusable, otherwise none of its children will be navigable
+  // via swipe nav. It's role wlil be a generic container (see above).
+  return focusable_flags || (HasLabelHint() && !GetLabelHint().empty() &&
+                             node_ptr_->child_node_ids().size() == 0);
+}
+
+bool FlutterSemanticsNodeWrapper::HasLabelHint() const {
+  return node_ptr_->has_label() || node_ptr_->has_hint();
+}
+
+std::string FlutterSemanticsNodeWrapper::GetLabelHint() const {
+  // TODO(rmrossi): Find out whether this order of precedence makes sense
+  if (node_ptr_->has_label() && node_ptr_->label().length() > 0) {
+    return node_ptr_->label();
+  }
+  if (node_ptr_->has_hint() && node_ptr_->hint().length() > 0) {
+    return node_ptr_->hint();
+  }
+  return "";
+}
+
+void FlutterSemanticsNodeWrapper::ComputeNameFromContents(
+    std::vector<std::string>* names) const {
+  DCHECK(names);
+  std::string name;
+  if (HasLabelHint()) {
+    name = GetLabelHint();
+    if (!name.empty()) {
+      names->push_back(name);
+      return;
+    }
+  }
+
+  std::vector<FlutterSemanticsNode*> children;
+  GetChildren(&children);
+  for (const FlutterSemanticsNode* child : children) {
+    child->ComputeNameFromContents(names);
+  }
+}
+
+bool FlutterSemanticsNodeWrapper::HasValue() const {
+  return node_ptr_->has_value();
+}
+
+std::string FlutterSemanticsNodeWrapper::GetValue() const {
+  return node_ptr_->value();
+}
+
+}  // namespace accessibility
+}  // namespace chromecast

chromecast/browser/accessibility/flutter/flutter_semantics_node_wrapper.h

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_FLUTTER_SEMANTICS_NODE_WRAPPER_H_
+#define CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_FLUTTER_SEMANTICS_NODE_WRAPPER_H_
+
+#include <string>
+#include <vector>
+
+#include "chromecast/browser/accessibility/flutter/flutter_semantics_node.h"
+#include "chromecast/browser/accessibility/proto/cast_server_accessibility.pb.h"
+#include "ui/accessibility/ax_enum_util.h"
+#include "ui/accessibility/ax_node_data.h"
+
+namespace chromecast {
+namespace accessibility {
+
+using gallium::castos::SemanticsNode;
+
+class AXTreeSourceFlutter;
+
+// A wrapper class for a SemanticsNode proto object.
+// This is used by AXTreeSourceFlutter to create accessibility tree updates
+// from semantics trees sent to us from the flutter process.
+class FlutterSemanticsNodeWrapper : public FlutterSemanticsNode {
+ public:
+  FlutterSemanticsNodeWrapper(AXTreeSourceFlutter* tree_source,
+                              const SemanticsNode* node);
+  FlutterSemanticsNodeWrapper(const FlutterSemanticsNodeWrapper&) = delete;
+  FlutterSemanticsNodeWrapper& operator=(const FlutterSemanticsNodeWrapper&) =
+      delete;
+
+  // FlutterSemanticsNode implementation:
+  int32_t GetId() const override;
+  const gfx::Rect GetBounds() const override;
+  bool IsVisibleToUser() const override;
+  bool IsFocused() const override;
+  bool IsLiveRegion() const override;
+  bool IsRapidChangingSlider() const override;
+  bool CanBeAccessibilityFocused() const override;
+  void PopulateAXRole(ui::AXNodeData* out_data) const override;
+  void PopulateAXState(ui::AXNodeData* out_data) const override;
+  void Serialize(ui::AXNodeData* out_data) const override;
+  void GetChildren(std::vector<FlutterSemanticsNode*>* children) const override;
+  bool HasLabelHint() const override;
+  std::string GetLabelHint() const override;
+  bool HasValue() const override;
+  std::string GetValue() const override;
+
+  const SemanticsNode* node() { return node_ptr_; }
+
+ private:
+  bool AnyChildIsActionable() const;
+  bool HasTapOrPress() const;
+  bool IsActionable() const;
+  bool IsScrollable() const;
+  bool IsFocusable() const;
+  void ComputeNameFromContents(std::vector<std::string>* names) const override;
+  void GetActionableChildren(
+      std::vector<FlutterSemanticsNodeWrapper*>* out_children) const;
+  // Check if this is a list item and return the node of its ancestor whose role
+  // is kList
+  FlutterSemanticsNodeWrapper* IsListItem() const;
+  bool IsDescendant(FlutterSemanticsNodeWrapper* ancestor) const;
+
+  AXTreeSourceFlutter* const tree_source_;
+  const SemanticsNode* const node_ptr_;
+};
+
+}  // namespace accessibility
+}  // namespace chromecast
+
+#endif  // CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_FLUTTER_SEMANTICS_NODE_WRAPPER_H_

chromecast/browser/accessibility/proto/BUILD.gn

+import("//third_party/protobuf/proto_library.gni")
+
+proto_library("gallium_accessibility_proto") {
+  sources = [
+    "cast_server_accessibility.proto",
+    "gallium_server_accessibility.proto",
+  ]
+
+  deps = [
+    "//third_party/grpc:grpcpp",
+    "//third_party/protobuf:protobuf_lite",
+  ]
+
+  generator_plugin_label = "//third_party/grpc:grpc_cpp_plugin"
+  generator_plugin_suffix = ".grpc.pb"
+}

chromecast/browser/accessibility/proto/cast_server_accessibility.proto

+syntax = "proto2";
+
+package gallium.castos;
+
+option optimize_for = LITE_RUNTIME;
+
+message Rect {
+  optional int32 left = 1;
+  optional int32 top = 2;
+  optional int32 right = 3;
+  optional int32 bottom = 4;
+}
+
+message BooleanProperties {
+  optional bool has_checked_state = 1;
+  optional bool is_checked = 2;
+  optional bool is_selected = 3;
+  optional bool is_button = 4;
+  optional bool is_text_field = 5;
+  optional bool is_focused = 6;
+  optional bool has_enabled_state = 7;
+  optional bool is_enabled = 8;
+  optional bool is_in_mutually_exclusive_group = 9;
+  optional bool is_header = 10;
+  optional bool is_obscured = 11;
+  optional bool scopes_route = 12;
+  optional bool names_route = 13;
+  optional bool is_hidden = 14;
+  optional bool is_image = 15;
+  optional bool is_live_region = 16;
+  optional bool has_toggled_state = 17;
+  optional bool is_toggled = 18;
+  optional bool has_implicit_scrolling = 19;
+}
+
+message ActionProperties {
+  optional bool tap = 1;
+  optional bool long_press = 2;
+  optional bool scroll_left = 3;
+  optional bool scroll_right = 4;
+  optional bool scroll_up = 5;
+  optional bool scroll_down = 6;
+  optional bool increase = 7;
+  optional bool decrease = 8;
+  optional bool show_on_screen = 9;
+  optional bool move_cursor_forward_by_character = 10;
+  optional bool move_cursor_backward_by_character = 11;
+  optional bool set_selection = 12;
+  optional bool copy = 13;
+  optional bool cut = 14;
+  optional bool paste = 15;
+  optional bool did_gain_accessibility_focus = 16;
+  optional bool did_lose_accessibility_focus = 17;
+  optional bool custom_action = 18;
+  optional bool dismiss = 19;
+  optional bool move_cursor_forward_by_word = 20;
+  optional bool move_cursor_backward_by_word = 21;
+};
+
+message CustomAction {
+  optional int32 id = 1;
+  optional string label = 2;
+}
+
+message SemanticsNode {
+  optional int32 node_id = 1;
+  optional Rect bounds_in_screen = 2;
+  optional int32 window_id = 3;
+  repeated int32 child_node_ids = 4;
+  optional BooleanProperties boolean_properties = 5;
+  optional ActionProperties action_properties = 6;
+  optional string value = 7;
+  optional string label = 8;
+  optional string hint = 9;
+  optional string text_direction = 10;
+  optional int32 resource_id = 11;
+  optional int32 text_selection_base = 12;
+  optional int32 text_selection_extent = 13;
+  optional int32 scroll_children = 14;
+  optional int32 scroll_index = 15;
+  optional float scroll_position = 16;
+  optional int32 scroll_extent_max = 17;
+  optional int32 scroll_extent_min = 18;
+  repeated CustomAction custom_actions = 19;
+  // If set, indicates the semantics information for this node
+  // is provided by a platform plugin or some other native backed
+  // entity.
+  optional string plugin_id = 20;
+}
+
+message OnAccessibilityEventRequest {
+  enum EventType {
+    UNSPECIFIED = 0;
+    FOCUSED = 1;
+    FOCUS_CLEARED = 2;
+    HOVER_ENTER = 3;
+    HOVER_EXIT = 4;
+    CLICKED = 5;
+    LONG_CLICKED = 6;
+    WINDOW_STATE_CHANGED = 7;
+    TEXT_CHANGED = 8;
+    TEXT_SELECTION_CHANGED = 9;
+    CONTENT_CHANGED = 10;
+    SCROLLED = 11;
+  }
+  // Type of event this represents.
+  optional EventType event_type = 1;
+  // The node responsible for triggering this event.
+  optional int32 source_id = 2;
+  // Unique identifier for flutter window.
+  optional int32 window_id = 3;
+  // The 'flattened' semantic tree nodes.
+  repeated SemanticsNode node_data = 4;
+}
+
+message OnAccessibilityEventResponse {}

chromecast/browser/accessibility/proto/gallium_server_accessibility.proto

+syntax = "proto2";
+
+package gallium.castos;
+
+option optimize_for = LITE_RUNTIME;
+
+message OnAccessibilityStateChangedRequest {
+  optional bool enabled = 1;
+}
+
+message OnAccessibilityStateChangedResponse {}
+
+message OnAccessibilityActionRequest {
+  enum AccessibilityActionType {
+    UNSPECIFIED = 0;
+    TAP = 1;
+    LONG_PRESS = 2;
+    SCROLL_LEFT = 3;
+    SCROLL_RIGHT = 4;
+    SCROLL_UP = 5;
+    SCROLL_DOWN = 6;
+    INCREASE = 7;
+    DECREASE = 8;
+    SHOW_ON_SCREEN = 9;
+    MOVE_CURSOR_FORWARD_BY_CHARACTER = 10;
+    MOVE_CURSOR_BACKWARD_BY_CHARACTER = 11;
+    SET_SELECTION = 12;
+    COPY = 13;
+    CUT = 14;
+    PASTE = 15;
+    DID_GAIN_ACCESSIBILITY_FOCUS = 16;
+    DID_LOSE_ACCESSIBILITY_FOCUS = 17;
+    CUSTOM_ACTION = 18;
+    DISMISS = 19;
+    MOVE_CURSOR_FORWARD_BY_WORD = 20;
+    MOVE_CURSOR_BACKWARD_BY_WORD = 21;
+  }
+
+  optional int32 node_id = 1;
+
+  optional AccessibilityActionType action_type = 2;
+
+  // custom_action_id must be set if action_type is CUSTOM_ACTION.
+  optional int32 custom_action_id = 3;
+
+  // window_id where the action is performed.
+  optional int32 window_id = 4;
+
+  // Parameters specifying indices to get text location of node.
+  optional int32 start_index = 5;
+  optional int32 end_index = 6;
+}
+
+message OnAccessibilityActionResponse {}