Upstream chromecast gallium accessibility bridge

Patch set 1 is baseline of unmodified downstream
files.  Downstream refs will change to point to
this ax bridge once this lands.

Test: Unittest
Bug: None
Change-Id: I5157c0a9caa1f662998e1f3222b519008369d621
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2386102Reviewed-by: Daniel Nicoara <dnicoara@chromium.org>
Reviewed-by: Avi Drissman <avi@chromium.org>
Reviewed-by: Mitsuru Oshima (Slow: gardener) <oshima@chromium.org>
Reviewed-by: Devlin <rdevlin.cronin@chromium.org>
Commit-Queue: Randy Rossi <rmrossi@chromium.org>
Cr-Commit-Position: refs/heads/master@{#805757}

chromecast/BUILD.gn

@@ -94,6 +94,12 @@ cast_test_group("cast_tests") {
     "//ui/base:ui_base_unittests",
   ]
 
+  if (enable_chromecast_extensions) {
+    tests += [
+      "//chromecast/browser/accessibility/flutter:cast_accessibility_unittests",
+    ]
+  }
+
   if (!is_cast_audio_only) {
     tests += [ "//gpu:gpu_unittests" ]
 

chromecast/browser/accessibility/flutter/BUILD.gn

+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//chromecast/build/tests/cast_test.gni")
+import("//chromecast/chromecast.gni")
+
+cast_source_set("accessibility") {
+  sources = [
+    "ax_tree_source_flutter.cc",
+    "ax_tree_source_flutter.h",
+    "flutter_accessibility_helper_bridge.cc",
+    "flutter_accessibility_helper_bridge.h",
+    "flutter_semantics_node.h",
+    "flutter_semantics_node_wrapper.cc",
+    "flutter_semantics_node_wrapper.h",
+  ]
+  public_deps = [ "//ui/accessibility" ]
+  deps = [
+    "//base",
+    "//chromecast/base",
+    "//chromecast/browser",
+    "//chromecast/browser/accessibility/proto:gallium_accessibility_proto",
+    "//chromecast/common",
+    "//components/exo",
+    "//content/public/browser",
+    "//extensions/browser/api",
+    "//skia",
+    "//third_party/grpc:grpcpp",
+    "//ui/views",
+  ]
+}
+
+test("cast_accessibility_unittests") {
+  sources = [ "ax_tree_source_flutter_unittest.cc" ]
+
+  deps = [
+    ":accessibility",
+    "//base",
+    "//base/test:run_all_unittests",
+    "//chromecast/browser",
+    "//extensions/browser/api",
+    "//skia",
+    "//testing/gmock",
+    "//testing/gtest",
+    "//third_party/protobuf:protobuf_lite",
+    "//ui/views",
+  ]
+}

chromecast/browser/accessibility/flutter/DEPS

+include_rules = [
+  "+chromecast/browser",
+  "+chromecast/common/extensions_api",
+  "+components/exo",
+  "+content/browser",
+  "+content/public/browser",
+  "+extensions/browser",
+  '+third_party/grpc',
+  "+ui/accessibility",
+  "+ui/aura",
+  "+ui/gfx",
+  "+ui/views",
+]

chromecast/browser/accessibility/flutter/ax_tree_source_flutter.h

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_AX_TREE_SOURCE_FLUTTER_H_
+#define CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_AX_TREE_SOURCE_FLUTTER_H_
+
+#include <stdint.h>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "base/containers/flat_map.h"
+#include "chromecast/browser/accessibility/proto/cast_server_accessibility.pb.h"
+#include "chromecast/browser/cast_web_contents.h"
+#include "content/public/browser/web_contents.h"
+#include "content/public/browser/web_contents_observer.h"
+#include "ui/accessibility/ax_action_handler.h"
+#include "ui/accessibility/ax_node.h"
+#include "ui/accessibility/ax_node_data.h"
+#include "ui/accessibility/ax_tree_data.h"
+#include "ui/accessibility/ax_tree_serializer.h"
+#include "ui/accessibility/ax_tree_source.h"
+
+namespace aura {
+class Window;
+}  // namespace aura
+
+namespace content {
+class BrowserContext;
+}  // namespace content
+
+namespace extensions {
+class AutomationEventRouterInterface;
+}  // namespace extensions
+
+namespace ui {
+struct AXEvent;
+}  // namespace ui
+
+namespace chromecast {
+namespace accessibility {
+
+class FlutterSemanticsNode;
+
+// This class translates accessibility trees found in the gallium accessibility
+// OnAccessibilityEventRequest proto into a tree update Chrome's accessibility
+// API can work with.
+class AXTreeSourceFlutter : public ui::AXTreeSource<FlutterSemanticsNode*,
+                                                    ui::AXNodeData,
+                                                    ui::AXTreeData>,
+                            public CastWebContents::Observer,
+                            public ui::AXActionHandler {
+ public:
+  class Delegate {
+   public:
+    virtual ~Delegate() {}
+    virtual void OnAction(const ui::AXActionData& data) = 0;
+  };
+
+  AXTreeSourceFlutter(
+      Delegate* delegate,
+      content::BrowserContext* browser_context,
+      extensions::AutomationEventRouterInterface* event_router = nullptr);
+  AXTreeSourceFlutter(const AXTreeSourceFlutter&) = delete;
+  ~AXTreeSourceFlutter() override;
+  AXTreeSourceFlutter& operator=(const AXTreeSourceFlutter&) = delete;
+
+  // AXTreeSource implementation.
+  bool GetTreeData(ui::AXTreeData* data) const override;
+
+  // AXTreeSource implementation used by FlutterAccessibilityInfoData
+  // subclasses.
+  FlutterSemanticsNode* GetRoot() const override;
+  FlutterSemanticsNode* GetFromId(int32_t id) const override;
+  void SerializeNode(FlutterSemanticsNode* node,
+                     ui::AXNodeData* out_data) const override;
+  FlutterSemanticsNode* GetParent(FlutterSemanticsNode* node) const override;
+
+  // Notifies automation of an accessibility event.
+  void NotifyAccessibilityEvent(
+      const ::gallium::castos::OnAccessibilityEventRequest* event_data);
+
+  // Notifies automation of a result to an action.
+  void NotifyActionResult(const ui::AXActionData& data, bool result);
+
+  // Attaches tree to an aura window and gives it system focus.
+  void Focus(aura::Window* window);
+
+  // Gets the window id of this tree.
+  int32_t window_id() const { return window_id_; }
+
+  // Returns bounds of a node which can be passed to AXNodeData.location. Bounds
+  // are returned in the following coordinates depending on whether it's root or
+  // not.
+  // - Root node is relative to its container.
+  // - Non-root node is relative to the root node of this tree.
+  const gfx::Rect GetBounds(FlutterSemanticsNode* node) const;
+
+  void UpdateTree();
+
+  // CastWebContents::Observer
+  void OnPageStopped(CastWebContents* cast_web_contents,
+                     int error_code) override;
+
+ private:
+  class AXTreeWebContentsObserver : public content::WebContentsObserver {
+   public:
+    AXTreeWebContentsObserver(
+        content::WebContents* web_contents,
+        chromecast::accessibility::AXTreeSourceFlutter* ax_tree_source);
+
+    void RenderFrameHostChanged(content::RenderFrameHost* old_host,
+                                content::RenderFrameHost* new_host) override;
+
+   private:
+    chromecast::accessibility::AXTreeSourceFlutter* ax_tree_source_;
+
+    DISALLOW_COPY_AND_ASSIGN(AXTreeWebContentsObserver);
+  };
+
+  using AXTreeFlutterSerializer = ui::
+      AXTreeSerializer<FlutterSemanticsNode*, ui::AXNodeData, ui::AXTreeData>;
+
+  friend class AXTreeSourceFlutterTest;
+
+  // AXTreeSource overrides.
+  int32_t GetId(FlutterSemanticsNode* node) const override;
+  void GetChildren(
+      FlutterSemanticsNode* node,
+      std::vector<FlutterSemanticsNode*>* out_children) const override;
+  bool IsValid(FlutterSemanticsNode* node) const override;
+  bool IsIgnored(FlutterSemanticsNode* node) const override;
+  bool IsEqual(FlutterSemanticsNode* node1,
+               FlutterSemanticsNode* node2) const override;
+  FlutterSemanticsNode* GetNull() const override;
+
+  // Computes the smallest rect that encloses all of the descendants of |node|.
+  gfx::Rect ComputeEnclosingBounds(FlutterSemanticsNode* node) const;
+
+  // Helper to recursively compute bounds for |node|. Returns true if non-empty
+  // bounds were encountered.
+  void ComputeEnclosingBoundsInternal(FlutterSemanticsNode* node,
+                                      gfx::Rect* computed_bounds) const;
+
+  // AXHostDelegate implementation.
+  void PerformAction(const ui::AXActionData& data) override;
+
+  // Resets tree state.
+  void Reset();
+
+  // Detects live region changes and generates events for them.
+  void HandleLiveRegions(std::vector<ui::AXEvent>* events);
+
+  // Detects rapidly changing nodes and use native TTS instead.
+  void HandleNativeTTS();
+
+  std::unique_ptr<AXTreeFlutterSerializer> current_tree_serializer_;
+  int32_t root_id_;
+  int32_t window_id_;
+  int32_t focused_id_;
+
+  // A delegate that handles accessibility actions on behalf of this tree. The
+  // delegate is valid during the lifetime of this tree.
+  Delegate* delegate_;
+  content::BrowserContext* const browser_context_;
+  extensions::AutomationEventRouterInterface* const event_router_;
+
+  // Maps a node id to its tree data.
+  base::flat_map<int32_t /* node_id */, std::unique_ptr<FlutterSemanticsNode>>
+      tree_map_;
+
+  // Maps a node id to its parent.
+  base::flat_map<int32_t /* node_id */, int32_t /* parent_node_id */>
+      parent_map_;
+
+  // Mapping from ArcAccessibilityInfoData ID to its cached computed bounds.
+  // This simplifies bounds calculations.
+  base::flat_map<int32_t, gfx::Rect> cached_computed_bounds_;
+
+  // Cache from node id to computed name for live region.
+  std::map<int32_t, std::string> live_region_name_cache_;
+
+  // Cache form node id to tts string for native tts components.
+  std::map<int32_t, std::string> native_tts_name_cache_;
+
+  std::vector<int32_t> reparented_children_;
+  std::vector<std::string> child_trees_;
+
+  // Maps web contents id to the web contents observer
+  base::flat_map<int32_t, std::unique_ptr<AXTreeWebContentsObserver>>
+      child_tree_observers_;
+
+  // Copy of most recent tree data
+  gallium::castos::OnAccessibilityEventRequest last_event_data_;
+};
+
+}  // namespace accessibility
+}  // namespace chromecast
+
+#endif  // CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_AX_TREE_SOURCE_FLUTTER_H_

chromecast/browser/accessibility/flutter/flutter_accessibility_helper_bridge.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chromecast/browser/accessibility/flutter/flutter_accessibility_helper_bridge.h"
+
+#include <utility>
+
+#include "base/task/post_task.h"
+#include "chromecast/browser/accessibility/accessibility_manager.h"
+#include "chromecast/browser/accessibility/proto/gallium_server_accessibility.grpc.pb.h"
+#include "chromecast/browser/cast_browser_process.h"
+#include "components/exo/fullscreen_shell_surface.h"
+#include "components/exo/shell_surface_util.h"
+#include "components/exo/surface.h"
+#include "content/public/browser/browser_context.h"
+#include "content/public/browser/browser_task_traits.h"
+#include "content/public/browser/browser_thread.h"
+#include "ui/accessibility/aura/aura_window_properties.h"
+#include "ui/accessibility/ax_action_data.h"
+#include "ui/aura/window.h"
+#include "ui/aura/window_tree_host.h"
+#include "ui/views/view.h"
+#include "ui/views/widget/widget.h"
+
+namespace chromecast {
+namespace gallium {
+namespace accessibility {
+
+using ::gallium::castos::
+    OnAccessibilityActionRequest_AccessibilityActionType_CUSTOM_ACTION;
+using ::gallium::castos::
+    // NOLINTNEXTLINE(whitespace/line_length)
+    OnAccessibilityActionRequest_AccessibilityActionType_DID_GAIN_ACCESSIBILITY_FOCUS;
+using ::gallium::castos::
+    // NOLINTNEXTLINE(whitespace/line_length)
+    OnAccessibilityActionRequest_AccessibilityActionType_DID_LOSE_ACCESSIBILITY_FOCUS;
+using ::gallium::castos::
+    OnAccessibilityActionRequest_AccessibilityActionType_SCROLL_DOWN;
+using ::gallium::castos::
+    OnAccessibilityActionRequest_AccessibilityActionType_SCROLL_LEFT;
+using ::gallium::castos::
+    OnAccessibilityActionRequest_AccessibilityActionType_SCROLL_RIGHT;
+using ::gallium::castos::
+    OnAccessibilityActionRequest_AccessibilityActionType_SCROLL_UP;
+using ::gallium::castos::
+    OnAccessibilityActionRequest_AccessibilityActionType_SET_SELECTION;
+using ::gallium::castos::
+    OnAccessibilityActionRequest_AccessibilityActionType_SHOW_ON_SCREEN;
+using ::gallium::castos::
+    OnAccessibilityActionRequest_AccessibilityActionType_TAP;
+
+FlutterAccessibilityHelperBridge::FlutterAccessibilityHelperBridge(
+    Delegate* bridge_delegate,
+    content::BrowserContext* browser_context)
+    : tree_source_(
+          std::make_unique<AXTreeSourceFlutter>(this, browser_context)),
+      bridge_delegate_(bridge_delegate) {}
+
+FlutterAccessibilityHelperBridge::~FlutterAccessibilityHelperBridge() = default;
+
+void FlutterAccessibilityHelperBridge::AccessibilityStateChanged(bool value) {
+  if (value) {
+    aura::Window* window = chromecast::shell::CastBrowserProcess::GetInstance()
+                               ->accessibility_manager()
+                               ->window_tree_host()
+                               ->window();
+
+    // Find the full screen shell surface for the exo::Surface representing
+    // the ui.  We must ensure our tree id is the child ax tree id for
+    // that view so when the root window is serialized, the flutter ax tree
+    // will be parented by that view.
+    bool found = false;
+    if (window) {
+      for (aura::Window* child : window->children()) {
+        exo::Surface* surface = exo::GetShellMainSurface(child);
+        if (surface) {
+          views::Widget* widget =
+              views::Widget::GetWidgetForNativeWindow(child);
+          if (widget) {
+            exo::FullscreenShellSurface* full_screen_shell_surface =
+                static_cast<exo::FullscreenShellSurface*>(
+                    widget->widget_delegate());
+            full_screen_shell_surface->SetChildAxTreeId(
+                tree_source_->ax_tree_id());
+            full_screen_shell_surface->GetContentsView()
+                ->NotifyAccessibilityEvent(ax::mojom::Event::kChildrenChanged,
+                                           false);
+            child->Focus();
+            found = true;
+          }
+          break;
+        }
+      }
+    }
+
+    if (!found) {
+      LOG(ERROR) << "Could not find full screen shell surface for ax tree.";
+    }
+  }
+}
+
+void FlutterAccessibilityHelperBridge::OnAccessibilityEventRequestInternal(
+    std::unique_ptr<::gallium::castos::OnAccessibilityEventRequest>
+        event_data) {
+  // Tell the tree source to serialize these changes.
+  tree_source_->NotifyAccessibilityEvent(event_data.get());
+}
+
+bool FlutterAccessibilityHelperBridge::OnAccessibilityEventRequest(
+    const ::gallium::castos::OnAccessibilityEventRequest* event_data) {
+  std::unique_ptr<::gallium::castos::OnAccessibilityEventRequest> event =
+      std::make_unique<::gallium::castos::OnAccessibilityEventRequest>(
+          *event_data);
+
+  base::PostTask(FROM_HERE, {content::BrowserThread::UI},
+                 base::BindOnce(&FlutterAccessibilityHelperBridge::
+                                    OnAccessibilityEventRequestInternal,
+                                base::Unretained(this), std::move(event)));
+  return true;
+}
+
+void FlutterAccessibilityHelperBridge::OnAction(const ui::AXActionData& data) {
+  // Called by tree source to dispatch ax action to flutter. Translate this
+  // to gallium accessibility proto and forward to the delegate for
+  // dispatching.
+  ::gallium::castos::OnAccessibilityActionRequest request;
+  request.set_node_id(data.target_node_id);
+
+  switch (data.action) {
+    case ax::mojom::Action::kDoDefault:
+      request.set_action_type(
+          OnAccessibilityActionRequest_AccessibilityActionType_TAP);
+      break;
+    case ax::mojom::Action::kScrollToMakeVisible:
+      request.set_action_type(
+          OnAccessibilityActionRequest_AccessibilityActionType_SHOW_ON_SCREEN);
+      break;
+    case ax::mojom::Action::kScrollBackward:
+      request.set_action_type(
+          OnAccessibilityActionRequest_AccessibilityActionType_SCROLL_LEFT);
+      break;
+    case ax::mojom::Action::kScrollForward:
+      request.set_action_type(
+          OnAccessibilityActionRequest_AccessibilityActionType_SCROLL_RIGHT);
+      break;
+    case ax::mojom::Action::kScrollUp:
+      request.set_action_type(
+          OnAccessibilityActionRequest_AccessibilityActionType_SCROLL_UP);
+      break;
+    case ax::mojom::Action::kScrollDown:
+      request.set_action_type(
+          OnAccessibilityActionRequest_AccessibilityActionType_SCROLL_DOWN);
+      break;
+    case ax::mojom::Action::kScrollLeft:
+      request.set_action_type(
+          OnAccessibilityActionRequest_AccessibilityActionType_SCROLL_LEFT);
+      break;
+    case ax::mojom::Action::kScrollRight:
+      request.set_action_type(
+          OnAccessibilityActionRequest_AccessibilityActionType_SCROLL_RIGHT);
+      break;
+    case ax::mojom::Action::kCustomAction:
+      request.set_action_type(
+          OnAccessibilityActionRequest_AccessibilityActionType_CUSTOM_ACTION);
+      request.set_custom_action_id(data.custom_action_id);
+      break;
+    case ax::mojom::Action::kSetAccessibilityFocus:
+      request.set_action_type(
+          // NOLINTNEXTLINE(whitespace/line_length)
+          OnAccessibilityActionRequest_AccessibilityActionType_DID_GAIN_ACCESSIBILITY_FOCUS);
+      break;
+    case ax::mojom::Action::kClearAccessibilityFocus:
+      request.set_action_type(
+          // NOLINTNEXTLINE(whitespace/line_length)
+          OnAccessibilityActionRequest_AccessibilityActionType_DID_LOSE_ACCESSIBILITY_FOCUS);
+      break;
+    case ax::mojom::Action::kGetTextLocation:
+      request.set_action_type(
+          OnAccessibilityActionRequest_AccessibilityActionType_SET_SELECTION);
+      request.set_start_index(data.start_index);
+      request.set_end_index(data.end_index);
+      break;
+    default:
+      LOG(WARNING) << "Cast ax action " << data.action
+                   << " not mapped to flutter action - dropped.";
+
+      return;
+  }
+
+  bridge_delegate_->SendAccessibilityAction(request);
+}
+
+}  // namespace accessibility
+}  // namespace gallium
+}  // namespace chromecast

chromecast/browser/accessibility/flutter/flutter_accessibility_helper_bridge.h

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_FLUTTER_ACCESSIBILITY_HELPER_BRIDGE_H_
+#define CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_FLUTTER_ACCESSIBILITY_HELPER_BRIDGE_H_
+
+#include <memory>
+
+#include "chromecast/browser/accessibility/flutter/ax_tree_source_flutter.h"
+
+using chromecast::accessibility::AXTreeSourceFlutter;
+
+namespace content {
+class BrowserContext;
+}  // namespace content
+
+namespace gallium {
+namespace castos {
+class OnAccessibilityEventRequest;
+class OnAccessibilityActionRequest;
+}  // namespace castos
+}  // namespace gallium
+
+namespace chromecast {
+namespace gallium {
+namespace accessibility {
+
+// FlutterAccessibilityHelperBridge receives Flutter accessibility
+// events from gallium, translates them to chrome tree updates and dispatches
+// them to chromecast accessibility services.
+class FlutterAccessibilityHelperBridge : public AXTreeSourceFlutter::Delegate {
+ public:
+  class Delegate {
+   public:
+    virtual void SendAccessibilityAction(
+        ::gallium::castos::OnAccessibilityActionRequest request) = 0;
+
+   protected:
+    virtual ~Delegate() = default;
+  };
+
+  FlutterAccessibilityHelperBridge(Delegate* bridge_delegate,
+                                   content::BrowserContext* browser_context);
+  FlutterAccessibilityHelperBridge(const FlutterAccessibilityHelperBridge&) =
+      delete;
+  ~FlutterAccessibilityHelperBridge() override;
+  FlutterAccessibilityHelperBridge& operator=(
+      const FlutterAccessibilityHelperBridge&) = delete;
+
+  // Receive an accessibility event from flutter.
+  bool OnAccessibilityEventRequest(
+      const ::gallium::castos::OnAccessibilityEventRequest* event_data);
+
+  // AXTreeSourceArc::Delegate implementation:
+  // Dispatch a chrome accessibility action to flutter.
+  void OnAction(const ui::AXActionData& data) override;
+
+  void AccessibilityStateChanged(bool value);
+
+ private:
+  void OnAccessibilityEventRequestInternal(
+      std::unique_ptr<::gallium::castos::OnAccessibilityEventRequest>
+          event_data);
+
+  std::unique_ptr<AXTreeSourceFlutter> tree_source_;
+  Delegate* bridge_delegate_;
+};
+
+}  // namespace accessibility
+}  // namespace gallium
+}  // namespace chromecast
+
+#endif  // CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_FLUTTER_ACCESSIBILITY_HELPER_BRIDGE_H_

chromecast/browser/accessibility/flutter/flutter_semantics_node.h

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_FLUTTER_SEMANTICS_NODE_H_
+#define CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_FLUTTER_SEMANTICS_NODE_H_
+
+#include <ui/gfx/geometry/rect.h>
+
+#include <string>
+#include <vector>
+
+namespace ui {
+struct AXNodeData;
+}  // namespace ui
+
+namespace chromecast {
+namespace accessibility {
+
+// FlutterSemanticsNode represents a single flutter semantics object from
+// flutter. This class is used by AXTreeSourceFlutter to encapsulate
+// flutter information which maps to a single AXNodeData.
+class FlutterSemanticsNode {
+ public:
+  virtual ~FlutterSemanticsNode() = default;
+
+  virtual int32_t GetId() const = 0;
+  virtual const gfx::Rect GetBounds() const = 0;
+  virtual bool IsVisibleToUser() const = 0;
+  virtual bool IsFocused() const = 0;
+  virtual bool IsLiveRegion() const = 0;
+  virtual bool IsRapidChangingSlider() const = 0;
+  virtual bool CanBeAccessibilityFocused() const = 0;
+  virtual void PopulateAXRole(ui::AXNodeData* out_data) const = 0;
+  virtual void PopulateAXState(ui::AXNodeData* out_data) const = 0;
+  virtual void Serialize(ui::AXNodeData* out_data) const = 0;
+  virtual void GetChildren(
+      std::vector<FlutterSemanticsNode*>* children) const = 0;
+  virtual void ComputeNameFromContents(
+      std::vector<std::string>* names) const = 0;
+  virtual bool HasLabelHint() const = 0;
+  virtual std::string GetLabelHint() const = 0;
+  virtual bool HasValue() const = 0;
+  virtual std::string GetValue() const = 0;
+};
+
+}  // namespace accessibility
+}  // namespace chromecast
+
+#endif  // CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_FLUTTER_SEMANTICS_NODE_H_

chromecast/browser/accessibility/flutter/flutter_semantics_node_wrapper.h

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_FLUTTER_SEMANTICS_NODE_WRAPPER_H_
+#define CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_FLUTTER_SEMANTICS_NODE_WRAPPER_H_
+
+#include <string>
+#include <vector>
+
+#include "chromecast/browser/accessibility/flutter/flutter_semantics_node.h"
+#include "chromecast/browser/accessibility/proto/cast_server_accessibility.pb.h"
+#include "ui/accessibility/ax_enum_util.h"
+#include "ui/accessibility/ax_node_data.h"
+
+namespace chromecast {
+namespace accessibility {
+
+using gallium::castos::SemanticsNode;
+
+class AXTreeSourceFlutter;
+
+// A wrapper class for a SemanticsNode proto object.
+// This is used by AXTreeSourceFlutter to create accessibility tree updates
+// from semantics trees sent to us from the flutter process.
+class FlutterSemanticsNodeWrapper : public FlutterSemanticsNode {
+ public:
+  FlutterSemanticsNodeWrapper(AXTreeSourceFlutter* tree_source,
+                              const SemanticsNode* node);
+  FlutterSemanticsNodeWrapper(const FlutterSemanticsNodeWrapper&) = delete;
+  FlutterSemanticsNodeWrapper& operator=(const FlutterSemanticsNodeWrapper&) =
+      delete;
+
+  // FlutterSemanticsNode implementation:
+  int32_t GetId() const override;
+  const gfx::Rect GetBounds() const override;
+  bool IsVisibleToUser() const override;
+  bool IsFocused() const override;
+  bool IsLiveRegion() const override;
+  bool IsRapidChangingSlider() const override;
+  bool CanBeAccessibilityFocused() const override;
+  void PopulateAXRole(ui::AXNodeData* out_data) const override;
+  void PopulateAXState(ui::AXNodeData* out_data) const override;
+  void Serialize(ui::AXNodeData* out_data) const override;
+  void GetChildren(std::vector<FlutterSemanticsNode*>* children) const override;
+  bool HasLabelHint() const override;
+  std::string GetLabelHint() const override;
+  bool HasValue() const override;
+  std::string GetValue() const override;
+
+  const SemanticsNode* node() { return node_ptr_; }
+
+ private:
+  bool AnyChildIsActionable() const;
+  bool HasTapOrPress() const;
+  bool IsActionable() const;
+  bool IsScrollable() const;
+  bool IsFocusable() const;
+  void ComputeNameFromContents(std::vector<std::string>* names) const override;
+  void GetActionableChildren(
+      std::vector<FlutterSemanticsNodeWrapper*>* out_children) const;
+  // Check if this is a list item and return the node of its ancestor whose role
+  // is kList
+  FlutterSemanticsNodeWrapper* IsListItem() const;
+  bool IsDescendant(FlutterSemanticsNodeWrapper* ancestor) const;
+
+  AXTreeSourceFlutter* const tree_source_;
+  const SemanticsNode* const node_ptr_;
+};
+
+}  // namespace accessibility
+}  // namespace chromecast
+
+#endif  // CHROMECAST_BROWSER_ACCESSIBILITY_FLUTTER_FLUTTER_SEMANTICS_NODE_WRAPPER_H_

chromecast/browser/accessibility/proto/BUILD.gn

+import("//third_party/protobuf/proto_library.gni")
+
+proto_library("gallium_accessibility_proto") {
+  sources = [
+    "cast_server_accessibility.proto",
+    "gallium_server_accessibility.proto",
+  ]
+
+  deps = [
+    "//third_party/grpc:grpcpp",
+    "//third_party/protobuf:protobuf_lite",
+  ]
+
+  generator_plugin_label = "//third_party/grpc:grpc_cpp_plugin"
+  generator_plugin_suffix = ".grpc.pb"
+}

chromecast/browser/accessibility/proto/cast_server_accessibility.proto

+syntax = "proto2";
+
+package gallium.castos;
+
+option optimize_for = LITE_RUNTIME;
+
+message Rect {
+  optional int32 left = 1;
+  optional int32 top = 2;
+  optional int32 right = 3;
+  optional int32 bottom = 4;
+}
+
+message BooleanProperties {
+  optional bool has_checked_state = 1;
+  optional bool is_checked = 2;
+  optional bool is_selected = 3;
+  optional bool is_button = 4;
+  optional bool is_text_field = 5;
+  optional bool is_focused = 6;
+  optional bool has_enabled_state = 7;
+  optional bool is_enabled = 8;
+  optional bool is_in_mutually_exclusive_group = 9;
+  optional bool is_header = 10;
+  optional bool is_obscured = 11;
+  optional bool scopes_route = 12;
+  optional bool names_route = 13;
+  optional bool is_hidden = 14;
+  optional bool is_image = 15;
+  optional bool is_live_region = 16;
+  optional bool has_toggled_state = 17;
+  optional bool is_toggled = 18;
+  optional bool has_implicit_scrolling = 19;
+}
+
+message ActionProperties {
+  optional bool tap = 1;
+  optional bool long_press = 2;
+  optional bool scroll_left = 3;
+  optional bool scroll_right = 4;
+  optional bool scroll_up = 5;
+  optional bool scroll_down = 6;
+  optional bool increase = 7;
+  optional bool decrease = 8;
+  optional bool show_on_screen = 9;
+  optional bool move_cursor_forward_by_character = 10;
+  optional bool move_cursor_backward_by_character = 11;
+  optional bool set_selection = 12;
+  optional bool copy = 13;
+  optional bool cut = 14;
+  optional bool paste = 15;
+  optional bool did_gain_accessibility_focus = 16;
+  optional bool did_lose_accessibility_focus = 17;
+  optional bool custom_action = 18;
+  optional bool dismiss = 19;
+  optional bool move_cursor_forward_by_word = 20;
+  optional bool move_cursor_backward_by_word = 21;
+};
+
+message CustomAction {
+  optional int32 id = 1;
+  optional string label = 2;
+}
+
+message SemanticsNode {
+  optional int32 node_id = 1;
+  optional Rect bounds_in_screen = 2;
+  optional int32 window_id = 3;
+  repeated int32 child_node_ids = 4;
+  optional BooleanProperties boolean_properties = 5;
+  optional ActionProperties action_properties = 6;
+  optional string value = 7;
+  optional string label = 8;
+  optional string hint = 9;
+  optional string text_direction = 10;
+  optional int32 resource_id = 11;
+  optional int32 text_selection_base = 12;
+  optional int32 text_selection_extent = 13;
+  optional int32 scroll_children = 14;
+  optional int32 scroll_index = 15;
+  optional float scroll_position = 16;
+  optional int32 scroll_extent_max = 17;
+  optional int32 scroll_extent_min = 18;
+  repeated CustomAction custom_actions = 19;
+  // If set, indicates the semantics information for this node
+  // is provided by a platform plugin or some other native backed
+  // entity.
+  optional string plugin_id = 20;
+}
+
+message OnAccessibilityEventRequest {
+  enum EventType {
+    UNSPECIFIED = 0;
+    FOCUSED = 1;
+    FOCUS_CLEARED = 2;
+    HOVER_ENTER = 3;
+    HOVER_EXIT = 4;
+    CLICKED = 5;
+    LONG_CLICKED = 6;
+    WINDOW_STATE_CHANGED = 7;
+    TEXT_CHANGED = 8;
+    TEXT_SELECTION_CHANGED = 9;
+    CONTENT_CHANGED = 10;
+    SCROLLED = 11;
+  }
+  // Type of event this represents.
+  optional EventType event_type = 1;
+  // The node responsible for triggering this event.
+  optional int32 source_id = 2;
+  // Unique identifier for flutter window.
+  optional int32 window_id = 3;
+  // The 'flattened' semantic tree nodes.
+  repeated SemanticsNode node_data = 4;
+}
+
+message OnAccessibilityEventResponse {}

chromecast/browser/accessibility/proto/gallium_server_accessibility.proto

+syntax = "proto2";
+
+package gallium.castos;
+
+option optimize_for = LITE_RUNTIME;
+
+message OnAccessibilityStateChangedRequest {
+  optional bool enabled = 1;
+}
+
+message OnAccessibilityStateChangedResponse {}
+
+message OnAccessibilityActionRequest {
+  enum AccessibilityActionType {
+    UNSPECIFIED = 0;
+    TAP = 1;
+    LONG_PRESS = 2;
+    SCROLL_LEFT = 3;
+    SCROLL_RIGHT = 4;
+    SCROLL_UP = 5;
+    SCROLL_DOWN = 6;
+    INCREASE = 7;
+    DECREASE = 8;
+    SHOW_ON_SCREEN = 9;
+    MOVE_CURSOR_FORWARD_BY_CHARACTER = 10;
+    MOVE_CURSOR_BACKWARD_BY_CHARACTER = 11;
+    SET_SELECTION = 12;
+    COPY = 13;
+    CUT = 14;
+    PASTE = 15;
+    DID_GAIN_ACCESSIBILITY_FOCUS = 16;
+    DID_LOSE_ACCESSIBILITY_FOCUS = 17;
+    CUSTOM_ACTION = 18;
+    DISMISS = 19;
+    MOVE_CURSOR_FORWARD_BY_WORD = 20;
+    MOVE_CURSOR_BACKWARD_BY_WORD = 21;
+  }
+
+  optional int32 node_id = 1;
+
+  optional AccessibilityActionType action_type = 2;
+
+  // custom_action_id must be set if action_type is CUSTOM_ACTION.
+  optional int32 custom_action_id = 3;
+
+  // window_id where the action is performed.
+  optional int32 window_id = 4;
+
+  // Parameters specifying indices to get text location of node.
+  optional int32 start_index = 5;
+  optional int32 end_index = 6;
+}
+
+message OnAccessibilityActionResponse {}