chromevox: speak content on mouse hover

Pass mouse events to chromevox's mouse_handler.js, and fire hover
events on mousemove. This allows mousemove events to cause spoken
feedback using the same functions as touch exploration. Mouse events
are only rewritten if the chromevox option speakTextUnderMouse is
enabled.

Note: chromevox only receives mousemove events when the mouse is down.
Temporarily, this functionality is used by holding the mouse button
down and moving the pointer around the page.

Bug: 853581

Cq-Include-Trybots: luci.chromium.try:closure_compilation
Change-Id: I2feb14a1f5d765296fde9a923b862c99f45b41bb
Reviewed-on: https://chromium-review.googlesource.com/c/1234260Reviewed-by: Devlin <rdevlin.cronin@chromium.org>
Reviewed-by: Tom Sepez <tsepez@chromium.org>
Reviewed-by: Scott Violet <sky@chromium.org>
Reviewed-by: David Tseng <dtseng@chromium.org>
Commit-Queue: James Wallace-Lee <jamwalla@chromium.org>
Cr-Commit-Position: refs/heads/master@{#597240}

ash/events/event_rewriter_controller.cc

@@ -89,6 +89,10 @@ void EventRewriterController::CaptureAllKeysForSpokenFeedback(bool capture) {
   spoken_feedback_event_rewriter_->set_capture_all_keys(capture);
 }
 
+void EventRewriterController::SetSendMouseEventsToDelegate(bool value) {
+  spoken_feedback_event_rewriter_->set_send_mouse_events(value);
+}
+
 void EventRewriterController::OnHostInitialized(aura::WindowTreeHost* host) {
   for (const auto& rewriter : rewriters_)
     host->GetEventSource()->AddEventRewriter(rewriter.get());

ash/events/event_rewriter_controller.h

@@ -48,6 +48,7 @@ class ASH_EXPORT EventRewriterController
   void OnUnhandledSpokenFeedbackEvent(
       std::unique_ptr<ui::Event> event) override;
   void CaptureAllKeysForSpokenFeedback(bool capture) override;
+  void SetSendMouseEventsToDelegate(bool value) override;
 
   // aura::EnvObserver:
   void OnWindowInitialized(aura::Window* window) override {}

ash/events/spoken_feedback_event_rewriter.cc

@@ -40,12 +40,11 @@ void SpokenFeedbackEventRewriter::OnUnhandledSpokenFeedbackEvent(
 ui::EventRewriteStatus SpokenFeedbackEventRewriter::RewriteEvent(
     const ui::Event& event,
     std::unique_ptr<ui::Event>* new_event) {
-  if (!delegate_.is_bound() || !event.IsKeyEvent())
-    return ui::EVENT_REWRITE_CONTINUE;
-
-  if (!Shell::Get()->accessibility_controller()->IsSpokenFeedbackEnabled())
+  if (!delegate_.is_bound() ||
+      !Shell::Get()->accessibility_controller()->IsSpokenFeedbackEnabled())
     return ui::EVENT_REWRITE_CONTINUE;
 
+  if (event.IsKeyEvent()) {
     const ui::KeyEvent* key_event = event.AsKeyEvent();
 
     bool capture = capture_all_keys_;
@@ -55,13 +54,20 @@ ui::EventRewriteStatus SpokenFeedbackEventRewriter::RewriteEvent(
 
     // Don't capture tab as it gets consumed by Blink so never comes back
     // unhandled. In third_party/WebKit/Source/core/input/EventHandler.cpp, a
-  // default tab handler consumes tab even when no focusable nodes are found; it
-  // sets focus to Chrome and eats the event.
+    // default tab handler consumes tab even when no focusable nodes are found;
+    // it sets focus to Chrome and eats the event.
     if (key_event->GetDomKey() == ui::DomKey::TAB)
       capture = false;
 
     delegate_->DispatchKeyEventToChromeVox(ui::Event::Clone(event), capture);
     return capture ? ui::EVENT_REWRITE_DISCARD : ui::EVENT_REWRITE_CONTINUE;
+  }
+
+  if (send_mouse_events_ && event.IsMouseEvent()) {
+    delegate_->DispatchMouseEventToChromeVox(ui::Event::Clone(event));
+  }
+
+  return ui::EVENT_REWRITE_CONTINUE;
 }
 
 ui::EventRewriteStatus SpokenFeedbackEventRewriter::NextDispatchEvent(

ash/events/spoken_feedback_event_rewriter.h

@@ -31,6 +31,7 @@ class ASH_EXPORT SpokenFeedbackEventRewriter : public ui::EventRewriter {
   void OnUnhandledSpokenFeedbackEvent(std::unique_ptr<ui::Event> event) const;
 
   void set_capture_all_keys(bool value) { capture_all_keys_ = value; }
+  void set_send_mouse_events(bool value) { send_mouse_events_ = value; }
 
  private:
   // ui::EventRewriter:
@@ -44,6 +45,9 @@ class ASH_EXPORT SpokenFeedbackEventRewriter : public ui::EventRewriter {
   // The delegate used to send key events to the ChromeVox extension.
   mojom::SpokenFeedbackEventRewriterDelegatePtr delegate_;
 
+  // Whether to send mouse events to the ChromeVox extension.
+  bool send_mouse_events_ = false;
+
   // Whether to capture all keys.
   bool capture_all_keys_ = false;
 

ash/events/spoken_feedback_event_rewriter_unittest.cc

@@ -77,6 +77,11 @@ class TestDelegate : public mojom::SpokenFeedbackEventRewriterDelegate {
       captured_event_count_++;
   }
 
+  void DispatchMouseEventToChromeVox(
+      std::unique_ptr<ui::Event> event) override {
+    recorded_event_count_++;
+  }
+
   // The binding that backs the interface pointer held by the event rewriter.
   mojo::Binding<ash::mojom::SpokenFeedbackEventRewriterDelegate> binding_;
 

ash/public/interfaces/event_rewriter_controller.mojom

@@ -12,6 +12,9 @@ interface SpokenFeedbackEventRewriterDelegate {
   // the rewriter discarded the event, false if the rewriter continues event
   // propagation.
   DispatchKeyEventToChromeVox(ui.mojom.Event event, bool capture);
+
+  // Used to send mouse events to the ChromeVox extension.
+  DispatchMouseEventToChromeVox(ui.mojom.Event event);
 };
 
 // Allows clients to toggle some event rewriting behavior.
@@ -34,4 +37,7 @@ interface EventRewriterController {
 
   // Discards key events and sends to spoken feedback when true.
   CaptureAllKeysForSpokenFeedback(bool capture);
+
+  // Sends mouse events to ChromeVox when true.
+  SetSendMouseEventsToDelegate(bool value);
 };

chrome/browser/accessibility/accessibility_extension_api.cc

@@ -284,6 +284,23 @@ AccessibilityPrivateSendSyntheticKeyEventFunction::Run() {
   return RespondNow(NoArguments());
 }
 
+ExtensionFunction::ResponseAction
+AccessibilityPrivateEnableChromeVoxMouseEventsFunction::Run() {
+#if defined(OS_CHROMEOS)
+  bool enabled = false;
+  EXTENSION_FUNCTION_VALIDATE(args_->GetBoolean(0, &enabled));
+  ash::mojom::EventRewriterControllerPtr event_rewriter_controller_ptr;
+  content::ServiceManagerConnection* connection =
+      content::ServiceManagerConnection::GetForProcess();
+  connection->GetConnector()->BindInterface(ash::mojom::kServiceName,
+                                            &event_rewriter_controller_ptr);
+  event_rewriter_controller_ptr->SetSendMouseEventsToDelegate(enabled);
+  return RespondNow(NoArguments());
+#else
+  return RespondNow(Error(kErrorNotSupported));
+#endif
+}
+
 ExtensionFunction::ResponseAction
 AccessibilityPrivateOnSelectToSpeakStateChangedFunction::Run() {
   std::unique_ptr<accessibility_private::OnSelectToSpeakStateChanged::Params>

chrome/browser/accessibility/accessibility_extension_api.h

@@ -89,6 +89,15 @@ class AccessibilityPrivateSendSyntheticKeyEventFunction
                              ACCESSIBILITY_PRIVATE_SENDSYNTHETICKEYEVENT)
 };
 
+// API function that enables or disables mouse events in ChromeVox.
+class AccessibilityPrivateEnableChromeVoxMouseEventsFunction
+    : public UIThreadExtensionFunction {
+  ~AccessibilityPrivateEnableChromeVoxMouseEventsFunction() override {}
+  ResponseAction Run() override;
+  DECLARE_EXTENSION_FUNCTION("accessibilityPrivate.enableChromeVoxMouseEvents",
+                             ACCESSIBILITY_PRIVATE_ENABLECHROMEVOXMOUSEEVENTS)
+};
+
 // API function that is called when the Select-to-Speak extension state changes.
 class AccessibilityPrivateOnSelectToSpeakStateChangedFunction
     : public UIThreadExtensionFunction {

chrome/browser/chromeos/accessibility/event_handler_common.cc

@@ -11,6 +11,8 @@
 #include "extensions/browser/extension_host.h"
 #include "extensions/browser/extension_registry.h"
 #include "extensions/browser/process_manager.h"
+#include "third_party/blink/public/platform/web_mouse_event.h"
+#include "ui/events/blink/web_input_event.h"
 
 namespace chromeos {
 
@@ -52,4 +54,33 @@ void ForwardKeyToExtension(const ui::KeyEvent& key_event,
   // Don't forward latency info, as these are getting forwarded to an extension.
   rvh->GetWidget()->ForwardKeyboardEvent(web_event);
 }
+
+void ForwardMouseToExtension(const ui::MouseEvent& mouse_event,
+                             extensions::ExtensionHost* host) {
+  if (!host) {
+    VLOG(2) << "Unable to forward mouse to extension";
+    return;
+  }
+
+  content::RenderViewHost* rvh = host->render_view_host();
+  if (!rvh) {
+    VLOG(3) << "Unable to forward mouse to extension";
+    return;
+  }
+
+  if (mouse_event.type() == ui::ET_MOUSE_EXITED) {
+    VLOG(3) << "Couldn't forward unsupported mouse event to extension";
+    return;
+  }
+
+  const blink::WebMouseEvent& web_event = ui::MakeWebMouseEvent(mouse_event);
+
+  if (web_event.GetType() == blink::WebInputEvent::kUndefined) {
+    VLOG(3) << "Couldn't forward unsupported mouse event to extension";
+    return;
+  }
+
+  // Don't forward latency info, as these are getting forwarded to an extension.
+  rvh->GetWidget()->ForwardMouseEvent(web_event);
+}
 }  // namespace chromeos

chrome/browser/chromeos/accessibility/event_handler_common.h

@@ -19,6 +19,11 @@ extensions::ExtensionHost* GetAccessibilityExtensionHost(
 void ForwardKeyToExtension(const ui::KeyEvent& key_event,
                            extensions::ExtensionHost* host);
 
+// Forwards the mouse event to the extension background page for the
+// corresponding host.
+void ForwardMouseToExtension(const ui::MouseEvent& mouse_event,
+                             extensions::ExtensionHost* host);
+
 }  // namespace chromeos
 
 #endif  // CHROME_BROWSER_CHROMEOS_ACCESSIBILITY_EVENT_HANDLER_COMMON_H_

chrome/browser/chromeos/accessibility/spoken_feedback_event_rewriter_delegate.cc

@@ -43,6 +43,14 @@ void SpokenFeedbackEventRewriterDelegate::DispatchKeyEventToChromeVox(
   chromeos::ForwardKeyToExtension(*(event->AsKeyEvent()), host);
 }
 
+void SpokenFeedbackEventRewriterDelegate::DispatchMouseEventToChromeVox(
+    std::unique_ptr<ui::Event> event) {
+  extensions::ExtensionHost* host = chromeos::GetAccessibilityExtensionHost(
+      extension_misc::kChromeVoxExtensionId);
+  // Forward the event to ChromeVox's background page.
+  chromeos::ForwardMouseToExtension(*(event->AsMouseEvent()), host);
+}
+
 bool SpokenFeedbackEventRewriterDelegate::ShouldDispatchKeyEventToChromeVox(
     const ui::Event* event) const {
   chromeos::AccessibilityManager* accessibility_manager =

chrome/browser/chromeos/accessibility/spoken_feedback_event_rewriter_delegate.h

@@ -25,6 +25,7 @@ class SpokenFeedbackEventRewriterDelegate
   // ui::mojom::SpokenFeedbackEventRewriterDelegate:
   void DispatchKeyEventToChromeVox(std::unique_ptr<ui::Event> event,
                                    bool capture) override;
+  void DispatchMouseEventToChromeVox(std::unique_ptr<ui::Event> event) override;
 
  private:
   // Returns whether the event should be dispatched to the ChromeVox extension.

chrome/browser/resources/chromeos/chromevox/BUILD.gn

@@ -143,6 +143,7 @@ chromevox_modules = [
   "cvox2/background/log_store.js",
   "cvox2/background/math_handler.js",
   "cvox2/background/media_automation_handler.js",
+  "cvox2/background/mouse_handler.js",
   "cvox2/background/next_earcons.js",
   "cvox2/background/notifications.js",
   "cvox2/background/output.js",

chrome/browser/resources/chromeos/chromevox/chromevox/background/options.html

@@ -40,6 +40,14 @@
     </label>
   </div>
 
+  <div class="option">
+    <input id="speakTextUnderMouse" type="checkbox" class="checkbox pref"
+           name="speakTextUnderMouse">
+    <label for="speakTextUnderMouse" class="i18n" msgid="options_speak_mouse">
+      Speak text under the mouse.
+    </label>
+  </div>
+
   <h2 class="i18n description" msgid="options_audio_description"
       id="audioDescription">
     When playing audio

chrome/browser/resources/chromeos/chromevox/chromevox/background/options.js

@@ -138,6 +138,13 @@ cvox.OptionsPage.init = function() {
   document.addEventListener('click', cvox.OptionsPage.eventListener, false);
   document.addEventListener('keydown', cvox.OptionsPage.eventListener, false);
 
+  window.addEventListener('storage', (event) => {
+    if (event.key == 'speakTextUnderMouse') {
+      chrome.accessibilityPrivate.enableChromeVoxMouseEvents(
+          event.newValue == String(true));
+    }
+  });
+
   cvox.ExtensionBridge.addMessageListener(function(message) {
     if (message['prefs']) {
       cvox.OptionsPage.update();

chrome/browser/resources/chromeos/chromevox/chromevox/background/prefs.js

@@ -87,6 +87,7 @@ cvox.ChromeVoxPrefs.DEFAULT_PREFS = {
       'https://ssl.gstatic.com/accessibility/javascript/ext/',
   'siteSpecificScriptLoader':
       'https://ssl.gstatic.com/accessibility/javascript/ext/loader.js',
+  'speakTextUnderMouse': false,
   'sticky': false,
   'typingEcho': 0,
   'useIBeamCursor': cvox.ChromeVox.isMac,

chrome/browser/resources/chromeos/chromevox/cvox2/background/background.js

@@ -12,6 +12,7 @@ goog.provide('Background');
 goog.require('AutomationPredicate');
 goog.require('AutomationUtil');
 goog.require('BackgroundKeyboardHandler');
+goog.require('BackgroundMouseHandler');
 goog.require('BrailleCommandData');
 goog.require('BrailleCommandHandler');
 goog.require('ChromeVoxState');
@@ -121,6 +122,13 @@ Background = function() {
   /** @type {!BackgroundKeyboardHandler} @private */
   this.keyboardHandler_ = new BackgroundKeyboardHandler();
 
+  /** @type {!BackgroundMouseHandler} @private */
+  this.mouseHandler_ = new BackgroundMouseHandler();
+
+  if (localStorage['speakTextUnderMouse'] == String(true)) {
+    chrome.accessibilityPrivate.enableChromeVoxMouseEvents(true);
+  }
+
   /** @type {!LiveRegions} @private */
   this.liveRegions_ = new LiveRegions(this);
 
@@ -462,7 +470,7 @@ Background.prototype = {
     // the next or previous focusable node from |start|.
     if (!start.state[StateType.OFFSCREEN])
       start.setSequentialFocusNavigationStartingPoint();
-  }
+  },
 };
 
 /**

chrome/browser/resources/chromeos/chromevox/cvox2/background/mouse_handler.js

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+/**
+ * @fileoverview ChromeVox mouse handler.
+ */
+
+goog.provide('BackgroundMouseHandler');
+
+var EventType = chrome.automation.EventType;
+
+/** @constructor */
+BackgroundMouseHandler = function() {
+  document.addEventListener('mousemove', this.onMouseMove.bind(this));
+
+  chrome.automation.getDesktop(function(desktop) {
+    /**
+     *
+     * The desktop node.
+     *
+     * @private {!chrome.automation.AutomationNode}
+     */
+    this.desktop_ = desktop;
+  }.bind(this));
+};
+
+BackgroundMouseHandler.prototype = {
+  /**
+   * Handles mouse move events.
+   * @param {Event} evt The mouse move event to process.
+   * @return {boolean} True if the default action should be performed.
+   */
+  onMouseMove: function(evt) {
+    // Immediately save the most recent mouse coordinates.
+    this.desktop_.hitTest(evt.screenX, evt.screenY, EventType.HOVER);
+    return false;
+  },
+};

chrome/browser/resources/chromeos/chromevox/strings/chromevox_strings.grd

@@ -514,6 +514,9 @@
       <message desc="An option to show the cursor between characters." name="IDS_CHROMEVOX_OPTIONS_CURSOR_BETWEEN_CHARACTERS">
         Place cursor between characters when editing text (like Mac OS X).
       </message>
+      <message desc="An option to speak text under the mouse." name="IDS_CHROMEVOX_OPTIONS_SPEAK_MOUSE">
+        Speak text under the mouse.
+      </message>
       <message desc="An options page section header for options about the ChromeVox voice. This section lets users change the voice by selecting a different voice from a listbox." name="IDS_CHROMEVOX_OPTIONS_VOICES">
         Voices
       </message>

chrome/common/extensions/api/accessibility_private.json

@@ -214,6 +214,19 @@
         ],
         "platforms": ["chromeos"]
       },
+      {
+	"name": "enableChromeVoxMouseEvents",
+	"type": "function",
+	"description": "Enables or disables mouse events in ChromeVox.",
+	"parameters": [
+	  {
+	    "name": "enabled",
+	    "type": "boolean",
+	    "description": "True if ChromeVox should receive mouse events."
+	  }
+	],
+        "platforms": ["chromeos"]
+      },
       {
         "name": "onSelectToSpeakStateChanged",
         "type": "function",

extensions/browser/extension_function_histogram_value.h

@@ -1345,6 +1345,7 @@ enum HistogramValue {
   AUTOTESTPRIVATE_RUNCROSTINIUNINSTALLER = 1282,
   AUTOTESTPRIVATE_TAKESCREENSHOT = 1283,
   ACCESSIBILITY_PRIVATE_TOGGLEDICTATION = 1284,
+  ACCESSIBILITY_PRIVATE_ENABLECHROMEVOXMOUSEEVENTS = 1285,
   // Last entry: Add new entries above, then run:
   // python tools/metrics/histograms/update_extension_histograms.py
   ENUM_BOUNDARY

third_party/closure_compiler/externs/accessibility_private.js

@@ -167,6 +167,12 @@ chrome.accessibilityPrivate.setNativeChromeVoxArcSupportForCurrentApp = function
  */
 chrome.accessibilityPrivate.sendSyntheticKeyEvent = function(keyEvent) {};
 
+/**
+ * Enables or disables mouse events in ChromeVox.
+ * @param {boolean} enabled True if ChromeVox should receive mouse events.
+ */
+chrome.accessibilityPrivate.enableChromeVoxMouseEvents = function(enabled) {};
+
 /**
  * Fired whenever ChromeVox should output introduction.
  * @type {!ChromeEvent}

tools/metrics/histograms/enums.xml

@@ -16934,6 +16934,7 @@ Called by update_net_error_codes.py.-->
   <int value="1282" label="AUTOTESTPRIVATE_RUNCROSTINIUNINSTALLER"/>
   <int value="1283" label="AUTOTESTPRIVATE_TAKESCREENSHOT"/>
   <int value="1284" label="ACCESSIBILITY_PRIVATE_TOGGLEDICTATION"/>
+  <int value="1285" label="ACCESSIBILITY_PRIVATE_ENABLECHROMEVOXMOUSEEVENTS"/>
 </enum>
 
 <enum name="ExtensionIconState">