Add AccessibilityPrivate methods to communicate Select-to-Speak extension state.

This is part of go/chromeos-sts-on-screen-ui, which requires the chrome process
to be able to communicate bi-directionally with the Select-to-Speak extension:
Chrome must be able to request Select-to-Speak to change states, and must get
notified whenever Select-to-Speak's state changes.

1. This change hooks up the extension receiving requests from Chrome to change
state, although the implementation of changing state logic in the extension is
currently a TODO.

2. This change also hooks up the extension communicating its state back to
Chrome, although the accessibility_extension_api method that receives the call
is implemented as a TODO after converting accessibility_private state into a
more universal mojom::SelectToSpeakState state.

Note: We need mojom::SelectToSpeakState because chrome needs to communicate
state changes through to the shelf in Ash, so the enum needs to be usable across
processes.

The next changes will complete the TODOs in 1. and 2. above.

Bug: 753018
Cq-Include-Trybots: master.tryserver.chromium.linux:closure_compilation
Change-Id: Ia1785a85adea02b3e60ed2e60525012d166e7ad2
Reviewed-on: https://chromium-review.googlesource.com/1030901
Commit-Queue: Katie Dektar <katie@chromium.org>
Reviewed-by: David Tseng <dtseng@chromium.org>
Reviewed-by: Steven Bennetts <stevenjb@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Ilya Sherman <isherman@chromium.org>
Cr-Commit-Position: refs/heads/master@{#555147}

ash/public/interfaces/accessibility_controller.mojom

@@ -36,6 +36,18 @@ enum AccessibilityAlert {
   WINDOW_OVERVIEW_MODE_ENTERED
 };
 
+enum SelectToSpeakState {
+  // Select to Speak is not actively selecting text or speaking.
+  kSelectToSpeakStateInactive,
+
+  // Select to Speak is being used to actively select a new region. Note that
+  // it might also be speaking, but the selecting state takes precedence.
+  kSelectToSpeakStateSelecting,
+
+  // Select to Speak is currently speaking.
+  kSelectToSpeakStateSpeaking,
+};
+
 // Interface for ash client (e.g. Chrome) to control and query accessibility
 // features.
 interface AccessibilityController {

chrome/browser/accessibility/accessibility_extension_api.cc

@@ -37,6 +37,7 @@
 #include "ash/shell.h"
 #include "chrome/browser/chromeos/accessibility/accessibility_manager.h"
 #include "chrome/browser/chromeos/arc/accessibility/arc_accessibility_helper_bridge.h"
+#include "services/ui/public/interfaces/accessibility_manager.mojom.h"
 #include "ui/aura/window_tree_host.h"
 #include "ui/events/event_sink.h"
 #endif
@@ -261,4 +262,34 @@ AccessibilityPrivateSendSyntheticKeyEventFunction::Run() {
   return RespondNow(NoArguments());
 }
 
+ExtensionFunction::ResponseAction
+AccessibilityPrivateOnSelectToSpeakStateChangedFunction::Run() {
+  std::unique_ptr<accessibility_private::OnSelectToSpeakStateChanged::Params>
+      params =
+          accessibility_private::OnSelectToSpeakStateChanged::Params::Create(
+              *args_);
+  EXTENSION_FUNCTION_VALIDATE(params);
+  accessibility_private::SelectToSpeakState params_state = params->state;
+  ash::mojom::SelectToSpeakState state;
+  switch (params_state) {
+    case accessibility_private::SelectToSpeakState::
+        SELECT_TO_SPEAK_STATE_SELECTING:
+      state = ash::mojom::SelectToSpeakState::kSelectToSpeakStateSelecting;
+      break;
+    case accessibility_private::SelectToSpeakState::
+        SELECT_TO_SPEAK_STATE_SPEAKING:
+      state = ash::mojom::SelectToSpeakState::kSelectToSpeakStateSpeaking;
+      break;
+    case accessibility_private::SelectToSpeakState::
+        SELECT_TO_SPEAK_STATE_INACTIVE:
+    case accessibility_private::SelectToSpeakState::SELECT_TO_SPEAK_STATE_NONE:
+      state = ash::mojom::SelectToSpeakState::kSelectToSpeakStateInactive;
+  }
+
+  // TODO(katie): Plumb state updates into AccessibilityManager for
+  // http://crbug.com/753018.
+
+  return RespondNow(NoArguments());
+}
+
 #endif  // defined (OS_CHROMEOS)

chrome/browser/accessibility/accessibility_extension_api.h

@@ -88,6 +88,15 @@ class AccessibilityPrivateSendSyntheticKeyEventFunction
   DECLARE_EXTENSION_FUNCTION("accessibilityPrivate.sendSyntheticKeyEvent",
                              ACCESSIBILITY_PRIVATE_SENDSYNTHETICKEYEVENT)
 };
+
+// API function that is called when the Select-to-Speak extension state changes.
+class AccessibilityPrivateOnSelectToSpeakStateChangedFunction
+    : public UIThreadExtensionFunction {
+  ~AccessibilityPrivateOnSelectToSpeakStateChangedFunction() override {}
+  ResponseAction Run() override;
+  DECLARE_EXTENSION_FUNCTION("accessibilityPrivate.onSelectToSpeakStateChanged",
+                             ACCESSIBILITY_PRIVATE_ONSELECTTOSPEAKSTATECHANGED)
+};
 #endif  // defined (OS_CHROMEOS)
 
 #endif  // CHROME_BROWSER_ACCESSIBILITY_ACCESSIBILITY_EXTENSION_API_H_

chrome/browser/resources/chromeos/select_to_speak/select_to_speak.js

@@ -5,6 +5,7 @@
 var AutomationEvent = chrome.automation.AutomationEvent;
 var EventType = chrome.automation.EventType;
 var RoleType = chrome.automation.RoleType;
+var SelectToSpeakState = chrome.accessibilityPrivate.SelectToSpeakState;
 
 // CrosSelectToSpeakStartSpeechMethod enums.
 // These values are persited to logs and should not be renumbered or re-used.
@@ -47,8 +48,12 @@ function getDriveAppRoot(node) {
  * @constructor
  */
 var SelectToSpeak = function() {
-  /** @private {AutomationNode} */
-  this.node_ = null;
+  /**
+   * The current state of the SelectToSpeak extension, from
+   * SelectToSpeakState.
+   * @private {!SelectToSpeakState}
+   */
+  this.state_ = SelectToSpeakState.INACTIVE;
 
   /** @private {boolean} */
   this.trackingMouse_ = false;
@@ -189,6 +194,8 @@ SelectToSpeak.prototype = {
     if (!this.isSearchKeyDown_ || this.isSelectionKeyDown_)
       return false;
 
+    this.onStateChanged_(SelectToSpeakState.SELECTING);
+
     this.trackingMouse_ = true;
     this.didTrackMouse_ = true;
     this.mouseStart_ = {x: evt.screenX, y: evt.screenY};
@@ -564,6 +571,7 @@ SelectToSpeak.prototype = {
   stopAll_: function() {
     chrome.tts.stop();
     this.clearFocusRing_();
+    this.onStateChanged_(SelectToSpeakState.INACTIVE);
   },
 
   /**
@@ -605,6 +613,20 @@ SelectToSpeak.prototype = {
     chrome.clipboard.onClipboardDataChanged.addListener(
         this.onClipboardDataChanged_.bind(this));
     document.addEventListener('paste', this.onClipboardPaste_.bind(this));
+    chrome.accessibilityPrivate.onSelectToSpeakStateChangeRequested.addListener(
+        this.onStateChangeRequested_.bind(this));
+    // Initialize the state to SelectToSpeakState.INACTIVE.
+    chrome.accessibilityPrivate.onSelectToSpeakStateChanged(this.state_);
+  },
+
+  /**
+   * Called when Chrome OS is requesting Select-to-Speak to switch states.
+   */
+  onStateChangeRequested_: function() {
+    // TODO(katie): Switch Select-to-Speak states on request.
+    // We will need to track the current state and toggle from one state to
+    // the next when this function is called, and then call
+    // accessibilityPrivate.onSelectToSpeakStateChanged with the new state.
   },
 
   /**
@@ -618,11 +640,12 @@ SelectToSpeak.prototype = {
     let options = this.speechOptions_();
     options.onEvent = (event) => {
       if (event.type == 'start') {
+        this.onStateChanged_(SelectToSpeakState.SPEAKING);
         this.testCurrentNode_();
       } else if (
           event.type == 'end' || event.type == 'interrupted' ||
           event.type == 'cancelled') {
-        this.clearFocusRingAndNode_();
+        this.onStateChanged_(SelectToSpeakState.INACTIVE);
       }
     };
     chrome.tts.speak(text, options);
@@ -684,6 +707,7 @@ SelectToSpeak.prototype = {
       let options = this.speechOptions_();
       options.onEvent = (event) => {
         if (event.type == 'start' && nodeGroup.nodes.length > 0) {
+          this.onStateChanged_(SelectToSpeakState.SPEAKING);
           this.currentBlockParent_ = nodeGroup.blockParent;
           this.currentNodeGroupIndex_ = 0;
           this.currentNode_ = nodeGroup.nodes[this.currentNodeGroupIndex_];
@@ -701,11 +725,10 @@ SelectToSpeak.prototype = {
             this.testCurrentNode_();
           }
         } else if (event.type == 'interrupted' || event.type == 'cancelled') {
-          this.clearFocusRingAndNode_();
+          this.onStateChanged_(SelectToSpeakState.INACTIVE);
         } else if (event.type == 'end') {
-          if (isLast) {
-            this.clearFocusRingAndNode_();
-          }
+          if (isLast)
+            this.onStateChanged_(SelectToSpeakState.INACTIVE);
         } else if (event.type == 'word') {
           console.debug(nodeGroup.text + ' (index ' + event.charIndex + ')');
           console.debug('-'.repeat(event.charIndex) + '^');
@@ -754,6 +777,29 @@ SelectToSpeak.prototype = {
         SelectToSpeak.NODE_STATE_TEST_INTERVAL_MS);
   },
 
+  /**
+   * Updates the state.
+   * @param {!SelectToSpeakState} state
+   */
+  onStateChanged_: function(state) {
+    if (this.state_ != state) {
+      if (this.state_ == SelectToSpeakState.SELECTING &&
+          state == SelectToSpeakState.INACTIVE && this.trackingMouse_) {
+        // If we are tracking the mouse actively, then we have requested tts
+        // to stop speaking just before mouse tracking began, so we
+        // shouldn't transition into the inactive state now: The call to stop
+        // speaking created an async 'cancel' event from the TTS engine that
+        // is now resulting in an attempt to set the state inactive.
+        return;
+      }
+      if (state == SelectToSpeakState.INACTIVE)
+        this.clearFocusRingAndNode_();
+      // Send state change event to Chrome.
+      chrome.accessibilityPrivate.onSelectToSpeakStateChanged(state);
+      this.state_ = state;
+    }
+  },
+
   /**
    * Generates the basic speech options for Select-to-Speak based on user
    * preferences. Call for each chrome.tts.speak.

chrome/browser/resources/chromeos/select_to_speak/test_support.js

@@ -39,3 +39,11 @@ chrome.metricsPrivate = {
 chrome.commandLinePrivate = {
   hasSwitch: function() {}
 };
+
+chrome.accessibilityPrivate = {};
+
+chrome.accessibilityPrivate.SelectToSpeakState = {
+  INACTIVE: 'inactive',
+  SELECTING: 'selecting',
+  SPEAKING: 'speaking'
+};

chrome/common/extensions/api/accessibility_private.json

@@ -82,6 +82,12 @@
             "description": "Contains all active modifiers."
           }
         }
+      },
+      {
+        "id": "SelectToSpeakState",
+        "type": "string",
+        "description": "The state of the Select-to-Speak extension",
+        "enum": ["selecting", "speaking", "inactive"]
       }
     ],
     "functions": [
@@ -207,6 +213,18 @@
           }
         ],
         "platforms": ["chromeos"]
+      },
+      {
+        "name": "onSelectToSpeakStateChanged",
+        "type": "function",
+        "description": "Called by the Select-to-Speak extension when Select-to-Speak has changed states, between selecting with the mouse, speaking, and inactive.",
+        "parameters": [
+          {
+            "name": "state",
+            "$ref": "SelectToSpeakState"
+          }
+        ],
+        "platforms": ["chromeos"]
       }
     ],
     "events": [
@@ -235,8 +253,15 @@
       {
         "name": "onTwoFingerTouchStop",
         "type": "function",
-        "description": "Fired when  the user is no longer holding down two fingers (including releasing one, holding down three, or moving them).",
+        "description": "Fired when the user is no longer holding down two fingers (including releasing one, holding down three, or moving them).",
         "parameters": []
+      },
+      {
+        "name": "onSelectToSpeakStateChangeRequested",
+        "type": "function",
+        "description": "Called when Chrome OS wants to change the Select-to-Speak state, between selecting with the mouse, speaking, and inactive.",
+        "parameters": [],
+        "platforms": ["chromeos"]
       }
     ]
   }

extensions/browser/extension_event_histogram_value.h

@@ -436,6 +436,7 @@ enum HistogramValue {
   SAFE_BROWSING_PRIVATE_ON_DANGEROUS_DOWNLOAD_OPENED,
   SAFE_BROWSING_PRIVATE_ON_SECURITY_INTERSTITIAL_SHOWN,
   SAFE_BROWSING_PRIVATE_ON_SECURITY_INTERSTITIAL_PROCEEDED,
+  ACCESSIBILITY_PRIVATE_ON_SELECT_TO_SPEAK_STATE_CHANGE_REQUESTED,
   // Last entry: Add new entries above, then run:
   // python tools/metrics/histograms/update_extension_histograms.py
   ENUM_BOUNDARY

extensions/browser/extension_function_histogram_value.h

@@ -1308,6 +1308,7 @@ enum HistogramValue {
   AUTOTESTPRIVATE_UPDATEPRINTER,
   AUTOTESTPRIVATE_REMOVEPRINTER,
   WALLPAPERPRIVATE_GETCURRENTWALLPAPERTHUMBNAIL,
+  ACCESSIBILITY_PRIVATE_ONSELECTTOSPEAKSTATECHANGED,
   // Last entry: Add new entries above, then run:
   // python tools/metrics/histograms/update_extension_histograms.py
   ENUM_BOUNDARY

third_party/closure_compiler/externs/accessibility_private.js

@@ -100,6 +100,16 @@ chrome.accessibilityPrivate.KeyboardMode = {
   FLOATING: 'FLOATING',
 };
 
+/**
+ * @enum {string}
+ * @see https://developer.chrome.com/extensions/accessibilityPrivate#type-SelectToSpeakState
+ */
+chrome.accessibilityPrivate.SelectToSpeakState = {
+  SELECTING: 'selecting',
+  SPEAKING: 'speaking',
+  INACTIVE: 'inactive',
+};
+
 /**
  * Enables or disables native accessibility support. Once disabled, it is up to
  * the calling extension to provide accessibility for web contents.
@@ -203,3 +213,20 @@ chrome.accessibilityPrivate.onTwoFingerTouchStart;
  * @see https://developer.chrome.com/extensions/accessibilityPrivate#event-onTwoFingerTouchStop
  */
 chrome.accessibilityPrivate.onTwoFingerTouchStop;
+
+/**
+ * Called by the Select-to-Speak extension when Select-to-Speak has changed states,
+ * between selecting with the mouse, speaking, and inactive.
+ * @param {!chrome.accessibilityPrivate.SelectToSpeakState} state The current
+ *    state of the Select-to-Speak extension
+ * @see https://developer.chrome.com/extensions/accessibilityPrivate#method-onSelectToSpeakStateChanged
+ */
+chrome.accessibilityPrivate.onSelectToSpeakStateChanged = function(state) {};
+
+/**
+ * Called when Chrome OS wants to toggle the Select-to-Speak state, between selecting
+ * with the mouse, speaking, and inactive
+ * @type {!ChromeEvent}
+ * @see https://developer.chrome.com/extensions/accessibilityPrivate#event-onSelectToSpeakStateChangeRequested
+ */
+chrome.accessibilityPrivate.onSelectToSpeakStateChangeRequested;

tools/metrics/histograms/enums.xml

@@ -14055,6 +14055,8 @@ Called by update_net_error_codes.py.-->
       label="SAFE_BROWSING_PRIVATE_ON_SECURITY_INTERSTITIAL_SHOWN"/>
   <int value="420"
       label="SAFE_BROWSING_PRIVATE_ON_SECURITY_INTERSTITIAL_PROCEEDED"/>
+  <int value="421"
+      label="ACCESSIBILITY_PRIVATE_ON_SELECT_TO_SPEAK_STATE_CHANGE_REQUESTED"/>
 </enum>
 
 <enum name="ExtensionFileWriteResult">
@@ -15357,6 +15359,9 @@ Called by update_net_error_codes.py.-->
   <int value="1245" label="AUTOTESTPRIVATE_UPDATEPRINTER"/>
   <int value="1246" label="AUTOTESTPRIVATE_REMOVEPRINTER"/>
   <int value="1247" label="WALLPAPERPRIVATE_GETCURRENTWALLPAPERTHUMBNAIL"/>
+  <int value="1248" label="ACCESSIBILITY_PRIVATE_ONSELECTTOSPEAKSTATECHANGED"/>
+  <int value="1249"
+      label="ACCESSIBILITY_PRIVATE_ONSELECTTOSPEAKSTATECHANGEREQUESTED"/>
 </enum>
 
 <enum name="ExtensionIconState">