Improve Learn Mode

This change:
- adds test coverage for Learn Mode with regard to keyboard, gesture, and braille
- adds live descriptions of each gesture (e.g. swipe right with one finger) in addition to describing the command
- adds the touchExplore gesture as something described (with special handling due to how frequently it fires)
- is more careful about queueing up too much speech; added robust tests for this throughout

R=akihiroota@chromium.org

Fixed: 1106462
AX-Relnotes: only announce a modifier once, when holding down modifiers like Search in ChromeVox Learn Mode. Also, adds descriptions for gestures like swipe right with one finger, in addition to the command triggered while in Learn Mode.
Change-Id: Ib600330c63cf3c412aa62c34050a4281e07cb816
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2309981
Commit-Queue: David Tseng <dtseng@chromium.org>
Reviewed-by: Akihiro Ota <akihiroota@chromium.org>
Cr-Commit-Position: refs/heads/master@{#790591}

chrome/browser/resources/chromeos/accessibility/chromevox/BUILD.gn

@@ -462,6 +462,7 @@ if (is_chromeos) {
       "braille/braille_translator_manager_test.js",
       "braille/liblouis_test.js",
       "common/tts_background_test.js",
+      "learn_mode/learn_mode_test.js",
       "options/options_test.js",
       "panel/i_search_test.js",
       "panel/panel_test.js",

chrome/browser/resources/chromeos/accessibility/chromevox/background/gesture_command_data.js

@@ -11,7 +11,7 @@ goog.provide('GestureGranularity');
  * @type {!Object<string,
  *     {
  *     msgId: string,
- *     command: string,
+ *     command: (string|undefined),
  *     menuKeyOverride: ({keyCode: number, modifiers: ({ctrl:
  * boolean}|undefined)}|undefined)
  *    }>}
@@ -48,6 +48,8 @@ GestureCommandData.GESTURE_COMMAND_MAP = {
   'swipeLeft3': {msgId: 'swipeleft3_gesture', command: 'previousGranularity'},
   'swipeRight3': {msgId: 'swiperight3_gesture', command: 'nextGranularity'},
 
+  'touchExplore': {msgId: 'touch_explore_gesture'},
+
   'tap2': {msgId: 'tap2_gesture', command: 'stopSpeech'},
   'tap4': {msgId: 'tap4_gesture', command: 'showPanelMenuMostRecent'},
 };

chrome/browser/resources/chromeos/accessibility/chromevox/learn_mode/kbexplorer.js

@@ -59,6 +59,7 @@ KbExplorer = class {
     $('instruction').focus();
 
     KbExplorer.output(Msgs.getMsg('learn_mode_intro'));
+    KbExplorer.shouldFlushSpeech_ = true;
   }
 
   /**
@@ -68,23 +69,21 @@ KbExplorer = class {
    * @return {boolean} True if the default action should be performed.
    */
   static onKeyDown(evt) {
-    if (KbExplorer.keydownWithoutKeyupEvents_.size == 0) {
-      ChromeVox.tts.stop();
-    }
-    KbExplorer.keydownWithoutKeyupEvents_.add(evt.keyCode);
-    ChromeVox.tts.speak(
-        KeyUtil.getReadableNameForKeyCode(evt.keyCode),
-        window.backgroundWindow.QueueMode.QUEUE,
-        AbstractTts.PERSONALITY_ANNOTATION);
+    // Process this event only once; it isn't a repeat (i.e. a user is holding a
+    // key down).
+    if (!evt.repeat) {
+      KbExplorer.output(KeyUtil.getReadableNameForKeyCode(evt.keyCode));
 
-    // Allow Ctrl+W or escape to be handled.
-    if ((evt.key == 'w' && evt.ctrlKey) || evt.key == 'Escape') {
-      KbExplorer.close_();
-      return true;
+      // Allow Ctrl+W or escape to be handled.
+      if ((evt.key == 'w' && evt.ctrlKey) || evt.key == 'Escape') {
+        KbExplorer.close_();
+        return true;
+      }
+
+      ChromeVoxKbHandler.basicKeyDownActionsListener(evt);
+      KbExplorer.clearRange();
     }
 
-    ChromeVoxKbHandler.basicKeyDownActionsListener(evt);
-    KbExplorer.clearRange();
     evt.preventDefault();
     evt.stopPropagation();
     return false;
@@ -95,7 +94,7 @@ KbExplorer = class {
    * @param {Event} evt key event.
    */
   static onKeyUp(evt) {
-    KbExplorer.keydownWithoutKeyupEvents_.delete(evt.keyCode);
+    KbExplorer.shouldFlushSpeech_ = true;
     KbExplorer.maybeClose_();
     KbExplorer.clearRange();
     evt.preventDefault();
@@ -116,6 +115,7 @@ KbExplorer = class {
    * @param {BrailleKeyEvent} evt The key event.
    */
   static onBrailleKeyEvent(evt) {
+    KbExplorer.shouldFlushSpeech_ = true;
     KbExplorer.maybeClose_();
     let msgid;
     const msgArgs = [];
@@ -211,10 +211,26 @@ KbExplorer = class {
    *     ax::mojom::Gesture enum defined in ui/accessibility/ax_enums.mojom
    */
   static onAccessibilityGesture(gesture) {
+    KbExplorer.shouldFlushSpeech_ = true;
     KbExplorer.maybeClose_();
+
+    if (gesture == 'touchExplore') {
+      if ((new Date() - KbExplorer.lastTouchExplore_) <
+          KbExplorer.MIN_TOUCH_EXPLORE_OUTPUT_TIME_MS_) {
+        return;
+      }
+      KbExplorer.lastTouchExplore_ = new Date();
+    }
+
+
     const gestureData = GestureCommandData.GESTURE_COMMAND_MAP[gesture];
     if (gestureData) {
-      KbExplorer.onCommand(gestureData.command);
+      if (gestureData.msgId) {
+        KbExplorer.output(Msgs.getMsg(gestureData.msgId));
+      }
+      if (gestureData.command) {
+        KbExplorer.onCommand(gestureData.command);
+      }
     }
   }
 
@@ -238,9 +254,14 @@ KbExplorer = class {
    * @param {string=} opt_braille If different from text.
    */
   static output(text, opt_braille) {
-    ChromeVox.tts.speak(text, window.backgroundWindow.QueueMode.QUEUE);
+    ChromeVox.tts.speak(
+        text,
+        KbExplorer.shouldFlushSpeech_ ?
+            window.backgroundWindow.QueueMode.FLUSH :
+            window.backgroundWindow.QueueMode.QUEUE);
     ChromeVox.braille.write(
         new NavBraille({text: new Spannable(opt_braille || text)}));
+    KbExplorer.shouldFlushSpeech_ = false;
   }
 
   /** Clears ChromeVox range. */
@@ -292,8 +313,19 @@ KbExplorer = class {
 };
 
 /**
- * Tracks all keydown events (keyed by key code) for which keyup events have
- * yet to be received.
- * @type {!Set<number>}
+ * Indicates when speech output should flush previous speech.
+ * @private {boolean}
+ */
+KbExplorer.shouldFlushSpeech_ = false;
+
+/**
+ * Last time a touch explore gesture was described.
+ * @private {!Date}
+ */
+KbExplorer.lastTouchExplore_ = new Date();
+
+/**
+ * The minimum time to wait before describing another touch explore gesture.
+ * @private {number}
  */
-KbExplorer.keydownWithoutKeyupEvents_ = new Set();
+KbExplorer.MIN_TOUCH_EXPLORE_OUTPUT_TIME_MS_ = 1000;

chrome/browser/resources/chromeos/accessibility/chromevox/learn_mode/learn_mode_test.js

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Include test fixture.
+GEN_INCLUDE([
+  '../testing/chromevox_next_e2e_test_base.js',
+  '//chrome/browser/resources/chromeos/accessibility/chromevox/testing/mock_feedback.js'
+]);
+
+/**
+ * Test fixture for ChromeVox Learn Mode page.
+ */
+ChromeVoxLearnModeTest = class extends ChromeVoxNextE2ETest {
+  constructor() {
+    super();
+    window.doKeyDown = this.doKeyDown.bind(this);
+    window.doKeyUp = this.doKeyUp.bind(this);
+    window.doGesture = this.doGesture.bind(this);
+    window.doBrailleKeyEvent = this.doBrailleKeyEvent.bind(this);
+  }
+
+  runOnLearnModePage(callback) {
+    const mockFeedback = this.createMockFeedback();
+    chrome.automation.getDesktop((desktop) => {
+      desktop.addEventListener(
+          chrome.automation.EventType.LOAD_COMPLETE, (evt) => {
+            if (evt.target.docUrl.indexOf('learn_mode/kbexplorer.html') == -1 ||
+                !evt.target.docLoaded) {
+              return;
+            }
+
+            mockFeedback.expectSpeech(/Starting Learn Mode/);
+            callback(mockFeedback, evt);
+          });
+      CommandHandler.onCommand('showKbExplorerPage');
+    });
+  }
+
+  /** @return {!MockFeedback} */
+  createMockFeedback() {
+    const mockFeedback =
+        new MockFeedback(this.newCallback(), this.newCallback.bind(this));
+    mockFeedback.install();
+    return mockFeedback;
+  }
+
+  getLearnModeWindow() {
+    let learnModeWindow = null;
+    while (!learnModeWindow) {
+      learnModeWindow = chrome.extension.getViews().find(function(view) {
+        return view.location.href.indexOf(
+                   'chromevox/learn_mode/kbexplorer.html') > 0;
+      });
+    }
+    return learnModeWindow;
+  }
+
+  makeMockKeyEvent(params) {
+    // Fake out these functions.
+    params.preventDefault = () => {};
+    params.stopPropagation = () => {};
+
+    // Set defaults if not defined.
+    params.repeat = params.repeat || false;
+
+    return params;
+  }
+
+  doKeyDown(evt) {
+    return () => {
+      this.getLearnModeWindow().KbExplorer.onKeyDown(
+          this.makeMockKeyEvent(evt));
+    };
+  }
+
+  doKeyUp(evt) {
+    return () => {
+      this.getLearnModeWindow().KbExplorer.onKeyUp(this.makeMockKeyEvent(evt));
+    };
+  }
+
+  doGesture(gesture) {
+    return () => {
+      this.getLearnModeWindow().KbExplorer.onAccessibilityGesture(gesture);
+    };
+  }
+
+  doBrailleKeyEvent(evt) {
+    return () => {
+      this.getLearnModeWindow().KbExplorer.onBrailleKeyEvent(evt);
+    };
+  }
+};
+
+TEST_F('ChromeVoxLearnModeTest', 'KeyboardInput', function() {
+  this.runOnLearnModePage((mockFeedback, evt) => {
+    // Press Search+Right.
+    mockFeedback.call(doKeyDown({keyCode: 91, metaKey: true}))
+        .expectSpeechWithQueueMode('Search', QueueMode.FLUSH)
+        .call(doKeyDown({keyCode: 39, metaKey: true}))
+        .expectSpeechWithQueueMode('Right arrow', QueueMode.QUEUE)
+        .expectSpeechWithQueueMode('Next Object', QueueMode.QUEUE)
+        .call(doKeyUp({keyCode: 39, metaKey: true}))
+
+        // Hit 'Right' again. We should get flushed output.
+        .call(doKeyDown({keyCode: 39, metaKey: true}))
+        .expectSpeechWithQueueMode('Right arrow', QueueMode.FLUSH)
+        .expectSpeechWithQueueMode('Next Object', QueueMode.QUEUE)
+        .call(doKeyUp({keyCode: 39, metaKey: true}))
+
+        .replay();
+  });
+});
+
+TEST_F('ChromeVoxLearnModeTest', 'KeyboardInputRepeat', function() {
+  this.runOnLearnModePage((mockFeedback, evt) => {
+    // Press Search repeatedly.
+    mockFeedback.call(doKeyDown({keyCode: 91, metaKey: true}))
+        .expectSpeechWithQueueMode('Search', QueueMode.FLUSH)
+
+        // This in theory should never happen (no repeat set).
+        .call(doKeyDown({keyCode: 91, metaKey: true}))
+        .expectSpeechWithQueueMode('Search', QueueMode.QUEUE)
+
+        // Hit Search again with the right underlying data. Then hit Control to
+        // generate some speech.
+        .call(doKeyDown({keyCode: 91, metaKey: true, repeat: true}))
+        .call(doKeyDown({keyCode: 17, ctrlKey: true}))
+        .expectNextSpeechUtteranceIsNot('Search')
+        .expectSpeechWithQueueMode('Control', QueueMode.QUEUE)
+
+        .replay();
+  });
+});
+
+TEST_F('ChromeVoxLearnModeTest', 'Gesture', function() {
+  this.runOnLearnModePage((mockFeedback, evt) => {
+    this.getLearnModeWindow().KbExplorer.MIN_TOUCH_EXPLORE_OUTPUT_TIME_MS_ = 0;
+    mockFeedback.call(doGesture('swipeRight1'))
+        .expectSpeechWithQueueMode('Swipe one finger right', QueueMode.FLUSH)
+        .expectSpeechWithQueueMode('Next Object', QueueMode.QUEUE)
+
+        .call(doGesture('swipeLeft1'))
+        .expectSpeechWithQueueMode('Swipe one finger left', QueueMode.FLUSH)
+        .expectSpeechWithQueueMode('Previous Object', QueueMode.QUEUE)
+
+        .call(doGesture('touchExplore'))
+        .expectSpeechWithQueueMode('Touch explore', QueueMode.FLUSH)
+
+        .replay();
+  });
+});
+
+TEST_F('ChromeVoxLearnModeTest', 'Braille', function() {
+  this.runOnLearnModePage((mockFeedback, evt) => {
+    // Hit the left panning hardware key on a braille display.
+    mockFeedback.call(doBrailleKeyEvent({command: BrailleKeyCommand.PAN_LEFT}))
+        .expectSpeechWithQueueMode('Pan backward', QueueMode.FLUSH)
+        .expectBraille('Pan backward')
+
+        // Hit the backspace chord on a Perkins braille keyboard (aka dot 7).
+        .call(doBrailleKeyEvent(
+            {command: BrailleKeyCommand.CHORD, brailleDots: 0b1000000}))
+        .expectSpeechWithQueueMode('Backspace', QueueMode.FLUSH)
+        .expectBraille('Backspace')
+
+        // Hit a 'd' chord (Perkins keys dot 1-4-5).
+        .call(doBrailleKeyEvent(
+            {command: BrailleKeyCommand.CHORD, brailleDots: 0b011001}))
+        .expectSpeechWithQueueMode('dots 1-4-5 chord', QueueMode.FLUSH)
+        .expectBraille('dots 1-4-5 chord')
+
+        .replay();
+  });
+});

chrome/browser/resources/chromeos/accessibility/strings/chromevox_strings.grdp

@@ -2984,5 +2984,8 @@
   <message desc="Describes the current web page position of the ChromeVox focus." is_accessibility_with_no_ui="true" name="IDS_CHROMEVOX_DESCRIBE_POS_BY_PAGE">
     Page <ph name="currentPage">$1<ex>1</ex></ph> of <ph name="totalPages">$2<ex>10</ex></ph>
   </message>
+  <message desc="Describes a gesture to be performed on a touch screen. The gesture is to touch and drag with one finger and have items under the finger read using text-to-speech. No associated UI with this string which is spoken using text-to-speech." name="IDS_CHROMEVOX_TOUCH_EXPLORE_GESTURE" is_accessibility_with_no_ui="true">
+    Touch explore
+  </message>
 
 </grit-part>