ChromeVox: Interactive OOBE lessons using UserActionMonitor.

This change:
1. Adds a new class called UserActionMonitor. This class stores a queue
of actions and blocks all other ChromeVox execution until the actions
are matched and consumed. The initial version of the
class intercepts key presses and keyboard commands; this is done by
hooking into the CommandHandler and BackgroundKeyboardHandler.
Later iterations will listen to range changes, mouse events, gestures,
and braille commands in a similar fashion.
2. Adds some simple tests for UserActionMonitor.
3. Adds tutorial lessons for the OOBE. These lessons are geared toward
brand new users. When showing one of these lessons, the tutorial
initializes UserActionMonitor with expectations, which allows us to
provide interactive walk-throughs. The lessons cover the search key,
basic navigation, and using search + space.
4. Wires up communication between UserActionMonitor and the tutorial.
Most of this wiring is done through the panel.

Bug: 1075752
AX-Relnotes: N/A

Change-Id: I521016192929d9655234b055cc63ca6d83aa9efa
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2253021
Commit-Queue: Akihiro Ota <akihiroota@chromium.org>
Reviewed-by: David Tseng <dtseng@chromium.org>
Cr-Commit-Position: refs/heads/master@{#792957}

chrome/browser/resources/chromeos/accessibility/chromevox/BUILD.gn

@@ -77,6 +77,7 @@ chromevox_modules = [
   "background/recovery_strategy.js",
   "background/smart_sticky_mode.js",
   "background/tabs_api_handler.js",
+  "background/user_action_monitor.js",
   "braille/bluetooth_braille_display_manager.js",
   "braille/bluetooth_braille_display_ui.js",
   "braille/braille_display_manager.js",
@@ -458,6 +459,7 @@ if (is_chromeos) {
       "background/portals_test.js",
       "background/recovery_strategy_test.js",
       "background/smart_sticky_mode_test.js",
+      "background/user_action_monitor_test.js",
       "braille/braille_table_test.js",
       "braille/braille_translator_manager_test.js",
       "braille/liblouis_test.js",

chrome/browser/resources/chromeos/accessibility/chromevox/background/chromevox_state.js

@@ -14,6 +14,7 @@ goog.provide('ChromeVoxStateObserver');
 goog.require('cursors.Cursor');
 goog.require('cursors.Range');
 goog.require('BrailleKeyEvent');
+goog.require('UserActionMonitor');
 
 /**
  * An interface implemented by objects that want to observe ChromeVox state
@@ -54,6 +55,8 @@ ChromeVoxState = function() {
 
   /** @private {!Array<!chrome.accessibilityPrivate.ScreenRect>} */
   this.focusBounds_ = [];
+  /** @private {UserActionMonitor} */
+  this.userActionMonitor_ = null;
 };
 
 /**
@@ -138,7 +141,34 @@ ChromeVoxState.prototype = {
       type: chrome.accessibilityPrivate.FocusType.GLOW,
       color: constants.FOCUS_COLOR
     }]);
-  }
+  },
+
+  /**
+   * Gets the user action monitor.
+   * @return {UserActionMonitor}
+   */
+  getUserActionMonitor() {
+    return this.userActionMonitor_;
+  },
+
+  /**
+   * Creates a new user action monitor.
+   * @param {!Array<{
+   *    type: string,
+   *    value: (string|Object),
+   *    beforeActionMsg: (string|undefined),
+   *    afterActionMsg: (string|undefined)
+   * }>} actions
+   * @param {function(): void} callback
+   */
+  createUserActionMonitor(actions, callback) {
+    this.userActionMonitor_ = new UserActionMonitor(actions, callback);
+  },
+
+  /** Destroys the user action monitor */
+  destroyUserActionMonitor() {
+    this.userActionMonitor_ = null;
+  },
 };
 
 /** @type {!Array<ChromeVoxStateObserver>} */

chrome/browser/resources/chromeos/accessibility/chromevox/background/keyboard_handler.js

@@ -100,7 +100,7 @@ BackgroundKeyboardHandler = class {
 
   /**
    * Handles key up events.
-   * @param {Event} evt The key down event to process.
+   * @param {Event} evt The key up event to process.
    * @return {boolean} This value has no effect since we ignore it in
    *     SpokenFeedbackEventRewriterDelegate::HandleKeyboardEvent.
    */

chrome/browser/resources/chromeos/accessibility/chromevox/background/user_action_monitor.js

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+/**
+ * @fileoverview Monitors user actions.
+ */
+
+goog.provide('UserActionMonitor');
+
+goog.require('KeySequence');
+goog.require('Output');
+
+/**
+ * The types of actions we want to monitor.
+ * @enum {string}
+ */
+const ActionType = {
+  BRAILLE: 'braille',
+  GESTURE: 'gesture',
+  KEY_SEQUENCE: 'key_sequence',
+  MOUSE_EVENT: 'mouse_event',
+  RANGE_CHANGE: 'range_change',
+};
+
+/**
+ * Monitors user actions. Receives a queue of expected actions upon construction
+ * and blocks ChromeVox execution until each action is matched. Hooks into
+ * various handlers to intercept user actions before they are processed by the
+ * rest of ChromeVox.
+ */
+UserActionMonitor = class {
+  /**
+   * @param {!Array<UserActionMonitor.ActionInfo>} actionInfos A queue of
+   *     expected actions.
+   * @param {function():void} onFinishedCallback Runs once after all expected
+   *     actions have been matched.
+   */
+  constructor(actionInfos, onFinishedCallback) {
+    if (actionInfos.length === 0) {
+      throw new Error(`UserActionMonitor: actionInfos can't be empty`);
+    }
+
+    /** @private {number} */
+    this.actionIndex_ = 0;
+    /** @private {!Array<!UserActionMonitor.Action>} */
+    this.actions_ = [];
+    /** @private {function():void} */
+    this.onFinishedCallback_ = onFinishedCallback;
+
+    for (let i = 0; i < actionInfos.length; ++i) {
+      this.actions_.push(
+          UserActionMonitor.Action.fromActionInfo(actionInfos[i]));
+    }
+    if (this.actions_[0].opt_beforeActionCallback) {
+      this.actions_[0].opt_beforeActionCallback();
+    }
+  }
+
+  // Public methods.
+
+  /**
+   * Returns true if the key sequence was matched. Returns false otherwise.
+   * @param {!KeySequence} actualSequence
+   * @return {boolean}
+   */
+  onKeySequence(actualSequence) {
+    const expectedAction = this.getExpectedAction_();
+    if (expectedAction.type !== ActionType.KEY_SEQUENCE) {
+      return false;
+    }
+
+    const expectedSequence = expectedAction.value;
+    if (!expectedSequence.equals(actualSequence)) {
+      return false;
+    }
+
+    this.expectedActionMatched_();
+    return true;
+  }
+
+  // Private methods.
+
+  /** @private */
+  expectedActionMatched_() {
+    const action = this.getExpectedAction_();
+    if (action.opt_afterActionCallback) {
+      action.opt_afterActionCallback();
+    }
+
+    this.nextAction_();
+  }
+
+  /** @private */
+  nextAction_() {
+    if (this.actionIndex_ < 0 || this.actionIndex_ >= this.actions_.length) {
+      throw new Error(
+          `UserActionMonitor: can't call nextAction_(), invalid index`);
+    }
+
+    this.actionIndex_ += 1;
+    if (this.actionIndex_ === this.actions_.length) {
+      this.onAllMatched_();
+      return;
+    }
+
+    const action = this.getExpectedAction_();
+    if (action.opt_beforeActionCallback) {
+      action.opt_beforeActionCallback();
+    }
+  }
+
+  /** @private */
+  onAllMatched_() {
+    this.onFinishedCallback_();
+  }
+
+  /**
+   * @return {!UserActionMonitor.Action}
+   * @private
+   */
+  getExpectedAction_() {
+    if (this.actionIndex_ >= 0 && this.actionIndex_ < this.actions_.length) {
+      return this.actions_[this.actionIndex_];
+    }
+
+    throw new Error('UserActionMonitor: actionIndex_ is invalid.');
+  }
+};
+
+/**
+ * Defines an object that is used to create a UserActionMonitor.Action.
+ * @typedef {{
+ *    type: ActionType,
+ *    value: (string|Object),
+ *    beforeActionMsg: (string|undefined),
+ *    afterActionMsg: (string|undefined)
+ * }}
+ */
+UserActionMonitor.ActionInfo;
+
+// Represents an expected action.
+UserActionMonitor.Action = class {
+  /**
+   * @param {ActionType} type
+   * @param {string|!KeySequence} value
+   * @param {(function():void)=} opt_beforeActionCallback Runs once before this
+   *     action is seen.
+   * @param {(function():void)=} opt_afterActionCallback Runs once after this
+   *     action is seen.
+   */
+  constructor(type, value, opt_beforeActionCallback, opt_afterActionCallback) {
+    switch (type) {
+      case ActionType.KEY_SEQUENCE:
+        if (!(value instanceof KeySequence)) {
+          throw new Error(
+              'UserActionMonitor: Must provide a KeySequence value for ' +
+              'Actions of type ActionType.KEY_SEQUENCE');
+        }
+        break;
+      default:
+        if (typeof value !== 'string') {
+          throw new Error(
+              'UserActionMonitor: Must provide a string value for Actions ' +
+              'if type is other than ActionType.KEY_SEQUENCE');
+        }
+    }
+
+    /** @type {ActionType} */
+    this.type = type;
+    /** @type {string|!KeySequence} */
+    this.value = value;
+    /** @type {(function():void)|undefined} */
+    this.opt_beforeActionCallback = opt_beforeActionCallback;
+    /** @type {(function():void)|undefined} */
+    this.opt_afterActionCallback = opt_afterActionCallback;
+  }
+
+  /**
+   * Constructs a new Action given an ActionInfo object.
+   * @param {!UserActionMonitor.ActionInfo} info
+   * @return {!UserActionMonitor.Action}
+   */
+  static fromActionInfo(info) {
+    switch (info.type) {
+      case ActionType.KEY_SEQUENCE:
+        if (typeof info.value !== 'object') {
+          throw new Error(
+              'UserActionMonitor: Must provide an object resembling a ' +
+              'KeySequence for Actions of type ActionType.KEY_SEQUENCE');
+        }
+        break;
+
+      default:
+        if (typeof info.value !== 'string') {
+          throw new Error(
+              'UserActionMonitor: Must provide a string value for Actions if ' +
+              'type is other than ActionType.KEY_SEQUENCE');
+        }
+    }
+
+    const type = info.type;
+    const value = (typeof info.value === 'object') ?
+        KeySequence.deserialize(info.value) :
+        info.value;
+    const beforeActionMsg = info.beforeActionMsg;
+    const afterActionMsg = info.afterActionMsg;
+
+    const beforeActionCallback = () => {
+      if (!beforeActionMsg) {
+        return;
+      }
+
+      UserActionMonitor.Action.output_(beforeActionMsg);
+    };
+
+    const afterActionCallback = () => {
+      if (!afterActionMsg) {
+        return;
+      }
+
+      UserActionMonitor.Action.output_(afterActionMsg);
+    };
+
+    return new UserActionMonitor.Action(
+        type, value, beforeActionCallback, afterActionCallback);
+  }
+
+  /**
+   * @param {!UserActionMonitor.Action} other
+   * @return {boolean}
+   */
+  equals(other) {
+    if (this.type !== other.type) {
+      return false;
+    }
+
+    if (this.type === ActionType.KEY_SEQUENCE) {
+      // For KeySequences, use the built-in equals method.
+      return this.value.equals(/** @type {!KeySequence} */ (other.value));
+    }
+
+    return this.value === other.value;
+  }
+
+  // Static methods.
+
+  /**
+   * Uses Output module to provide speech and braille feedback.
+   * @param {string} message
+   * @private
+   */
+  static output_(message) {
+    new Output().withString(message).withQueueMode(QueueMode.FLUSH).go();
+  }
+};
\ No newline at end of file

chrome/browser/resources/chromeos/accessibility/chromevox/common/key_sequence.js

@@ -391,7 +391,7 @@ KeySequence = class {
   /**
    * Creates a KeySequence event from a generic object.
    * @param {Object} sequenceObject The object.
-   * @return {KeySequence} The created KeySequence object.
+   * @return {!KeySequence} The created KeySequence object.
    */
   static deserialize(sequenceObject) {
     const firstSequenceEvent = {};

chrome/browser/resources/chromeos/accessibility/chromevox/common/key_util.js

@@ -33,7 +33,7 @@ KeyUtil = class {
    * Convert a key event into a Key Sequence representation.
    *
    * @param {Event|SimpleKeyEvent} keyEvent The keyEvent to convert.
-   * @return {KeySequence} A key sequence representation of the key event.
+   * @return {!KeySequence} A key sequence representation of the key event.
    */
   static keyEventToKeySequence(keyEvent) {
     const util = KeyUtil;

chrome/browser/resources/chromeos/accessibility/chromevox/common/keyboard_handler.js

@@ -8,6 +8,7 @@ goog.require('ChromeVox');
 goog.require('KeyMap');
 goog.require('KeySequence');
 goog.require('KeyUtil');
+goog.require('ChromeVoxState');
 
 /**
  * @fileoverview Handles user keyboard input events.
@@ -85,6 +86,15 @@ ChromeVoxKbHandler.sortKeyToFunctionsTable_ = function(keyToFunctionsTable) {
  */
 ChromeVoxKbHandler.basicKeyDownActionsListener = function(evt) {
   const keySequence = KeyUtil.keyEventToKeySequence(evt);
+  const chromeVoxState = ChromeVoxState.instance;
+  const monitor = chromeVoxState ? chromeVoxState.getUserActionMonitor() : null;
+  if (monitor && !monitor.onKeySequence(keySequence)) {
+    // UserActionMonitor returns true if this key sequence was matched. If a
+    // key sequence is matched by the UserActionMonitor, allow it to process.
+    // Otherwise, prevent the default action.
+    return false;
+  }
+
   let functionName;
   if (ChromeVoxKbHandler.handlerKeyMap != undefined) {
     functionName = ChromeVoxKbHandler.handlerKeyMap.commandForKey(keySequence);

chrome/browser/resources/chromeos/accessibility/chromevox/i_tutorial/components/i_tutorial.html

@@ -101,6 +101,7 @@ h1 {
     hidden$="[[ shouldHideMainMenu(activeScreen) ]]">
   <h1 id="mainMenuHeader" tabindex="-1">[[ chooseYourExperience ]]</h1>
   <div id="mainMenuButtons">
+    <cr-button id="oobeButton" on-click="chooseCurriculum">OOBE</cr-button>
     <cr-button id="newUserButton" on-click="chooseCurriculum">
       [[ newUser ]]
     </cr-button>
@@ -138,6 +139,8 @@ h1 {
     practice-state="[[ lesson.practiceState ]]"
     hints="[[ lesson.hints ]]"
     events="[[ lesson.events ]]"
+    actions="[[ lesson.actions ]]"
+    auto-interactive="[[ lesson.autoInteractive ]]"
     active-lesson-num="[[ activeLessonNum ]]">
     </tutorial-lesson>
   </template>

chrome/browser/resources/chromeos/accessibility/chromevox/i_tutorial/components/i_tutorial.js

@@ -92,6 +92,83 @@ Polymer({
     lessonData: {
       type: Array,
       value: [
+        {
+          content:
+              ['Welcome to the ChromeVox tutorial. We noticed this might be ' +
+               'your first time using ChromeVox, so let\'s quickly cover the ' +
+               `basics. When you're ready, use the space bar to move to the ` +
+               'next lesson.'],
+          medium: InteractionMedium.KEYBOARD,
+          curriculums: [Curriculum.OOBE],
+          actions: [
+            {type: 'key_sequence', value: {keys: {keyCode: [32 /* Space */]}}}
+          ],
+          autoInteractive: true,
+        },
+
+        {
+          content: [
+            `Let's learn how to navigate the tutorial first. ` +
+                'In ChromeVox, the Search key is the modifier key. ' +
+                'Most ChromeVox shortcuts start with the Search key. ' +
+                'On the Chromebook, the Search key is immediately above the ' +
+                `left Shift key. When you're ready, try finding and ` +
+                'pressing the search key on your keyboard',
+          ],
+          medium: InteractionMedium.KEYBOARD,
+          curriculums: [Curriculum.OOBE],
+          actions: [{
+            type: 'key_sequence',
+            value: {skipStripping: false, keys: {keyCode: [91 /* Search*/]}},
+            afterActionMsg: 'You found the search key!',
+          }],
+          autoInteractive: true,
+        },
+
+        {
+          content: [
+            `Now that you know where the search key is, let's learn ` +
+                'some basic navigation. While holding search, use the arrow ' +
+                'keys to move ChromeVox around the screen. Press Search + ' +
+                'right arrow to move to the next instruction',
+            'You can use Search + right arrow and search + left arrow to ' +
+                'navigate the tutorial. Now press Search + right arrow again.',
+            'If you reach an item you want to click, press Search + Space. ' +
+                'Try doing so now to move to the next lesson',
+          ],
+          medium: InteractionMedium.KEYBOARD,
+          curriculums: [Curriculum.OOBE],
+          actions: [
+            {
+              type: 'key_sequence',
+              value: {cvoxModifier: true, keys: {keyCode: [39]}},
+              afterActionMsg: 'Great! You pressed Search + right arrow.'
+            },
+            {
+              type: 'key_sequence',
+              value: {cvoxModifier: true, keys: {keyCode: [39]}},
+            },
+            {
+              type: 'key_sequence',
+              value: {cvoxModifier: true, keys: {keyCode: [32]}},
+              afterActionMsg: 'Great! You pressed Search + space',
+            }
+          ],
+          autoInteractive: true,
+        },
+
+        {
+          title: 'Basic orientation complete!',
+          content: [
+            'Well done! You have learned the basics of ChromeVox. You can ' +
+                `now continue browsing the tutorial with what you've ` +
+                'learned, or exit by finding and pressing the Quit Tutorial ' +
+                'button',
+          ],
+          medium: InteractionMedium.KEYBOARD,
+          curriculums: [Curriculum.OOBE],
+        },
+
         {
           title: 'On, Off, and Stop',
           content: [
@@ -100,7 +177,7 @@ Polymer({
             'To turn ChromeVox on or off, use Control+Alt+Z.',
           ],
           medium: InteractionMedium.KEYBOARD,
-          curriculums: [Curriculum.OOBE, Curriculum.NEW_USER],
+          curriculums: [Curriculum.NEW_USER],
         },
 
         {
@@ -113,7 +190,7 @@ Polymer({
                 'left Shift key.',
           ],
           medium: InteractionMedium.KEYBOARD,
-          curriculums: [Curriculum.OOBE, Curriculum.NEW_USER],
+          curriculums: [Curriculum.NEW_USER],
         },
 
         {
@@ -126,7 +203,7 @@ Polymer({
             'If you reach an item you want to click, press Search + Space.',
           ],
           medium: InteractionMedium.KEYBOARD,
-          curriculums: [Curriculum.OOBE, Curriculum.NEW_USER],
+          curriculums: [Curriculum.NEW_USER],
           practiceTitle: 'Basic Navigation Practice',
           practiceInstructions:
               'Try using basic navigation to navigate through the items ' +
@@ -289,6 +366,8 @@ Polymer({
       this.curriculum = Curriculum.EXPERIENCED_USER;
     } else if (id === 'developerButton') {
       this.curriculum = Curriculum.DEVELOPER;
+    } else if (id === 'oobeButton') {
+      this.curriculum = Curriculum.OOBE;
     } else {
       throw new Error('Invalid target for chooseCurriculum: ' + evt.target.id);
     }
@@ -320,6 +399,11 @@ Polymer({
     // Lessons observe activeLessonNum. When updated, lessons automatically
     // update their visibility.
     this.activeLessonNum = this.includedLessons[index].lessonNum;
+
+    const lesson = this.includedLessons[this.activeLessonIndex];
+    if (lesson.autoInteractive) {
+      this.startInteractiveMode(lesson.actions);
+    }
   },
 
 
@@ -465,6 +549,13 @@ Polymer({
 
   /** @private */
   exit() {
-    this.dispatchEvent(new CustomEvent('tutorial-close', {}));
+    this.dispatchEvent(new CustomEvent('closetutorial', {}));
+  },
+
+  // Interactive mode.
+
+  startInteractiveMode(actions) {
+    this.dispatchEvent(new CustomEvent(
+        'startinteractivemode', {composed: true, detail: {actions}}));
   },
 });

chrome/browser/resources/chromeos/accessibility/chromevox/i_tutorial/components/tutorial_lesson.html

@@ -23,10 +23,12 @@
 </style>
 
 <div id="container" hidden>
-  <h1 id="title" tabindex="-1">[[ title ]]</h1>
+  <template is="dom-if" if="[[ title ]]">
+    <h1 id="title" tabindex="-1">[[ title ]]</h1>
+  </template>
   <div id="content">
     <template is="dom-repeat" items="[[ content ]]" as="text">
-      <p>[[ text ]]</p>
+      <p tabindex="-1">[[ text ]]</p>
     </template>
   </div>
   <cr-dialog id="practice" close-text="Exit practice area"

chrome/browser/resources/chromeos/accessibility/chromevox/i_tutorial/components/tutorial_lesson.js

@@ -41,6 +41,10 @@ export const TutorialLesson = Polymer({
 
     goalStateReached: {type: Boolean, value: false},
 
+    actions: {type: Array},
+
+    autoInteractive: {type: Boolean, value: false},
+
     // Observed properties.
 
     activeLessonNum: {type: Number, observer: 'setVisibility'},
@@ -73,7 +77,13 @@ export const TutorialLesson = Polymer({
   /** @private */
   show() {
     this.$.container.hidden = false;
-    this.$.title.focus();
+    // Shorthand for Polymer.dom(this.root).querySelector(...).
+    const focus = this.$$('[tabindex]');
+    if (!focus) {
+      throw new Error(
+          'A lesson must have an element which specifies tabindex.');
+    }
+    focus.focus();
   },
 
   /** @private */
@@ -220,7 +230,7 @@ export const TutorialLesson = Polymer({
    */
   requestSpeech(text) {
     this.dispatchEvent(
-        new CustomEvent('request-speech', {composed: true, detail: {text}}));
+        new CustomEvent('requestspeech', {composed: true, detail: {text}}));
   },
 
   /**

chrome/browser/resources/chromeos/accessibility/chromevox/panel/panel.js

@@ -111,6 +111,20 @@ Panel = class {
      */
     Panel.tutorial_ = new Tutorial();
 
+    /**
+     * @type {Object}
+     * @private
+     */
+    Panel.iTutorial = {
+      // Closure needs this to know about the showNextLesson function.
+      // Otherwise, Closure fires an error whenever calling showNextLesson().
+      showNextLesson: () => {
+        throw new Error(
+            'iTutorial should be assigned to the <i-tutorial> ' +
+            'element before calling showNextLesson()');
+      }
+    };
+
     Panel.setPendingCallback(null);
     Panel.updateFromPrefs();
 
@@ -1116,17 +1130,34 @@ Panel = class {
         tutorialElement.setAttribute('id', 'i-tutorial');
         tutorialContainer.appendChild(tutorialElement);
         document.body.appendChild(tutorialContainer);
+        Panel.iTutorial = tutorialElement;
 
         // Add listeners. These are custom events fired from custom components.
-        $('i-tutorial')
-            .addEventListener('tutorial-close', Panel.onCloseTutorial);
-        $('i-tutorial').addEventListener('request-speech', (evt) => {
+        $('i-tutorial').addEventListener('closetutorial', (evt) => {
+          // Ensure UserActionMonitor is destroyed before closing tutorial.
+          const background =
+              chrome.extension
+                  .getBackgroundPage()['ChromeVoxState']['instance'];
+          background.destroyUserActionMonitor();
+          Panel.onCloseTutorial();
+        });
+        $('i-tutorial').addEventListener('requestspeech', (evt) => {
           const text = evt.detail.text;
           const background = chrome.extension.getBackgroundPage();
           const cvox = background['ChromeVox'];
           cvox.tts.speak(
               text, background.QueueMode.FLUSH, {'doNotInterrupt': true});
         });
+        $('i-tutorial').addEventListener('startinteractivemode', (evt) => {
+          const actions = evt.detail.actions;
+          const background =
+              chrome.extension
+                  .getBackgroundPage()['ChromeVoxState']['instance'];
+          background.createUserActionMonitor(actions, () => {
+            background.destroyUserActionMonitor();
+            Panel.iTutorial.showNextLesson();
+          });
+        });
       }
 
       Panel.setMode(Panel.Mode.FULLSCREEN_I_TUTORIAL);