Enable Select to Speak to continue reading across boundaries

When reading user-selected content (e.g., from mouse selection), STS pauses at the end of the selected content. This CL enables the user to use resume to continue reading from the pause location till the end of the paragraph.

When resuming from the content processed by the navigation feature (e.g., the user hits pause), STS will continue reading from the start of the current sentence (implemented in 2572905).

In both cases, if there is no content left in the paragraph, STS will continue reading content from the next paragraph.

AX-Relnotes: N/A
Bug: 1143817
Change-Id: Idb48219fd805c8b5167f1b30850fea904a06259e
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2582524
Commit-Queue: Lei Shi <leileilei@google.com>
Reviewed-by: David Tseng <dtseng@chromium.org>
Reviewed-by: Dominic Mazzoni <dmazzoni@chromium.org>
Reviewed-by: Josiah Krutz <josiahk@google.com>
Cr-Commit-Position: refs/heads/master@{#838255}

chrome/browser/resources/chromeos/accessibility/select_to_speak/node_utils.js

@@ -546,6 +546,64 @@ class NodeUtils {
         });
   }
 
+  /**
+   * Gets the remaining content of a paragraph with an assigned position. The
+   * position is defined by the |charIndex| to the text of |nodeGroup|. The
+   * remaining content is returned as a list of nodes with offset.
+   * @param {!ParagraphUtils.NodeGroup} nodeGroup
+   * @param {number} charIndex The char index of the position. The index is
+   *     relative to the text content of the |nodeGroup|. This index is
+   *     inclusive: if we set |charIndex| to 0, we will get the remaining
+   *     content of the paragraph including all the content in the input
+   *     |nodeGroup|.
+   * @return {!{nodes: !Array<!AutomationNode>,
+   *          offset: number}}
+   *     nodes: the nodes that have the remaining content.
+   *     offset: the offset for the nodes. The offset is into the name of the
+   * first node, and marks the start index of the remaining content in the first
+   * node, inclusively. For example, "Hello" with an offset 3 indicates that the
+   * remaining content is "lo".
+   */
+  static getNextNodesInParagraphFromNodeGroupWithOffset(nodeGroup, charIndex) {
+    if (nodeGroup.nodes.length === 0) {
+      return {nodes: [], offset: -1};
+    }
+    let {node: startNode, offset} =
+        ParagraphUtils.findNodeFromNodeGroupByCharIndex(nodeGroup, charIndex);
+    // If we did not find a node for the charIndex, we use the last node of the
+    // current node group as the starting node and set offset to the length of
+    // the node's name. This enables us to get all the content within this
+    // paragraph but after the current node group.
+    if (!startNode) {
+      const lastNode = nodeGroup.nodes[nodeGroup.nodes.length - 1].node;
+      const lastChildOfLastNode = NodeUtils.getLastLeafChild(lastNode);
+      startNode = lastChildOfLastNode || lastNode;
+      offset = ParagraphUtils.getNodeName(startNode).length;
+    }
+
+    // Gets all the trailing nodes after the startNode. We include the start
+    // node if it is not empty. Note that this is based on the assumption that
+    // this function will still include nodes with overflow text.
+    const trailingNodes =
+        NodeUtils.getNextNodesInParagraph(startNode, constants.Dir.FORWARD);
+    const textInStartNode =
+        ParagraphUtils.getNodeName(startNode).substr(offset);
+    if (!ParagraphUtils.isWhitespace(textInStartNode)) {
+      const nodes = [startNode, ...trailingNodes];
+      return {nodes, offset};
+    }
+    if (trailingNodes.length === 0) {
+      return {nodes: [], offset: -1};
+    }
+    // Returns all the nodes once we find a valid one among them.
+    for (const node of trailingNodes) {
+      if (NodeUtils.isValidLeafNode(node)) {
+        return {nodes: trailingNodes, offset: 0};
+      }
+    }
+    return {nodes: [], offset: -1};
+  }
+
   /**
    * @param {!AutomationNode} node
    * @return {boolean} Whether the given node is a valid leaf node that is can

chrome/browser/resources/chromeos/accessibility/select_to_speak/node_utils_unittest.js

@@ -830,6 +830,90 @@ TEST_F('SelectToSpeakNodeUtilsUnitTest', 'getAllNodesInParagraph', function() {
   assertEquals(result[2], text5);
 });
 
+TEST_F(
+    'SelectToSpeakNodeUtilsUnitTest',
+    'getNextNodesInParagraphFromNodeGroupWithOffset', function() {
+      // The nodeGroup has four inline text nodes and one static text node.
+      // Their starting indexes are 0, 9, 20, 30, and 51.
+      const nodeGroup = generateTestNodeGroup();
+
+      let result = NodeUtils.getNextNodesInParagraphFromNodeGroupWithOffset(
+          nodeGroup, 0 /* charIndex */);
+      assertEquals(result.nodes.length, 5);
+      assertEquals(result.offset, 0);
+
+      result = NodeUtils.getNextNodesInParagraphFromNodeGroupWithOffset(
+          nodeGroup, 5 /* charIndex */);
+      assertEquals(result.nodes.length, 5);
+      assertEquals(result.offset, 5);
+
+      result = NodeUtils.getNextNodesInParagraphFromNodeGroupWithOffset(
+          nodeGroup, 9 /* charIndex */);
+      assertEquals(result.nodes.length, 4);
+      assertEquals(result.offset, 0);
+
+      result = NodeUtils.getNextNodesInParagraphFromNodeGroupWithOffset(
+          nodeGroup, 25 /* charIndex */);
+      assertEquals(result.nodes.length, 3);
+      assertEquals(result.offset, 5);
+
+      result = NodeUtils.getNextNodesInParagraphFromNodeGroupWithOffset(
+          nodeGroup, 51 /* charIndex */);
+      assertEquals(result.nodes.length, 1);
+      assertEquals(result.offset, 0);
+
+      result = NodeUtils.getNextNodesInParagraphFromNodeGroupWithOffset(
+          nodeGroup, 100 /* charIndex */);
+      assertEquals(result.nodes.length, 0);
+    });
+
+TEST_F(
+    'SelectToSpeakNodeUtilsUnitTest',
+    'getNextNodesInParagraphFromNodeGroupWithOffsetWithEmptyTail', function() {
+      // The nodeGroup consists of three inline text nodes: "Hello", "world ",
+      // and " ".
+      const nodeGroup = generateTestNodeGroupWithEmptyTail();
+
+      // We can find the non-empty node at this charIndex but there is actually
+      // no text content afterwards.
+      const nodeWithOffset = ParagraphUtils.findNodeFromNodeGroupByCharIndex(
+          nodeGroup, 11 /* charIndex */);
+      assertEquals(nodeWithOffset.node.name, 'world ');
+      assertEquals(nodeWithOffset.offset, 5);
+
+      let result = NodeUtils.getNextNodesInParagraphFromNodeGroupWithOffset(
+          nodeGroup, 11 /* charIndex */);
+      assertEquals(result.nodes.length, 0);
+
+      // If we decrease the charIndex, we will get the node with 'world ' but
+      // not any empty node.
+      result = NodeUtils.getNextNodesInParagraphFromNodeGroupWithOffset(
+          nodeGroup, 10 /* charIndex */);
+      assertEquals(result.nodes.length, 1);
+      assertEquals(result.nodes[0].name, 'world ');
+    });
+
+TEST_F(
+    'SelectToSpeakNodeUtilsUnitTest',
+    'getNextNodesInParagraphFromNodeGroupWithOffsetFromPartialParagraph',
+    function() {
+      // The nodeGroup consists only one static text node, which is "one". The
+      // entire paragraph has three static text nodes: "Sentence", "one",
+      // "here".
+      const nodeGroup = generateTestNodeGroupFromPartialParagraph();
+
+      // After reading "one", the TTS events will set charIndex to 3. Finding
+      // the static text node from the node group will return null.
+      const nodeWithOffset = ParagraphUtils.findNodeFromNodeGroupByCharIndex(
+          nodeGroup, 3 /* charIndex */);
+      assertEquals(nodeWithOffset.node, null);
+
+      const result = NodeUtils.getNextNodesInParagraphFromNodeGroupWithOffset(
+          nodeGroup, 3 /* charIndex */);
+      assertEquals(result.nodes.length, 1);
+      assertEquals(result.offset, 0);
+    });
+
 /**
  * Creates a AutomationNode-like object.
  * @param {!Object} properties
@@ -840,7 +924,8 @@ function createMockNode(properties) {
         htmlAttributes: [],
         state: {},
         children: [],
-        location: {},
+        unclippedLocation: {left: 20, top: 10, width: 100, height: 50},
+        location: {left: 20, top: 10, width: 100, height: 50},
       },
       properties);
 
@@ -859,3 +944,99 @@ function createMockNode(properties) {
   }
   return node;
 }
+
+/**
+ * Creates a nodeGroup for test purpose.
+ * @return {!ParagraphUtils.NodeGroup}
+ */
+function generateTestNodeGroup() {
+  const root = createMockNode({role: 'rootWebArea'});
+  const paragraph =
+      createMockNode({role: 'paragraph', display: 'block', parent: root, root});
+  const text1 = createMockNode(
+      {name: 'The first sentence.', role: 'staticText', parent: paragraph});
+  const inlineText1 = createMockNode({
+    role: 'inlineTextBox',
+    name: 'The first',
+    indexInParent: 0,
+    parent: text1
+  });
+  const inlineText2 = createMockNode({
+    role: 'inlineTextBox',
+    name: ' sentence.',
+    indexInParent: 1,
+    parent: text1
+  });
+
+  const text2 = createMockNode({
+    name: 'The second sentence is longer.',
+    role: 'staticText',
+    parent: paragraph
+  });
+  const inlineText3 = createMockNode({
+    role: 'inlineTextBox',
+    name: 'The second',
+    indexInParent: 0,
+    parent: text2
+  });
+  const inlineText4 = createMockNode({
+    role: 'inlineTextBox',
+    name: ' sentence is longer.',
+    indexInParent: 1,
+    parent: text2
+  });
+
+  const text3 = createMockNode(
+      {name: 'No child sentence.', role: 'staticText', parent: paragraph});
+
+  return ParagraphUtils.buildNodeGroup(
+      [inlineText1, inlineText2, inlineText3, inlineText4, text3], 0,
+      false /* do not split on language */);
+}
+
+/**
+ * Creates a nodeGroup that has an empty tail (i.e., "Hello world  ").
+ * @return {!ParagraphUtils.NodeGroup}
+ */
+function generateTestNodeGroupWithEmptyTail() {
+  const root = createMockNode({role: 'rootWebArea'});
+  const paragraph =
+      createMockNode({role: 'paragraph', display: 'block', parent: root, root});
+  const text1 =
+      createMockNode({name: 'Hello', role: 'staticText', parent: paragraph});
+  const inlineText1 = createMockNode(
+      {role: 'inlineTextBox', name: 'Hello', indexInParent: 0, parent: text1});
+
+  const text2 =
+      createMockNode({name: 'world  ', role: 'staticText', parent: paragraph});
+  const inlineText2 = createMockNode(
+      {role: 'inlineTextBox', name: 'world ', indexInParent: 0, parent: text2});
+  const inlineText3 = createMockNode(
+      {role: 'inlineTextBox', name: ' ', indexInParent: 1, parent: text2});
+
+  return ParagraphUtils.buildNodeGroup(
+      [inlineText1, inlineText2, inlineText3], 0,
+      false /* do not split on language */);
+}
+
+/**
+ * Creates a nodeGroup that only has a part of the paragraph (e.g., the "one" in
+ * <p>Sentence <span>one</span> here</p>).
+ * @return {!ParagraphUtils.NodeGroup}
+ */
+function generateTestNodeGroupFromPartialParagraph() {
+  const root = createMockNode({role: 'rootWebArea'});
+  const paragraph =
+      createMockNode({role: 'paragraph', display: 'block', parent: root, root});
+  const text1 =
+      createMockNode({name: 'Sentence', role: 'staticText', parent: paragraph});
+
+  const text2 =
+      createMockNode({name: 'one', role: 'staticText', parent: paragraph});
+
+  const text3 =
+      createMockNode({name: 'here', role: 'staticText', parent: paragraph});
+
+  return ParagraphUtils.buildNodeGroup(
+      [text2], 0, false /* do not split on language */);
+}
\ No newline at end of file

chrome/browser/resources/chromeos/accessibility/select_to_speak/paragraph_utils.js

@@ -244,6 +244,8 @@ class ParagraphUtils {
   /**
    * Determines the index into the parent name at which the inlineTextBox
    * node name begins.
+   * TODO(leileilei@google.com): Corrects the annotation of |inlineTextNode|
+   * to non-nullable.
    * @param {AutomationNode} inlineTextNode An inlineTextBox type node.
    * @return {number} The character index into the parent node at which
    *     this node begins.
@@ -417,6 +419,53 @@ class ParagraphUtils {
     result.endIndex = index;
     return result;
   }
+
+  /**
+   * Finds the AutomationNode that appears at the given character index within
+   * the |nodeGroup|.
+   * @param {!ParagraphUtils.NodeGroup} nodeGroup The nodeGroup that has the
+   *     nodeGroupItem.
+   * @param {number} charIndex The char index into the nodeGroup's text. The
+   *     index is relative to the start of the |nodeGroup|.
+   * @return {!{node: ?AutomationNode,
+   *          offset: number}}
+   *     node: the AutomationNode within the |nodeGroup| that appears at
+   * |charIndex|. For a static text node that has inline text nodes, we will
+   * return the inline text node corresponding to the |charIndex|.
+   *     offset: the offset indicating the position of the |charIndex| within
+   * the found nodeGroupItem. The offset is relative to the start of the |node|.
+   */
+  static findNodeFromNodeGroupByCharIndex(nodeGroup, charIndex) {
+    for (const currentNodeGroupItem of nodeGroup.nodes) {
+      const currentNode = currentNodeGroupItem.node;
+      const currentNodeNameLength =
+          ParagraphUtils.getNodeName(currentNode).length;
+      // We iterate over each nodeGroupItem until the |charIndex| is pointing
+      // before the current node's name.
+      if (currentNodeGroupItem.startChar + currentNodeNameLength > charIndex) {
+        // The currentNodeOffset is the position of the |charIndex| within the
+        // current nodeGroupItem.
+        const currentNodeOffset = charIndex - currentNodeGroupItem.startChar;
+        if (!currentNodeGroupItem.hasInlineText) {
+          // If the nodeGroupItem does not have inline text, we return the
+          // corresponding node and the current offset.
+          return {node: currentNode, offset: currentNodeOffset};
+        }
+
+        const inlineTextNode =
+            ParagraphUtils.findInlineTextNodeByCharacterIndex(
+                currentNode, currentNodeOffset);
+
+        return inlineTextNode ? {
+          node: inlineTextNode,
+          offset: currentNodeOffset -
+              ParagraphUtils.getStartCharIndexInParent(inlineTextNode)
+        } :
+                                {node: currentNode, offset: currentNodeOffset};
+      }
+    }
+    return {node: null, offset: 0};
+  }
 }
 
 
@@ -438,7 +487,7 @@ ParagraphUtils.NodeGroup = class {
 
     /**
      * List of nodes in this paragraph in order.
-     * @type {Array<ParagraphUtils.NodeGroupItem>}
+     * @type {!Array<ParagraphUtils.NodeGroupItem>}
      */
     this.nodes = [];
 

chrome/browser/resources/chromeos/accessibility/select_to_speak/paragraph_utils_unittest.js

@@ -484,3 +484,106 @@ TEST_F('SelectToSpeakParagraphUnitTest', 'BuildNodeGroupWithSvg', function() {
       [inline1, inline2], 0, {splitOnLanguage: false});
   assertEquals('Hello, world! ', result.text);
 });
+
+TEST_F(
+    'SelectToSpeakParagraphUnitTest', 'findNodeFromNodeGroupByCharIndex',
+    function() {
+      // The array has four inline text nodes and one static text node.
+      // Their starting indexes are 0, 9, 20, 30, and 51.
+      const nodeGroup = ParagraphUtils.buildNodeGroup(
+          generateNodesForParagraph(), 0, false /* do not split on language */);
+      const nodes = generateNodesForParagraph();
+      const firstInline = 'The first';
+      const secondInline = ' sentence.';
+      const thirdInline = 'The second';
+      const fourthInline = ' sentence is longer.';
+      const thirdStatic = 'No child sentence.';
+
+      let result = ParagraphUtils.findNodeFromNodeGroupByCharIndex(
+          nodeGroup, 0 /* charIndex */);
+      assertEquals(result.node.name, firstInline);
+      assertEquals(result.offset, 0);
+
+      result = ParagraphUtils.findNodeFromNodeGroupByCharIndex(
+          nodeGroup, 3 /* charIndex */);
+      assertEquals(result.node.name, firstInline);
+      assertEquals(result.offset, 3);
+
+      result = ParagraphUtils.findNodeFromNodeGroupByCharIndex(
+          nodeGroup, 10 /* charIndex */);
+      assertEquals(result.node.name, secondInline);
+      assertEquals(result.offset, 1);
+
+      result = ParagraphUtils.findNodeFromNodeGroupByCharIndex(
+          nodeGroup, 20 /* charIndex */);
+      assertEquals(result.node.name, thirdInline);
+      assertEquals(result.offset, 0);
+
+      result = ParagraphUtils.findNodeFromNodeGroupByCharIndex(
+          nodeGroup, 33 /* charIndex */);
+      assertEquals(result.node.name, fourthInline);
+      assertEquals(result.offset, 3);
+
+      result = ParagraphUtils.findNodeFromNodeGroupByCharIndex(
+          nodeGroup, 52 /* charIndex */);
+      assertEquals(result.node.name, thirdStatic);
+      assertEquals(result.offset, 1);
+
+      result = ParagraphUtils.findNodeFromNodeGroupByCharIndex(
+          nodeGroup, 100 /* charIndex */);
+      assertEquals(result.node, null);
+    });
+
+/**
+ * Creates an array of nodes that represents a paragraph.
+ * @return {Array<AutomationNode>}
+ */
+function generateNodesForParagraph() {
+  const root = {role: 'rootWebArea'};
+  const paragraph = {role: 'paragraph', display: 'block', parent: root, root};
+  const text1 = {
+    name: 'The first sentence.',
+    role: 'staticText',
+    parent: paragraph
+  };
+  const inlineText1 = {
+    role: 'inlineTextBox',
+    name: 'The first',
+    indexInParent: 0,
+    parent: text1
+  };
+  const inlineText2 = {
+    role: 'inlineTextBox',
+    name: ' sentence.',
+    indexInParent: 1,
+    parent: text1
+  };
+  text1.children = [inlineText1, inlineText2];
+
+  const text2 = {
+    name: 'The second sentence is longer.',
+    role: 'staticText',
+    parent: paragraph
+  };
+  const inlineText3 = {
+    role: 'inlineTextBox',
+    name: 'The second',
+    indexInParent: 0,
+    parent: text2
+  };
+  const inlineText4 = {
+    role: 'inlineTextBox',
+    name: ' sentence is longer.',
+    indexInParent: 1,
+    parent: text2
+  };
+  text2.children = [inlineText3, inlineText4];
+
+  const text3 = {
+    name: 'No child sentence.',
+    role: 'staticText',
+    parent: paragraph
+  };
+
+  return [inlineText1, inlineText2, inlineText3, inlineText4, text3];
+}
\ No newline at end of file

chrome/browser/resources/chromeos/accessibility/select_to_speak/select_to_speak_navigation_control_test.js

@@ -323,8 +323,7 @@ TEST_F(
                 this.mockTts.pendingUtterances()[0], 'Paragraph 1');
             assertEquals(this.mockTts.getOptions().rate, 1.2);
 
-            // Changing speed will resume where we left off with selected speech
-            // rate.
+            // Changing speed will resume at the start of the current sentence.
             selectToSpeak.onSelectToSpeakPanelAction_(
                 chrome.accessibilityPrivate.SelectToSpeakPanelAction
                     .CHANGE_SPEED,
@@ -385,3 +384,51 @@ TEST_F(
             }, 0));
           });
     });
+
+TEST_F(
+    'SelectToSpeakNavigationControlTest', 'ResumeAtTheEndOfParagraph',
+    function() {
+      const bodyHtml = `
+        <p id="p1">Paragraph 1</p>
+        <p id="p2">Paragraph 2</p>
+      `;
+      this.runWithLoadedTree(
+          this.generateHtmlWithSelectedElement('p1', bodyHtml), () => {
+            this.triggerReadSelectedText();
+
+            // Finishes the current utterance.
+            this.mockTts.finishPendingUtterance();
+
+            // Hitting resume will start the next paragraph.
+            selectToSpeak.onSelectToSpeakPanelAction_(
+                chrome.accessibilityPrivate.SelectToSpeakPanelAction.RESUME);
+            assertTrue(this.mockTts.currentlySpeaking());
+            assertEquals(this.mockTts.pendingUtterances().length, 1);
+            this.assertEqualsCollapseWhitespace(
+                this.mockTts.pendingUtterances()[0], 'Paragraph 2');
+          });
+    });
+
+TEST_F(
+    'SelectToSpeakNavigationControlTest', 'ResumeAtTheEndOfUserSelection',
+    function() {
+      const bodyHtml = `
+        <p id="p1">Sentence <span id="s1">one</span>. Sentence two.</p>
+        <p id="p2">Paragraph 2</p>
+      `;
+      this.runWithLoadedTree(
+          this.generateHtmlWithSelectedElement('s1', bodyHtml), () => {
+            this.triggerReadSelectedText();
+
+            // Finishes the current utterance.
+            this.mockTts.finishPendingUtterance();
+
+            // Hitting resume will start the remaining content.
+            selectToSpeak.onSelectToSpeakPanelAction_(
+                chrome.accessibilityPrivate.SelectToSpeakPanelAction.RESUME);
+            assertTrue(this.mockTts.currentlySpeaking());
+            assertEquals(this.mockTts.pendingUtterances().length, 1);
+            this.assertEqualsCollapseWhitespace(
+                this.mockTts.pendingUtterances()[0], '. Sentence two.');
+          });
+    });
\ No newline at end of file