Have select-to-speak read SVG text nodes in visual reading order.

Select-to-speak will re-order SVG children nodes by reading order based on position, instead of DOM order. SVGs have no concept of reflow, but are instead absolutely positioned, so DOM order can easily be out of sync with visual order.

Re-order logic will keep original groupings intact (nodes within SVG <g> element), as these groupings can be semantic in nature, for instance, columns of text.

Note: This is LTR only for now.

AX-Relnotes: Select-to-speak will read SVG content in LTR reading order on ChromeOS. Example SVG content is Google Slides.

Bug: 892822
Change-Id: Ia9092557442451b52c8ac4d10643cf720dc8d3ca
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2391266
Commit-Queue: Joel Riley <joelriley@google.com>
Reviewed-by: Anastasia Helfinstein <anastasi@google.com>
Reviewed-by: Dominic Mazzoni <dmazzoni@chromium.org>
Cr-Commit-Position: refs/heads/master@{#808792}

chrome/browser/resources/chromeos/accessibility/BUILD.gn

@@ -151,8 +151,11 @@ js2gtest("misc_unit_tests_js") {
     "select_to_speak/select_to_speak_unittest.js",
     "select_to_speak/word_utils_unittest.js",
   ]
+  gen_include_files = [ "common/closure_shim.js" ]
   extra_js_files = [
     "braille_ime/braille_ime.js",
+    "common/constants.js",
+    "common/automation_util.js",
     "common/rect_util.js",
     "select_to_speak/paragraph_utils.js",
     "select_to_speak/select_to_speak.js",

chrome/browser/resources/chromeos/accessibility/common/automation_util.js

@@ -171,6 +171,21 @@ AutomationUtil = class {
     return ret.reverse();
   }
 
+  /**
+   * Finds the lowest ancestor with a given role.
+   * @param {!AutomationNode} node
+   * @param {!RoleType} role
+   */
+  static getFirstAncestorWithRole(node, role) {
+    if (!node.parent) {
+      return null;
+    }
+    if (node.parent.role === role) {
+      return node.parent;
+    }
+    return AutomationUtil.getFirstAncestorWithRole(node.parent, role);
+  }
+
   /**
    * Gets the first index where the two input arrays differ. Returns -1 if they
    * do not.

chrome/browser/resources/chromeos/accessibility/common/automation_util_test.js

@@ -74,6 +74,30 @@ TEST_F(
       });
     });
 
+TEST_F(
+    'AccessibilityExtensionAutomationUtilE2ETest', 'GetFirstAncestorWithRole',
+    function() {
+      this.runWithLoadedTree(
+          `
+    <div aria-label="x">
+      <div aria-label="y">
+        <p>
+          <button>Hello world</div>
+        </p>
+      </div>
+    </div>`,
+          function(root) {
+            const buttonNode = root.firstChild.firstChild.firstChild;
+            const containerNode = AutomationUtil.getFirstAncestorWithRole(
+                buttonNode, RoleType.GENERIC_CONTAINER);
+            assertEquals(containerNode.name, 'y');
+
+            const parentContainerNode = AutomationUtil.getFirstAncestorWithRole(
+                containerNode, RoleType.GENERIC_CONTAINER);
+            assertEquals(parentContainerNode.name, 'x');
+          });
+    });
+
 TEST_F(
     'AccessibilityExtensionAutomationUtilE2ETest', 'GetUniqueAncestors',
     function() {

chrome/browser/resources/chromeos/accessibility/common/closure_shim.js

@@ -9,12 +9,7 @@
 var goog = {};
 
 goog.provide = function(n) {
-  // Skip setting window property in non-browser environments (e.g. pure V8)
-  if (typeof window === 'undefined') {
-    return;
-  }
-
-  window[n] = {};
+  globalThis[n] = {};
 };
 
 goog.require = function() {};

chrome/browser/resources/chromeos/accessibility/select_to_speak/node_utils.js

@@ -352,6 +352,95 @@ class NodeUtils {
     }
     return {node, offset: NodeUtils.nameLength(node)};
   }
+
+  /**
+   * Sorts given nodes by visual reading order. Expects nodes to be leaf nodes
+   * with text.
+   * @param {!Array<!AutomationNode>} nodes
+   */
+  static sortNodesByReadingOrder(nodes) {
+    // Pre-compute ancestors for each node.
+    const nodeAncestorMap = new Map();
+    for (const node of nodes) {
+      nodeAncestorMap.set(node, AutomationUtil.getAncestors(node));
+    }
+
+    // Sort nodes by bounds of their divergent ancestors. This will ensure all
+    // nodes with the same parent are grouped together.
+    nodes.sort((a, b) => {
+      const ancestorsA = nodeAncestorMap.get(a);
+      const ancestorsB = nodeAncestorMap.get(b);
+      const divergence = AutomationUtil.getDivergence(ancestorsA, ancestorsB);
+      if (divergence === -1 || divergence >= ancestorsA.length ||
+          divergence >= ancestorsB.length) {
+        // Nodes do not have any ancestors in common (different trees) or one
+        // node is the ancestor of another.
+        console.warn(
+            'Nodes are directly related or have no common ancestors', a, b);
+        return 0;
+      }
+      const divA = ancestorsA[divergence];
+      const divB = ancestorsB[divergence];
+
+      if (RectUtil.sameRow(divA.unclippedLocation, divB.unclippedLocation)) {
+        // Nodes are on the same line, sort by LTR reading order.
+        // TODO(joelriley@google.com): Handle RTL.
+        if (divA.unclippedLocation.left < divB.unclippedLocation.left) {
+          return -1;
+        }
+        if (divB.unclippedLocation.left < divA.unclippedLocation.left) {
+          return 1;
+        }
+        return 0;
+      }
+      // Nodes are on different lines, sort top-to-bottom.
+      if (divA.unclippedLocation.top < divB.unclippedLocation.top) {
+        return -1;
+      }
+      if (divB.unclippedLocation.top < divA.unclippedLocation.top) {
+        return 1;
+      }
+      return 0;
+    });
+  }
+
+  /**
+   * Sorts a specific range of a given array of nodes by visual reading order.
+   * Expects nodes to be leaf nodes with text.
+   * @param {!Array<!AutomationNode>} nodes
+   * @param {number} startIndex Index specifying start of range.
+   * @param {number} endIndex  Index specifying end of range, non-inclusive.
+   */
+  static sortNodeRangeByReadingOrder(nodes, startIndex, endIndex) {
+    const nodesToSort = nodes.slice(startIndex, endIndex);
+    NodeUtils.sortNodesByReadingOrder(nodesToSort);
+    nodes.splice(startIndex, endIndex - startIndex, ...nodesToSort);
+  }
+
+  /**
+   * Sorts SVG nodes with the same SVG root parent by visual reading order.
+   * @param {!Array<!AutomationNode>} nodes
+   */
+  static sortSvgNodesByReadingOrder(nodes) {
+    let lastSvgRoot = null;
+    let startIndex = 0;
+    for (let i = 0; i < nodes.length; i++) {
+      const node = nodes[i];
+      const svgRoot =
+          AutomationUtil.getFirstAncestorWithRole(node, RoleType.SVG_ROOT);
+      if (svgRoot !== lastSvgRoot) {
+        if (lastSvgRoot !== null) {
+          NodeUtils.sortNodeRangeByReadingOrder(nodes, startIndex, i);
+        } else if (svgRoot !== null) {
+          startIndex = i;
+        }
+        lastSvgRoot = svgRoot;
+      }
+    }
+    if (lastSvgRoot !== null) {
+      NodeUtils.sortNodeRangeByReadingOrder(nodes, startIndex, nodes.length);
+    }
+  }
 }
 
 /**

chrome/browser/resources/chromeos/accessibility/select_to_speak/node_utils_unittest.js

@@ -13,6 +13,9 @@ SelectToSpeakNodeUtilsUnitTest.prototype.extraLibraries = [
   'paragraph_utils.js',
   'node_utils.js',
   'word_utils.js',
+  '../common/closure_shim.js',
+  '../common/constants.js',
+  '../common/automation_util.js',
   '../common/rect_util.js',
 ];
 
@@ -397,3 +400,90 @@ TEST_F(
       assertEquals(child3, result.node);
       assertEquals(0, result.offset);
     });
+
+TEST_F(
+    'SelectToSpeakNodeUtilsUnitTest', 'sortSvgNodesByReadingOrder', function() {
+      const svgRootNode = {role: 'svgRoot'};
+      const gNode1 = {
+        role: 'genericContainer',
+        parent: svgRootNode,
+        unclippedLocation: {left: 300, top: 10, width: 100, height: 50}
+      };
+      const gNode2 = {
+        role: 'genericContainer',
+        parent: svgRootNode,
+        unclippedLocation: {left: 20, top: 10, width: 100, height: 50}
+      };
+      const textNode1 = {
+        role: 'staticText',
+        parent: gNode2,
+        unclippedLocation: {left: 50, top: 10, width: 20, height: 50},
+        name: 'one'
+      };
+      const textNode2 = {
+        role: 'staticText',
+        parent: gNode1,
+        unclippedLocation: {left: 300, top: 10, width: 20, height: 50},
+        name: 'two'
+      };
+      const textNode3 = {
+        role: 'staticText',
+        parent: gNode1,
+        unclippedLocation: {left: 350, top: 10, width: 20, height: 50},
+        name: 'three'
+      };
+
+      const nodes = [textNode3, textNode2, textNode1];
+      NodeUtils.sortSvgNodesByReadingOrder(nodes);
+      assertEquals(nodes[0].name, 'one');
+      assertEquals(nodes[1].name, 'two');
+      assertEquals(nodes[2].name, 'three');
+    });
+
+TEST_F(
+    'SelectToSpeakNodeUtilsUnitTest', 'sortNodesByReadingOrderMultipleSVGs',
+    function() {
+      const textNode1 = {role: 'staticText', name: 'Text Node 1'};
+      const svg1RootNode = {role: 'svgRoot'};
+      const svg1Node1 = {
+        role: 'staticText',
+        parent: svg1RootNode,
+        unclippedLocation: {left: 0, top: 10, width: 20, height: 50},
+        name: 'SVG 1 Node 1'
+      };
+      const svg1Node2 = {
+        role: 'staticText',
+        parent: svg1RootNode,
+        unclippedLocation: {left: 50, top: 10, width: 20, height: 50},
+        name: 'SVG 1 Node 2'
+      };
+      const textNode2 = {role: 'staticText', name: 'Text Node 2'};
+      const svg2RootNode = {role: 'svgRoot'};
+      const svg2Node1 = {
+        role: 'staticText',
+        parent: svg2RootNode,
+        unclippedLocation: {left: 300, top: 10, width: 20, height: 50},
+        name: 'SVG 2 Node 1'
+      };
+      const svg2Node2 = {
+        role: 'staticText',
+        parent: svg2RootNode,
+        unclippedLocation: {left: 350, top: 10, width: 20, height: 50},
+        name: 'SVG 2 Node 2'
+      };
+      const textNode3 = {role: 'staticText', name: 'Text Node 3'};
+
+      const nodes = [
+        textNode1, svg1Node2, svg1Node1, textNode2, svg2Node2, svg2Node1,
+        textNode3
+      ];
+      NodeUtils.sortSvgNodesByReadingOrder(nodes);
+
+      assertEquals(nodes[0].name, 'Text Node 1');
+      assertEquals(nodes[1].name, 'SVG 1 Node 1');
+      assertEquals(nodes[2].name, 'SVG 1 Node 2');
+      assertEquals(nodes[3].name, 'Text Node 2');
+      assertEquals(nodes[4].name, 'SVG 2 Node 1');
+      assertEquals(nodes[5].name, 'SVG 2 Node 2');
+      assertEquals(nodes[6].name, 'Text Node 3');
+    });
\ No newline at end of file

chrome/browser/resources/chromeos/accessibility/select_to_speak/select_to_speak.js

@@ -586,6 +586,28 @@ class SelectToSpeak {
    */
   startSpeechQueue_(nodes, opt_startIndex, opt_endIndex) {
     this.prepareForSpeech_();
+
+    if (nodes.length === 0) {
+      return;
+    }
+
+    // Remember the original first and last node in the given list, as
+    // opt_startIndex and opt_endIndex pertain to them. If, after SVG
+    // resorting, the first or last nodes are re-ordered, do not clip them.
+    const originalFirstNode = nodes[0];
+    const originalLastNode = nodes[nodes.length - 1];
+
+    // Sort any SVG child nodes, if present, by visual reading order.
+    NodeUtils.sortSvgNodesByReadingOrder(nodes);
+
+    // Override start or end index if original nodes were sorted.
+    if (originalFirstNode !== nodes[0]) {
+      opt_startIndex = undefined;
+    }
+    if (originalLastNode !== nodes[nodes.length - 1]) {
+      opt_endIndex = undefined;
+    }
+
     for (var i = 0; i < nodes.length; i++) {
       const nodeGroup = ParagraphUtils.buildNodeGroup(
           nodes, i, this.enableLanguageDetectionIntegration_);
@@ -595,18 +617,15 @@ class SelectToSpeak {
         // the start index so that it is not spoken.
         // Backfill with spaces so that index counting functions don't get
         // confused.
-        // Must check opt_startIndex in its own if statement to make the
-        // Closure compiler happy.
-        if (opt_startIndex !== undefined) {
-          if (nodeGroup.nodes.length > 0 && nodeGroup.nodes[0].hasInlineText) {
-            // The first node is inlineText type. Find the start index in
-            // its staticText parent.
-            const startIndexInParent =
-                ParagraphUtils.getStartCharIndexInParent(nodes[0]);
-            opt_startIndex += startIndexInParent;
-            nodeGroup.text = ' '.repeat(opt_startIndex) +
-                nodeGroup.text.substr(opt_startIndex);
-          }
+        if (opt_startIndex !== undefined && nodeGroup.nodes.length > 0 &&
+            nodeGroup.nodes[0].hasInlineText) {
+          // The first node is inlineText type. Find the start index in
+          // its staticText parent.
+          const startIndexInParent =
+              ParagraphUtils.getStartCharIndexInParent(nodes[0]);
+          opt_startIndex += startIndexInParent;
+          nodeGroup.text = ' '.repeat(opt_startIndex) +
+              nodeGroup.text.substr(opt_startIndex);
         }
       }
       const isFirst = i == 0;

chrome/browser/resources/chromeos/accessibility/select_to_speak/select_to_speak_keystroke_selection_test.js

@@ -570,3 +570,102 @@ TEST_F(
                 this.mockTts.pendingUtterances()[1], 'd e f');
           });
     });
+
+TEST_F(
+    'SelectToSpeakKeystrokeSelectionTest', 'ReordersSvgSingleLine', function() {
+      const selectionCode =
+          'let body = document.getElementsByTagName("body")[0];' +
+          'range.setStart(body, 0);' +
+          'range.setEnd(body, 1);';
+      this.runWithLoadedTree(
+          this.generateHtmlWithSelection(
+              selectionCode,
+              '<svg viewBox="0 0 240 80" xmlns="http://www.w3.org/2000/svg">' +
+                  '  <text x="65" y="55">Grumpy!</text>' +
+                  '  <text x="20" y="35">My</text>' +
+                  '  <text x="40" y="35">cat</text>' +
+                  '  <text x="55" y="55">is</text>' +
+                  '</svg>'),
+          function() {
+            this.triggerReadSelectedText();
+            assertTrue(this.mockTts.currentlySpeaking());
+            this.assertEqualsCollapseWhitespace(
+                this.mockTts.pendingUtterances()[0], 'My cat is Grumpy!');
+          });
+    });
+
+TEST_F(
+    'SelectToSpeakKeystrokeSelectionTest', 'ReordersSvgWithGroups', function() {
+      const selectionCode =
+          'let body = document.getElementsByTagName("body")[0];' +
+          'range.setStart(body, 0);' +
+          'range.setEnd(body, 1);';
+      this.runWithLoadedTree(
+          this.generateHtmlWithSelection(
+              selectionCode,
+              '<svg viewBox="0 0 240 80" xmlns="http://www.w3.org/2000/svg">' +
+                  '  <g>' +
+                  '    <text x="65" y="0">Column 2, Text 1</text>' +
+                  '    <text x="65" y="50">Column 2, Text 2</text>' +
+                  '  </g>' +
+                  '  <g>' +
+                  '    <text x="0" y="50">Column 1, Text 2</text>' +
+                  '    <text x="0" y="0">Column 1, Text 1</text>' +
+                  '  </g>' +
+                  '</svg>'),
+          function() {
+            this.triggerReadSelectedText();
+            assertTrue(this.mockTts.currentlySpeaking());
+            this.assertEqualsCollapseWhitespace(
+                this.mockTts.pendingUtterances()[0], 'Column 1, Text 1');
+            this.assertEqualsCollapseWhitespace(
+                this.mockTts.pendingUtterances()[1], 'Column 1, Text 2');
+            this.assertEqualsCollapseWhitespace(
+                this.mockTts.pendingUtterances()[2], 'Column 2, Text 1');
+            this.assertEqualsCollapseWhitespace(
+                this.mockTts.pendingUtterances()[3], 'Column 2, Text 2');
+          });
+    });
+TEST_F(
+    'SelectToSpeakKeystrokeSelectionTest',
+    'NonReorderedSvgPreservesSelectionStartEnd', function() {
+      const selectionCode = 'const t1 = document.getElementById("t1");' +
+          'const t2 = document.getElementById("t2");' +
+          'range.setStart(t1.childNodes[0], 3);' +
+          'range.setEnd(t2.childNodes[0], 2);';
+      this.runWithLoadedTree(
+          this.generateHtmlWithSelection(
+              selectionCode,
+              '<svg viewBox="0 0 240 80" xmlns="http://www.w3.org/2000/svg">' +
+                  '  <text id="t1" x="0" y="55">My cat</text>' +
+                  '  <text id="t2" x="100" y="55">is Grumpy!</text>' +
+                  '</svg>'),
+          function() {
+            this.triggerReadSelectedText();
+            assertTrue(this.mockTts.currentlySpeaking());
+            this.assertEqualsCollapseWhitespace(
+                this.mockTts.pendingUtterances()[0], 'cat is');
+          });
+    });
+
+TEST_F(
+    'SelectToSpeakKeystrokeSelectionTest',
+    'ReorderedSvgIgnoresSelectionStartEnd', function() {
+      const selectionCode = 'const t1 = document.getElementById("t1");' +
+          'const t2 = document.getElementById("t2");' +
+          'range.setStart(t1.childNodes[0], 3);' +
+          'range.setEnd(t2.childNodes[0], 2);';
+      this.runWithLoadedTree(
+          this.generateHtmlWithSelection(
+              selectionCode,
+              '<svg viewBox="0 0 240 80" xmlns="http://www.w3.org/2000/svg">' +
+                  '  <text id="t1" x="100" y="55">is Grumpy!</text>' +
+                  '  <text id="t2" x="0" y="55">My cat</text>' +
+                  '</svg>'),
+          function() {
+            this.triggerReadSelectedText();
+            assertTrue(this.mockTts.currentlySpeaking());
+            this.assertEqualsCollapseWhitespace(
+                this.mockTts.pendingUtterances()[0], 'My cat is Grumpy!');
+          });
+    });
\ No newline at end of file