[Switch Access] Group virtual keyboard keys into rows

go/cros-switch-navigation

Bug: 1011630
Change-Id: I27d974be1c549f176c9ccd2c4d1814b02e5498c8
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1876867
Commit-Queue: Anastasia Helfinstein <anastasi@google.com>
Reviewed-by: Aran Gilman <gilmanmh@google.com>
Cr-Commit-Position: refs/heads/master@{#711021}

chrome/browser/resources/chromeos/switch_access/BUILD.gn

@@ -70,6 +70,7 @@ run_jsbundler("switch_access_copied_files") {
     "menu_panel_interface.js",
     "navigation_manager.js",
     "nodes/back_button_node.js",
+    "nodes/group_node.js",
     "nodes/keyboard_node.js",
     "nodes/node_wrapper.js",
     "nodes/switch_access_node.js",
@@ -182,6 +183,7 @@ js_type_check("closure_compile") {
     ":commands",
     ":event_helper",
     ":focus_ring_manager",
+    ":group_node",
     ":keyboard_node",
     ":menu_manager",
     ":menu_panel",
@@ -252,12 +254,30 @@ js_library("focus_ring_manager") {
   externs_list = [ "$externs_path/accessibility_private.js" ]
 }
 
+js_library("group_node") {
+  sources = [
+    "nodes/group_node.js",
+  ]
+  deps = [
+    ":back_button_node",
+    ":node_wrapper",
+    ":rect_helper",
+    ":switch_access_constants",
+    ":switch_access_node",
+  ]
+  externs_list = [
+    "$externs_path/accessibility_private.js",
+    "$externs_path/automation.js",
+  ]
+}
+
 js_library("keyboard_node") {
   sources = [
     "nodes/keyboard_node.js",
   ]
   deps = [
     ":event_helper",
+    ":group_node",
     ":node_wrapper",
     ":rect_helper",
     ":switch_access_constants",

chrome/browser/resources/chromeos/switch_access/manifest.json.jinja2

@@ -22,6 +22,7 @@
       "nodes/switch_access_node.js",
       "nodes/node_wrapper.js",
       "nodes/back_button_node.js",
+      "nodes/group_node.js",
       "nodes/keyboard_node.js",
       "nodes/system_menu_node.js",
       "preferences.js",

chrome/browser/resources/chromeos/switch_access/nodes/group_node.js

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+/**
+ * This class handles the grouping of nodes that are not grouped in the
+ *     automation tree. They are defined by their parent and child nodes.
+ * Ex: Nodes in the virtual keyboard have no intermediate grouping, but should
+ *     be grouped by row.
+ */
+class GroupNode extends SAChildNode {
+  /**
+   * @param {!Array<!SAChildNode>} children The nodes that this group contains.
+   *     Should not include the back button.
+   * @private
+   */
+  constructor(children) {
+    super(true /* isGroup */);
+
+    /** @type {!Array<!SAChildNode>} */
+    this.children_ = children;
+  }
+
+  /** @override */
+  equals(other) {
+    if (!(other instanceof GroupNode)) return false;
+
+    other = /** @type {GroupNode} */ (other);
+    if (other.children_.length !== this.children_.length) return false;
+    for (let i = 0; i < this.children_.length; i++) {
+      if (other.children_[i].equals(this.children_[i])) return false;
+    }
+    return true;
+  }
+
+  /** @override */
+  get role() {
+    return chrome.automation.RoleType.GROUP;
+  }
+
+  /** @override */
+  get location() {
+    const childLocations = this.children_.map(c => c.location);
+    return RectHelper.unionAll(childLocations);
+  }
+
+  /** @override */
+  get automationNode() {
+    return null;
+  }
+
+  /** @override */
+  get actions() {
+    return [];
+  }
+
+  /** @override */
+  performAction(action) {
+    return true;
+  }
+
+  /** @override */
+  isEquivalentTo(node) {
+    return false;
+  }
+
+  /** @override */
+  asRootNode() {
+    const root = new SARootNode();
+
+    let children = [];
+    for (const child of this.children_) {
+      children.push(child);
+    }
+
+    const backButton = new BackButtonNode(root);
+    children.push(backButton);
+
+    SARootNode.connectChildren(children);
+    root.setChildren(children);
+
+    return root;
+  }
+
+  /**
+   * Assumes nodes are visually in rows.
+   * @param {!Array<!SAChildNode>} nodes
+   * @return {!Array<!GroupNode>}
+   */
+  static separateByRow(nodes) {
+    let result = [];
+
+    for (let i = 0; i < nodes.length;) {
+      let children = [];
+      children.push(nodes[i]);
+      i++;
+
+      while (i < nodes.length &&
+             RectHelper.sameRow(children[0].location, nodes[i].location)) {
+        children.push(nodes[i]);
+        i++;
+      }
+      if (children.length <= 1) {
+        throw new Error('Cannot group row with only one element.');
+      }
+
+      result.push(new GroupNode(children));
+    }
+
+    return result;
+  }
+}

chrome/browser/resources/chromeos/switch_access/nodes/keyboard_node.js

@@ -46,21 +46,30 @@ class KeyboardNode extends NodeWrapper {
       throw new TypeError('Keyboard nodes must have an automation node.');
     }
 
-    return KeyboardNode.buildTree_(node);
+    const root = new RootNodeWrapper(node);
+    KeyboardNode.buildHelper(root);
+    return root;
   }
 
   /**
    * Builds a tree of KeyboardNodes.
-   * @param {!chrome.automation.AutomationNode} automationNode
-   * @return {!SARootNode}
-   * @private
+   * @param {!RootNodeWrapper} root
    */
-  static buildTree_(automationNode) {
-    const root = new RootNodeWrapper(automationNode);
+  static buildHelper(root) {
     const childConstructor = (node) => new KeyboardNode(node, root);
 
-    RootNodeWrapper.buildHelper(root, childConstructor);
-    return root;
+    /** @type {!Array<!chrome.automation.AutomationNode>} */
+    let interestingChildren = RootNodeWrapper.getInterestingChildren(root);
+    let children = interestingChildren.map(childConstructor);
+    if (interestingChildren.length > SAConstants.KEYBOARD_MAX_ROW_LENGTH) {
+      children = GroupNode.separateByRow(children);
+    }
+
+    const backButton = new BackButtonNode(root);
+    children.push(backButton);
+
+    SARootNode.connectChildren(children);
+    root.setChildren(children);
   }
 }
 
@@ -107,9 +116,7 @@ class KeyboardRootNode extends RootNodeWrapper {
 
     const root = new KeyboardRootNode(keyboard);
     root.onEnter_();
-    const childConstructor = (node) => new KeyboardNode(node, root);
-
-    RootNodeWrapper.buildHelper(root, childConstructor);
+    KeyboardNode.buildHelper(root);
     return root;
   }
 }

chrome/browser/resources/chromeos/switch_access/nodes/node_wrapper.js

@@ -203,16 +203,13 @@ class RootNodeWrapper extends SARootNode {
     if (interestingChildren.length < 1) {
       throw new Error('Root node must have at least 1 interesting child.');
     }
+    let children = interestingChildren.map(childConstructor);
 
     const backButton = new BackButtonNode(root);
-
-    let children = interestingChildren.map(childConstructor);
     children.push(backButton);
 
     SARootNode.connectChildren(children);
-    root.setChildren_(children);
-
-    return;
+    root.setChildren(children);
   }
 
   /**
@@ -231,7 +228,7 @@ class RootNodeWrapper extends SARootNode {
     let children = interestingChildren.map(childConstructor);
 
     SARootNode.connectChildren(children);
-    root.setChildren_(children);
+    root.setChildren(children);
 
     return root;
   }

chrome/browser/resources/chromeos/switch_access/nodes/switch_access_node.js

@@ -185,11 +185,8 @@ class SARootNode {
     this.children_ = [];
   }
 
-  /**
-   * @param {!Array<!SAChildNode>} children
-   * @protected
-   */
-  setChildren_(children) {
+  /** @param {!Array<!SAChildNode>} children */
+  setChildren(children) {
     this.children_ = children;
   }
 
@@ -240,7 +237,8 @@ class SARootNode {
 
   /** @return {!chrome.accessibilityPrivate.ScreenRect} */
   get location() {
-    let childLocations = this.children_.map((c) => c.location);
+    let children = this.children_.filter((c) => !(c instanceof BackButtonNode));
+    let childLocations = children.map((c) => c.location);
     return RectHelper.unionAll(childLocations);
   }
 

chrome/browser/resources/chromeos/switch_access/rect_helper.js

@@ -100,6 +100,19 @@ const RectHelper = {
     return str;
   },
 
+  /*
+   * @param {chrome.accessibilityPrivate.ScreenRect=} rect1
+   * @param {chrome.accessibilityPrivate.ScreenRect=} rect2
+   */
+  sameRow: (rect1, rect2) => {
+    if (!rect1 || !rect2) return false;
+    const halfHeight = Math.round(rect1.height / 2);
+    const topBound = rect1.top - halfHeight;
+    const bottomBound = rect1.top + halfHeight;
+
+    return topBound <= rect2.top && bottomBound >= rect2.top;
+  },
+
   /**
    * Increases the size of |outer| to entirely enclose |inner|, with |padding|
    * buffer on each side.

chrome/browser/resources/chromeos/switch_access/switch_access_constants.js

@@ -92,6 +92,13 @@ SAConstants.Focus.BUFFER = 4;
  */
 SAConstants.VK_KEY_PRESS_DURATION_MS = 100;
 
+/**
+ * The maximum length of a row in the Virtual Keyboard.
+ * @type {number}
+ * @const
+ */
+SAConstants.KEYBOARD_MAX_ROW_LENGTH = 14;
+
 /**
  * Preferences that are configurable in Switch Access.
  * @enum {string}

chrome/browser/resources/chromeos/switch_access/switch_access_predicate.js

@@ -6,8 +6,6 @@ const StateType = chrome.automation.StateType;
 const RoleType = chrome.automation.RoleType;
 const DefaultActionVerb = chrome.automation.DefaultActionVerb;
 
-const GROUP_INTERESTING_CHILD_THRESHOLD = 2;
-
 /**
  * Contains predicates for the chrome automation API. The following basic
  * predicates are available:
@@ -23,6 +21,8 @@ const GROUP_INTERESTING_CHILD_THRESHOLD = 2;
  * restrictions required by TreeWalker for specific traversal situations.
  */
 const SwitchAccessPredicate = {
+  GROUP_INTERESTING_CHILD_THRESHOLD: 2,
+
   /**
    * Returns true if |node| is actionable, meaning that a user can interact with
    * it in some way.
@@ -119,7 +119,8 @@ const SwitchAccessPredicate = {
       if (SwitchAccessPredicate.isInterestingSubtree(child)) {
         interestingBranchesCount += 1;
       }
-      if (interestingBranchesCount >= GROUP_INTERESTING_CHILD_THRESHOLD) {
+      if (interestingBranchesCount >=
+          SwitchAccessPredicate.GROUP_INTERESTING_CHILD_THRESHOLD) {
         return true;
       }
       child = child.nextSibling;