Adds friendlier api for markerTypes in chrome.automation

Also, adds additional PRESUBMIT checks to ensure automation keeps in sync with
ax_enums for MarkerType.

Test: existing browser_tests --gtest_filter=ChromeVoxEditing*.*
Change-Id: I58556cc8317cbca62f00aed8d9f3afbcf7feeee2
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1940868
Commit-Queue: David Tseng <dtseng@chromium.org>
Reviewed-by: Akihiro Ota <akihiroota@chromium.org>
Cr-Commit-Position: refs/heads/master@{#721368}

chrome/browser/resources/chromeos/accessibility/chromevox/background/editing.js

@@ -81,11 +81,13 @@ editing.TextEditHandler.prototype = {
     if (evt.type !== EventType.TEXT_CHANGED &&
         evt.type !== EventType.TEXT_SELECTION_CHANGED &&
         evt.type !== EventType.DOCUMENT_SELECTION_CHANGED &&
-        evt.type !== EventType.VALUE_CHANGED && evt.type !== EventType.FOCUS)
+        evt.type !== EventType.VALUE_CHANGED && evt.type !== EventType.FOCUS) {
       return;
+    }
     if (!evt.target.state.focused || !evt.target.state.editable ||
-        evt.target != this.node_)
+        evt.target != this.node_) {
       return;
+    }
 
     this.editableText_.onUpdate(evt.eventFrom);
   },
@@ -273,8 +275,9 @@ function AutomationRichEditableText(node) {
   var root = this.node_.root;
   if (!root || !root.selectionStartObject || !root.selectionEndObject ||
       root.selectionStartOffset === undefined ||
-      root.selectionEndOffset === undefined)
+      root.selectionEndOffset === undefined) {
     return;
+  }
 
   this.startLine_ = new editing.EditableLine(
       root.selectionStartObject, root.selectionStartOffset,
@@ -356,8 +359,9 @@ AutomationRichEditableText.prototype = {
     var root = this.node_.root;
     if (!root.selectionStartObject || !root.selectionEndObject ||
         root.selectionStartOffset === undefined ||
-        root.selectionEndOffset === undefined)
+        root.selectionEndOffset === undefined) {
       return;
+    }
 
     var startLine = new editing.EditableLine(
         root.selectionStartObject, root.selectionStartOffset,
@@ -607,22 +611,27 @@ AutomationRichEditableText.prototype = {
    * @private
    */
   speakTextMarker_: function(container, selStart, selEnd) {
-    var markersWithinSelection = 0;
-    if (container.markerTypes) {
-      for (var i = 0; i < container.markerTypes.length; i++) {
+    var markersWithinSelection = {};
+    var markers = container.markers;
+    if (markers) {
+      for (var marker of markers) {
         // See if our selection intersects with this marker.
-        if (container.markerStarts[i] >= selStart ||
-            selEnd < container.markerEnds[i])
-          markersWithinSelection |= container.markerTypes[i];
+        if (marker.startOffset >= selStart || selEnd < marker.endOffset) {
+          for (var key in marker.flags) {
+            markersWithinSelection[key] = true;
+          }
+        }
       }
     }
 
     var msgs = [];
-    if (this.misspelled == !(markersWithinSelection & 1)) {
+    if (this.misspelled ==
+        !(markersWithinSelection[chrome.automation.MarkerType.SPELLING])) {
       this.misspelled = !this.misspelled;
       msgs.push(this.misspelled ? 'misspelling_start' : 'misspelling_end');
     }
-    if (this.grammarError == !(markersWithinSelection & 2)) {
+    if (this.grammarError ==
+        !(markersWithinSelection[chrome.automation.MarkerType.GRAMMAR])) {
       this.grammarError = !this.grammarError;
       msgs.push(this.grammarError ? 'grammar_start' : 'grammar_end');
     }

chrome/browser/resources/chromeos/accessibility/chromevox/background/find_handler.js

@@ -32,10 +32,11 @@ FindHandler.uninit_ = function() {
  * @private
  */
 FindHandler.onTextMatch_ = function(evt) {
-  if (!evt.target.markerTypes.some(function(markerType) {
-        return markerType & 4 /* Text match */;
-      }))
+  if (!evt.target.markers.some(function(marker) {
+        return marker.flags[chrome.automation.MarkerType.TEXT_MATCH];
+      })) {
     return;
+  }
 
   var range = cursors.Range.fromNode(evt.target);
   ChromeVoxState.instance.setCurrentRange(range);

extensions/common/api/automation.idl

@@ -426,6 +426,15 @@
     uncheck
 };
 
+  // Types of markers on text. See <code>AutomationNode.markerTypes</code>.
+enum MarkerType {
+  spelling,
+  grammar,
+  textMatch,
+  activeSuggestion,
+  suggestion
+};
+
   dictionary Rect {
     long left;
     long top;
@@ -523,8 +532,7 @@
   callback PerformActionCallbackWithNode = void(AutomationNode node);
   callback BoundsForRangeCallback = void(Rect bounds);
 
-
-  dictionary CustomAction {
+      [nocompile, noinline_doc] dictionary CustomAction {
     long id;
     DOMString description;
   };
@@ -534,7 +542,7 @@
   // for details on how to associate this object with a string.
   // Also, the start and end indices always point to the first code unit of a
   // valid code-point.
-  dictionary LanguageSpan {
+    [nocompile, noinline_doc] dictionary LanguageSpan {
     // Inclusive start index of substring that contains language.
     long startIndex;
     // Exclusive end index of substring that contains language.
@@ -545,6 +553,19 @@
     double probability;
   };
 
+    // A marker associated with an AutomationNode.
+  [nocompile, noinline_doc] dictionary Marker {
+    // The start offset within the text of the associated node.
+    long startOffset;
+
+    // The end offset within the text of the associated node.
+    long endOffset;
+
+    // A mapping of MarkerType to true or undefined indicating the marker types
+    // for this marker.
+    object flags;
+  };
+
   // A single node in an Automation tree.
   [nocompile, noinline_doc] dictionary AutomationNode {
     // The root node of the tree containing this AutomationNode.
@@ -745,15 +766,8 @@
     // The input type, like email or number.
     DOMString? textInputType;
 
-    // An array of indexes of the start position of each text marker.
-    long[] markerStarts;
-
-    // An array of indexes of the end position of each text marker.
-    long[] markerEnds;
-
-    // An array of numerical types indicating the type of each text marker,
-    // such as a spelling error.
-    long[] markerTypes;
+    // An array of Marker objects for this node.
+    Marker[]? markers;
 
     //
     // Tree selection attributes (available on root nodes only)

extensions/renderer/api/automation/automation_internal_custom_bindings.cc

@@ -79,6 +79,24 @@ v8::Local<v8::Object> RectToV8Object(v8::Isolate* isolate,
       .Build();
 }
 
+api::automation::MarkerType ConvertMarkerTypeFromAXToAutomation(
+    ax::mojom::MarkerType ax) {
+  switch (ax) {
+    case ax::mojom::MarkerType::kNone:
+      return api::automation::MARKER_TYPE_NONE;
+    case ax::mojom::MarkerType::kSpelling:
+      return api::automation::MARKER_TYPE_SPELLING;
+    case ax::mojom::MarkerType::kGrammar:
+      return api::automation::MARKER_TYPE_GRAMMAR;
+    case ax::mojom::MarkerType::kTextMatch:
+      return api::automation::MARKER_TYPE_TEXTMATCH;
+    case ax::mojom::MarkerType::kActiveSuggestion:
+      return api::automation::MARKER_TYPE_ACTIVESUGGESTION;
+    case ax::mojom::MarkerType::kSuggestion:
+      return api::automation::MARKER_TYPE_SUGGESTION;
+  }
+}
+
 // Adjust the bounding box of a node from local to global coordinates,
 // walking up the parent hierarchy to offset by frame offsets and
 // scroll offsets.
@@ -855,6 +873,50 @@ void AutomationInternalCustomBindings::AddRoutes() {
             node->GetString16Attribute(ax::mojom::StringAttribute::kName));
         result.Set(gin::ConvertToV8(isolate, word_ends));
       });
+  RouteNodeIDFunction("GetMarkers", [](v8::Isolate* isolate,
+                                       v8::ReturnValue<v8::Value> result,
+                                       AutomationAXTreeWrapper* tree_wrapper,
+                                       ui::AXNode* node) {
+    if (!node->HasIntListAttribute(
+            ax::mojom::IntListAttribute::kMarkerStarts) ||
+        !node->HasIntListAttribute(ax::mojom::IntListAttribute::kMarkerEnds) ||
+        !node->HasIntListAttribute(ax::mojom::IntListAttribute::kMarkerTypes)) {
+      return;
+    }
+
+    const std::vector<int32_t>& marker_starts =
+        node->GetIntListAttribute(ax::mojom::IntListAttribute::kMarkerStarts);
+    const std::vector<int32_t>& marker_ends =
+        node->GetIntListAttribute(ax::mojom::IntListAttribute::kMarkerEnds);
+    const std::vector<int32_t>& marker_types =
+        node->GetIntListAttribute(ax::mojom::IntListAttribute::kMarkerTypes);
+
+    std::vector<v8::Local<v8::Object>> markers;
+    for (size_t i = 0; i < marker_types.size(); ++i) {
+      gin::DataObjectBuilder marker_obj(isolate);
+      marker_obj.Set("startOffset", marker_starts[i]);
+      marker_obj.Set("endOffset", marker_ends[i]);
+
+      gin::DataObjectBuilder flags(isolate);
+      int32_t marker_type = marker_types[i];
+      int32_t marker_pos = 1;
+      while (marker_type) {
+        if (marker_type & 1) {
+          flags.Set(
+              api::automation::ToString(ConvertMarkerTypeFromAXToAutomation(
+                  static_cast<ax::mojom::MarkerType>(marker_pos))),
+              true);
+        }
+        marker_type = marker_type >> 1;
+        marker_pos = marker_pos << 1;
+      }
+
+      marker_obj.Set("flags", flags.Build());
+      markers.push_back(marker_obj.Build());
+    }
+
+    result.Set(gin::ConvertToV8(isolate, markers));
+  });
   // Bindings that take a Tree ID and Node ID and string attribute name
   // and return a property of the node.
 

extensions/renderer/resources/automation/automation_node.js

@@ -515,6 +515,13 @@ var EventListenerAdded = natives.EventListenerAdded;
  */
 var EventListenerRemoved = natives.EventListenerRemoved;
 
+/**
+ * @param {string} axTreeID The id of the accessibility tree.
+ * @param {number} nodeID The id of a node.
+ * @return {Array}
+ */
+var GetMarkers = natives.GetMarkers;
+
 var logging = requireNative('logging');
 var utils = require('utils');
 
@@ -778,6 +785,10 @@ AutomationNodeImpl.prototype = {
     return GetWordEndOffsets(this.treeID, this.id);
   },
 
+  get markers() {
+    return GetMarkers(this.treeID, this.id);
+  },
+
   doDefault: function() {
     this.performAction_('doDefault');
   },
@@ -1244,9 +1255,6 @@ var nodeRefAttributes = [
 
 var intListAttributes = [
     'lineBreaks',
-    'markerEnds',
-    'markerStarts',
-    'markerTypes',
     'wordEnds',
     'wordStarts'];
 
@@ -1760,43 +1768,44 @@ utils.expose(AutomationNode, AutomationNodeImpl, {
   readonly: $Array.concat(
       publicAttributes,
       [
-        'parent',
-        'firstChild',
-        'lastChild',
-        'children',
-        'previousSibling',
-        'nextSibling',
-        'isRootNode',
-        'role',
+        'bold',
         'checked',
+        'children',
+        'customActions',
         'defaultActionVerb',
+        'descriptionFrom',
+        'detectedLanguage',
+        'firstChild',
         'hasPopup',
-        'restriction',
-        'state',
-        'location',
+        'htmlAttributes',
         'imageAnnotation',
         'indexInParent',
-        'lineStartOffsets',
-        'root',
-        'htmlAttributes',
-        'nameFrom',
-        'descriptionFrom',
-        'bold',
+        'isRootNode',
         'italic',
-        'underline',
+        'lastChild',
+        'lineStartOffsets',
         'lineThrough',
-        'detectedLanguage',
-        'customActions',
+        'location',
+        'markers',
+        'nameFrom',
+        'nextSibling',
+        'nonInlineTextWordEnds',
+        'nonInlineTextWordStarts',
+        'parent',
+        'previousSibling',
+        'restriction',
+        'role',
+        'root',
         'standardActions',
-        'unclippedLocation',
+        'state',
+        'tableCellAriaColumnIndex',
+        'tableCellAriaRowIndex',
         'tableCellColumnHeaders',
-        'tableCellRowHeaders',
         'tableCellColumnIndex',
+        'tableCellRowHeaders',
         'tableCellRowIndex',
-        'tableCellAriaRowIndex',
-        'tableCellAriaColumnIndex',
-        'nonInlineTextWordStarts',
-        'nonInlineTextWordEnds',
+        'unclippedLocation',
+        'underline',
       ]),
 });
 

third_party/closure_compiler/externs/automation.js

@@ -421,6 +421,18 @@ chrome.automation.DefaultActionVerb = {
   UNCHECK: 'uncheck',
 };
 
+/**
+ * @enum {string}
+ * @see https://developer.chrome.com/extensions/automation#type-MarkerType
+ */
+chrome.automation.MarkerType = {
+  SPELLING: 'spelling',
+  GRAMMAR: 'grammar',
+  TEXT_MATCH: 'textMatch',
+  ACTIVE_SUGGESTION: 'activeSuggestion',
+  SUGGESTION: 'suggestion',
+};
+
 /**
  * @typedef {{
  *   left: number,
@@ -541,6 +553,16 @@ chrome.automation.CustomAction;
  */
 chrome.automation.LanguageSpan;
 
+/**
+ * @typedef {{
+ *   startOffset: number,
+ *   endOffset: number,
+ *   flags: Object
+ * }}
+ * @see https://developer.chrome.com/extensions/automation#type-Marker
+ */
+chrome.automation.Marker;
+
 /**
  * @constructor
  * @private
@@ -940,25 +962,11 @@ chrome.automation.AutomationNode.prototype.textSelEnd;
 chrome.automation.AutomationNode.prototype.textInputType;
 
 /**
- * An array of indexes of the start position of each text marker.
- * @type {!Array<number>}
- * @see https://developer.chrome.com/extensions/automation#type-markerStarts
- */
-chrome.automation.AutomationNode.prototype.markerStarts;
-
-/**
- * An array of indexes of the end position of each text marker.
- * @type {!Array<number>}
- * @see https://developer.chrome.com/extensions/automation#type-markerEnds
- */
-chrome.automation.AutomationNode.prototype.markerEnds;
-
-/**
- * An array of numerical types indicating the type of each text marker, such as a spelling error.
- * @type {!Array<number>}
- * @see https://developer.chrome.com/extensions/automation#type-markerTypes
+ * An array of Marker objects for this node.
+ * @type {(!Array<!chrome.automation.Marker>|undefined)}
+ * @see https://developer.chrome.com/extensions/automation#type-markers
  */
-chrome.automation.AutomationNode.prototype.markerTypes;
+chrome.automation.AutomationNode.prototype.markers;
 
 /**
  * If a selection is present, whether the anchor of the selection comes after its focus in the accessibility tree.

ui/accessibility/PRESUBMIT.py

@@ -141,12 +141,14 @@ def CheckEnumsMatch(input_api, output_api):
                     output_api)
   CheckMatchingEnum(ax_enums, 'NameFrom', automation_enums, 'NameFromType',
                     errs, output_api)
-  CheckMatchingEnum(ax_enums, 'DescriptionFrom', automation_enums, 'DescriptionFromType',
-                    errs, output_api)
+  CheckMatchingEnum(ax_enums, 'DescriptionFrom', automation_enums,
+                    'DescriptionFromType', errs, output_api)
   CheckMatchingEnum(ax_enums, 'Restriction', automation_enums,
                    'Restriction', errs, output_api)
   CheckMatchingEnum(ax_enums, 'DefaultActionVerb', automation_enums,
                    'DefaultActionVerb', errs, output_api)
+  CheckMatchingEnum(ax_enums, 'MarkerType', automation_enums,
+                   'MarkerType', errs, output_api)
   return errs
 
 # Given a full path to c++ header, return an array of the first static