Make ChromeVox send mouse moves from js for touch explore events

This change is a speculative response to the bug below.

The bug observes that touch exploration is broken on CFM's main UI.

Assuming that observation is true, this change reverts back to sending mouse moves from js. This should be functionally the same as sending mouse moves from C++ in TouchExplorationController as an EventRewriter.

R=dmazzoni@chromium.org

Bug: b/161161531
Change-Id: I3ae0cfe421708e3630d2d5a0d99d402ec40fa7af
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2316451
Commit-Queue: David Tseng <dtseng@chromium.org>
Reviewed-by: Istiaque Ahmed <lazyboy@chromium.org>
Reviewed-by: Dominic Mazzoni <dmazzoni@chromium.org>
Cr-Commit-Position: refs/heads/master@{#792797}

chrome/browser/accessibility/accessibility_extension_api.cc

@@ -308,7 +308,7 @@ AccessibilityPrivateSendSyntheticMouseEventFunction::Run() {
   EXTENSION_FUNCTION_VALIDATE(params);
   accessibility_private::SyntheticMouseEvent* mouse_data = &params->mouse_event;
 
-  ui::EventType type;
+  ui::EventType type = ui::ET_UNKNOWN;
   switch (mouse_data->type) {
     case accessibility_private::SYNTHETIC_MOUSE_EVENT_TYPE_PRESS:
       type = ui::ET_MOUSE_PRESSED;
@@ -332,7 +332,14 @@ AccessibilityPrivateSendSyntheticMouseEventFunction::Run() {
       NOTREACHED();
   }
 
-  int flags = ui::EF_LEFT_MOUSE_BUTTON;
+  int flags = 0;
+  if (type != ui::ET_MOUSE_MOVED)
+    flags |= ui::EF_LEFT_MOUSE_BUTTON;
+
+  int changed_button_flags = flags;
+
+  if (mouse_data->touch_accessibility && *(mouse_data->touch_accessibility))
+    flags |= ui::EF_TOUCH_ACCESSIBILITY;
 
   // Locations are assumed to be display relative (and in DIPs).
   // TODO(crbug/893752) Choose correct display
@@ -341,7 +348,7 @@ AccessibilityPrivateSendSyntheticMouseEventFunction::Run() {
   std::unique_ptr<ui::MouseEvent> synthetic_mouse_event =
       std::make_unique<ui::MouseEvent>(type, location, location,
                                        ui::EventTimeForNow(), flags,
-                                       flags /* changed_button_flags */);
+                                       changed_button_flags);
 
   auto* host = ash::GetWindowTreeHostForDisplay(display.id());
   DCHECK(host);

chrome/browser/resources/chromeos/accessibility/chromevox/background/background_test.js

@@ -2560,15 +2560,13 @@ TEST_F('ChromeVoxBackgroundTest', 'HitTestOnExoSurface', function() {
               assertEquals(fakeWindow.location.top, evt.y);
             });
 
-        // Fake a touch explore gesture on the real text field. This should not
+        // Fake a mouse explore event on the real text field. This should not
         // trigger the above mouse path.
-        GestureCommandHandler.onAccessibilityGesture_(
-            'touchExplore', realTextField.location.left,
-            realTextField.location.top);
+        GestureCommandHandler.pointerHandler_.onMouseMove(
+            realTextField.location.left, realTextField.location.top);
 
-        // Fake a touch explore gesture event on the fake window. This does a
-        // real hit test on the fake window resulting in the fake window which
-        // should trigger the mouse path above.
+        // Fake a touch explore gesture event on the fake window which should
+        // trigger a mouse move.
         GestureCommandHandler.onAccessibilityGesture_(
             'touchExplore', fakeWindow.location.left, fakeWindow.location.top);
       });

chrome/browser/resources/chromeos/accessibility/chromevox/background/gesture_command_handler.js

@@ -46,7 +46,7 @@ GestureCommandHandler.onAccessibilityGesture_ = function(gesture, x, y) {
   EventSourceState.set(EventSourceType.TOUCH_GESTURE);
 
   if (gesture == 'touchExplore') {
-    GestureCommandHandler.pointerHandler_.onMove(x, y);
+    GestureCommandHandler.pointerHandler_.onTouchMove(x, y);
     return;
   }
 

chrome/browser/resources/chromeos/accessibility/chromevox/background/pointer_handler.js

@@ -57,18 +57,29 @@ PointerHandler = class extends BaseAutomationHandler {
    *
    * Note that runHitTest only ever does a hit test when |hasPendingEvents| is
    * true.
+   * @param {boolean} isTouch
    */
-  runHitTest() {
+  runHitTest(isTouch = false) {
     if (this.mouseX_ === undefined || this.mouseY_ === undefined) {
       return;
     }
     if (!this.hasPendingEvents_) {
       return;
     }
-    this.node_.hitTestWithReply(this.mouseX_, this.mouseY_, (target) => {
-      this.handleHitTestResult(target);
-      this.runHitTest();
-    });
+
+    if (isTouch) {
+      // TODO(accessibility): hit testing seems to be broken in some cases e.g.
+      // on the main CFM UI. Synthesize mouse moves with the touch
+      // accessibility flag for now for touch-based user gestures. Eliminate
+      // this branch once hit testing is fixed.
+      this.synthesizeMouseMove();
+    } else {
+      // Otherwise, use hit testing.
+      this.node_.hitTestWithReply(this.mouseX_, this.mouseY_, (target) => {
+        this.handleHitTestResult(target);
+        this.runHitTest();
+      });
+    }
     this.hasPendingEvents_ = false;
   }
 
@@ -80,16 +91,26 @@ PointerHandler = class extends BaseAutomationHandler {
     this.onMove(evt.mouseX, evt.mouseY);
   }
 
+  /**
+   * Handles touch move events.
+   * @param {number} x
+   * @param {number} y
+   */
+  onTouchMove(x, y) {
+    this.onMove(x, y, true);
+  }
+
   /**
    * Inform this handler of a move to (x, y).
    * @param {number} x
    * @param {number} y
+   * @param {boolean} isTouch
    */
-  onMove(x, y) {
+  onMove(x, y, isTouch = false) {
     this.mouseX_ = x;
     this.mouseY_ = y;
     this.hasPendingEvents_ = true;
-    this.runHitTest();
+    this.runHitTest(isTouch);
   }
 
   /**
@@ -103,7 +124,8 @@ PointerHandler = class extends BaseAutomationHandler {
     chrome.accessibilityPrivate.sendSyntheticMouseEvent({
       type: chrome.accessibilityPrivate.SyntheticMouseEventType.MOVE,
       x: this.mouseX_,
-      y: this.mouseY_
+      y: this.mouseY_,
+      touchAccessibility: true
     });
   }
 

chrome/common/extensions/api/accessibility_private.json

@@ -113,7 +113,12 @@
         "properties": {
           "type": {"$ref": "SyntheticMouseEventType"},
           "x": {"type": "integer", "description": "X coordinate for mouse event in global screen coordinates"},
-          "y": {"type": "integer", "description": "Y coordinate for mouse event in global screen coordinates"}
+          "y": {"type": "integer", "description": "Y coordinate for mouse event in global screen coordinates"},
+          "touchAccessibility": {
+            "type": "boolean",
+            "description": "True if the touch accessibility flag should be set.",
+            "optional": true
+          }
         }
       },
       {

third_party/closure_compiler/externs/accessibility_private.js

@@ -150,7 +150,8 @@ chrome.accessibilityPrivate.SyntheticMouseEventType = {
  * @typedef {{
  *   type: !chrome.accessibilityPrivate.SyntheticMouseEventType,
  *   x: number,
- *   y: number
+ *   y: number,
+ *   touchAccessibility: (boolean|undefined)
  * }}
  */
 chrome.accessibilityPrivate.SyntheticMouseEvent;
@@ -297,10 +298,10 @@ chrome.accessibilityPrivate.sendSyntheticMouseEvent = function(mouseEvent) {};
 chrome.accessibilityPrivate.onSelectToSpeakStateChanged = function(state) {};
 
 /**
- * Called by the Accessibility Common extension when findScrollableBoundsForPoint
- * has found a scrolling container. |rect| will be the bounds of the nearest
- * scrollable ancestor of the node at the point requested using
- * findScrollableBoundsForPoint.
+ * Called by the Accessibility Common extension when
+ * findScrollableBoundsForPoint has found a scrolling container. |rect| will be
+ * the bounds of the nearest scrollable ancestor of the node at the point
+ * requested using findScrollableBoundsForPoint.
  * @param {!chrome.accessibilityPrivate.ScreenRect} rect
  */
 chrome.accessibilityPrivate.onScrollableBoundsForPointFound = function(rect) {};