Supports drag and drop actions with Autoclick.

This works by starting the drag on the first dwell
and ending the drag on the second dwell, and rewriting
mouse moved events to mouse dragged events in between.

This is still hidden behind the accessibility flag.

Bug: 894907
Change-Id: Ia2cd72053e3d1023711764e2b581caa686f2fb98
Reviewed-on: https://chromium-review.googlesource.com/c/1295250
Commit-Queue: Katie Dektar <katie@chromium.org>
Reviewed-by: James Cook <jamescook@chromium.org>
Reviewed-by: David Tseng <dtseng@chromium.org>
Cr-Commit-Position: refs/heads/master@{#602512}

ash/BUILD.gn

@@ -248,6 +248,8 @@ component("ash") {
     "assistant/util/views_util.h",
     "autoclick/autoclick_controller.cc",
     "autoclick/autoclick_controller.h",
+    "autoclick/autoclick_drag_event_rewriter.cc",
+    "autoclick/autoclick_drag_event_rewriter.h",
     "autoclick/autoclick_ring_handler.cc",
     "autoclick/autoclick_ring_handler.h",
     "bluetooth_devices_observer.cc",
@@ -1729,6 +1731,7 @@ test("ash_unittests") {
     "assistant/assistant_screen_context_controller_unittest.cc",
     "assistant/ui/assistant_container_view_unittest.cc",
     "assistant/util/deep_link_util_unittest.cc",
+    "autoclick/autoclick_drag_event_rewriter_unittest.cc",
     "autoclick/autoclick_unittest.cc",
     "cursor_unittest.cc",
     "detachable_base/detachable_base_handler_unittest.cc",

ash/autoclick/autoclick_controller.cc

@@ -4,6 +4,7 @@
 
 #include "ash/autoclick/autoclick_controller.h"
 
+#include "ash/autoclick/autoclick_drag_event_rewriter.h"
 #include "ash/autoclick/autoclick_ring_handler.h"
 #include "ash/public/cpp/ash_constants.h"
 #include "ash/public/cpp/shell_window_ids.h"
@@ -72,12 +73,19 @@ AutoclickController::AutoclickController()
       delay_(GetDefaultAutoclickDelay()),
       mouse_event_flags_(ui::EF_NONE),
       anchor_location_(-kMovementThreshold, -kMovementThreshold),
-      autoclick_ring_handler_(std::make_unique<AutoclickRingHandler>()) {
+      autoclick_ring_handler_(std::make_unique<AutoclickRingHandler>()),
+      drag_event_rewriter_(std::make_unique<AutoclickDragEventRewriter>()) {
+  Shell::GetPrimaryRootWindow()->GetHost()->GetEventSource()->AddEventRewriter(
+      drag_event_rewriter_.get());
   InitClickTimer();
 }
 
 AutoclickController::~AutoclickController() {
   SetTapDownTarget(nullptr);
+  Shell::GetPrimaryRootWindow()
+      ->GetHost()
+      ->GetEventSource()
+      ->RemoveEventRewriter(drag_event_rewriter_.get());
 }
 
 void AutoclickController::SetTapDownTarget(aura::Window* target) {
@@ -171,24 +179,39 @@ void AutoclickController::DoAutoclickAction() {
   aura::WindowTreeHost* host = root_window->GetHost();
   host->ConvertDIPToPixels(&location_in_pixels);
 
-  // TODO(katie): Implement drag-and-drop.
+  bool drag_start = event_type_ == mojom::AutoclickEventType::kDragAndDrop &&
+                    !drag_event_rewriter_->IsEnabled();
+  bool drag_stop = event_type_ == mojom::AutoclickEventType::kDragAndDrop &&
+                   drag_event_rewriter_->IsEnabled();
+
   if (event_type_ == mojom::AutoclickEventType::kLeftClick ||
       event_type_ == mojom::AutoclickEventType::kRightClick ||
-      event_type_ == mojom::AutoclickEventType::kDoubleClick) {
+      event_type_ == mojom::AutoclickEventType::kDoubleClick || drag_start ||
+      drag_stop) {
     int button = event_type_ == mojom::AutoclickEventType::kRightClick
                      ? ui::EF_RIGHT_MOUSE_BUTTON
                      : ui::EF_LEFT_MOUSE_BUTTON;
-    ui::MouseEvent press_event(ui::ET_MOUSE_PRESSED, location_in_pixels,
-                               location_in_pixels, ui::EventTimeForNow(),
-                               mouse_event_flags_ | button, button);
+
+    ui::EventDispatchDetails details;
+    if (!drag_stop) {
+      // Left click, right click, double click, and beginning of a drag have
+      // a pressed event next.
+      ui::MouseEvent press_event(ui::ET_MOUSE_PRESSED, location_in_pixels,
+                                 location_in_pixels, ui::EventTimeForNow(),
+                                 mouse_event_flags_ | button, button);
+      details = host->event_sink()->OnEventFromSource(&press_event);
+      if (drag_start) {
+        drag_event_rewriter_->SetEnabled(true);
+        return;
+      }
+      if (details.dispatcher_destroyed)
+        return;
+    }
+    if (drag_stop)
+      drag_event_rewriter_->SetEnabled(false);
     ui::MouseEvent release_event(ui::ET_MOUSE_RELEASED, location_in_pixels,
                                  location_in_pixels, ui::EventTimeForNow(),
                                  mouse_event_flags_ | button, button);
-
-    ui::EventDispatchDetails details =
-        host->event_sink()->OnEventFromSource(&press_event);
-    if (details.dispatcher_destroyed)
-      return;
     details = host->event_sink()->OnEventFromSource(&release_event);
 
     // Now a single click has been completed.
@@ -216,6 +239,7 @@ void AutoclickController::CancelAutoclickAction() {
     autoclick_timer_->Stop();
   }
   autoclick_ring_handler_->StopGesture();
+  drag_event_rewriter_->SetEnabled(false);
   SetTapDownTarget(nullptr);
 }
 
@@ -240,8 +264,10 @@ void AutoclickController::OnMouseEvent(ui::MouseEvent* event) {
   if (event_type_ == mojom::AutoclickEventType::kNoAction)
     return;
   gfx::Point point_in_screen = event->target()->GetScreenLocation(*event);
-  if (event->type() == ui::ET_MOUSE_MOVED &&
-      !(event->flags() & ui::EF_IS_SYNTHESIZED)) {
+  if (!(event->flags() & ui::EF_IS_SYNTHESIZED) &&
+      (event->type() == ui::ET_MOUSE_MOVED ||
+       (event->type() == ui::ET_MOUSE_DRAGGED &&
+        drag_event_rewriter_->IsEnabled()))) {
     mouse_event_flags_ = event->flags();
     UpdateRingWidget(point_in_screen);
 
@@ -260,7 +286,8 @@ void AutoclickController::OnMouseEvent(ui::MouseEvent* event) {
     } else if (autoclick_timer_->IsRunning()) {
       autoclick_ring_handler_->SetGestureCenter(point_in_screen, widget_.get());
     }
-  } else if (event->type() == ui::ET_MOUSE_PRESSED) {
+  } else if (event->type() == ui::ET_MOUSE_PRESSED ||
+             event->type() == ui::ET_MOUSE_RELEASED) {
     CancelAutoclickAction();
   } else if (event->type() == ui::ET_MOUSEWHEEL &&
              autoclick_timer_->IsRunning()) {

ash/autoclick/autoclick_controller.h

@@ -23,6 +23,7 @@ class Widget;
 
 namespace ash {
 
+class AutoclickDragEventRewriter;
 class AutoclickRingHandler;
 
 // Autoclick is one of the accessibility features. If enabled, two circles will
@@ -84,6 +85,7 @@ class ASH_EXPORT AutoclickController : public ui::EventHandler,
   // the distance the mouse has moved.
   gfx::Point anchor_location_;
   std::unique_ptr<AutoclickRingHandler> autoclick_ring_handler_;
+  std::unique_ptr<AutoclickDragEventRewriter> drag_event_rewriter_;
 
   DISALLOW_COPY_AND_ASSIGN(AutoclickController);
 };

ash/autoclick/autoclick_drag_event_rewriter.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "ash/autoclick/autoclick_drag_event_rewriter.h"
+
+namespace ash {
+
+void AutoclickDragEventRewriter::SetEnabled(bool enabled) {
+  enabled_ = enabled;
+}
+
+bool AutoclickDragEventRewriter::IsEnabled() const {
+  return enabled_;
+}
+
+ui::EventRewriteStatus AutoclickDragEventRewriter::RewriteEvent(
+    const ui::Event& event,
+    std::unique_ptr<ui::Event>* new_event) {
+  // Only rewrite mouse moved events to drag events when enabled.
+  if (!enabled_ || event.type() != ui::ET_MOUSE_MOVED)
+    return ui::EVENT_REWRITE_CONTINUE;
+  // TODO(katie): Should this have an ui::EF_LEFT_MOUSE_BUTTON flag for drag?
+  const ui::MouseEvent* mouse_event = event.AsMouseEvent();
+  ui::MouseEvent* rewritten_event = new ui::MouseEvent(
+      ui::ET_MOUSE_DRAGGED, mouse_event->location(),
+      mouse_event->root_location(), mouse_event->time_stamp(),
+      mouse_event->flags(), mouse_event->changed_button_flags(),
+      mouse_event->pointer_details());
+  new_event->reset(rewritten_event);
+  return ui::EVENT_REWRITE_REWRITTEN;
+}
+
+ui::EventRewriteStatus AutoclickDragEventRewriter::NextDispatchEvent(
+    const ui::Event& last_event,
+    std::unique_ptr<ui::Event>* new_event) {
+  // Unused.
+  return ui::EVENT_REWRITE_CONTINUE;
+}
+
+}  // namespace ash
\ No newline at end of file

ash/autoclick/autoclick_drag_event_rewriter.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef ASH_AUTOCLICK_AUTOCLICK_DRAG_EVENT_REWRITER_H_
+#define ASH_AUTOCLICK_AUTOCLICK_DRAG_EVENT_REWRITER_H_
+
+#include "ash/ash_export.h"
+#include "ui/events/event_rewriter.h"
+
+namespace ash {
+
+// EventRewriter to change move events into drag events during drag-and-drop
+// actions from Autoclick.
+class ASH_EXPORT AutoclickDragEventRewriter : public ui::EventRewriter {
+ public:
+  AutoclickDragEventRewriter() = default;
+  ~AutoclickDragEventRewriter() override = default;
+
+  void SetEnabled(bool enabled);
+  bool IsEnabled() const;
+
+  // ui::EventRewriter (visible for testing):
+  ui::EventRewriteStatus RewriteEvent(
+      const ui::Event& event,
+      std::unique_ptr<ui::Event>* new_event) override;
+
+ private:
+  // ui::EventRewriter:
+  ui::EventRewriteStatus NextDispatchEvent(
+      const ui::Event& last_event,
+      std::unique_ptr<ui::Event>* new_event) override;
+
+  bool enabled_ = false;
+
+  DISALLOW_COPY_AND_ASSIGN(AutoclickDragEventRewriter);
+};
+
+}  // namespace ash
+
+#endif  // ASH_AUTOCLICK_AUTOCLICK_DRAG_EVENT_REWRITER_H_

ash/autoclick/autoclick_drag_event_rewriter_unittest.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "ash/autoclick/autoclick_drag_event_rewriter.h"
+#include "ash/test/ash_test_base.h"
+#include "ui/aura/window.h"
+#include "ui/aura/window_tree_host.h"
+#include "ui/events/base_event_utils.h"
+#include "ui/events/event_rewriter.h"
+#include "ui/events/keycodes/keyboard_codes.h"
+#include "ui/events/test/event_generator.h"
+
+namespace ash {
+
+class EventRecorder : public ui::EventRewriter {
+ public:
+  EventRecorder() = default;
+  ~EventRecorder() override = default;
+
+  // ui::EventRewriter:
+  ui::EventRewriteStatus RewriteEvent(
+      const ui::Event& event,
+      std::unique_ptr<ui::Event>* new_event) override {
+    recorded_event_count_++;
+    last_recorded_event_type_ = event.type();
+    return ui::EVENT_REWRITE_CONTINUE;
+  }
+  ui::EventRewriteStatus NextDispatchEvent(
+      const ui::Event& last_event,
+      std::unique_ptr<ui::Event>* new_event) override {
+    NOTREACHED();
+    return ui::EVENT_REWRITE_CONTINUE;
+  }
+
+  // Count of events sent to the rewriter.
+  size_t recorded_event_count_ = 0;
+  ui::EventType last_recorded_event_type_ = ui::EventType::ET_UNKNOWN;
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(EventRecorder);
+};
+
+class AutoclickDragEventRewriterTest : public AshTestBase {
+ public:
+  AutoclickDragEventRewriterTest() = default;
+
+  ~AutoclickDragEventRewriterTest() override = default;
+
+  void SetUp() override {
+    AshTestBase::SetUp();
+    generator_ = AshTestBase::GetEventGenerator();
+    CurrentContext()->GetHost()->GetEventSource()->AddEventRewriter(
+        &drag_event_rewriter_);
+    CurrentContext()->GetHost()->GetEventSource()->AddEventRewriter(
+        &event_recorder_);
+  }
+
+  void TearDown() override {
+    CurrentContext()->GetHost()->GetEventSource()->RemoveEventRewriter(
+        &event_recorder_);
+    CurrentContext()->GetHost()->GetEventSource()->RemoveEventRewriter(
+        &drag_event_rewriter_);
+    generator_ = nullptr;
+    AshTestBase::TearDown();
+  }
+
+ protected:
+  // Generates ui::Events from simulated user input.
+  ui::test::EventGenerator* generator_ = nullptr;
+  // Records events delivered to the next event rewriter after
+  // AutoclickDragEventRewriter.
+  EventRecorder event_recorder_;
+
+  AutoclickDragEventRewriter drag_event_rewriter_;
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(AutoclickDragEventRewriterTest);
+};
+
+TEST_F(AutoclickDragEventRewriterTest, EventsNotConsumedWhenDisabled) {
+  drag_event_rewriter_.SetEnabled(false);
+  // Events are not consume.
+  generator_->PressKey(ui::VKEY_A, ui::EF_NONE);
+  EXPECT_EQ(1U, event_recorder_.recorded_event_count_);
+  generator_->ReleaseKey(ui::VKEY_A, ui::EF_NONE);
+  EXPECT_EQ(2U, event_recorder_.recorded_event_count_);
+  generator_->PressLeftButton();
+  EXPECT_EQ(3U, event_recorder_.recorded_event_count_);
+  EXPECT_EQ(ui::EventType::ET_MOUSE_PRESSED,
+            event_recorder_.last_recorded_event_type_);
+  generator_->MoveMouseTo(gfx::Point(200, 200), 1);
+  EXPECT_EQ(4U, event_recorder_.recorded_event_count_);
+  EXPECT_EQ(ui::EventType::ET_MOUSE_DRAGGED,
+            event_recorder_.last_recorded_event_type_);
+  generator_->ReleaseLeftButton();
+  EXPECT_EQ(5U, event_recorder_.recorded_event_count_);
+  EXPECT_EQ(ui::EventType::ET_MOUSE_RELEASED,
+            event_recorder_.last_recorded_event_type_);
+
+  // Move events are not consumed either.
+  generator_->MoveMouseTo(gfx::Point(100, 100), 1);
+  EXPECT_EQ(6U, event_recorder_.recorded_event_count_);
+  EXPECT_EQ(ui::EventType::ET_MOUSE_MOVED,
+            event_recorder_.last_recorded_event_type_);
+}
+
+TEST_F(AutoclickDragEventRewriterTest, OnlyMouseMoveEventsConsumedWhenEnabled) {
+  drag_event_rewriter_.SetEnabled(true);
+  // Most events are still not consumed.
+  generator_->PressKey(ui::VKEY_A, ui::EF_NONE);
+  EXPECT_EQ(1U, event_recorder_.recorded_event_count_);
+  generator_->ReleaseKey(ui::VKEY_A, ui::EF_NONE);
+  EXPECT_EQ(2U, event_recorder_.recorded_event_count_);
+  generator_->PressLeftButton();
+  EXPECT_EQ(3U, event_recorder_.recorded_event_count_);
+  EXPECT_EQ(ui::EventType::ET_MOUSE_PRESSED,
+            event_recorder_.last_recorded_event_type_);
+  generator_->MoveMouseTo(gfx::Point(200, 200), 1);
+  EXPECT_EQ(4U, event_recorder_.recorded_event_count_);
+  EXPECT_EQ(ui::EventType::ET_MOUSE_DRAGGED,
+            event_recorder_.last_recorded_event_type_);
+  generator_->ReleaseLeftButton();
+  EXPECT_EQ(5U, event_recorder_.recorded_event_count_);
+  EXPECT_EQ(ui::EventType::ET_MOUSE_RELEASED,
+            event_recorder_.last_recorded_event_type_);
+
+  // Mouse move events are consumed and changed into drag events.
+  generator_->MoveMouseTo(gfx::Point(100, 100), 1);
+  EXPECT_EQ(5U, event_recorder_.recorded_event_count_);
+  generator_->MoveMouseTo(gfx::Point(150, 150), 1);
+  EXPECT_EQ(5U, event_recorder_.recorded_event_count_);
+}
+
+TEST_F(AutoclickDragEventRewriterTest, RewritesMouseMovesToDrags) {
+  drag_event_rewriter_.SetEnabled(true);
+  base::TimeTicks time_stamp = ui::EventTimeForNow();
+  gfx::Point location(100, 100);
+  gfx::Point root_location(150, 150);
+  int flags = ui::EF_SHIFT_DOWN;                        // Set a random flag.
+  int changed_button_flags = ui::EF_LEFT_MOUSE_BUTTON;  // Set a random flag.
+  ui::MouseEvent event(ui::EventType::ET_MOUSE_MOVED, location, root_location,
+                       time_stamp, flags, changed_button_flags);
+  std::unique_ptr<ui::Event> rewritten_event;
+  ui::EventRewriteStatus status =
+      drag_event_rewriter_.RewriteEvent(event, &rewritten_event);
+  EXPECT_EQ(ui::EVENT_REWRITE_REWRITTEN, status);
+  // The type should be a drag.
+  ASSERT_EQ(ui::ET_MOUSE_DRAGGED, rewritten_event->type());
+
+  // Everything else should be the same as the original.
+  ui::MouseEvent* rewritten_mouse_event = rewritten_event->AsMouseEvent();
+  EXPECT_EQ(location, rewritten_mouse_event->location());
+  EXPECT_EQ(root_location, rewritten_mouse_event->root_location());
+  EXPECT_EQ(time_stamp, rewritten_mouse_event->time_stamp());
+  EXPECT_EQ(flags, rewritten_mouse_event->flags());
+  EXPECT_EQ(changed_button_flags,
+            rewritten_mouse_event->changed_button_flags());
+}
+
+}  // namespace ash

ash/autoclick/autoclick_unittest.cc

@@ -26,24 +26,33 @@ class MouseEventCapturer : public ui::EventHandler {
   void Reset() { events_.clear(); }
 
   void OnMouseEvent(ui::MouseEvent* event) override {
-    // Only track left and right mouse button events, ensuring that we get
-    // left-click, right-click and double-click.
-    if (!(event->flags() & ui::EF_LEFT_MOUSE_BUTTON) &&
-        (!(event->flags() & ui::EF_RIGHT_MOUSE_BUTTON)))
-      return;
+    bool save_event = false;
+    bool stop_event = false;
     // Filter out extraneous mouse events like mouse entered, exited,
     // capture changed, etc.
     ui::EventType type = event->type();
-    if (type == ui::ET_MOUSE_MOVED || type == ui::ET_MOUSE_PRESSED ||
-        type == ui::ET_MOUSE_RELEASED) {
+    if (type == ui::ET_MOUSE_PRESSED || type == ui::ET_MOUSE_RELEASED) {
+      // Only track left and right mouse button events, ensuring that we get
+      // left-click, right-click and double-click.
+      if (!(event->flags() & ui::EF_LEFT_MOUSE_BUTTON) &&
+          (!(event->flags() & ui::EF_RIGHT_MOUSE_BUTTON)))
+        return;
+      save_event = true;
+      // Stop event propagation so we don't click on random stuff that
+      // might break test assumptions.
+      stop_event = true;
+    } else if (type == ui::ET_MOUSE_DRAGGED) {
+      save_event = true;
+      stop_event = false;
+    }
+    if (save_event) {
       events_.push_back(ui::MouseEvent(event->type(), event->location(),
                                        event->root_location(),
                                        ui::EventTimeForNow(), event->flags(),
                                        event->changed_button_flags()));
-      // Stop event propagation so we don't click on random stuff that
-      // might break test assumptions.
-      event->StopPropagation();
     }
+    if (stop_event)
+      event->StopPropagation();
 
     // If there is a possibility that we're in an infinite loop, we should
     // exit early with a sensible error rather than letting the test time out.
@@ -89,15 +98,21 @@ class AutoclickTest : public AshTestBase {
   }
 
   const std::vector<ui::MouseEvent>& WaitForMouseEvents() {
-    mouse_event_capturer_.Reset();
+    ClearMouseEvents();
     RunAllPendingInMessageLoop();
-    return mouse_event_capturer_.captured_events();
+    return GetMouseEvents();
   }
 
   AutoclickController* GetAutoclickController() {
     return Shell::Get()->autoclick_controller();
   }
 
+  void ClearMouseEvents() { mouse_event_capturer_.Reset(); }
+
+  const std::vector<ui::MouseEvent>& GetMouseEvents() {
+    return mouse_event_capturer_.captured_events();
+  }
+
  private:
   MouseEventCapturer mouse_event_capturer_;
 
@@ -359,4 +374,35 @@ TEST_F(AutoclickTest, AutoclickChangeEventTypes) {
   EXPECT_EQ(0u, events.size());
 }
 
+TEST_F(AutoclickTest, AutoclickDragAndDropEvents) {
+  GetAutoclickController()->SetEnabled(true);
+  GetAutoclickController()->SetAutoclickEventType(
+      mojom::AutoclickEventType::kDragAndDrop);
+  std::vector<ui::MouseEvent> events;
+
+  GetEventGenerator()->MoveMouseTo(30, 30);
+  events = WaitForMouseEvents();
+  ASSERT_EQ(1u, events.size());
+  EXPECT_EQ(ui::ET_MOUSE_PRESSED, events[0].type());
+  EXPECT_TRUE(ui::EF_LEFT_MOUSE_BUTTON & events[0].flags());
+
+  ClearMouseEvents();
+  GetEventGenerator()->MoveMouseTo(60, 60);
+  events = GetMouseEvents();
+  ASSERT_EQ(1u, events.size());
+  EXPECT_EQ(ui::ET_MOUSE_DRAGGED, events[0].type());
+
+  // Another move creates a drag
+  ClearMouseEvents();
+  GetEventGenerator()->MoveMouseTo(90, 90);
+  events = GetMouseEvents();
+  ASSERT_EQ(1u, events.size());
+  EXPECT_EQ(ui::ET_MOUSE_DRAGGED, events[0].type());
+
+  // Waiting in place creates the released event.
+  events = WaitForMouseEvents();
+  ASSERT_EQ(1u, events.size());
+  EXPECT_EQ(ui::ET_MOUSE_RELEASED, events[0].type());
+}
+
 }  // namespace ash

ash/shell.cc

@@ -832,6 +832,7 @@ Shell::~Shell() {
   // These need a valid Shell instance to clean up properly, so explicitly
   // delete them before invalidating the instance.
   // Alphabetical. TODO(oshima): sort.
+  autoclick_controller_.reset();
   magnification_controller_.reset();
   tooltip_controller_.reset();
   event_client_.reset();