Exo: Support multiple touch surfaces

Exo previously dispatched all touch events to one focused surface: the
surface where the first touch point landed. For ARC++, this does not work
for supporting multiple displays because the display for the touch is
determined from the surface of the touch event. Therefore each touch
down event needs to be sent to the correct surface that it was directed
to.

This CL adds two unordered_maps to track the touch pointers and their
target surfaces: one mapping the touch pointer id to its surface, and
mapping the surface to the number of pointers on the surface. The latter
is needed to know when to stop tracking the surface's lifecycle.

Bug: b/145027838
Change-Id: I4121dac9c39bd9fb569df94330107ed382fe7234
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1931584
Commit-Queue: Prabir Pradhan <prabirmsp@chromium.org>
Auto-Submit: Prabir Pradhan <prabirmsp@chromium.org>
Reviewed-by: Mitsuru Oshima <oshima@chromium.org>
Cr-Commit-Position: refs/heads/master@{#732700}

components/exo/touch.cc

@@ -19,19 +19,6 @@
 namespace exo {
 namespace {
 
-// Helper function that returns an iterator to the first item in |vector|
-// with |value|.
-template <typename T, typename U>
-typename T::iterator FindVectorItem(T& vector, U value) {
-  return std::find(vector.begin(), vector.end(), value);
-}
-
-// Helper function that returns true if |vector| contains an item with |value|.
-template <typename T, typename U>
-bool VectorContainsItem(T& vector, U value) {
-  return FindVectorItem(vector, value) != vector.end();
-}
-
 gfx::PointF EventLocationInWindow(ui::TouchEvent* event, aura::Window* window) {
   ui::Layer* root = window->GetRootWindow()->layer();
   ui::Layer* target = window->layer();
@@ -57,8 +44,7 @@ Touch::~Touch() {
   delegate_->OnTouchDestroying(this);
   if (HasStylusDelegate())
     stylus_delegate_->OnTouchDestroying(this);
-  if (focus_)
-    focus_->RemoveSurfaceObserver(this);
+  CancelAllTouches();
   WMHelper::GetInstance()->RemovePreTargetHandler(this);
 }
 
@@ -80,30 +66,33 @@ void Touch::OnTouchEvent(ui::TouchEvent* event) {
   switch (event->type()) {
     case ui::ET_TOUCH_PRESSED: {
       // Early out if event doesn't contain a valid target for touch device.
+      // TODO(b/147848270): Verify GetEffectiveTargetForEvent gets the correct
+      // surface when input is captured.
       Surface* target = GetEffectiveTargetForEvent(event);
       if (!target)
         return;
 
       TRACE_EXO_INPUT_EVENT(event);
-
-      // If this is the first touch point then target becomes the focus surface
-      // until all touch points have been released.
-      if (touch_points_.empty()) {
-        DCHECK(!focus_);
-        focus_ = target;
-        focus_->AddSurfaceObserver(this);
+      DCHECK(touch_points_surface_map_.find(touch_pointer_id) ==
+             touch_points_surface_map_.end());
+
+      touch_points_surface_map_.emplace(touch_pointer_id, target);
+
+      // Update the count of pointers on the target surface.
+      auto it = surface_touch_count_map_.find(target);
+      if (it == surface_touch_count_map_.end()) {
+        target->AddSurfaceObserver(this);
+        surface_touch_count_map_.emplace(target, 1);
+      } else {
+        it->second++;
       }
 
-      DCHECK(!VectorContainsItem(touch_points_, touch_pointer_id));
-      touch_points_.push_back(touch_pointer_id);
+      // Convert location to target surface coordinate space.
+      const gfx::PointF location =
+          EventLocationInWindow(event, target->window());
 
-      // Convert location to focus surface coordinate space.
-      DCHECK(focus_);
-      gfx::PointF location = EventLocationInWindow(event, focus_->window());
-
-      // Generate a touch down event for the focus surface. Note that this can
-      // be different from the target surface.
-      delegate_->OnTouchDown(focus_, event->time_stamp(), touch_pointer_id,
+      // Generate a touch down event for the target surface.
+      delegate_->OnTouchDown(target, event->time_stamp(), touch_pointer_id,
                              location);
       if (stylus_delegate_ && event->pointer_details().pointer_type !=
                                   ui::EventPointerType::POINTER_TYPE_TOUCH) {
@@ -113,51 +102,51 @@ void Touch::OnTouchEvent(ui::TouchEvent* event) {
       send_details = true;
     } break;
     case ui::ET_TOUCH_RELEASED: {
-      auto it = FindVectorItem(touch_points_, touch_pointer_id);
-      if (it == touch_points_.end())
+      auto it = touch_points_surface_map_.find(touch_pointer_id);
+      if (it == touch_points_surface_map_.end())
         return;
 
+      Surface* target = it->second;
+      DCHECK(target);
+
       TRACE_EXO_INPUT_EVENT(event);
 
-      touch_points_.erase(it);
+      touch_points_surface_map_.erase(it);
 
-      // Reset focus surface if this is the last touch point.
-      if (touch_points_.empty()) {
-        DCHECK(focus_);
-        focus_->RemoveSurfaceObserver(this);
-        focus_ = nullptr;
+      // Update the count of pointers on the target surface.
+      auto count_it = surface_touch_count_map_.find(target);
+      if (count_it == surface_touch_count_map_.end())
+        return;
+      if ((--count_it->second) <= 0) {
+        surface_touch_count_map_.erase(target);
+        target->RemoveSurfaceObserver(this);
       }
 
       delegate_->OnTouchUp(event->time_stamp(), touch_pointer_id);
       seat_->AbortPendingDragOperation();
     } break;
     case ui::ET_TOUCH_MOVED: {
-      auto it = FindVectorItem(touch_points_, touch_pointer_id);
-      if (it == touch_points_.end())
+      auto it = touch_points_surface_map_.find(touch_pointer_id);
+      if (it == touch_points_surface_map_.end())
         return;
 
+      Surface* target = it->second;
+      DCHECK(target);
+
       TRACE_EXO_INPUT_EVENT(event);
 
-      DCHECK(focus_);
       // Convert location to focus surface coordinate space.
-      gfx::PointF location = EventLocationInWindow(event, focus_->window());
+      gfx::PointF location = EventLocationInWindow(event, target->window());
       delegate_->OnTouchMotion(event->time_stamp(), touch_pointer_id, location);
       send_details = true;
     } break;
     case ui::ET_TOUCH_CANCELLED: {
-      auto it = FindVectorItem(touch_points_, touch_pointer_id);
-      if (it == touch_points_.end())
-        return;
-
       TRACE_EXO_INPUT_EVENT(event);
 
-      DCHECK(focus_);
-      focus_->RemoveSurfaceObserver(this);
-      focus_ = nullptr;
-
       // Cancel the full set of touch sequences as soon as one is canceled.
-      touch_points_.clear();
+      CancelAllTouches();
       delegate_->OnTouchCancel();
+
       seat_->AbortPendingDragOperation();
     } break;
     default:
@@ -194,13 +183,9 @@ void Touch::OnTouchEvent(ui::TouchEvent* event) {
 // SurfaceObserver overrides:
 
 void Touch::OnSurfaceDestroying(Surface* surface) {
-  DCHECK(surface == focus_);
-  focus_ = nullptr;
-  surface->RemoveSurfaceObserver(this);
-
-  // Cancel touch sequences.
-  DCHECK_NE(touch_points_.size(), 0u);
-  touch_points_.clear();
+  // TODO(b/147848407): Do not cancel touches on surfaces of different clients
+  // when this surface dies.
+  CancelAllTouches();
   delegate_->OnTouchCancel();
 }
 
@@ -216,4 +201,12 @@ Surface* Touch::GetEffectiveTargetForEvent(ui::LocatedEvent* event) const {
   return delegate_->CanAcceptTouchEventsForSurface(target) ? target : nullptr;
 }
 
+void Touch::CancelAllTouches() {
+  std::for_each(surface_touch_count_map_.begin(),
+                surface_touch_count_map_.end(),
+                [this](auto& it) { it.first->RemoveSurfaceObserver(this); });
+  touch_points_surface_map_.clear();
+  surface_touch_count_map_.clear();
+}
+
 }  // namespace exo

components/exo/touch.h

@@ -5,8 +5,7 @@
 #ifndef COMPONENTS_EXO_TOUCH_H_
 #define COMPONENTS_EXO_TOUCH_H_
 
-#include <vector>
-
+#include "base/containers/flat_map.h"
 #include "base/macros.h"
 #include "components/exo/surface_observer.h"
 #include "ui/events/event_handler.h"
@@ -45,6 +44,9 @@ class Touch : public ui::EventHandler, public SurfaceObserver {
   // Returns the effective target for |event|.
   Surface* GetEffectiveTargetForEvent(ui::LocatedEvent* event) const;
 
+  // Cancels touches on all the surfaces.
+  void CancelAllTouches();
+
   // The delegate instance that all events are dispatched to.
   TouchDelegate* const delegate_;
 
@@ -53,11 +55,11 @@ class Touch : public ui::EventHandler, public SurfaceObserver {
   // The delegate instance that all stylus related events are dispatched to.
   TouchStylusDelegate* stylus_delegate_ = nullptr;
 
-  // The current focus surface for the touch device.
-  Surface* focus_ = nullptr;
+  // Map of touch points to its focus surface.
+  base::flat_map<int, Surface*> touch_points_surface_map_;
 
-  // Vector of touch points in focus surface.
-  std::vector<int> touch_points_;
+  // Map of a touched surface to the count of touch pointers on that surface.
+  base::flat_map<Surface*, int> surface_touch_count_map_;
 
   DISALLOW_COPY_AND_ASSIGN(Touch);
 };

components/exo/touch_unittest.cc

@@ -245,6 +245,79 @@ TEST_F(TouchTest, OnTouchCancel) {
   touch.reset();
 }
 
+TEST_F(TouchTest, OnTouchCancelWhenSurfaceDestroying) {
+  auto surface = std::make_unique<Surface>();
+  auto shell_surface = std::make_unique<ShellSurface>(surface.get());
+  gfx::Size buffer_size(10, 10);
+  auto buffer = std::make_unique<Buffer>(
+      exo_test_helper()->CreateGpuMemoryBuffer(buffer_size));
+  surface->Attach(buffer.get());
+  surface->Commit();
+
+  MockTouchDelegate delegate;
+  Seat seat;
+  auto touch = std::make_unique<Touch>(&delegate, &seat);
+  ui::test::EventGenerator generator(ash::Shell::GetPrimaryRootWindow());
+
+  EXPECT_CALL(delegate, OnTouchShape(testing::_, testing::_, testing::_))
+      .Times(testing::AnyNumber());
+  EXPECT_CALL(delegate, CanAcceptTouchEventsForSurface(surface.get()))
+      .WillRepeatedly(testing::Return(true));
+  EXPECT_CALL(delegate,
+              OnTouchDown(surface.get(), testing::_, 1, gfx::PointF()));
+  EXPECT_CALL(delegate, OnTouchFrame());
+  const gfx::Point origin = surface->window()->GetBoundsInScreen().origin();
+  generator.set_current_screen_location(origin);
+  generator.PressTouchId(1);
+
+  // Since there is an active touch pointer on the surface, destroying the
+  // surface should cancel touches.
+  EXPECT_CALL(delegate, OnTouchCancel());
+  EXPECT_CALL(delegate, OnTouchFrame());
+  surface.reset();
+
+  EXPECT_CALL(delegate, OnTouchDestroying(touch.get()));
+  touch.reset();
+}
+
+TEST_F(TouchTest, OnTouchCancelNotTriggeredAfterTouchReleased) {
+  auto surface = std::make_unique<Surface>();
+  auto shell_surface = std::make_unique<ShellSurface>(surface.get());
+  gfx::Size buffer_size(10, 10);
+  auto buffer = std::make_unique<Buffer>(
+      exo_test_helper()->CreateGpuMemoryBuffer(buffer_size));
+  surface->Attach(buffer.get());
+  surface->Commit();
+
+  MockTouchDelegate delegate;
+  Seat seat;
+  auto touch = std::make_unique<Touch>(&delegate, &seat);
+  ui::test::EventGenerator generator(ash::Shell::GetPrimaryRootWindow());
+
+  EXPECT_CALL(delegate, OnTouchShape(testing::_, testing::_, testing::_))
+      .Times(testing::AnyNumber());
+  EXPECT_CALL(delegate, CanAcceptTouchEventsForSurface(surface.get()))
+      .WillRepeatedly(testing::Return(true));
+  EXPECT_CALL(delegate,
+              OnTouchDown(surface.get(), testing::_, 1, gfx::PointF()));
+  EXPECT_CALL(delegate, OnTouchFrame());
+  const gfx::Point origin = surface->window()->GetBoundsInScreen().origin();
+  generator.set_current_screen_location(origin);
+  generator.PressTouchId(1);
+
+  EXPECT_CALL(delegate, OnTouchUp(testing::_, 1));
+  EXPECT_CALL(delegate, OnTouchFrame());
+  generator.ReleaseTouchId(1);
+
+  // Since the surface no longer has any active touch pointers, destroying the
+  // surface should not cancel any touches.
+  EXPECT_CALL(delegate, OnTouchCancel()).Times(0);
+  surface.reset();
+
+  EXPECT_CALL(delegate, OnTouchDestroying(touch.get()));
+  touch.reset();
+}
+
 TEST_F(TouchTest, IgnoreTouchEventDuringModal) {
   auto window = exo_test_helper()->CreateWindow(10, 10, false);
   auto modal = exo_test_helper()->CreateWindow(5, 5, true);
@@ -465,5 +538,77 @@ TEST_F(TouchTest, DragDropAbort) {
   touch.reset();
 }
 
+TEST_F(TouchTest, TouchMultipleSurfaces) {
+  auto surface = std::make_unique<Surface>();
+  auto shell_surface = std::make_unique<ShellSurface>(surface.get());
+  gfx::Size buffer_size(10, 10);
+  auto buffer = std::make_unique<Buffer>(
+      exo_test_helper()->CreateGpuMemoryBuffer(buffer_size));
+  surface->Attach(buffer.get());
+  surface->Commit();
+
+  auto child_surface = std::make_unique<Surface>();
+  auto child_shell_surface = std::make_unique<ShellSurface>(
+      child_surface.get(), gfx::Point(), true, false,
+      ash::desks_util::GetActiveDeskContainerId());
+  child_shell_surface->DisableMovement();
+  child_shell_surface->SetParent(shell_surface.get());
+  gfx::Size child_buffer_size(15, 15);
+  auto child_buffer = std::make_unique<Buffer>(
+      exo_test_helper()->CreateGpuMemoryBuffer(child_buffer_size));
+  child_surface->Attach(child_buffer.get());
+  child_surface->Commit();
+
+  MockTouchDelegate delegate;
+  Seat seat;
+  auto touch = std::make_unique<Touch>(&delegate, &seat);
+  ui::test::EventGenerator generator(ash::Shell::GetPrimaryRootWindow());
+
+  // Touch down on the two surfaces.
+  EXPECT_CALL(delegate, OnTouchShape(testing::_, testing::_, testing::_))
+      .Times(testing::AnyNumber());
+  EXPECT_CALL(delegate, CanAcceptTouchEventsForSurface(surface.get()))
+      .WillRepeatedly(testing::Return(true));
+  EXPECT_CALL(delegate,
+              OnTouchDown(surface.get(), testing::_, 1, gfx::PointF()));
+  EXPECT_CALL(delegate, OnTouchFrame());
+  const gfx::Point origin = surface->window()->GetBoundsInScreen().origin();
+  generator.set_current_screen_location(origin);
+  generator.PressTouchId(1);
+
+  EXPECT_CALL(delegate, CanAcceptTouchEventsForSurface(child_surface.get()))
+      .WillRepeatedly(testing::Return(true));
+  EXPECT_CALL(delegate,
+              OnTouchDown(child_surface.get(), testing::_, 2, gfx::PointF()));
+  EXPECT_CALL(delegate, OnTouchFrame());
+  const gfx::Point child_origin =
+      child_surface->window()->GetBoundsInScreen().origin();
+  generator.set_current_screen_location(child_origin);
+  generator.PressTouchId(2);
+
+  // Move the two touch pointers.
+  for (int i = 1; i <= 10; i++) {
+    EXPECT_CALL(delegate, OnTouchMotion(testing::_, 1, gfx::PointF(i, i)));
+    EXPECT_CALL(delegate, OnTouchFrame());
+    generator.MoveTouchId(origin + gfx::Vector2d(i, i), 1);
+
+    EXPECT_CALL(delegate, OnTouchMotion(testing::_, 2, gfx::PointF(i, i)));
+    EXPECT_CALL(delegate, OnTouchFrame());
+    generator.MoveTouchId(child_origin + gfx::Vector2d(i, i), 2);
+  }
+
+  // Release the two touch pointers.
+  EXPECT_CALL(delegate, OnTouchUp(testing::_, 2));
+  EXPECT_CALL(delegate, OnTouchFrame());
+  generator.ReleaseTouchId(2);
+
+  EXPECT_CALL(delegate, OnTouchUp(testing::_, 1));
+  EXPECT_CALL(delegate, OnTouchFrame());
+  generator.ReleaseTouchId(1);
+
+  EXPECT_CALL(delegate, OnTouchDestroying(touch.get()));
+  touch.reset();
+}
+
 }  // namespace
 }  // namespace exo