exo: Add pointer_stylus interface to zcr_stylus

A stylus device can either report MouseEvents or TouchEvents. Styli used
on built-in displays should report TouchEvents, whereas those used on
external digitizers should report MouseEvents. This way, moving the
stylus on an opaque digitizer will also result in mouse/hover movements.

The zcr_stylus interface did not previously have a way to report stylus
events that were MouseEvents. We add a zcr_pointer_stylus_v2 interface
that extends the wl_pointer protocol so that the following additional
details can be reported with MouseEvents:
- tool change (mouse, touch, pen, eraser)
- force change (pressure)
- tool tilt change

BUG=b:169094254
TEST=exo_unittests

Change-Id: I0c4ac8412e1d5f7f01911ce3e0acbe6f8fbcad4a
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2426981
Auto-Submit: Prabir Pradhan <prabirmsp@chromium.org>
Reviewed-by: Mitsuru Oshima <oshima@chromium.org>
Commit-Queue: Prabir Pradhan <prabirmsp@chromium.org>
Cr-Commit-Position: refs/heads/master@{#810791}

components/exo/pointer.cc

@@ -13,6 +13,7 @@
 #include "components/exo/pointer_constraint_delegate.h"
 #include "components/exo/pointer_delegate.h"
 #include "components/exo/pointer_gesture_pinch_delegate.h"
+#include "components/exo/pointer_stylus_delegate.h"
 #include "components/exo/relative_pointer_delegate.h"
 #include "components/exo/seat.h"
 #include "components/exo/shell_surface_util.h"
@@ -71,6 +72,14 @@ bool SameLocation(const gfx::PointF& location_in_target,
   return offset.LengthSquared() < (2 * kLocatedEventEpsilonSquared);
 }
 
+// Granularity for reporting force/pressure values coming from styli or other
+// devices that are normalized from 0 to 1, used to limit sending noisy values.
+const float kForceGranularity = 1e-2f;
+
+// Granularity for reporting tilt values coming from styli or other devices in
+// degrees, used to limit sending noisy values.
+const float kTiltGranularity = 1.f;
+
 display::ManagedDisplayInfo GetCaptureDisplayInfo() {
   display::ManagedDisplayInfo capture_info;
   for (const auto& display : display::Screen::GetScreen()->GetAllDisplays()) {
@@ -115,9 +124,8 @@ Pointer::Pointer(PointerDelegate* delegate, Seat* seat)
 
 Pointer::~Pointer() {
   delegate_->OnPointerDestroying(this);
-  if (focus_surface_) {
+  if (focus_surface_)
     focus_surface_->RemoveSurfaceObserver(this);
-  }
   if (pinch_delegate_)
     pinch_delegate_->OnPointerDestroying(this);
   if (relative_pointer_delegate_)
@@ -128,6 +136,8 @@ Pointer::~Pointer() {
     VLOG(1) << "Pointer constraint broken by pointer destruction";
     pointer_constraint_delegate_->OnConstraintBroken();
   }
+  if (stylus_delegate_)
+    stylus_delegate_->OnPointerDestroying(this);
   WMHelper* helper = WMHelper::GetInstance();
   helper->RemovePreTargetHandler(this);
   // TODO(sky): CursorClient does not exist in mash
@@ -302,6 +312,19 @@ void Pointer::DisablePointerCapture() {
   UpdateCursor();
 }
 
+void Pointer::SetStylusDelegate(PointerStylusDelegate* delegate) {
+  stylus_delegate_ = delegate;
+
+  // Reset last reported values to default.
+  last_pointer_type_ = ui::EventPointerType::kUnknown;
+  last_force_ = std::numeric_limits<float>::quiet_NaN();
+  last_tilt_ = gfx::Vector2dF();
+}
+
+bool Pointer::HasStylusDelegate() const {
+  return !!stylus_delegate_;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // SurfaceDelegate overrides:
 
@@ -367,6 +390,13 @@ void Pointer::OnMouseEvent(ui::MouseEvent* event) {
 
   TRACE_EXO_INPUT_EVENT(event);
 
+  const auto& details = event->pointer_details();
+  if (stylus_delegate_ && last_pointer_type_ != details.pointer_type) {
+    last_pointer_type_ = details.pointer_type;
+    stylus_delegate_->OnPointerToolChange(details.pointer_type);
+    delegate_->OnPointerFrame();
+  }
+
   if (event->IsMouseEvent()) {
     // Generate motion event if location changed. We need to check location
     // here as mouse movement can generate both "moved" and "entered" events
@@ -459,6 +489,31 @@ void Pointer::OnMouseEvent(ui::MouseEvent* event) {
       break;
   }
 
+  if (stylus_delegate_) {
+    bool needs_frame = false;
+    // Report the force value when either:
+    // - switching from a device that supports force to one that doesn't or
+    //   vice-versa (since force is NaN if the device doesn't support it), OR
+    // - the force value differs from the last reported force by greater than
+    //   the granularity.
+    // Using std::isgreaterequal for quiet error handling for NaNs.
+    if (std::isnan(last_force_) != std::isnan(details.force) ||
+        std::isgreaterequal(abs(last_force_ - details.force),
+                            kForceGranularity)) {
+      last_force_ = details.force;
+      stylus_delegate_->OnPointerForce(event->time_stamp(), details.force);
+      needs_frame = true;
+    }
+    if (abs(last_tilt_.x() - details.tilt_x) >= kTiltGranularity ||
+        abs(last_tilt_.y() - details.tilt_y) >= kTiltGranularity) {
+      last_tilt_ = gfx::Vector2dF(details.tilt_x, details.tilt_y);
+      stylus_delegate_->OnPointerTilt(event->time_stamp(), last_tilt_);
+      needs_frame = true;
+    }
+    if (needs_frame)
+      delegate_->OnPointerFrame();
+  }
+
   last_event_type_ = event->type();
 
   // Consume all mouse events when pointer capture is enabled.

components/exo/pointer.h

@@ -38,6 +38,7 @@ namespace exo {
 class PointerConstraintDelegate;
 class PointerDelegate;
 class PointerGesturePinchDelegate;
+class PointerStylusDelegate;
 class RelativePointerDelegate;
 class Seat;
 class Surface;
@@ -106,6 +107,10 @@ class Pointer : public SurfaceTreeHost,
   // delegate.
   void UnconstrainPointer();
 
+  // Set the stylus delegate for handling stylus events.
+  void SetStylusDelegate(PointerStylusDelegate* delegate);
+  bool HasStylusDelegate() const;
+
  private:
   // Capture the pointer for the given surface. Returns true iff the capture
   // succeeded.
@@ -167,6 +172,9 @@ class Pointer : public SurfaceTreeHost,
   // The delegate instance that controls when to lock/unlock this pointer.
   PointerConstraintDelegate* pointer_constraint_delegate_ = nullptr;
 
+  // The delegate instance that stylus/pen events are dispatched to.
+  PointerStylusDelegate* stylus_delegate_ = nullptr;
+
   // The current focus surface for the pointer.
   Surface* focus_surface_ = nullptr;
 
@@ -209,6 +217,11 @@ class Pointer : public SurfaceTreeHost,
   // Last received event type.
   ui::EventType last_event_type_ = ui::ET_UNKNOWN;
 
+  // Last reported stylus values.
+  ui::EventPointerType last_pointer_type_ = ui::EventPointerType::kUnknown;
+  float last_force_ = std::numeric_limits<float>::quiet_NaN();
+  gfx::Vector2dF last_tilt_;
+
   // Weak pointer factory used for cursor capture callbacks.
   base::WeakPtrFactory<Pointer> cursor_capture_weak_ptr_factory_{this};
 

components/exo/pointer_stylus_delegate.h

@@ -27,14 +27,16 @@ class PointerStylusDelegate {
   virtual void OnPointerToolChange(ui::EventPointerType type) = 0;
 
   // Called when the force (pressure) of the pointer changes.
-  // Normalized to be [0, 1].
+  // Normalized to be [0, 1]. NaN means pressure is not supported by the
+  // input device.
   virtual void OnPointerForce(base::TimeTicks time_stamp, float force) = 0;
 
   // Called when the tilt of a pen/stylus changes. Measured from surface normal
   // as plane angle in degrees, values lie in [-90,90]. A positive x is to the
-  // right and a positive y is towards the user.
+  // right and a positive y is towards the user. Always 0 if the device does
+  // not support it.
   virtual void OnPointerTilt(base::TimeTicks time_stamp,
-                             gfx::Vector2dF tilt) = 0;
+                             const gfx::Vector2dF& tilt) = 0;
 
  protected:
   virtual ~PointerStylusDelegate() {}

components/exo/pointer_unittest.cc

@@ -16,6 +16,7 @@
 #include "components/exo/data_source_delegate.h"
 #include "components/exo/pointer_constraint_delegate.h"
 #include "components/exo/pointer_delegate.h"
+#include "components/exo/pointer_stylus_delegate.h"
 #include "components/exo/relative_pointer_delegate.h"
 #include "components/exo/seat.h"
 #include "components/exo/shell_surface.h"
@@ -80,6 +81,17 @@ class MockPointerConstraintDelegate : public PointerConstraintDelegate {
   MOCK_METHOD0(GetConstrainedSurface, Surface*());
 };
 
+class MockPointerStylusDelegate : public PointerStylusDelegate {
+ public:
+  MockPointerStylusDelegate() {}
+
+  // Overridden from PointerStylusDelegate:
+  MOCK_METHOD(void, OnPointerDestroying, (Pointer*));
+  MOCK_METHOD(void, OnPointerToolChange, (ui::EventPointerType));
+  MOCK_METHOD(void, OnPointerForce, (base::TimeTicks, float));
+  MOCK_METHOD(void, OnPointerTilt, (base::TimeTicks, const gfx::Vector2dF&));
+};
+
 class TestDataSourceDelegate : public DataSourceDelegate {
  public:
   TestDataSourceDelegate() {}
@@ -1267,5 +1279,41 @@ TEST_F(PointerTest, UnconstrainPointerWhenWindowLosesFocus) {
 }
 #endif
 
+TEST_F(PointerTest, PointerStylus) {
+  std::unique_ptr<Surface> surface(new Surface);
+  std::unique_ptr<ShellSurface> shell_surface(new ShellSurface(surface.get()));
+  gfx::Size buffer_size(10, 10);
+  std::unique_ptr<Buffer> buffer(
+      new Buffer(exo_test_helper()->CreateGpuMemoryBuffer(buffer_size)));
+  surface->Attach(buffer.get());
+  surface->Commit();
+
+  MockPointerDelegate delegate;
+  MockPointerStylusDelegate stylus_delegate;
+  Seat seat;
+  std::unique_ptr<Pointer> pointer(new Pointer(&delegate, &seat));
+  ui::test::EventGenerator generator(ash::Shell::GetPrimaryRootWindow());
+
+  pointer->SetStylusDelegate(&stylus_delegate);
+
+  EXPECT_CALL(delegate, CanAcceptPointerEventsForSurface(surface.get()))
+      .WillRepeatedly(testing::Return(true));
+
+  {
+    testing::InSequence sequence;
+    EXPECT_CALL(delegate, OnPointerEnter(surface.get(), gfx::PointF(), 0));
+    EXPECT_CALL(delegate, OnPointerFrame());
+    EXPECT_CALL(stylus_delegate,
+                OnPointerToolChange(ui::EventPointerType::kMouse));
+    EXPECT_CALL(delegate, OnPointerFrame());
+  }
+
+  generator.MoveMouseTo(surface->window()->GetBoundsInScreen().origin());
+
+  EXPECT_CALL(delegate, OnPointerDestroying(pointer.get()));
+  EXPECT_CALL(stylus_delegate, OnPointerDestroying(pointer.get()));
+  pointer.reset();
+}
+
 }  // namespace
 }  // namespace exo

components/exo/wayland/server.cc

@@ -159,8 +159,8 @@ Server::Server(Display* display)
                    display_, bind_secure_output);
   wl_global_create(wl_display_.get(), &zcr_alpha_compositing_v1_interface, 1,
                    display_, bind_alpha_compositing);
-  wl_global_create(wl_display_.get(), &zcr_stylus_v2_interface, 1, display_,
-                   bind_stylus_v2);
+  wl_global_create(wl_display_.get(), &zcr_stylus_v2_interface,
+                   zcr_stylus_v2_interface.version, display_, bind_stylus_v2);
 
   seat_data_ =
       std::make_unique<WaylandSeat>(display_->seat(), serial_tracker_.get());

components/exo/wayland/zcr_stylus.cc

@@ -6,6 +6,8 @@
 
 #include <stylus-unstable-v2-server-protocol.h>
 
+#include "components/exo/pointer.h"
+#include "components/exo/pointer_stylus_delegate.h"
 #include "components/exo/touch.h"
 #include "components/exo/touch_stylus_delegate.h"
 #include "components/exo/wayland/server_util.h"
@@ -15,7 +17,7 @@ namespace wayland {
 namespace {
 
 ////////////////////////////////////////////////////////////////////////////////
-// touch_stylus interface:
+// zcr_touch_stylus_v2 interface:
 
 class WaylandTouchStylusDelegate : public TouchStylusDelegate {
  public:
@@ -66,7 +68,75 @@ const struct zcr_touch_stylus_v2_interface touch_stylus_implementation = {
     touch_stylus_destroy};
 
 ////////////////////////////////////////////////////////////////////////////////
-// stylus_v2 interface:
+// zcr_pointer_stylus_v2 interface:
+
+class WaylandPointerStylusDelegate : public PointerStylusDelegate {
+ public:
+  WaylandPointerStylusDelegate(wl_resource* resource, Pointer* pointer)
+      : resource_(resource), pointer_(pointer) {
+    pointer_->SetStylusDelegate(this);
+  }
+  WaylandPointerStylusDelegate(const WaylandPointerStylusDelegate&) = delete;
+  const WaylandPointerStylusDelegate& operator=(
+      const WaylandPointerStylusDelegate&) = delete;
+  ~WaylandPointerStylusDelegate() override {
+    if (pointer_ != nullptr)
+      pointer_->SetStylusDelegate(nullptr);
+  }
+  void OnPointerDestroying(Pointer* pointer_) override { pointer_ = nullptr; }
+  void OnPointerToolChange(ui::EventPointerType type) override {
+    uint wayland_type = ZCR_POINTER_STYLUS_V2_TOOL_TYPE_NONE;
+    if (type == ui::EventPointerType::kTouch)
+      wayland_type = ZCR_POINTER_STYLUS_V2_TOOL_TYPE_TOUCH;
+    else if (type == ui::EventPointerType::kPen)
+      wayland_type = ZCR_POINTER_STYLUS_V2_TOOL_TYPE_PEN;
+    else if (type == ui::EventPointerType::kEraser)
+      wayland_type = ZCR_POINTER_STYLUS_V2_TOOL_TYPE_ERASER;
+    zcr_pointer_stylus_v2_send_tool(resource_, wayland_type);
+    supports_force_ = false;
+    supports_tilt_ = false;
+  }
+  void OnPointerForce(base::TimeTicks time_stamp, float force) override {
+    // Set the force as 0 if the current tool previously reported a valid
+    // force, but is now reporting a NaN value indicating that force is not
+    // supported.
+    if (std::isnan(force)) {
+      if (supports_force_)
+        force = 0;
+      else
+        return;
+    }
+    supports_force_ = true;
+    zcr_pointer_stylus_v2_send_force(resource_,
+                                     TimeTicksToMilliseconds(time_stamp),
+                                     wl_fixed_from_double(force));
+  }
+  void OnPointerTilt(base::TimeTicks time_stamp,
+                     const gfx::Vector2dF& tilt) override {
+    if (!supports_tilt_ && tilt.IsZero())
+      return;
+    supports_tilt_ = true;
+    zcr_pointer_stylus_v2_send_tilt(
+        resource_, TimeTicksToMilliseconds(time_stamp),
+        wl_fixed_from_double(tilt.x()), wl_fixed_from_double(tilt.y()));
+  }
+
+ private:
+  wl_resource* resource_;
+  Pointer* pointer_;
+  bool supports_force_ = false;
+  bool supports_tilt_ = false;
+};
+
+void pointer_stylus_destroy(wl_client* client, wl_resource* resource) {
+  wl_resource_destroy(resource);
+}
+
+const struct zcr_pointer_stylus_v2_interface pointer_stylus_implementation = {
+    pointer_stylus_destroy};
+
+////////////////////////////////////////////////////////////////////////////////
+// zcr_stylus_v2 interface:
 
 void stylus_get_touch_stylus(wl_client* client,
                              wl_resource* resource,
@@ -81,15 +151,37 @@ void stylus_get_touch_stylus(wl_client* client,
   }
 
   wl_resource* stylus_resource =
-      wl_resource_create(client, &zcr_touch_stylus_v2_interface, 1, id);
+      wl_resource_create(client, &zcr_touch_stylus_v2_interface,
+                         wl_resource_get_version(resource), id);
 
   SetImplementation(
       stylus_resource, &touch_stylus_implementation,
       std::make_unique<WaylandTouchStylusDelegate>(stylus_resource, touch));
 }
 
+void stylus_get_pointer_stylus(wl_client* client,
+                               wl_resource* resource,
+                               uint32_t id,
+                               wl_resource* pointer_resource) {
+  Pointer* pointer = GetUserDataAs<Pointer>(pointer_resource);
+  if (pointer->HasStylusDelegate()) {
+    wl_resource_post_error(
+        resource, ZCR_STYLUS_V2_ERROR_POINTER_STYLUS_EXISTS,
+        "pointer has already been associated with a stylus object");
+    return;
+  }
+
+  wl_resource* stylus_resource =
+      wl_resource_create(client, &zcr_pointer_stylus_v2_interface,
+                         wl_resource_get_version(resource), id);
+
+  SetImplementation(
+      stylus_resource, &pointer_stylus_implementation,
+      std::make_unique<WaylandPointerStylusDelegate>(stylus_resource, pointer));
+}
+
 const struct zcr_stylus_v2_interface stylus_v2_implementation = {
-    stylus_get_touch_stylus};
+    stylus_get_touch_stylus, stylus_get_pointer_stylus};
 
 }  // namespace
 

third_party/wayland-protocols/unstable/stylus/stylus-unstable-v2.xml

@@ -24,15 +24,11 @@
     DEALINGS IN THE SOFTWARE.
   </copyright>
 
-  <interface name="zcr_stylus_v2" version="1">
-    <description summary="extends wl_touch with events for on-screen stylus">
-      Allows a wl_touch to report stylus specific information. The client can
-      interpret the on-screen stylus like any other touch event, and use
-      this protocol to obtain detail information about the type of stylus,
-      as well as the force and tilt of the tool.
-
-      These events are to be fired by the server within the same frame as other
-      wl_touch events.
+  <interface name="zcr_stylus_v2" version="2">
+    <description summary="extends wl_touch and wl_pointer for styli">
+      Allows a wl_touch or a wl_pointer to report stylus specific information.
+      The client can use this protocol to obtain detail information about the
+      type of stylus, as well as the force and tilt of the tool.
 
       Warning! The protocol described in this file is experimental and
       backward incompatible changes may be made. Backward compatible changes
@@ -47,23 +43,43 @@
     <enum name="error">
       <entry name="touch_stylus_exists" value="0"
              summary="the touch already has a touch_stylus object associated"/>
+
+      <!-- Version 2 Additions -->
+      <entry name="pointer_stylus_exists" value="1" since="2"
+             summary="the pointer already has a pointer_stylus object associated"/>
     </enum>
 
     <request name="get_touch_stylus">
       <description summary="get stylus interface for touch">
-        Create touch_stylus object. See zcr_touch_stylus_v1 interface for
+        Create touch_stylus object. See zcr_touch_stylus_v2 interface for
         details. If the given wl_touch already has a touch_stylus object
         associated, the touch_stylus_exists protocol error is raised.
       </description>
       <arg name="id" type="new_id" interface="zcr_touch_stylus_v2"/>
       <arg name="touch" type="object" interface="wl_touch"/>
     </request>
+
+    <!-- Version 2 Additions -->
+    <request name="get_pointer_stylus" since="2">
+      <description summary="get stylus interface for pointer">
+        Create pointer_stylus object. See zcr_pointer_stylus_v2 interface for
+        details. If the given wl_pointer already has a pointer_stylus object
+        associated, the pointer_stylus_exists protocol error is raised.
+      </description>
+      <arg name="id" type="new_id" interface="zcr_pointer_stylus_v2"/>
+      <arg name="pointer" type="object" interface="wl_pointer"/>
+    </request>
   </interface>
 
   <interface name="zcr_touch_stylus_v2" version="1">
     <description summary="stylus extension for touch">
-      The zcr_touch_stylus_v1 interface extends the wl_touch interface with
-      events to describe details about a stylus.
+      The zcr_touch_stylus_v2 interface extends the wl_touch interface with
+      events to describe details about a stylus. A stylus that reports events
+      through this interface is likely an on-screen stylus, where the user
+      interacts with the stylus directly on a display.
+
+      These events are to be fired by the server within the same frame as other
+      wl_touch events.
     </description>
 
     <request name="destroy" type="destructor">
@@ -113,4 +129,72 @@
     </event>
   </interface>
 
+  <!-- Version 2 Additions -->
+
+  <interface name="zcr_pointer_stylus_v2" version="2">
+    <description summary="stylus extension for touch">
+      The zcr_pointer_stylus_v2 interface extends the wl_pointer interface with
+      events to describe details about a stylus. A stylus that reports events
+      through this interface also moves the mouse cursor. The type of the
+      device reporting values through this interface is described by the
+      tool_type. When the tool changes, the values previously reported through
+      this interface are assumed to be reset.
+
+      These events are to be fired by the server within the same frame as other
+      wl_pointer events.
+    </description>
+
+    <request name="destroy" type="destructor">
+      <description summary="destroy stylus object"/>
+    </request>
+
+    <enum name="tool_type">
+      <description summary="the tool type of the device"/>
+      <entry name="none" value="0" summary="Default pointer device"/>
+      <entry name="touch" value="1" summary="Touch"/>
+      <entry name="pen" value="2" summary="Pen"/>
+      <entry name="eraser" value="3" summary="Eraser"/>
+    </enum>
+
+    <event name="tool">
+      <description summary="tool change event">
+        Notification that the user is using a tool type other than touch. There
+        can only be one tool in use at a time.
+      </description>
+      <arg name="type" type="uint" enum="tool_type" summary="type of tool in use"/>
+    </event>
+
+    <event name="force">
+      <description summary="force change event">
+        Notification of the physical force of the stylus on the surface.
+        The force is calibrated and normalized to the 0 to 1 range.
+
+        The client should assume that the force value is reset when the
+        tool changes, and that the tool does not support force detection
+        until the first force event is sent. That force value will persist
+        until the next force update or tool change.
+      </description>
+      <arg name="time" type="uint" summary="timestamp with millisecond granularity"/>
+      <arg name="force" type="fixed" summary="new value of force"/>
+    </event>
+
+    <event name="tilt">
+      <description summary="tilt change event">
+        Notification of a change in tilt of a stylus.
+
+        Measured from surface normal as plane angle in degrees, values lie in
+        [-90,90]. A positive x is to the right and a positive y is towards the
+        user.
+
+        The client should assume that the tilt value is reset when the
+        tool changes, and that the tool does not support tilt detection
+        until the first tilt event is sent. That value will persist
+        until the next tilt update or tool change.
+      </description>
+      <arg name="time" type="uint" summary="timestamp with millisecond granularity"/>
+      <arg name="tilt_x" type="fixed" summary="tilt in x direction"/>
+      <arg name="tilt_y" type="fixed" summary="tilt in y direction"/>
+    </event>
+  </interface>
+
 </protocol>