Added support of relative mouse motion in Chromoting.

This CL makes the client plugin treat a completely transparent mouse cursor as an indication that the host switched to relative mouse mode.

If the browser rejects or cancels the mouse lock for any reason, the plugin sets the cursor to the standard arrow pointer until a new cursor is set by the host.

The webapp has to send the 'allowMouseLock' message to enable this behavior.

BUG=132613

Review URL: https://chromiumcodereview.appspot.com/23484015

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@222110 0039d316-1c4b-4281-b951-d872f2087c98

remoting/client/plugin/chromoting_instance.cc

@@ -4,6 +4,7 @@
 
 #include "remoting/client/plugin/chromoting_instance.h"
 
+#include <algorithm>
 #include <string>
 #include <vector>
 
@@ -23,8 +24,8 @@
 #include "net/socket/ssl_server_socket.h"
 #include "ppapi/cpp/completion_callback.h"
 #include "ppapi/cpp/dev/url_util_dev.h"
+#include "ppapi/cpp/image_data.h"
 #include "ppapi/cpp/input_event.h"
-#include "ppapi/cpp/mouse_cursor.h"
 #include "ppapi/cpp/rect.h"
 #include "remoting/base/constants.h"
 #include "remoting/base/util.h"
@@ -55,6 +56,12 @@ namespace {
 // 32-bit BGRA is 4 bytes per pixel.
 const int kBytesPerPixel = 4;
 
+#if defined(ARCH_CPU_LITTLE_ENDIAN)
+const uint32_t kPixelAlphaMask = 0xff000000;
+#else  // !defined(ARCH_CPU_LITTLE_ENDIAN)
+const uint32_t kPixelAlphaMask = 0x000000ff;
+#endif  // !defined(ARCH_CPU_LITTLE_ENDIAN)
+
 // Default DPI to assume for old clients that use notifyClientResolution.
 const int kDefaultDPI = 96;
 
@@ -125,6 +132,16 @@ std::string ConnectionErrorToString(protocol::ErrorCode error) {
   return std::string();
 }
 
+// Returns true if |pixel| is not completely transparent.
+bool IsVisiblePixel(uint32_t pixel) {
+  return (pixel & kPixelAlphaMask) != 0;
+}
+
+// Returns true if there is at least one visible pixel in the given range.
+bool IsVisibleRow(const uint32_t* begin, const uint32_t* end) {
+  return std::find_if(begin, end, &IsVisiblePixel) != end;
+}
+
 // This flag blocks LOGs to the UI if we're already in the middle of logging
 // to the UI. This prevents a potential infinite loop if we encounter an error
 // while sending the log message to the UI.
@@ -144,7 +161,7 @@ logging::LogMessageHandlerFunction g_logging_old_handler = NULL;
 const char ChromotingInstance::kApiFeatures[] =
     "highQualityScaling injectKeyEvent sendClipboardItem remapKey trapKey "
     "notifyClientResolution pauseVideo pauseAudio asyncPin thirdPartyAuth "
-    "pinlessAuth extensionMessage";
+    "pinlessAuth extensionMessage allowMouseLock";
 
 const char ChromotingInstance::kRequestedCapabilities[] = "";
 const char ChromotingInstance::kSupportedCapabilities[] = "desktopShape";
@@ -176,7 +193,7 @@ ChromotingInstance::ChromotingInstance(PP_Instance pp_instance)
       input_tracker_(&mouse_input_filter_),
       key_mapper_(&input_tracker_),
       normalizing_input_filter_(CreateNormalizingInputFilter(&key_mapper_)),
-      input_handler_(normalizing_input_filter_.get()),
+      input_handler_(this, normalizing_input_filter_.get()),
       use_async_pin_dialog_(false),
       weak_factory_(this) {
   RequestInputEvents(PP_INPUTEVENT_CLASS_MOUSE | PP_INPUTEVENT_CLASS_WHEEL);
@@ -304,9 +321,17 @@ void ChromotingInstance::HandleMessage(const pp::Var& message) {
     HandleRequestPairing(*data);
   } else if (method == "extensionMessage") {
     HandleExtensionMessage(*data);
+  } else if (method == "allowMouseLock") {
+    HandleAllowMouseLockMessage();
   }
 }
 
+void ChromotingInstance::DidChangeFocus(bool has_focus) {
+  DCHECK(plugin_task_runner_->BelongsToCurrentThread());
+
+  input_handler_.DidChangeFocus(has_focus);
+}
+
 void ChromotingInstance::DidChangeView(const pp::View& view) {
   DCHECK(plugin_task_runner_->BelongsToCurrentThread());
 
@@ -464,6 +489,8 @@ void ChromotingInstance::InjectClipboardEvent(
 
 void ChromotingInstance::SetCursorShape(
     const protocol::CursorShapeInfo& cursor_shape) {
+  COMPILE_ASSERT(sizeof(uint32_t) == kBytesPerPixel, rgba_pixels_are_32bit);
+
   if (!cursor_shape.has_data() ||
       !cursor_shape.has_width() ||
       !cursor_shape.has_height() ||
@@ -500,12 +527,17 @@ void ChromotingInstance::SetCursorShape(
 
   int hotspot_x = cursor_shape.hotspot_x();
   int hotspot_y = cursor_shape.hotspot_y();
-
   int bytes_per_row = width * kBytesPerPixel;
-  const uint8* src_row_data = reinterpret_cast<const uint8*>(
+  int src_stride = width;
+  const uint32_t* src_row_data = reinterpret_cast<const uint32_t*>(
       cursor_shape.data().data());
-  int stride = bytes_per_row;
+  const uint32_t* src_row_data_end = src_row_data + src_stride * height;
+
+  scoped_ptr<pp::ImageData> cursor_image;
+  pp::Point cursor_hotspot;
 
+  // Check if the cursor is visible.
+  if (IsVisibleRow(src_row_data, src_row_data_end)) {
     // If the cursor exceeds the size permitted by PPAPI then crop it, keeping
     // the hotspot as close to the center of the new cursor shape as possible.
     if (height > kMaxCursorHeight) {
@@ -513,7 +545,7 @@ void ChromotingInstance::SetCursorShape(
       y = std::max(y, 0);
       y = std::min(y, height - kMaxCursorHeight);
 
-    src_row_data += stride * y;
+      src_row_data += src_stride * y;
       height = kMaxCursorHeight;
       hotspot_y -= y;
     }
@@ -522,25 +554,25 @@ void ChromotingInstance::SetCursorShape(
       x = std::max(x, 0);
       x = std::min(x, height - kMaxCursorWidth);
 
-    src_row_data += x * kBytesPerPixel;
+      src_row_data += x;
       width = kMaxCursorWidth;
       bytes_per_row = width * kBytesPerPixel;
       hotspot_x -= x;
     }
 
-  pp::ImageData cursor_image(this, PP_IMAGEDATAFORMAT_BGRA_PREMUL,
-                             pp::Size(width, height), false);
+    cursor_image.reset(new pp::ImageData(this, PP_IMAGEDATAFORMAT_BGRA_PREMUL,
+                                          pp::Size(width, height), false));
+    cursor_hotspot = pp::Point(hotspot_x, hotspot_y);
 
-  uint8* dst_row_data = reinterpret_cast<uint8*>(cursor_image.data());
+    uint8* dst_row_data = reinterpret_cast<uint8*>(cursor_image->data());
     for (int row = 0; row < height; row++) {
       memcpy(dst_row_data, src_row_data, bytes_per_row);
-    src_row_data += stride;
-    dst_row_data += cursor_image.stride();
+      src_row_data += src_stride;
+      dst_row_data += cursor_image->stride();
+    }
   }
 
-  pp::MouseCursor::SetCursor(this, PP_MOUSECURSOR_TYPE_CUSTOM,
-                             cursor_image,
-                             pp::Point(hotspot_x, hotspot_y));
+  input_handler_.SetMouseCursor(cursor_image.Pass(), cursor_hotspot);
 }
 
 void ChromotingInstance::OnFirstFrameReceived() {
@@ -860,6 +892,10 @@ void ChromotingInstance::HandleExtensionMessage(
   host_connection_->host_stub()->DeliverClientMessage(message);
 }
 
+void ChromotingInstance::HandleAllowMouseLockMessage() {
+  input_handler_.AllowMouseLock();
+}
+
 ChromotingStats* ChromotingInstance::GetStats() {
   if (!client_.get())
     return NULL;

remoting/client/plugin/chromoting_instance.h

@@ -102,6 +102,7 @@ class ChromotingInstance :
   virtual ~ChromotingInstance();
 
   // pp::Instance interface.
+  virtual void DidChangeFocus(bool has_focus) OVERRIDE;
   virtual void DidChangeView(const pp::View& view) OVERRIDE;
   virtual bool Init(uint32_t argc, const char* argn[],
                     const char* argv[]) OVERRIDE;
@@ -195,6 +196,7 @@ class ChromotingInstance :
   void HandleOnThirdPartyTokenFetched(const base::DictionaryValue& data);
   void HandleRequestPairing(const base::DictionaryValue& data);
   void HandleExtensionMessage(const base::DictionaryValue& data);
+  void HandleAllowMouseLockMessage();
 
   // Helper method called from Connect() to connect with parsed config.
   void ConnectWithConfig(const ClientConfig& config,

remoting/client/plugin/pepper_input_handler.cc

@@ -6,15 +6,24 @@
 
 #include "base/logging.h"
 #include "ppapi/c/dev/ppb_keyboard_input_event_dev.h"
+#include "ppapi/cpp/image_data.h"
 #include "ppapi/cpp/input_event.h"
 #include "ppapi/cpp/module_impl.h"
+#include "ppapi/cpp/mouse_cursor.h"
 #include "ppapi/cpp/point.h"
 #include "remoting/proto/event.pb.h"
 
 namespace remoting {
 
-PepperInputHandler::PepperInputHandler(protocol::InputStub* input_stub)
-    : input_stub_(input_stub),
+PepperInputHandler::PepperInputHandler(
+    pp::Instance* instance,
+    protocol::InputStub* input_stub)
+    : pp::MouseLock(instance),
+      instance_(instance),
+      input_stub_(input_stub),
+      callback_factory_(this),
+      has_focus_(false),
+      mouse_lock_state_(MouseLockDisallowed),
       wheel_delta_x_(0),
       wheel_delta_y_(0),
       wheel_ticks_x_(0),
@@ -98,6 +107,14 @@ bool PepperInputHandler::HandleInputEvent(const pp::InputEvent& event) {
       protocol::MouseEvent mouse_event;
       mouse_event.set_x(pp_mouse_event.GetPosition().x());
       mouse_event.set_y(pp_mouse_event.GetPosition().y());
+
+      // Add relative movement if the mouse is locked.
+      if (mouse_lock_state_ == MouseLockOn) {
+        pp::Point delta = pp_mouse_event.GetMovement();
+        mouse_event.set_delta_x(delta.x());
+        mouse_event.set_delta_y(delta.y());
+      }
+
       input_stub_->InjectMouseEvent(mouse_event);
       return true;
     }
@@ -159,4 +176,115 @@ bool PepperInputHandler::HandleInputEvent(const pp::InputEvent& event) {
   return false;
 }
 
+void PepperInputHandler::AllowMouseLock() {
+  DCHECK_EQ(mouse_lock_state_, MouseLockDisallowed);
+  mouse_lock_state_ = MouseLockOff;
+}
+
+void PepperInputHandler::DidChangeFocus(bool has_focus) {
+  has_focus_ = has_focus;
+  if (has_focus_)
+    RequestMouseLock();
+}
+
+void PepperInputHandler::SetMouseCursor(scoped_ptr<pp::ImageData> image,
+                                        const pp::Point& hotspot) {
+  cursor_image_ = image.Pass();
+  cursor_hotspot_ = hotspot;
+
+  if (mouse_lock_state_ != MouseLockDisallowed && !cursor_image_) {
+    RequestMouseLock();
+  } else {
+    CancelMouseLock();
+  }
+}
+
+void PepperInputHandler::MouseLockLost() {
+  DCHECK(mouse_lock_state_ == MouseLockOn ||
+         mouse_lock_state_ == MouseLockCancelling);
+
+  mouse_lock_state_ = MouseLockOff;
+  UpdateMouseCursor();
+}
+
+void PepperInputHandler::RequestMouseLock() {
+  // Request mouse lock only if the plugin is focused, the host-supplied cursor
+  // is empty and no callback is pending.
+  if (has_focus_ && !cursor_image_ && mouse_lock_state_ == MouseLockOff) {
+    pp::CompletionCallback callback =
+        callback_factory_.NewCallback(&PepperInputHandler::OnMouseLocked);
+    int result = pp::MouseLock::LockMouse(callback);
+    DCHECK_EQ(result, PP_OK_COMPLETIONPENDING);
+
+    // Hide cursor to avoid it becoming a black square (see crbug.com/285809).
+    pp::MouseCursor::SetCursor(instance_, PP_MOUSECURSOR_TYPE_NONE);
+
+    mouse_lock_state_ = MouseLockRequestPending;
+  }
+}
+
+void PepperInputHandler::CancelMouseLock() {
+  switch (mouse_lock_state_) {
+    case MouseLockDisallowed:
+    case MouseLockOff:
+      UpdateMouseCursor();
+      break;
+
+    case MouseLockCancelling:
+      break;
+
+    case MouseLockRequestPending:
+      // The mouse lock request is pending. Delay UnlockMouse() call until
+      // the callback is called.
+      mouse_lock_state_ = MouseLockCancelling;
+      break;
+
+    case MouseLockOn:
+      pp::MouseLock::UnlockMouse();
+
+      // Note that mouse-lock has been cancelled. We will continue to receive
+      // locked events until MouseLockLost() is called back.
+      mouse_lock_state_ = MouseLockCancelling;
+      break;
+
+    default:
+      NOTREACHED();
+  }
+}
+
+void PepperInputHandler::UpdateMouseCursor() {
+  DCHECK(mouse_lock_state_ == MouseLockDisallowed ||
+         mouse_lock_state_ == MouseLockOff);
+
+  if (cursor_image_) {
+    pp::MouseCursor::SetCursor(instance_, PP_MOUSECURSOR_TYPE_CUSTOM,
+                               *cursor_image_,
+                               cursor_hotspot_);
+  } else {
+    // If there is no cursor shape stored, either because the host never
+    // supplied one, or we were previously in mouse-lock mode, then use
+    // a standard arrow pointer.
+    pp::MouseCursor::SetCursor(instance_, PP_MOUSECURSOR_TYPE_POINTER);
+  }
+}
+
+void PepperInputHandler::OnMouseLocked(int error) {
+  DCHECK(mouse_lock_state_ == MouseLockRequestPending ||
+         mouse_lock_state_ == MouseLockCancelling);
+
+  bool should_cancel = (mouse_lock_state_ == MouseLockCancelling);
+
+  // See if the operation succeeded.
+  if (error == PP_OK) {
+    mouse_lock_state_ = MouseLockOn;
+  } else {
+    mouse_lock_state_ = MouseLockOff;
+    UpdateMouseCursor();
+  }
+
+  // Cancel as needed.
+  if (should_cancel)
+    CancelMouseLock();
+}
+
 }  // namespace remoting

remoting/client/plugin/pepper_input_handler.h

@@ -6,10 +6,16 @@
 #define REMOTING_CLIENT_PLUGIN_PEPPER_INPUT_HANDLER_H_
 
 #include "base/compiler_specific.h"
+#include "base/memory/scoped_ptr.h"
+#include "ppapi/cpp/mouse_lock.h"
+#include "ppapi/cpp/point.h"
+#include "ppapi/utility/completion_callback_factory.h"
 #include "remoting/protocol/input_stub.h"
 
 namespace pp {
+class ImageData;
 class InputEvent;
+class Instance;
 }  // namespace pp
 
 namespace remoting {
@@ -18,16 +24,70 @@ namespace protocol {
 class InputStub;
 } // namespace protocol
 
-class PepperInputHandler {
+class PepperInputHandler : public pp::MouseLock {
  public:
-  explicit PepperInputHandler(protocol::InputStub* input_stub);
+  // |instance| must outlive |this|.
+  PepperInputHandler(pp::Instance* instance, protocol::InputStub* input_stub);
   virtual ~PepperInputHandler();
 
   bool HandleInputEvent(const pp::InputEvent& event);
 
+  // Enables locking the mouse when the host sets a completely transparent mouse
+  // cursor.
+  void AllowMouseLock();
+
+  // Called when the plugin receives or loses focus.
+  void DidChangeFocus(bool has_focus);
+
+  // Sets the mouse cursor image. Passing NULL image will lock the mouse if
+  // mouse lock is enabled.
+  void SetMouseCursor(scoped_ptr<pp::ImageData> image,
+                      const pp::Point& hotspot);
+
  private:
+  enum MouseLockState {
+    MouseLockDisallowed,
+    MouseLockOff,
+    MouseLockRequestPending,
+    MouseLockOn,
+    MouseLockCancelling
+  };
+
+  // pp::MouseLock interface.
+  virtual void MouseLockLost() OVERRIDE;
+
+  // Requests the browser to lock the mouse and hides the cursor.
+  void RequestMouseLock();
+
+  // Requests the browser to cancel mouse lock and restores the cursor once
+  // the lock is gone.
+  void CancelMouseLock();
+
+  // Applies |cursor_image_| as the custom pointer or uses the standard arrow
+  // pointer if |cursor_image_| is not available.
+  void UpdateMouseCursor();
+
+  // Handles completion of the mouse lock request issued by RequestMouseLock().
+  void OnMouseLocked(int error);
+
+  pp::Instance* instance_;
   protocol::InputStub* input_stub_;
 
+  pp::CompletionCallbackFactory<PepperInputHandler> callback_factory_;
+
+  // Custom cursor image sent by the host. |cursor_image_| is set to NULL when
+  // the cursor image is completely transparent. This can be interpreted as
+  // a mouse lock request if enabled by the webapp.
+  scoped_ptr<pp::ImageData> cursor_image_;
+
+  // Hot spot for |cursor_image_|.
+  pp::Point cursor_hotspot_;
+
+  // True if the plugin has focus.
+  bool has_focus_;
+
+  MouseLockState mouse_lock_state_;
+
   // Accumulated sub-pixel and sub-tick deltas from wheel events.
   float wheel_delta_x_;
   float wheel_delta_y_;

remoting/host/input_injector_linux.cc

@@ -299,7 +299,16 @@ void InputInjectorLinux::Core::InjectMouseEvent(const MouseEvent& event) {
     return;
   }
 
-  if (event.has_x() && event.has_y()) {
+  if (event.has_delta_x() &&
+      event.has_delta_y() &&
+      (event.delta_x() != 0 || event.delta_y() != 0)) {
+    latest_mouse_position_ = SkIPoint::Make(-1, -1);
+    VLOG(3) << "Moving mouse by " << event.delta_x() << "," << event.delta_y();
+    XTestFakeRelativeMotionEvent(display_,
+                                 event.delta_x(), event.delta_y(),
+                                 CurrentTime);
+
+  } else if (event.has_x() && event.has_y()) {
     // Injecting a motion event immediately before a button release results in
     // a MotionNotify even if the mouse position hasn't changed, which confuses
     // apps which assume MotionNotify implies movement. See crbug.com/138075.

remoting/host/input_injector_win.cc

@@ -219,20 +219,26 @@ void InputInjectorWin::Core::HandleMouse(const MouseEvent& event) {
   // Reset the system idle suspend timeout.
   SetThreadExecutionState(ES_SYSTEM_REQUIRED);
 
-  // TODO(garykac) Collapse mouse (x,y) and button events into a single
+  // TODO(garykac) Collapse mouse movement and button events into a single
   // input event when possible.
-  if (event.has_x() && event.has_y()) {
-    int x = event.x();
-    int y = event.y();
-
+  if (event.has_delta_x() && event.has_delta_y()) {
+    INPUT input;
+    input.type = INPUT_MOUSE;
+    input.mi.time = 0;
+    input.mi.dx = event.delta_x();
+    input.mi.dy = event.delta_y();
+    input.mi.dwFlags = MOUSEEVENTF_MOVE | MOUSEEVENTF_VIRTUALDESK;
+    if (SendInput(1, &input, sizeof(INPUT)) == 0)
+      LOG_GETLASTERROR(ERROR) << "Failed to inject a mouse move event";
+  } else if (event.has_x() && event.has_y()) {
     INPUT input;
     input.type = INPUT_MOUSE;
     input.mi.time = 0;
     SkISize screen_size(SkISize::Make(GetSystemMetrics(SM_CXVIRTUALSCREEN),
                                       GetSystemMetrics(SM_CYVIRTUALSCREEN)));
     if ((screen_size.width() > 1) && (screen_size.height() > 1)) {
-      x = std::max(0, std::min(screen_size.width(), x));
-      y = std::max(0, std::min(screen_size.height(), y));
+      int x = std::max(0, std::min(screen_size.width(), event.x()));
+      int y = std::max(0, std::min(screen_size.height(), event.y()));
       input.mi.dx = static_cast<int>((x * 65535) / (screen_size.width() - 1));
       input.mi.dy = static_cast<int>((y * 65535) / (screen_size.height() - 1));
       input.mi.dwFlags =

remoting/proto/event.proto

@@ -58,6 +58,10 @@ message MouseEvent {
   optional float wheel_delta_y = 8;
   optional float wheel_ticks_x = 9;
   optional float wheel_ticks_y = 10;
+
+  // Mouse movement information. Provided only when mouse lock is engaged.
+  optional int32 delta_x = 11;
+  optional int32 delta_y = 12;
 }
 
 // Defines an event that sends clipboard data between peers.