Send TextEvent message from Android client

The new message is currently used only for keys for which Android
generates ACTION_MULTIPLE events.

BUG=265945,270356
R=lambroslambrou@chromium.org

Review URL: https://codereview.chromium.org/214173002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@260088 0039d316-1c4b-4281-b951-d872f2087c98

remoting/android/java/src/org/chromium/chromoting/Desktop.java

@@ -103,10 +103,10 @@ public class Desktop extends Activity {
                         KeyEvent.KEYCODE_FORWARD_DEL,
                     };
                     for (int key : keys) {
-                        JniInterface.keyboardAction(key, true);
+                        JniInterface.sendKeyEvent(key, true);
                     }
                     for (int key : keys) {
-                        JniInterface.keyboardAction(key, false);
+                        JniInterface.sendKeyEvent(key, false);
                     }
                 }
                 return true;
@@ -127,32 +127,48 @@ public class Desktop extends Activity {
      */
     @Override
     public boolean dispatchKeyEvent(KeyEvent event) {
+        // Send ACTION_MULTIPLE event as TextEvent.
+        //
+        // TODO(sergeyu): For all keys on English keyboard Android generates
+        // ACTION_DOWN/ACTION_UP events, so they are sent as KeyEvent instead of
+        // TextEvent. As result the host may handle them as non-English chars
+        // when it has non-English layout selected, which might be confusing for
+        // the user. This code should be fixed to send all text input events as
+        // TextEvent, but it cannot be done now because not all hosts support
+        // TextEvent. Also, to handle keyboard shortcuts properly this code will
+        // need to track the state of modifier keys (such as Ctrl or Alt) and
+        // send KeyEvents in the case any of the modifier keys are pressed.
+        if (event.getAction() == KeyEvent.ACTION_MULTIPLE) {
+            JniInterface.sendTextEvent(event.getCharacters());
+            return super.dispatchKeyEvent(event);
+        }
+
         boolean depressed = event.getAction() == KeyEvent.ACTION_DOWN;
 
         switch (event.getKeyCode()) {
             case KeyEvent.KEYCODE_AT:
-                JniInterface.keyboardAction(KeyEvent.KEYCODE_SHIFT_LEFT, depressed);
-                JniInterface.keyboardAction(KeyEvent.KEYCODE_2, depressed);
+                JniInterface.sendKeyEvent(KeyEvent.KEYCODE_SHIFT_LEFT, depressed);
+                JniInterface.sendKeyEvent(KeyEvent.KEYCODE_2, depressed);
                 break;
 
             case KeyEvent.KEYCODE_POUND:
-                JniInterface.keyboardAction(KeyEvent.KEYCODE_SHIFT_LEFT, depressed);
-                JniInterface.keyboardAction(KeyEvent.KEYCODE_3, depressed);
+                JniInterface.sendKeyEvent(KeyEvent.KEYCODE_SHIFT_LEFT, depressed);
+                JniInterface.sendKeyEvent(KeyEvent.KEYCODE_3, depressed);
                 break;
 
             case KeyEvent.KEYCODE_STAR:
-                JniInterface.keyboardAction(KeyEvent.KEYCODE_SHIFT_LEFT, depressed);
-                JniInterface.keyboardAction(KeyEvent.KEYCODE_8, depressed);
+                JniInterface.sendKeyEvent(KeyEvent.KEYCODE_SHIFT_LEFT, depressed);
+                JniInterface.sendKeyEvent(KeyEvent.KEYCODE_8, depressed);
                 break;
 
             case KeyEvent.KEYCODE_PLUS:
-                JniInterface.keyboardAction(KeyEvent.KEYCODE_SHIFT_LEFT, depressed);
-                JniInterface.keyboardAction(KeyEvent.KEYCODE_EQUALS, depressed);
+                JniInterface.sendKeyEvent(KeyEvent.KEYCODE_SHIFT_LEFT, depressed);
+                JniInterface.sendKeyEvent(KeyEvent.KEYCODE_EQUALS, depressed);
                 break;
 
             default:
                 // We try to send all other key codes to the host directly.
-                JniInterface.keyboardAction(event.getKeyCode(), depressed);
+                JniInterface.sendKeyEvent(event.getKeyCode(), depressed);
         }
 
         return super.dispatchKeyEvent(event);

remoting/android/java/src/org/chromium/chromoting/DesktopView.java

@@ -344,7 +344,7 @@ public class DesktopView extends SurfaceView implements DesktopViewInterface,
             return;
         }
 
-        JniInterface.mouseAction(x, y, button, pressed);
+        JniInterface.sendMouseEvent(x, y, button, pressed);
         if (cursorMoved) {
             // TODO(lambroslambrou): Optimize this by only repainting the affected areas.
             requestRepaint();
@@ -353,7 +353,7 @@ public class DesktopView extends SurfaceView implements DesktopViewInterface,
 
     @Override
     public void injectMouseWheelDeltaEvent(int deltaX, int deltaY) {
-        JniInterface.mouseWheelDeltaAction(deltaX, deltaY);
+        JniInterface.sendMouseWheelEvent(deltaX, deltaY);
     }
 
     @Override

remoting/android/java/src/org/chromium/chromoting/jni/JniInterface.java

@@ -281,7 +281,7 @@ public class JniInterface {
      *                   is true.
      */
     private static native void nativeAuthenticationResponse(String pin, boolean createPair,
-                                                            String deviceName);
+            String deviceName);
 
     /** Saves newly-received pairing credentials to permanent storage. Called on the UI thread. */
     @CalledByNative
@@ -296,40 +296,53 @@ public class JniInterface {
      * Moves the mouse cursor, possibly while clicking the specified (nonnegative) button. Called
      * on the UI thread.
      */
-    public static void mouseAction(int x, int y, int whichButton, boolean buttonDown) {
+    public static void sendMouseEvent(int x, int y, int whichButton, boolean buttonDown) {
         if (!sConnected) {
             return;
         }
 
-        nativeMouseAction(x, y, whichButton, buttonDown);
+        nativeSendMouseEvent(x, y, whichButton, buttonDown);
     }
 
     /** Passes mouse information to the native handling code. */
-    private static native void nativeMouseAction(int x, int y, int whichButton, boolean buttonDown);
+    private static native void nativeSendMouseEvent(int x, int y, int whichButton,
+            boolean buttonDown);
 
     /** Injects a mouse-wheel event with delta values. Called on the UI thread. */
-    public static void mouseWheelDeltaAction(int deltaX, int deltaY) {
+    public static void sendMouseWheelEvent(int deltaX, int deltaY) {
         if (!sConnected) {
             return;
         }
 
-        nativeMouseWheelDeltaAction(deltaX, deltaY);
+        nativeSendMouseWheelEvent(deltaX, deltaY);
     }
 
     /** Passes mouse-wheel information to the native handling code. */
-    private static native void nativeMouseWheelDeltaAction(int deltaX, int deltaY);
+    private static native void nativeSendMouseWheelEvent(int deltaX, int deltaY);
 
-    /** Presses and releases the specified (nonnegative) key. Called on the UI thread. */
-    public static void keyboardAction(int keyCode, boolean keyDown) {
+    /** Presses or releases the specified (nonnegative) key. Called on the UI thread. */
+    public static void sendKeyEvent(int keyCode, boolean keyDown) {
         if (!sConnected) {
             return;
         }
 
-        nativeKeyboardAction(keyCode, keyDown);
+        nativeSendKeyEvent(keyCode, keyDown);
     }
 
     /** Passes key press information to the native handling code. */
-    private static native void nativeKeyboardAction(int keyCode, boolean keyDown);
+    private static native void nativeSendKeyEvent(int keyCode, boolean keyDown);
+
+    /** Sends TextEvent to the host. Called on the UI thread. */
+    public static void sendTextEvent(String text) {
+        if (!sConnected) {
+            return;
+        }
+
+        nativeSendTextEvent(text);
+    }
+
+    /** Passes text event information to the native handling code. */
+    private static native void nativeSendTextEvent(String text);
 
     /**
      * Sets the redraw callback to the provided functor. Provide a value of null whenever the

remoting/client/jni/chromoting_jni_instance.cc

@@ -128,62 +128,74 @@ void ChromotingJniInstance::RedrawDesktop() {
   jni_runtime_->RedrawCanvas();
 }
 
-void ChromotingJniInstance::PerformMouseAction(
+void ChromotingJniInstance::SendMouseEvent(
     int x, int y,
     protocol::MouseEvent_MouseButton button,
     bool button_down) {
   if (!jni_runtime_->network_task_runner()->BelongsToCurrentThread()) {
     jni_runtime_->network_task_runner()->PostTask(
-        FROM_HERE, base::Bind(&ChromotingJniInstance::PerformMouseAction,
+        FROM_HERE, base::Bind(&ChromotingJniInstance::SendMouseEvent,
                               this, x, y, button, button_down));
     return;
   }
 
-  protocol::MouseEvent action;
-  action.set_x(x);
-  action.set_y(y);
-  action.set_button(button);
+  protocol::MouseEvent event;
+  event.set_x(x);
+  event.set_y(y);
+  event.set_button(button);
   if (button != protocol::MouseEvent::BUTTON_UNDEFINED)
-    action.set_button_down(button_down);
+    event.set_button_down(button_down);
 
-  connection_->input_stub()->InjectMouseEvent(action);
+  connection_->input_stub()->InjectMouseEvent(event);
 }
 
-void ChromotingJniInstance::PerformMouseWheelDeltaAction(int delta_x,
-                                                         int delta_y) {
+void ChromotingJniInstance::SendMouseWheelEvent(int delta_x, int delta_y) {
   if (!jni_runtime_->network_task_runner()->BelongsToCurrentThread()) {
     jni_runtime_->network_task_runner()->PostTask(
         FROM_HERE,
-        base::Bind(&ChromotingJniInstance::PerformMouseWheelDeltaAction, this,
+        base::Bind(&ChromotingJniInstance::SendMouseWheelEvent, this,
                    delta_x, delta_y));
     return;
   }
 
-  protocol::MouseEvent action;
-  action.set_wheel_delta_x(delta_x);
-  action.set_wheel_delta_y(delta_y);
-  connection_->input_stub()->InjectMouseEvent(action);
+  protocol::MouseEvent event;
+  event.set_wheel_delta_x(delta_x);
+  event.set_wheel_delta_y(delta_y);
+  connection_->input_stub()->InjectMouseEvent(event);
 }
 
-void ChromotingJniInstance::PerformKeyboardAction(int key_code, bool key_down) {
+void ChromotingJniInstance::SendKeyEvent(int key_code, bool key_down) {
   if (!jni_runtime_->network_task_runner()->BelongsToCurrentThread()) {
     jni_runtime_->network_task_runner()->PostTask(
-        FROM_HERE, base::Bind(&ChromotingJniInstance::PerformKeyboardAction,
+        FROM_HERE, base::Bind(&ChromotingJniInstance::SendKeyEvent,
                               this, key_code, key_down));
     return;
   }
 
   uint32 usb_code = AndroidKeycodeToUsbKeycode(key_code);
   if (usb_code) {
-    protocol::KeyEvent action;
-    action.set_usb_keycode(usb_code);
-    action.set_pressed(key_down);
-    connection_->input_stub()->InjectKeyEvent(action);
+    protocol::KeyEvent event;
+    event.set_usb_keycode(usb_code);
+    event.set_pressed(key_down);
+    connection_->input_stub()->InjectKeyEvent(event);
   } else {
     LOG(WARNING) << "Ignoring unknown keycode: " << key_code;
   }
 }
 
+void ChromotingJniInstance::SendTextEvent(const std::string& text) {
+  if (!jni_runtime_->network_task_runner()->BelongsToCurrentThread()) {
+    jni_runtime_->network_task_runner()->PostTask(
+        FROM_HERE,
+        base::Bind(&ChromotingJniInstance::SendTextEvent, this, text));
+    return;
+  }
+
+  protocol::TextEvent event;
+  event.set_text(text);
+  connection_->input_stub()->InjectTextEvent(event);
+}
+
 void ChromotingJniInstance::RecordPaintTime(int64 paint_time_ms) {
   if (!jni_runtime_->network_task_runner()->BelongsToCurrentThread()) {
     jni_runtime_->network_task_runner()->PostTask(

remoting/client/jni/chromoting_jni_instance.h

@@ -65,14 +65,15 @@ class ChromotingJniInstance
 
   // Moves the host's cursor to the specified coordinates, optionally with some
   // mouse button depressed. If |button| is BUTTON_UNDEFINED, no click is made.
-  void PerformMouseAction(int x, int y,
-                          protocol::MouseEvent_MouseButton button,
-                          bool button_down);
-
-  void PerformMouseWheelDeltaAction(int delta_x, int delta_y);
+  void SendMouseEvent(int x, int y,
+                      protocol::MouseEvent_MouseButton button,
+                      bool button_down);
+  void SendMouseWheelEvent(int delta_x, int delta_y);
 
   // Sends the provided keyboard scan code to the host.
-  void PerformKeyboardAction(int key_code, bool key_down);
+  void SendKeyEvent(int key_code, bool key_down);
+
+  void SendTextEvent(const std::string& text);
 
   // Records paint time for statistics logging, if enabled. May be called from
   // any thread.

remoting/client/jni/chromoting_jni_runtime.cc

@@ -103,38 +103,42 @@ static void ScheduleRedraw(JNIEnv* env, jclass clazz) {
   remoting::ChromotingJniRuntime::GetInstance()->session()->RedrawDesktop();
 }
 
-static void MouseAction(JNIEnv* env,
-                        jclass clazz,
-                        jint x,
-                        jint y,
-                        jint whichButton,
-                        jboolean buttonDown) {
+static void SendMouseEvent(JNIEnv* env,
+                           jclass clazz,
+                           jint x,
+                           jint y,
+                           jint whichButton,
+                           jboolean buttonDown) {
   // Button must be within the bounds of the MouseEvent_MouseButton enum.
   DCHECK(whichButton >= 0 && whichButton < 5);
 
-  remoting::ChromotingJniRuntime::GetInstance()->session()->PerformMouseAction(
-      x,
-      y,
+  remoting::ChromotingJniRuntime::GetInstance()->session()->SendMouseEvent(
+      x, y,
       static_cast<remoting::protocol::MouseEvent_MouseButton>(whichButton),
       buttonDown);
 }
 
-static void MouseWheelDeltaAction(JNIEnv* env,
-                                  jclass clazz,
-                                  jint delta_x,
-                                  jint delta_y) {
-  remoting::ChromotingJniRuntime::GetInstance()
-      ->session()
-      ->PerformMouseWheelDeltaAction(delta_x, delta_y);
+static void SendMouseWheelEvent(JNIEnv* env,
+                                jclass clazz,
+                                jint delta_x,
+                                jint delta_y) {
+  remoting::ChromotingJniRuntime::GetInstance()->session()->SendMouseWheelEvent(
+      delta_x, delta_y);
 }
 
-static void KeyboardAction(JNIEnv* env,
-                           jclass clazz,
-                           jint keyCode,
-                           jboolean keyDown) {
-  remoting::ChromotingJniRuntime::GetInstance()
-      ->session()
-      ->PerformKeyboardAction(keyCode, keyDown);
+static void SendKeyEvent(JNIEnv* env,
+                         jclass clazz,
+                         jint keyCode,
+                         jboolean keyDown) {
+  remoting::ChromotingJniRuntime::GetInstance()->session()->SendKeyEvent(
+      keyCode, keyDown);
+}
+
+static void SendTextEvent(JNIEnv* env,
+                          jclass clazz,
+                          jstring text) {
+  remoting::ChromotingJniRuntime::GetInstance()->session()->SendTextEvent(
+      ConvertJavaStringToUTF8(env, text));
 }
 
 // ChromotingJniRuntime implementation.