[CRD iOS] Implement Char-to-Keycode mapping logic for US keyboard layout

Previously we don't bother to implement a keycode mapping logic and
we directly send texts to the host to inject. Hosts usually handle text
injection very differently than key injection and may end up texts being
directly injected to the host without passing through IME. In other
words, host side IME doesn't work with our current key injection
implementation.

The downside of this change is that non-US keyboard user will now get
the wrong keyboard output on the host again, which is what we had on the
v1 client.

For longer term, we can add a new host setting for changing the keyboard
layout.

NOTRY=true

Bug: 830959
Change-Id: Ia34b6894fd7c0a7a07a5b9657aca19c2a878bd4e
Reviewed-on: https://chromium-review.googlesource.com/1006065
Commit-Queue: Yuwei Huang <yuweih@chromium.org>
Reviewed-by: Gary Kacmarcik <garykac@chromium.org>
Reviewed-by: Jamie Walch <jamiewalch@chromium.org>
Cr-Commit-Position: refs/heads/master@{#549706}

remoting/client/input/BUILD.gn

@@ -12,6 +12,8 @@ source_set("input") {
     "keyboard_input_strategy.h",
     "keyboard_interpreter.cc",
     "keyboard_interpreter.h",
+    "keycode_map.cc",
+    "keycode_map.h",
     "native_device_keymap.cc",
     "native_device_keymap.h",
     "native_device_keymap_android.cc",
@@ -65,6 +67,7 @@ source_set("unit_tests") {
 
   sources = [
     "key_event_mapper_unittest.cc",
+    "keycode_map_unittest.cc",
     "touch_input_scaler_unittest.cc",
   ]
 

remoting/client/input/keyboard_interpreter.cc

@@ -5,6 +5,7 @@
 #include "remoting/client/input/keyboard_interpreter.h"
 
 #include "base/logging.h"
+#include "remoting/client/input/keycode_map.h"
 #include "remoting/client/input/text_keyboard_input_strategy.h"
 #include "ui/events/keycodes/dom/dom_code.h"
 
@@ -24,6 +25,24 @@ void KeyboardInterpreter::SetContext(ClientInputInjector* input_injector) {
   }
 }
 
+void KeyboardInterpreter::HandleKeypressEvent(const KeypressInfo& keypress) {
+  if (!input_strategy_) {
+    return;
+  }
+
+  DCHECK(keypress.dom_code != ui::DomCode::NONE);
+  base::queue<KeyEvent> keys;
+  if (keypress.modifiers & KeypressInfo::Modifier::SHIFT) {
+    keys.push({static_cast<uint32_t>(ui::DomCode::SHIFT_LEFT), true});
+  }
+  keys.push({static_cast<uint32_t>(keypress.dom_code), true});
+  keys.push({static_cast<uint32_t>(keypress.dom_code), false});
+  if (keypress.modifiers & KeypressInfo::Modifier::SHIFT) {
+    keys.push({static_cast<uint32_t>(ui::DomCode::SHIFT_LEFT), false});
+  }
+  input_strategy_->HandleKeysEvent(keys);
+}
+
 void KeyboardInterpreter::HandleTextEvent(const std::string& text,
                                           uint8_t modifiers) {
   if (!input_strategy_) {

remoting/client/input/keyboard_interpreter.h

@@ -15,6 +15,8 @@ namespace remoting {
 class KeyboardInputStrategy;
 class ClientInputInjector;
 
+struct KeypressInfo;
+
 // This is a class for interpreting raw keyboard input, it will delegate
 // handling of text events to the selected keyboard input strategy.
 class KeyboardInterpreter {
@@ -25,6 +27,9 @@ class KeyboardInterpreter {
   // If |input_injector| is nullptr, all methods below will have no effect.
   void SetContext(ClientInputInjector* input_injector);
 
+  // Assembles the key events and then delegates to |KeyboardInputStrategy| to
+  // send the keys.
+  void HandleKeypressEvent(const KeypressInfo& keypress);
   // Delegates to |KeyboardInputStrategy| to covert and send the input.
   void HandleTextEvent(const std::string& text, uint8_t modifiers);
   // Delegates to |KeyboardInputStrategy| to covert and send the delete.

remoting/client/input/keycode_map.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "remoting/client/input/keycode_map.h"
+
+#include <array>
+#include <limits>
+
+#include "base/logging.h"
+#include "base/no_destructor.h"
+
+namespace remoting {
+
+namespace {
+
+// TODO(yuweih): Using char to store the characters may not work if we decide
+// to support AZERTY or other keyboard.
+const size_t kMaxAsciiConvertibleLength =
+    std::numeric_limits<unsigned char>::max();
+
+// An array with character code as its index and KeycodeInfo as its value.
+using KeycodeMap = std::array<KeypressInfo, kMaxAsciiConvertibleLength>;
+
+struct KeycodeMapEntry {
+  ui::DomCode dom_code;
+  unsigned char normal_character;
+  // 0 if there is no shift character.
+  unsigned char shift_character;
+};
+
+constexpr KeycodeMapEntry kKeycodeMapEntriesQwerty[] = {
+    {ui::DomCode::US_A, 'a', 'A'},
+    {ui::DomCode::US_B, 'b', 'B'},
+    {ui::DomCode::US_C, 'c', 'C'},
+    {ui::DomCode::US_D, 'd', 'D'},
+    {ui::DomCode::US_E, 'e', 'E'},
+    {ui::DomCode::US_F, 'f', 'F'},
+    {ui::DomCode::US_G, 'g', 'G'},
+    {ui::DomCode::US_H, 'h', 'H'},
+    {ui::DomCode::US_I, 'i', 'I'},
+    {ui::DomCode::US_J, 'j', 'J'},
+    {ui::DomCode::US_K, 'k', 'K'},
+    {ui::DomCode::US_L, 'l', 'L'},
+    {ui::DomCode::US_M, 'm', 'M'},
+    {ui::DomCode::US_N, 'n', 'N'},
+    {ui::DomCode::US_O, 'o', 'O'},
+    {ui::DomCode::US_P, 'p', 'P'},
+    {ui::DomCode::US_Q, 'q', 'Q'},
+    {ui::DomCode::US_R, 'r', 'R'},
+    {ui::DomCode::US_S, 's', 'S'},
+    {ui::DomCode::US_T, 't', 'T'},
+    {ui::DomCode::US_U, 'u', 'U'},
+    {ui::DomCode::US_V, 'v', 'V'},
+    {ui::DomCode::US_W, 'w', 'W'},
+    {ui::DomCode::US_X, 'x', 'X'},
+    {ui::DomCode::US_Y, 'y', 'Y'},
+    {ui::DomCode::US_Z, 'z', 'Z'},
+    {ui::DomCode::DIGIT1, '1', '!'},
+    {ui::DomCode::DIGIT2, '2', '@'},
+    {ui::DomCode::DIGIT3, '3', '#'},
+    {ui::DomCode::DIGIT4, '4', '$'},
+    {ui::DomCode::DIGIT5, '5', '%'},
+    {ui::DomCode::DIGIT6, '6', '^'},
+    {ui::DomCode::DIGIT7, '7', '&'},
+    {ui::DomCode::DIGIT8, '8', '*'},
+    {ui::DomCode::DIGIT9, '9', '('},
+    {ui::DomCode::DIGIT0, '0', ')'},
+    {ui::DomCode::SPACE, ' ', 0},
+    {ui::DomCode::ENTER, '\n', 0},
+    {ui::DomCode::MINUS, '-', '_'},
+    {ui::DomCode::EQUAL, '=', '+'},
+    {ui::DomCode::BRACKET_LEFT, '[', '{'},
+    {ui::DomCode::BRACKET_RIGHT, ']', '}'},
+    {ui::DomCode::BACKSLASH, '\\', '|'},
+    {ui::DomCode::SEMICOLON, ';', ':'},
+    {ui::DomCode::QUOTE, '\'', '"'},
+    {ui::DomCode::BACKQUOTE, '`', '~'},
+    {ui::DomCode::COMMA, ',', '<'},
+    {ui::DomCode::PERIOD, '.', '>'},
+    {ui::DomCode::SLASH, '/', '?'},
+};
+
+template <size_t N>
+KeycodeMap CreateKeycodeMapFromMapEntries(const KeycodeMapEntry (&entries)[N]) {
+  KeycodeMap map;
+  // Initialize keycodes with NONE.
+  for (size_t i = 0; i < map.size(); i++) {
+    map[i] = {ui::DomCode::NONE, KeypressInfo::Modifier::NONE};
+  }
+  for (size_t i = 0; i < N; i++) {
+    const KeycodeMapEntry& entry = entries[i];
+    DCHECK(entry.dom_code != ui::DomCode::NONE);
+    DCHECK(entry.normal_character != 0);
+
+    DCHECK(map[entry.normal_character].dom_code == ui::DomCode::NONE)
+        << "Character " << entry.normal_character << " is already in the map.";
+    map[entry.normal_character].dom_code = entry.dom_code;
+
+    if (entry.shift_character != 0) {
+      DCHECK(map[entry.shift_character].dom_code == ui::DomCode::NONE)
+          << "Character " << entry.shift_character << " is already in the map.";
+      map[entry.shift_character].dom_code = entry.dom_code;
+      map[entry.shift_character].modifiers = KeypressInfo::Modifier::SHIFT;
+    }
+  }
+  return map;
+}
+
+const KeycodeMap& GetKeycodeMapQwerty() {
+  static const base::NoDestructor<KeycodeMap> map(
+      CreateKeycodeMapFromMapEntries(kKeycodeMapEntriesQwerty));
+  return *map;
+}
+
+}  // namespace
+
+KeypressInfo KeypressFromUnicode(unsigned int unicode) {
+  if (unicode >= kMaxAsciiConvertibleLength) {
+    return {ui::DomCode::NONE, KeypressInfo::Modifier::NONE};
+  } else {
+    return GetKeycodeMapQwerty()[unicode];
+  }
+}
+
+}  // namespace remoting

remoting/client/input/keycode_map.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef REMOTING_CLIENT_INPUT_KEYCODE_MAP_H_
+#define REMOTING_CLIENT_INPUT_KEYCODE_MAP_H_
+
+#include "ui/events/keycodes/dom/dom_code.h"
+
+namespace remoting {
+
+// A class representing a combination of keys (i.e. main key+modifiers) that
+// should be pressed at the same time.
+struct KeypressInfo {
+  enum Modifier {
+    NONE = 0,
+    SHIFT = 1 << 0,
+    // TODO(yuweih): Add more modifiers when needed.
+  };
+
+  ui::DomCode dom_code;
+  Modifier modifiers;
+};
+
+// Gets a keypress that can produce the given unicode on the given keyboard
+// layout. If no such a keypress can be found, a keypress with dom_code = NONE
+// will be returned.
+KeypressInfo KeypressFromUnicode(unsigned int unicode);
+
+}  // namespace remoting
+#endif  // REMOTING_CLIENT_INPUT_KEYCODE_MAP_H_

remoting/client/input/keycode_map_unittest.cc

+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "remoting/client/input/keycode_map.h"
+
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace remoting {
+
+TEST(KeycodeMapTest, KeypressFromUnicodeWithAsciiCharsOnQwertyLayout) {
+  KeypressInfo keypress = KeypressFromUnicode('a');
+  EXPECT_EQ(ui::DomCode::US_A, keypress.dom_code);
+  EXPECT_EQ(KeypressInfo::Modifier::NONE, keypress.modifiers);
+
+  keypress = KeypressFromUnicode('Q');
+  EXPECT_EQ(ui::DomCode::US_Q, keypress.dom_code);
+  EXPECT_EQ(KeypressInfo::Modifier::SHIFT, keypress.modifiers);
+
+  keypress = KeypressFromUnicode(' ');
+  EXPECT_EQ(ui::DomCode::SPACE, keypress.dom_code);
+  EXPECT_EQ(KeypressInfo::Modifier::NONE, keypress.modifiers);
+
+  keypress = KeypressFromUnicode('\n');
+  EXPECT_EQ(ui::DomCode::ENTER, keypress.dom_code);
+  EXPECT_EQ(KeypressInfo::Modifier::NONE, keypress.modifiers);
+
+  keypress = KeypressFromUnicode('!');
+  EXPECT_EQ(ui::DomCode::DIGIT1, keypress.dom_code);
+  EXPECT_EQ(KeypressInfo::Modifier::SHIFT, keypress.modifiers);
+
+  keypress = KeypressFromUnicode('`');
+  EXPECT_EQ(ui::DomCode::BACKQUOTE, keypress.dom_code);
+  EXPECT_EQ(KeypressInfo::Modifier::NONE, keypress.modifiers);
+}
+
+TEST(KeycodeMapTest, KeypressFromUnicodeWithUnmappableCharsOnQwertyLayout) {
+  // NULL
+  KeypressInfo keypress = KeypressFromUnicode(0);
+  EXPECT_EQ(ui::DomCode::NONE, keypress.dom_code);
+  EXPECT_EQ(KeypressInfo::Modifier::NONE, keypress.modifiers);
+
+  // BELL
+  keypress = KeypressFromUnicode(7);
+  EXPECT_EQ(ui::DomCode::NONE, keypress.dom_code);
+  EXPECT_EQ(KeypressInfo::Modifier::NONE, keypress.modifiers);
+
+  // ESC
+  keypress = KeypressFromUnicode(27);
+  EXPECT_EQ(ui::DomCode::NONE, keypress.dom_code);
+  EXPECT_EQ(KeypressInfo::Modifier::NONE, keypress.modifiers);
+
+  // Non-ASCII character
+  keypress = KeypressFromUnicode(128);
+  EXPECT_EQ(ui::DomCode::NONE, keypress.dom_code);
+  EXPECT_EQ(KeypressInfo::Modifier::NONE, keypress.modifiers);
+
+  // Non-ASCII character
+  keypress = KeypressFromUnicode(2000);
+  EXPECT_EQ(ui::DomCode::NONE, keypress.dom_code);
+  EXPECT_EQ(KeypressInfo::Modifier::NONE, keypress.modifiers);
+}
+
+}  // namespace remoting

remoting/ios/app/host_view_controller.mm

@@ -326,6 +326,10 @@ static NSString* const kFeedbackContext = @"InSessionFeedbackContext";
                                                0);
 }
 
+- (void)clientKeyboardShouldSendKey:(const remoting::KeypressInfo&)key {
+  _client.keyboardInterpreter->HandleKeypressEvent(key);
+}
+
 - (void)clientKeyboardShouldDelete {
   _client.keyboardInterpreter->HandleDeleteEvent(0);
 }

remoting/ios/client_keyboard.h

@@ -8,8 +8,13 @@
 #import <Foundation/Foundation.h>
 #import <UIKit/UIKit.h>
 
+namespace remoting {
+struct KeypressInfo;
+}  // namespace remoting
+
 @protocol ClientKeyboardDelegate<NSObject>
 - (void)clientKeyboardShouldSend:(NSString*)text;
+- (void)clientKeyboardShouldSendKey:(const remoting::KeypressInfo&)key;
 - (void)clientKeyboardShouldDelete;
 @end
 

remoting/ios/client_keyboard.mm

@@ -8,6 +8,8 @@
 
 #import "remoting/ios/client_keyboard.h"
 
+#include "remoting/client/input/keycode_map.h"
+
 // TODO(nicholss): Look into inputAccessoryView to get the top bar for sending
 // special keys.
 // TODO(nicholss): Look into inputView - The custom input view to display when
@@ -49,6 +51,16 @@
 #pragma mark - UIKeyInput
 
 - (void)insertText:(NSString*)text {
+  if (text.length == 1) {
+    // TODO(yuweih): KeyboardLayout should be configurable.
+    remoting::KeypressInfo keypress =
+        remoting::KeypressFromUnicode([text characterAtIndex:0]);
+    if (keypress.dom_code != ui::DomCode::NONE) {
+      [_delegate clientKeyboardShouldSendKey:keypress];
+      return;
+    }
+  }
+  // Fallback to text injection.
   [_delegate clientKeyboardShouldSend:text];
 }