Add mock keyboard input for VR instrumentation tests

Adds the ability to use a mock keyboard during VR instrumentation tests,
which allows arbitrary text input without having to actually click the
keys on the keyboard. This is achieved in a similar way to the mock
controller input, where the keyboard delegate is swapped to the mock
one when necessary.

Also automates one manual test for navigation on URL entry that uses
the new functionality.

Bug: 887523
Cq-Include-Trybots: luci.chromium.try:android_optional_gpu_tests_rel;luci.chromium.try:linux_optional_gpu_tests_rel;luci.chromium.try:linux_vr;luci.chromium.try:mac_optional_gpu_tests_rel;luci.chromium.try:win_optional_gpu_tests_rel
Change-Id: Ia1130df87a154a6771038182883521ede209be99
Reviewed-on: https://chromium-review.googlesource.com/c/1257665Reviewed-by: Aldo Culquicondor <acondor@chromium.org>
Reviewed-by: Michael Thiessen <mthiesse@chromium.org>
Commit-Queue: Brian Sheedy <bsheedy@chromium.org>
Cr-Commit-Position: refs/heads/master@{#596733}

chrome/android/java/src/org/chromium/chrome/browser/vr/VrShell.java

@@ -1251,6 +1251,12 @@ public class VrShell extends GvrLayout
                 mNativeVrShell, elementName, actionType, position.x, position.y);
     }
 
+    public void performKeyboardInputForTesting(int inputType, String inputString) {
+        ThreadUtils.runOnUiThreadBlocking(() -> {
+            nativePerformKeyboardInputForTesting(mNativeVrShell, inputType, inputString);
+        });
+    }
+
     public void registerUiOperationCallbackForTesting(
             int actionType, Runnable resultCallback, int timeoutMs, int elementName) {
         assert actionType < UiTestOperationType.NUM_UI_TEST_OPERATION_TYPES;
@@ -1339,6 +1345,8 @@ public class VrShell extends GvrLayout
     private native void nativeAcceptDoffPromptForTesting(long nativeVrShell);
     private native void nativePerformControllerActionForTesting(
             long nativeVrShell, int elementName, int actionType, float x, float y);
+    private native void nativePerformKeyboardInputForTesting(
+            long nativeVrShell, int inputType, String inputString);
     private native void nativeSetUiExpectingActivityForTesting(
             long nativeVrShell, int quiescenceTimeoutMs);
     private native void nativeSaveNextFrameBufferToDiskForTesting(

chrome/android/javatests/src/org/chromium/chrome/browser/vr/TestVrShellDelegate.java

@@ -94,6 +94,10 @@ public class TestVrShellDelegate extends VrShellDelegate {
         getVrShell().performControllerActionForTesting(elementName, actionType, position);
     }
 
+    public void performKeyboardInputForTesting(int inputType, String inputString) {
+        getVrShell().performKeyboardInputForTesting(inputType, inputString);
+    }
+
     public void registerUiOperationCallbackForTesting(
             int actionType, Runnable resultCallback, int timeoutMs, int elementName) {
         getVrShell().registerUiOperationCallbackForTesting(

chrome/android/javatests/src/org/chromium/chrome/browser/vr/VrBrowserNavigationTest.java

@@ -657,4 +657,27 @@ public class VrBrowserNavigationTest {
         Assert.assertTrue("Created native page is not a NTP",
                 mTestRule.getActivity().getActivityTab().getNativePage() instanceof NewTabPage);
     }
+
+    /**
+     * Tests that inputting a URL into the URL bar results in a successful navigation.
+     */
+    @Test
+    @MediumTest
+    public void testUrlEntryTriggersNavigation() throws InterruptedException {
+        NativeUiUtils.enableMockedKeyboard();
+        NativeUiUtils.clickElementAndWaitForUiQuiescence(UserFriendlyElementName.URL, new PointF());
+        // This is a roundabout solution for ensuring that the committing/pressing of enter actually
+        // results in a navigation - internally, this is handled by grabbing the first suggestion
+        // and navigating to its destination. So, we need to make sure we have suggestions before
+        // pressing enter, otherwise nothing happens. If this ever stops working, e.g. due to always
+        // having suggestions present, waiting for a number of frames before committing seemed
+        // stable. Adding an operation to listen for suggestion changes would also work, but
+        // would add some pretty niche test-only code to the native side.
+        NativeUiUtils.performActionAndWaitForVisibilityChange(
+                UserFriendlyElementName.SUGGESTION_BOX,
+                () -> { NativeUiUtils.inputString("chrome://version/"); });
+        NativeUiUtils.inputEnter();
+        ChromeTabUtils.waitForTabPageLoaded(
+                mTestRule.getActivity().getActivityTab(), "chrome://version/");
+    }
 }

chrome/android/javatests/src/org/chromium/chrome/browser/vr/util/NativeUiUtils.java

@@ -14,6 +14,7 @@ import org.junit.Assert;
 import org.chromium.base.ThreadUtils;
 import org.chromium.chrome.R;
 import org.chromium.chrome.browser.ChromeActivity;
+import org.chromium.chrome.browser.vr.KeyboardTestAction;
 import org.chromium.chrome.browser.vr.TestVrShellDelegate;
 import org.chromium.chrome.browser.vr.UiTestOperationResult;
 import org.chromium.chrome.browser.vr.UiTestOperationType;
@@ -46,6 +47,16 @@ public class NativeUiUtils {
                 new PointF() /* position, unused */);
     }
 
+    /**
+     * Enables the mock keyboard without sending any input. Should be called before performing an
+     * action that would trigger the keyboard for the first time to ensure that the mock keyboard
+     * is used instead of the real one.
+     */
+    public static void enableMockedKeyboard() {
+        TestVrShellDelegate.getInstance().performKeyboardInputForTesting(
+                KeyboardTestAction.ENABLE_MOCKED_KEYBOARD, "" /* unused */);
+    }
+
     /**
      * Clicks on a UI element as if done via a controller.
      *
@@ -58,6 +69,32 @@ public class NativeUiUtils {
                 elementName, VrControllerTestAction.CLICK, position);
     }
 
+    /**
+     * Inputs the given text as if done via the VR keyboard.
+     *
+     * @param inputString The String to input via the keyboard.
+     */
+    public static void inputString(String inputString) {
+        TestVrShellDelegate.getInstance().performKeyboardInputForTesting(
+                KeyboardTestAction.INPUT_TEXT, inputString);
+    }
+
+    /**
+     * Presses backspace as if done via the VR keyboard.
+     */
+    public static void inputBackspace() {
+        TestVrShellDelegate.getInstance().performKeyboardInputForTesting(
+                KeyboardTestAction.BACKSPACE, "" /* unused */);
+    }
+
+    /**
+     * Presses enter as if done via the VR keyboard.
+     */
+    public static void inputEnter() throws InterruptedException {
+        TestVrShellDelegate.getInstance().performKeyboardInputForTesting(
+                KeyboardTestAction.ENTER, "" /* unused */);
+    }
+
     /**
      * Clicks on a UI element as if done via a controller and waits until all resulting
      * animations have finished and propogated to the point of being visible in screenshots.

chrome/browser/android/vr/vr_gl_thread.cc

@@ -553,6 +553,15 @@ void VrGLThread::RemoveAllTabs() {
       base::BindOnce(&BrowserUiInterface::RemoveAllTabs, weak_browser_ui_));
 }
 
+void VrGLThread::PerformKeyboardInputForTesting(
+    KeyboardTestInput keyboard_input) {
+  DCHECK(OnMainThread());
+  task_runner()->PostTask(
+      FROM_HERE,
+      base::BindOnce(&BrowserUiInterface::PerformKeyboardInputForTesting,
+                     weak_browser_ui_, keyboard_input));
+}
+
 void VrGLThread::ReportUiOperationResultForTesting(
     const UiTestOperationType& action_type,
     const UiTestOperationResult& result) {

chrome/browser/android/vr/vr_gl_thread.h

@@ -37,6 +37,7 @@ namespace vr {
 
 class VrInputConnection;
 class VrShell;
+struct KeyboardTestInput;
 
 class VrGLThread : public base::android::JavaHandlerThread,
                    public PlatformInputHandler,
@@ -158,6 +159,8 @@ class VrGLThread : public base::android::JavaHandlerThread,
                       const base::string16& title) override;
   void RemoveTab(int id, bool incognito) override;
   void RemoveAllTabs() override;
+  void PerformKeyboardInputForTesting(
+      KeyboardTestInput keyboard_input) override;
 
  protected:
   void Init() override;

chrome/browser/android/vr/vr_shell.cc

@@ -1308,6 +1308,18 @@ void VrShell::PerformControllerActionForTesting(
                      gl_thread_->GetBrowserRenderer(), controller_input));
 }
 
+void VrShell::PerformKeyboardInputForTesting(
+    JNIEnv* env,
+    const base::android::JavaParamRef<jobject>& obj,
+    jint input_type,
+    jstring input_string) {
+  KeyboardTestInput keyboard_input;
+  keyboard_input.action = static_cast<KeyboardTestAction>(input_type);
+  keyboard_input.input_text =
+      base::android::ConvertJavaStringToUTF8(env, input_string);
+  ui_->PerformKeyboardInputForTesting(keyboard_input);
+}
+
 std::unique_ptr<PageInfo> VrShell::CreatePageInfo() {
   if (!web_contents_)
     return nullptr;

chrome/browser/android/vr/vr_shell.h

@@ -301,6 +301,12 @@ class VrShell : device::GvrGamepadDataProvider,
       jint action_type,
       jfloat x,
       jfloat y);
+
+  void PerformKeyboardInputForTesting(
+      JNIEnv* env,
+      const base::android::JavaParamRef<jobject>& obj,
+      jint input_type,
+      jstring input_string);
   gfx::AcceleratedWidget GetRenderSurface();
 
  private:

chrome/browser/vr/BUILD.gn

@@ -340,6 +340,8 @@ source_set("vr_base") {
     "input_event.cc",
     "input_event.h",
     "keyboard_delegate.h",
+    "keyboard_delegate_for_testing.cc",
+    "keyboard_delegate_for_testing.h",
     "keyboard_ui_interface.h",
     "macros.h",
     "metrics/metrics_helper.cc",

chrome/browser/vr/browser_ui_interface.h

@@ -20,6 +20,7 @@ class Version;
 namespace vr {
 
 struct Assets;
+struct KeyboardTestInput;
 struct OmniboxSuggestions;
 struct ToolbarState;
 
@@ -70,6 +71,8 @@ class VR_EXPORT BrowserUiInterface {
                               const base::string16& title) = 0;
   virtual void RemoveTab(int id, bool incognito) = 0;
   virtual void RemoveAllTabs() = 0;
+  virtual void PerformKeyboardInputForTesting(
+      KeyboardTestInput keyboard_input) = 0;
 };
 
 }  // namespace vr

chrome/browser/vr/keyboard_delegate_for_testing.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/vr/keyboard_delegate_for_testing.h"
+
+#include <algorithm>
+
+#include "base/strings/utf_string_conversions.h"
+#include "chrome/browser/vr/keyboard_ui_interface.h"
+#include "chrome/browser/vr/model/text_input_info.h"
+#include "chrome/browser/vr/ui_test_input.h"
+
+namespace vr {
+
+KeyboardDelegateForTesting::KeyboardDelegateForTesting() {
+  cached_keyboard_input_ = TextInputInfo();
+}
+
+KeyboardDelegateForTesting::~KeyboardDelegateForTesting() = default;
+
+void KeyboardDelegateForTesting::QueueKeyboardInputForTesting(
+    KeyboardTestInput keyboard_input) {
+  keyboard_input_queue_.push(keyboard_input);
+}
+
+bool KeyboardDelegateForTesting::IsQueueEmpty() const {
+  return keyboard_input_queue_.empty();
+}
+
+void KeyboardDelegateForTesting::SetUiInterface(KeyboardUiInterface* ui) {
+  ui_ = ui;
+}
+
+void KeyboardDelegateForTesting::ShowKeyboard() {
+  keyboard_shown_ = true;
+}
+void KeyboardDelegateForTesting::HideKeyboard() {
+  keyboard_shown_ = false;
+}
+void KeyboardDelegateForTesting::SetTransform(const gfx::Transform&) {}
+bool KeyboardDelegateForTesting::HitTest(const gfx::Point3F& ray_origin,
+                                         const gfx::Point3F& ray_target,
+                                         gfx::Point3F* touch_position) {
+  return false;
+}
+
+void KeyboardDelegateForTesting::OnBeginFrame() {
+  if (!keyboard_shown_ || IsQueueEmpty() || pause_keyboard_input_)
+    return;
+
+  DCHECK(ui_);
+  KeyboardTestInput input = keyboard_input_queue_.front();
+  keyboard_input_queue_.pop();
+  TextInputInfo next_info;
+  auto current_string = cached_keyboard_input_.text;
+  int cursor_start = std::min(cached_keyboard_input_.selection_start,
+                              cached_keyboard_input_.selection_end);
+  int new_selection_start;
+
+  switch (input.action) {
+    case KeyboardTestAction::kInputText:
+      // We can either be inputting text at a cursor position or replacing
+      // selected text.
+      if (cached_keyboard_input_.SelectionSize() == 0) {
+        // Inputting at cursor.
+        current_string.insert(cached_keyboard_input_.selection_start,
+                              base::UTF8ToUTF16(input.input_text));
+      } else {
+        // Replacing selected text.
+        current_string.replace(cursor_start,
+                               cached_keyboard_input_.SelectionSize(),
+                               base::UTF8ToUTF16(input.input_text));
+      }
+      new_selection_start = cursor_start + input.input_text.length();
+      next_info = TextInputInfo(current_string, new_selection_start,
+                                new_selection_start);
+      break;
+    case KeyboardTestAction::kBackspace:
+      // We can either be deleting at a cursor position or deleting selected
+      // text.
+      if (cached_keyboard_input_.SelectionSize() == 0) {
+        // Deleting at cursor.
+        // We can't delete if the cursor is at the start, so no-op.
+        if (cursor_start == 0) {
+          return;
+        }
+        current_string.erase(cursor_start - 1, 1);
+        new_selection_start = cursor_start - 1;
+      } else {
+        // Deleting selected text.
+        current_string.erase(cursor_start,
+                             cached_keyboard_input_.SelectionSize());
+        new_selection_start = cursor_start;
+      }
+      next_info = TextInputInfo(current_string, new_selection_start,
+                                new_selection_start);
+      break;
+    case KeyboardTestAction::kEnter:
+      ui_->OnInputCommitted(
+          EditedText(cached_keyboard_input_, cached_keyboard_input_));
+      return;
+    default:
+      NOTREACHED() << "Given unsupported controller action";
+  }
+  ui_->OnInputEdited(EditedText(next_info, cached_keyboard_input_));
+  pause_keyboard_input_ = true;
+}
+
+void KeyboardDelegateForTesting::Draw(const CameraModel&) {}
+
+bool KeyboardDelegateForTesting::SupportsSelection() {
+  return true;
+}
+
+void KeyboardDelegateForTesting::UpdateInput(const TextInputInfo& info) {
+  cached_keyboard_input_ = info;
+  pause_keyboard_input_ = false;
+}
+
+}  // namespace vr

chrome/browser/vr/keyboard_delegate_for_testing.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_VR_KEYBOARD_DELEGATE_FOR_TESTING_H_
+#define CHROME_BROWSER_VR_KEYBOARD_DELEGATE_FOR_TESTING_H_
+
+#include <queue>
+
+#include "base/macros.h"
+#include "chrome/browser/vr/keyboard_delegate.h"
+#include "chrome/browser/vr/model/text_input_info.h"
+#include "chrome/browser/vr/ui_test_input.h"
+
+namespace gfx {
+class Point3F;
+class Transform;
+}  // namespace gfx
+
+namespace vr {
+
+class KeyboardUiInterface;
+struct CameraModel;
+
+class KeyboardDelegateForTesting : public KeyboardDelegate {
+ public:
+  KeyboardDelegateForTesting();
+  ~KeyboardDelegateForTesting() override;
+
+  void QueueKeyboardInputForTesting(KeyboardTestInput keyboard_input);
+  bool IsQueueEmpty() const;
+
+  // KeyboardDelegate implementation.
+  void SetUiInterface(KeyboardUiInterface* ui) override;
+  void ShowKeyboard() override;
+  void HideKeyboard() override;
+  void SetTransform(const gfx::Transform&) override;
+  bool HitTest(const gfx::Point3F& ray_origin,
+               const gfx::Point3F& ray_target,
+               gfx::Point3F* touch_position) override;
+  void OnBeginFrame() override;
+  void Draw(const CameraModel&) override;
+  bool SupportsSelection() override;
+  void UpdateInput(const TextInputInfo& info) override;
+
+ private:
+  KeyboardUiInterface* ui_;
+  std::queue<KeyboardTestInput> keyboard_input_queue_;
+  TextInputInfo cached_keyboard_input_;
+  bool keyboard_shown_ = false;
+  bool pause_keyboard_input_ = false;
+
+  DISALLOW_COPY_AND_ASSIGN(KeyboardDelegateForTesting);
+};
+
+}  // namespace vr
+
+#endif  // CHROME_BROWSER_VR_KEYBOARD_DELEGATE_FOR_TESTING_H_

chrome/browser/vr/test/mock_browser_ui_interface.h

@@ -10,6 +10,7 @@
 #include "chrome/browser/vr/model/assets.h"
 #include "chrome/browser/vr/model/omnibox_suggestions.h"
 #include "chrome/browser/vr/model/toolbar_state.h"
+#include "chrome/browser/vr/ui_test_input.h"
 #include "testing/gmock/include/gmock/gmock.h"
 
 namespace vr {
@@ -56,6 +57,7 @@ class MockBrowserUiInterface : public BrowserUiInterface {
                void(int id, bool incognito, const base::string16& title));
   MOCK_METHOD2(RemoveTab, void(int id, bool incognito));
   MOCK_METHOD0(RemoveAllTabs, void());
+  MOCK_METHOD1(PerformKeyboardInputForTesting, void(KeyboardTestInput));
 
  private:
   DISALLOW_COPY_AND_ASSIGN(MockBrowserUiInterface);

chrome/browser/vr/ui.cc

@@ -17,8 +17,10 @@
 #include "base/threading/thread_task_runner_handle.h"
 #include "chrome/browser/vr/content_input_delegate.h"
 #include "chrome/browser/vr/elements/content_element.h"
+#include "chrome/browser/vr/elements/keyboard.h"
 #include "chrome/browser/vr/elements/text_input.h"
 #include "chrome/browser/vr/keyboard_delegate.h"
+#include "chrome/browser/vr/keyboard_delegate_for_testing.h"
 #include "chrome/browser/vr/model/assets.h"
 #include "chrome/browser/vr/model/model.h"
 #include "chrome/browser/vr/model/omnibox_suggestions.h"
@@ -69,6 +71,8 @@ UiElementName UserFriendlyElementNameToUiElementName(
       return kOverflowMenuCloseAllIncognitoTabsItem;
     case UserFriendlyElementName::kExitPrompt:
       return kExitPrompt;
+    case UserFriendlyElementName::kSuggestionBox:
+      return kOmniboxSuggestions;
     default:
       NOTREACHED();
       return kNone;
@@ -609,6 +613,44 @@ gfx::Point3F Ui::GetTargetPointForTesting(UserFriendlyElementName element_name,
          gfx::ScaleVector3d(direction, scene()->background_distance());
 }
 
+void Ui::PerformKeyboardInputForTesting(KeyboardTestInput keyboard_input) {
+  DCHECK(keyboard_delegate_);
+  if (keyboard_input.action == KeyboardTestAction::kRevertToRealKeyboard) {
+    if (using_keyboard_delegate_for_testing_) {
+      DCHECK(static_cast<KeyboardDelegateForTesting*>(keyboard_delegate_.get())
+                 ->IsQueueEmpty())
+          << "Attempted to revert to real keyboard with input still queued";
+      using_keyboard_delegate_for_testing_ = false;
+      keyboard_delegate_for_testing_.swap(keyboard_delegate_);
+      static_cast<Keyboard*>(
+          scene_->GetUiElementByName(UiElementName::kKeyboard))
+          ->SetKeyboardDelegate(keyboard_delegate_.get());
+      text_input_delegate_->SetUpdateInputCallback(
+          base::BindRepeating(&KeyboardDelegate::UpdateInput,
+                              base::Unretained(keyboard_delegate_.get())));
+    }
+    return;
+  }
+  if (!using_keyboard_delegate_for_testing_) {
+    using_keyboard_delegate_for_testing_ = true;
+    if (!keyboard_delegate_for_testing_) {
+      keyboard_delegate_for_testing_ =
+          std::make_unique<KeyboardDelegateForTesting>();
+      keyboard_delegate_for_testing_->SetUiInterface(this);
+    }
+    keyboard_delegate_for_testing_.swap(keyboard_delegate_);
+    static_cast<Keyboard*>(scene_->GetUiElementByName(UiElementName::kKeyboard))
+        ->SetKeyboardDelegate(keyboard_delegate_.get());
+    text_input_delegate_->SetUpdateInputCallback(
+        base::BindRepeating(&KeyboardDelegate::UpdateInput,
+                            base::Unretained(keyboard_delegate_.get())));
+  }
+  if (keyboard_input.action != KeyboardTestAction::kEnableMockedKeyboard) {
+    static_cast<KeyboardDelegateForTesting*>(keyboard_delegate_.get())
+        ->QueueKeyboardInputForTesting(keyboard_input);
+  }
+}
+
 ContentElement* Ui::GetContentElement() {
   if (!content_element_) {
     content_element_ =

chrome/browser/vr/ui.h

@@ -121,6 +121,8 @@ class VR_UI_EXPORT Ui : public UiInterface,
                       const base::string16& title) override;
   void RemoveTab(int id, bool incognito) override;
   void RemoveAllTabs() override;
+  void PerformKeyboardInputForTesting(
+      KeyboardTestInput keyboard_input) override;
 
   // UiInterface
   base::WeakPtr<BrowserUiInterface> GetBrowserUiWeakPtr() override;
@@ -225,6 +227,8 @@ class VR_UI_EXPORT Ui : public UiInterface,
   ContentElement* content_element_ = nullptr;
 
   std::unique_ptr<KeyboardDelegate> keyboard_delegate_;
+  std::unique_ptr<KeyboardDelegate> keyboard_delegate_for_testing_;
+  bool using_keyboard_delegate_for_testing_ = false;
   std::unique_ptr<TextInputDelegate> text_input_delegate_;
   std::unique_ptr<AudioDelegate> audio_delegate_;
 

chrome/browser/vr/ui_test_input.h

@@ -26,6 +26,7 @@ enum class UserFriendlyElementName : int {
   kCloseIncognitoTabs,  // Button to close all Incognito tabs in the overflow
                         // menu
   kExitPrompt,          // DOFF prompt/request to exit VR
+  kSuggestionBox,       // Box containing the omnibox suggestions
 };
 
 // These are the types of actions that Java can request callbacks for once
@@ -64,6 +65,17 @@ enum class VrControllerTestAction : int {
   kMove,
 };
 
+// These are used to specify what type of keyboard input should be performed
+// for a frame during testing.
+// GENERATED_JAVA_ENUM_PACKAGE: org.chromium.chrome.browser.vr
+enum class KeyboardTestAction : int {
+  kInputText,
+  kBackspace,
+  kEnter,
+  kEnableMockedKeyboard,
+  kRevertToRealKeyboard,
+};
+
 // Holds all information necessary to perform a simulated controller action on
 // a UI element.
 struct ControllerTestInput {
@@ -72,6 +84,12 @@ struct ControllerTestInput {
   gfx::PointF position;
 };
 
+// Holds all the information necessary to perform simulated keyboard input.
+struct KeyboardTestInput {
+  KeyboardTestAction action;
+  std::string input_text;
+};
+
 struct UiTestActivityExpectation {
   int quiescence_timeout_ms;
 };