Add app button to VR controller test actions

Adds the ability to press the app button via the Chrome VR native
controller input code and switches all uses of app button presses over
to it. This is should fix all the flakiness we've been seeing with
the app button-related tests, as we're no longer relying on the flaky
VrCore-side controller emulation.

As a side-effect, also adds gesture detection capabilities when using
the controller input code since app button short/long presses require
it. This currently only supports button click detection, but could
be easily extended later on to support scrolling, etc.

Bug: 902938
Change-Id: I36dcb3b0b9e34adb92ade8a1a08d0d53536148ce
Reviewed-on: https://chromium-review.googlesource.com/c/1324829Reviewed-by: Christopher Grant <cjgrant@chromium.org>
Commit-Queue: Brian Sheedy <bsheedy@chromium.org>
Cr-Commit-Position: refs/heads/master@{#608112}

chrome/android/javatests/src/org/chromium/chrome/browser/vr/VrBrowserControllerInputTest.java

@@ -339,7 +339,7 @@ public class VrBrowserControllerInputTest {
         Assert.assertTrue("Page did not enter fullscreen",
                 DOMUtils.isFullscreen(mVrBrowserTestFramework.getFirstTabWebContents()));
 
-        mController.pressReleaseAppButton();
+        NativeUiUtils.clickAppButton(UserFriendlyElementName.NONE, new PointF());
         CriteriaHelper.pollInstrumentationThread(
                 ()
                         -> {
@@ -406,10 +406,11 @@ public class VrBrowserControllerInputTest {
         // is not actually visible, we'll hit a DCHECK in the native code.
         NativeUiUtils.clickElementAndWaitForUiQuiescence(
                 UserFriendlyElementName.OMNIBOX_TEXT_FIELD, new PointF());
-        NativeUiUtils.revertToRealInput();
         // Wait for the URL bar to re-appear, which we take as a signal that we've exited omnibox
         // text input mode.
-        NativeUiUtils.performActionAndWaitForVisibilityStatus(UserFriendlyElementName.URL,
-                true /* visible */, () -> { mController.pressReleaseAppButton(); });
+        NativeUiUtils.performActionAndWaitForVisibilityStatus(
+                UserFriendlyElementName.URL, true /* visible */, () -> {
+                    NativeUiUtils.clickAppButton(UserFriendlyElementName.NONE, new PointF());
+                });
     }
 }

chrome/android/javatests/src/org/chromium/chrome/browser/vr/VrBrowserTransitionTest.java

@@ -15,6 +15,7 @@ import static org.chromium.chrome.test.util.ChromeRestriction.RESTRICTION_TYPE_V
 import android.app.Activity;
 import android.content.Context;
 import android.content.Intent;
+import android.graphics.PointF;
 import android.support.test.InstrumentationRegistry;
 import android.support.test.filters.LargeTest;
 import android.support.test.filters.MediumTest;
@@ -316,7 +317,6 @@ public class VrBrowserTransitionTest {
     private void reEntryFromVrBrowserImpl(String url, WebXrVrTestFramework framework)
             throws InterruptedException {
         VrBrowserTransitionUtils.forceEnterVrBrowserOrFail(POLL_TIMEOUT_LONG_MS);
-        EmulatedVrController controller = new EmulatedVrController(mTestRule.getActivity());
 
         framework.loadUrlAndAwaitInitialization(url, PAGE_LOAD_TIMEOUT_S);
         framework.enterSessionWithUserGestureOrFail();
@@ -324,7 +324,7 @@ public class VrBrowserTransitionTest {
         framework.executeStepAndWait("stepVerifyFirstPresent()");
         // The bug did not reproduce with vrDisplay.exitPresent(), so it might not reproduce with
         // session.end(). Instead, use the controller to exit.
-        controller.pressReleaseAppButton();
+        NativeUiUtils.clickAppButton(UserFriendlyElementName.NONE, new PointF());
         framework.executeStepAndWait("stepVerifyMagicWindow()");
 
         framework.enterSessionWithUserGestureOrFail();

chrome/android/javatests/src/org/chromium/chrome/browser/vr/VrBrowserWebInputEditingTest.java

@@ -404,12 +404,7 @@ public class VrBrowserWebInputEditingTest {
                 "Keyboard did not show from focusing a web input box", POLL_TIMEOUT_SHORT_MS,
                 POLL_CHECK_INTERVAL_SHORT_MS);
 
-        NativeUiUtils.revertToRealInput();
-        EmulatedVrController controller = new EmulatedVrController(mVrTestRule.getActivity());
-        // Do this several times to help combat the flakiness of the emulated controller.
-        for (int i = 0; i < 3; ++i) {
-            controller.pressReleaseAppButton();
-        }
+        NativeUiUtils.clickAppButton(UserFriendlyElementName.NONE, new PointF());
         CriteriaHelper.pollInstrumentationThread(
                 ()
                         -> {

chrome/android/javatests/src/org/chromium/chrome/browser/vr/WebXrVrInputTest.java

@@ -300,7 +300,6 @@ public class WebXrVrInputTest {
             @CommandLineFlags.Add({"enable-features=WebXR"})
             @XrActivityRestriction({XrActivityRestriction.SupportedActivity.ALL})
             public void testScreenTapsRegisteredOnCardboard_WebXr() throws InterruptedException {
-        EmulatedVrController controller = new EmulatedVrController(mTestRule.getActivity());
         mWebXrVrTestFramework.loadUrlAndAwaitInitialization(
                 WebXrVrTestFramework.getFileUrlForHtmlTestFile("test_webxr_input"),
                 PAGE_LOAD_TIMEOUT_S);
@@ -397,8 +396,7 @@ public class WebXrVrInputTest {
             throws InterruptedException {
         framework.loadUrlAndAwaitInitialization(url, PAGE_LOAD_TIMEOUT_S);
         framework.enterSessionWithUserGestureOrFail();
-        EmulatedVrController controller = new EmulatedVrController(mTestRule.getActivity());
-        controller.pressReleaseAppButton();
+        NativeUiUtils.clickAppButton(UserFriendlyElementName.NONE, new PointF());
         assertAppButtonEffect(true /* shouldHaveExited */, framework);
         framework.assertNoJavaScriptErrors();
     }
@@ -477,8 +475,7 @@ public class WebXrVrInputTest {
         MockVrDaydreamApi mockApi = new MockVrDaydreamApi();
         VrShellDelegateUtils.getDelegateInstance().overrideDaydreamApiForTesting(mockApi);
 
-        EmulatedVrController controller = new EmulatedVrController(mTestRule.getActivity());
-        controller.pressReleaseAppButton();
+        NativeUiUtils.clickAppButton(UserFriendlyElementName.NONE, new PointF());
         Assert.assertFalse("App button left Chrome",
                 ThreadUtils.runOnUiThreadBlocking(new Callable<Boolean>() {
                     @Override
@@ -557,8 +554,7 @@ public class WebXrVrInputTest {
         framework.enterSessionWithUserGestureOrFail();
         // Wait for page to stop submitting frames.
         framework.waitOnJavaScriptStep();
-        EmulatedVrController controller = new EmulatedVrController(mTestRule.getActivity());
-        controller.pressReleaseAppButton();
+        NativeUiUtils.clickAppButton(UserFriendlyElementName.NONE, new PointF());
         assertAppButtonEffect(true /* shouldHaveExited */, framework);
         framework.assertNoJavaScriptErrors();
     }
@@ -742,7 +738,6 @@ public class WebXrVrInputTest {
     private void testAppButtonLongPressDisplaysPermissionsImpl() throws InterruptedException {
         // Note that we need to pass in the WebContents to use throughout this because automatically
         // using the first tab's WebContents doesn't work in Incognito.
-        EmulatedVrController controller = new EmulatedVrController(mTestRule.getActivity());
         EmbeddedTestServer server = mTestRule.getTestServer();
         boolean teardownServer = false;
         if (server == null) {
@@ -767,12 +762,13 @@ public class WebXrVrInputTest {
         NativeUiUtils.performActionAndWaitForVisibilityStatus(
                 UserFriendlyElementName.WEB_XR_AUDIO_INDICATOR, false /* visible */, () -> {});
         NativeUiUtils.performActionAndWaitForVisibilityStatus(
-                UserFriendlyElementName.WEB_XR_AUDIO_INDICATOR, true /* visible */,
-                () -> { controller.sendAppButtonToggleEvent(); });
+                UserFriendlyElementName.WEB_XR_AUDIO_INDICATOR, true /* visible */, () -> {
+                    NativeUiUtils.pressAppButton(UserFriendlyElementName.NONE, new PointF());
+                });
         // The toast should automatically disappear after ~5 second after the button is pressed,
         // regardless of whether it's released or not.
         SystemClock.sleep(1000);
-        controller.sendAppButtonToggleEvent();
+        NativeUiUtils.releaseAppButton(UserFriendlyElementName.NONE, new PointF());
         SystemClock.sleep(3500);
         // Make sure it's still present shortly before we expect it to disappear.
         NativeUiUtils.performActionAndWaitForVisibilityStatus(
@@ -781,8 +777,9 @@ public class WebXrVrInputTest {
                 UserFriendlyElementName.WEB_XR_AUDIO_INDICATOR, false /* visible */, () -> {});
         // Do the same, but make sure the toast disappears even with the button still held.
         NativeUiUtils.performActionAndWaitForVisibilityStatus(
-                UserFriendlyElementName.WEB_XR_AUDIO_INDICATOR, true /* visible */,
-                () -> { controller.sendAppButtonToggleEvent(); });
+                UserFriendlyElementName.WEB_XR_AUDIO_INDICATOR, true /* visible */, () -> {
+                    NativeUiUtils.pressAppButton(UserFriendlyElementName.NONE, new PointF());
+                });
         SystemClock.sleep(4500);
         NativeUiUtils.performActionAndWaitForVisibilityStatus(
                 UserFriendlyElementName.WEB_XR_AUDIO_INDICATOR, true /* visible */, () -> {});

chrome/android/javatests/src/org/chromium/chrome/browser/vr/util/NativeUiUtils.java

@@ -82,7 +82,44 @@ public class NativeUiUtils {
      */
     public static void clickElement(int elementName, PointF position) {
         TestVrShellDelegate.getInstance().performControllerActionForTesting(
-                elementName, VrControllerTestAction.CLICK, position);
+                elementName, VrControllerTestAction.CLICK_DOWN, position);
+        TestVrShellDelegate.getInstance().performControllerActionForTesting(
+                elementName, VrControllerTestAction.CLICK_UP, position);
+    }
+
+    /**
+     * Clicks the app button while pointed at a UI element.
+     * @param elementName The UserFriendlyElementName that will be pointed at.
+     * @param position A PointF specifying where on the element to point at relative to a unit
+     *        square centered on (0, 0).
+     */
+    public static void clickAppButton(int elementName, PointF position) {
+        TestVrShellDelegate.getInstance().performControllerActionForTesting(
+                elementName, VrControllerTestAction.APP_DOWN, position);
+        TestVrShellDelegate.getInstance().performControllerActionForTesting(
+                elementName, VrControllerTestAction.APP_UP, position);
+    }
+
+    /**
+     * Presses the app button down while pointed at a UI element.
+     * @param elementName The UserFriendlyElementName that will be pointed at.
+     * @param position A PointF specifying where on the element to point at relative to a unit
+     *        square centered on (0, 0).
+     */
+    public static void pressAppButton(int elementName, PointF position) {
+        TestVrShellDelegate.getInstance().performControllerActionForTesting(
+                elementName, VrControllerTestAction.APP_DOWN, position);
+    }
+
+    /**
+     * Releases the app button while pointed at a UI element.
+     * @param elementName The UserFriendlyElementName that will be pointed at.
+     * @param position A PointF specifying where on the element to point at relative to a unit
+     *        square centered on (0, 0).
+     */
+    public static void releaseAppButton(int elementName, PointF position) {
+        TestVrShellDelegate.getInstance().performControllerActionForTesting(
+                elementName, VrControllerTestAction.APP_UP, position);
     }
 
     /**

chrome/browser/vr/BUILD.gn

@@ -235,6 +235,8 @@ component("vr_common") {
     "location_bar_helper.cc",
     "location_bar_helper.h",
     "platform_controller.h",
+    "platform_controller_for_testing.cc",
+    "platform_controller_for_testing.h",
     "sample_queue.cc",
     "sample_queue.h",
     "scheduler_browser_renderer_interface.h",

chrome/browser/vr/input_delegate_for_testing.cc

@@ -5,6 +5,7 @@
 #include "chrome/browser/vr/input_delegate_for_testing.h"
 
 #include "chrome/browser/vr/input_event.h"
+#include "chrome/browser/vr/platform_controller_for_testing.h"
 #include "chrome/browser/vr/ui_interface.h"
 #include "chrome/browser/vr/ui_scene_constants.h"
 #include "chrome/browser/vr/ui_test_input.h"
@@ -35,6 +36,7 @@ void SetOriginAndTransform(vr::ControllerModel* model) {
 namespace vr {
 
 InputDelegateForTesting::InputDelegateForTesting(UiInterface* ui) : ui_(ui) {
+  gesture_detector_ = std::make_unique<GestureDetector>();
   cached_controller_model_.laser_direction = kForwardVector;
   SetOriginAndTransform(&cached_controller_model_);
 }
@@ -54,23 +56,17 @@ void InputDelegateForTesting::QueueControllerActionForTesting(
   DCHECK_NE(controller_input.action,
             VrControllerTestAction::kRevertToRealInput);
   ControllerModel controller_model;
-  auto target_point = ui_->GetTargetPointForTesting(
-      controller_input.element_name, controller_input.position);
-  auto direction = (target_point - kStartControllerPosition) - kOrigin;
-  direction.GetNormalized(&controller_model.laser_direction);
+  if (controller_input.element_name == UserFriendlyElementName::kNone) {
+    controller_model.laser_direction = kForwardVector;
+  } else {
+    auto target_point = ui_->GetTargetPointForTesting(
+        controller_input.element_name, controller_input.position);
+    auto direction = (target_point - kStartControllerPosition) - kOrigin;
+    direction.GetNormalized(&controller_model.laser_direction);
+  }
   SetOriginAndTransform(&controller_model);
 
   switch (controller_input.action) {
-    case VrControllerTestAction::kClick:
-      // Add in the button down action.
-      controller_model.touchpad_button_state =
-          ControllerModel::ButtonState::kDown;
-      controller_model_queue_.push(controller_model);
-      // Add in the button up action.
-      controller_model.touchpad_button_state =
-          ControllerModel::ButtonState::kUp;
-      controller_model_queue_.push(controller_model);
-      break;
     case VrControllerTestAction::kHover:
       FALLTHROUGH;
     case VrControllerTestAction::kClickUp:
@@ -94,6 +90,14 @@ void InputDelegateForTesting::QueueControllerActionForTesting(
       }
       controller_model_queue_.push(controller_model);
       break;
+    case VrControllerTestAction::kAppDown:
+      controller_model.app_button_state = ControllerModel::ButtonState::kDown;
+      controller_model_queue_.push(controller_model);
+      break;
+    case VrControllerTestAction::kAppUp:
+      controller_model.app_button_state = ControllerModel::ButtonState::kUp;
+      controller_model_queue_.push(controller_model);
+      break;
     default:
       NOTREACHED() << "Given unsupported controller action";
   }
@@ -106,6 +110,7 @@ bool InputDelegateForTesting::IsQueueEmpty() const {
 void InputDelegateForTesting::UpdateController(const gfx::Transform& head_pose,
                                                base::TimeTicks current_time,
                                                bool is_webxr_frame) {
+  previous_controller_model_ = cached_controller_model_;
   if (!controller_model_queue_.empty()) {
     cached_controller_model_ = controller_model_queue_.front();
     controller_model_queue_.pop();
@@ -121,7 +126,9 @@ ControllerModel InputDelegateForTesting::GetControllerModel(
 
 InputEventList InputDelegateForTesting::GetGestures(
     base::TimeTicks current_time) {
-  return InputEventList();
+  PlatformControllerForTesting controller(&previous_controller_model_,
+                                          &cached_controller_model_);
+  return gesture_detector_->DetectGestures(controller, current_time);
 }
 
 device::mojom::XRInputSourceStatePtr

chrome/browser/vr/input_delegate_for_testing.h

@@ -8,6 +8,7 @@
 #include <queue>
 
 #include "base/macros.h"
+#include "chrome/browser/vr/gesture_detector.h"
 #include "chrome/browser/vr/input_delegate.h"
 #include "chrome/browser/vr/model/controller_model.h"
 
@@ -40,6 +41,8 @@ class InputDelegateForTesting : public InputDelegate {
   UiInterface* ui_;
   std::queue<ControllerModel> controller_model_queue_;
   ControllerModel cached_controller_model_;
+  ControllerModel previous_controller_model_;
+  std::unique_ptr<GestureDetector> gesture_detector_;
 
   DISALLOW_COPY_AND_ASSIGN(InputDelegateForTesting);
 };

chrome/browser/vr/platform_controller_for_testing.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/vr/platform_controller_for_testing.h"
+#include "base/time/time.h"
+#include "chrome/browser/vr/model/controller_model.h"
+#include "ui/gfx/geometry/point_f.h"
+
+namespace vr {
+
+PlatformControllerForTesting::PlatformControllerForTesting(
+    ControllerModel* prev_model,
+    ControllerModel* cur_model)
+    : prev_model_(prev_model), cur_model_(cur_model) {}
+
+bool PlatformControllerForTesting::IsButtonDown(
+    PlatformController::ButtonType type) const {
+  switch (type) {
+    case PlatformController::ButtonType::kButtonMenu:
+      return cur_model_->app_button_state ==
+             ControllerModel::ButtonState::kDown;
+    case PlatformController::ButtonType::kButtonSelect:
+      return cur_model_->touchpad_button_state ==
+             ControllerModel::ButtonState::kDown;
+    default:
+      return false;
+  }
+}
+
+bool PlatformControllerForTesting::ButtonUpHappened(
+    PlatformController::ButtonType type) const {
+  switch (type) {
+    case PlatformController::ButtonType::kButtonMenu:
+      return (cur_model_->app_button_state ==
+                  ControllerModel::ButtonState::kUp &&
+              cur_model_->app_button_state != prev_model_->app_button_state);
+    case PlatformController::ButtonType::kButtonSelect:
+      return (cur_model_->touchpad_button_state ==
+                  ControllerModel::ButtonState::kUp &&
+              cur_model_->touchpad_button_state !=
+                  prev_model_->touchpad_button_state);
+    default:
+      return false;
+  }
+}
+
+bool PlatformControllerForTesting::ButtonDownHappened(
+    PlatformController::ButtonType type) const {
+  switch (type) {
+    case PlatformController::ButtonType::kButtonMenu:
+      return (cur_model_->app_button_state ==
+                  ControllerModel::ButtonState::kDown &&
+              cur_model_->app_button_state != prev_model_->app_button_state);
+    case PlatformController::ButtonType::kButtonSelect:
+      return (cur_model_->touchpad_button_state ==
+                  ControllerModel::ButtonState::kDown &&
+              cur_model_->touchpad_button_state !=
+                  prev_model_->touchpad_button_state);
+    default:
+      return false;
+  }
+}
+
+bool PlatformControllerForTesting::IsTouchingTrackpad() const {
+  return false;
+}
+
+gfx::PointF PlatformControllerForTesting::GetPositionInTrackpad() const {
+  return gfx::PointF();
+}
+
+base::TimeTicks PlatformControllerForTesting::GetLastOrientationTimestamp()
+    const {
+  return prev_model_->last_orientation_timestamp;
+}
+
+base::TimeTicks PlatformControllerForTesting::GetLastTouchTimestamp() const {
+  return prev_model_->last_button_timestamp;
+}
+
+base::TimeTicks PlatformControllerForTesting::GetLastButtonTimestamp() const {
+  return prev_model_->last_button_timestamp;
+}
+
+ControllerModel::Handedness PlatformControllerForTesting::GetHandedness()
+    const {
+  return ControllerModel::Handedness::kRightHanded;
+}
+
+bool PlatformControllerForTesting::GetRecentered() const {
+  return false;
+}
+
+int PlatformControllerForTesting::GetBatteryLevel() const {
+  return 100;
+}
+
+}  // namespace vr

chrome/browser/vr/platform_controller_for_testing.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_VR_PLATFORM_CONTROLLER_FOR_TESTING_H_
+#define CHROME_BROWSER_VR_PLATFORM_CONTROLLER_FOR_TESTING_H_
+
+#include "base/macros.h"
+#include "base/time/time.h"
+#include "chrome/browser/vr/model/controller_model.h"
+#include "chrome/browser/vr/platform_controller.h"
+
+namespace vr {
+
+class PlatformControllerForTesting : public PlatformController {
+ public:
+  explicit PlatformControllerForTesting(ControllerModel* prev_model,
+                                        ControllerModel* cur_model);
+  ~PlatformControllerForTesting() override{};
+
+  bool IsButtonDown(PlatformController::ButtonType type) const override;
+  bool ButtonUpHappened(PlatformController::ButtonType type) const override;
+  bool ButtonDownHappened(PlatformController::ButtonType type) const override;
+  bool IsTouchingTrackpad() const override;
+  gfx::PointF GetPositionInTrackpad() const override;
+  base::TimeTicks GetLastOrientationTimestamp() const override;
+  base::TimeTicks GetLastTouchTimestamp() const override;
+  base::TimeTicks GetLastButtonTimestamp() const override;
+  ControllerModel::Handedness GetHandedness() const override;
+  bool GetRecentered() const override;
+  int GetBatteryLevel() const override;
+
+ private:
+  ControllerModel* prev_model_;
+  ControllerModel* cur_model_;
+  DISALLOW_COPY_AND_ASSIGN(PlatformControllerForTesting);
+};
+
+}  // namespace vr
+
+#endif  // CHROME_BROWSER_VR_PLATFORM_CONTROLLER_FOR_TESTING_H_

chrome/browser/vr/ui_test_input.h

@@ -14,7 +14,9 @@ namespace vr {
 // element names for interaction during testing.
 // GENERATED_JAVA_ENUM_PACKAGE: org.chromium.chrome.browser.vr
 enum class UserFriendlyElementName : int {
-  kUrl = 0,          // URL bar
+  kNone = 0,         // A special "element" that causes the controller to point
+                     // straight forward.
+  kUrl,              // URL bar
   kBackButton,       // Back button on the URL bar
   kForwardButton,    // Forward button in the overflow menu
   kReloadButton,     // Reload button in the overflow menu
@@ -68,13 +70,14 @@ enum class UiTestOperationResult : int {
 // element using simulated controller input during testing.
 // GENERATED_JAVA_ENUM_PACKAGE: org.chromium.chrome.browser.vr
 enum class VrControllerTestAction : int {
-  kClick,
   kHover,
   kEnableMockedInput,
   kRevertToRealInput,
   kClickDown,
   kClickUp,
   kMove,
+  kAppDown,
+  kAppUp,
 };
 
 // These are used to specify what type of keyboard input should be performed