WebXR: Add pause/resume for ARCoreDevice

Add pause/resume on focus changes in ARCoreDevice to correctly
pause/resume ARCore.

Bug: 838513, 837550

Cq-Include-Trybots: luci.chromium.try:android_optional_gpu_tests_rel;luci.chromium.try:linux_optional_gpu_tests_rel;luci.chromium.try:mac_optional_gpu_tests_rel;luci.chromium.try:win_optional_gpu_tests_rel;master.tryserver.chromium.linux:linux_vr
Change-Id: I40f11c48c6a9f15b3a558546e017545f0ab66a1d
Reviewed-on: https://chromium-review.googlesource.com/1046112Reviewed-by: David Dorwin <ddorwin@chromium.org>
Reviewed-by: Michael Thiessen <mthiesse@chromium.org>
Reviewed-by: Klaus Weidner <klausw@chromium.org>
Reviewed-by: Bill Orr <billorr@chromium.org>
Commit-Queue: Iker Jamardo <ijamardo@chromium.org>
Cr-Commit-Position: refs/heads/master@{#563168}

chrome/browser/android/vr/arcore_device/arcore.h

@@ -21,6 +21,8 @@ class ARCore {
  public:
   virtual ~ARCore() = default;
 
+  // Initializes the runtime and returns whether it was successful.
+  // If successful, the runtime must be paused when this method returns.
   virtual bool Initialize() = 0;
 
   virtual void SetDisplayGeometry(
@@ -38,6 +40,9 @@ class ARCore {
       const mojom::XRRayPtr& ray,
       const gfx::Size& image_size,
       std::vector<mojom::XRHitResultPtr>* hit_results) = 0;
+
+  virtual void Pause() = 0;
+  virtual void Resume() = 0;
 };
 
 }  // namespace device

chrome/browser/android/vr/arcore_device/arcore_device.cc

@@ -13,7 +13,6 @@
 #include "chrome/browser/android/vr/arcore_device/arcore_gl.h"
 #include "chrome/browser/android/vr/arcore_device/arcore_gl_thread.h"
 #include "chrome/browser/android/vr/mailbox_to_surface_bridge.h"
-#include "ui/display/display.h"
 
 using base::android::JavaRef;
 
@@ -72,9 +71,38 @@ ARCoreDevice::ARCoreDevice()
 }
 
 ARCoreDevice::~ARCoreDevice() {
-  if (arcore_gl_thread_) {
+  if (arcore_gl_thread_)
     arcore_gl_thread_->Stop();
-  }
+}
+
+void ARCoreDevice::PauseTracking() {
+  DCHECK(IsOnMainThread());
+
+  if (is_paused_)
+    return;
+
+  is_paused_ = true;
+
+  if (!is_arcore_gl_thread_initialized_)
+    return;
+
+  PostTaskToGlThread(base::BindOnce(
+      &ARCoreGl::Pause, arcore_gl_thread_->GetARCoreGl()->GetWeakPtr()));
+}
+
+void ARCoreDevice::ResumeTracking() {
+  DCHECK(IsOnMainThread());
+
+  if (!is_paused_)
+    return;
+
+  is_paused_ = false;
+
+  if (!is_arcore_gl_thread_initialized_)
+    return;
+
+  PostTaskToGlThread(base::BindOnce(
+      &ARCoreGl::Resume, arcore_gl_thread_->GetARCoreGl()->GetWeakPtr()));
 }
 
 void ARCoreDevice::OnMailboxBridgeReady() {
@@ -91,17 +119,19 @@ void ARCoreDevice::OnMailboxBridgeReady() {
 }
 
 void ARCoreDevice::OnARCoreGlThreadInitialized(bool success) {
+  DCHECK(IsOnMainThread());
+
   if (!success) {
     DLOG(ERROR) << "Failed to initialize ARCoreDevice/GL system!";
     return;
   }
 
   is_arcore_gl_thread_initialized_ = true;
-}
 
-void ARCoreDevice::PostTaskToGlThread(base::OnceClosure task) {
-  arcore_gl_thread_->GetARCoreGl()->GetGlThreadTaskRunner()->PostTask(
-      FROM_HERE, std::move(task));
+  if (!is_paused_) {
+    PostTaskToGlThread(base::BindOnce(
+        &ARCoreGl::Resume, arcore_gl_thread_->GetARCoreGl()->GetWeakPtr()));
+  }
 }
 
 void ARCoreDevice::RequestSession(
@@ -111,6 +141,10 @@ void ARCoreDevice::RequestSession(
   std::move(callback).Run(true);
 }
 
+bool ARCoreDevice::ShouldPauseAndResumeOnFocusChange() {
+  return true;
+}
+
 void ARCoreDevice::OnMagicWindowFrameDataRequest(
     const gfx::Size& frame_size,
     display::Display::Rotation display_rotation,
@@ -118,13 +152,11 @@ void ARCoreDevice::OnMagicWindowFrameDataRequest(
   TRACE_EVENT0("gpu", __FUNCTION__);
   DCHECK(IsOnMainThread());
 
-  // Check if ARCoreGl is ready.
-  // TODO(https://crbug.com/837944): Delay callback until ready.
-  if (!is_arcore_gl_thread_initialized_) {
-    // It is not safe to access arcore_gl_thread_->GetARCoreGl() until we are
-    // sure it has finished initializing / writing to that member variable.
-    // is_initialized_ is set by a callback we pass to the ARCoreGlThread
-    // constructor that is then run back here on the main thread.
+  // TODO(ijamardo): Do we need to queue requests to avoid breaking
+  // applications?
+  // TODO(https://crbug.com/837944): Ensure is_arcore_gl_thread_initialized_
+  // is always true by blocking requestDevice()'s callback until it is true
+  if (is_paused_ || !is_arcore_gl_thread_initialized_) {
     std::move(callback).Run(nullptr);
     return;
   }
@@ -145,6 +177,12 @@ void ARCoreDevice::RequestHitTest(
       std::move(ray), CreateMainThreadCallback(std::move(callback))));
 }
 
+void ARCoreDevice::PostTaskToGlThread(base::OnceClosure task) {
+  DCHECK(IsOnMainThread());
+  arcore_gl_thread_->GetARCoreGl()->GetGlThreadTaskRunner()->PostTask(
+      FROM_HERE, std::move(task));
+}
+
 bool ARCoreDevice::IsOnMainThread() {
   return main_thread_task_runner_->BelongsToCurrentThread();
 }

chrome/browser/android/vr/arcore_device/arcore_device.h

@@ -26,6 +26,8 @@ class ARCoreDevice : public VRDeviceBase {
   ~ARCoreDevice() override;
 
   // VRDeviceBase implementation.
+  void PauseTracking() override;
+  void ResumeTracking() override;
   void RequestSession(
       VRDisplayImpl* display,
       mojom::VRDisplayHost::RequestSessionCallback callback) override;
@@ -35,13 +37,16 @@ class ARCoreDevice : public VRDeviceBase {
   }
 
  private:
+  // VRDeviceBase implementation
+  bool ShouldPauseAndResumeOnFocusChange() override;
   void OnMagicWindowFrameDataRequest(
       const gfx::Size& frame_size,
-      display::Display::Rotation frame_rotation,
+      display::Display::Rotation display_rotation,
       mojom::VRMagicWindowProvider::GetFrameDataCallback callback) override;
   void RequestHitTest(
       mojom::XRRayPtr ray,
       mojom::VRMagicWindowProvider::RequestHitTestCallback callback) override;
+
   void OnMailboxBridgeReady();
   void OnARCoreGlThreadInitialized(bool success);
 
@@ -63,12 +68,19 @@ class ARCoreDevice : public VRDeviceBase {
   void PostTaskToGlThread(base::OnceClosure task);
 
   bool IsOnMainThread();
+
   scoped_refptr<base::SingleThreadTaskRunner> main_thread_task_runner_;
   std::unique_ptr<vr::MailboxToSurfaceBridge> mailbox_bridge_;
   std::unique_ptr<ARCoreGlThread> arcore_gl_thread_;
 
   bool is_arcore_gl_thread_initialized_ = false;
 
+  // This object is not paused when it is created. Although it is not
+  // necessarily running during initialization, it is not paused. If it is
+  // paused before initialization completes, then the underlying runtime will
+  // not be resumed.
+  bool is_paused_ = false;
+
   // Must be last.
   base::WeakPtrFactory<ARCoreDevice> weak_ptr_factory_;
   DISALLOW_COPY_AND_ASSIGN(ARCoreDevice);

chrome/browser/android/vr/arcore_device/arcore_gl.cc

@@ -128,7 +128,6 @@ bool ARCoreGl::Initialize() {
 
   if (!arcore_->Initialize()) {
     DLOG(ERROR) << "ARCore failed to initialize";
-
     return false;
   }
 
@@ -136,6 +135,7 @@ bool ARCoreGl::Initialize() {
     DLOG(ERROR) << "ARImageTransport failed to initialize";
     return false;
   }
+
   // Set the texture on ARCore to render the camera.
   arcore_->SetCameraTexture(ar_image_transport_->GetCameraTextureId());
 
@@ -264,6 +264,18 @@ void ARCoreGl::ProcessFrame(
   std::move(callback).Run(std::move(frame_data));
 }
 
+void ARCoreGl::Pause() {
+  DCHECK(IsOnGlThread());
+  DCHECK(is_initialized_);
+  arcore_->Pause();
+}
+
+void ARCoreGl::Resume() {
+  DCHECK(IsOnGlThread());
+  DCHECK(is_initialized_);
+  arcore_->Resume();
+}
+
 bool ARCoreGl::IsOnGlThread() const {
   return gl_thread_task_runner_->BelongsToCurrentThread();
 }

chrome/browser/android/vr/arcore_device/arcore_gl.h

@@ -51,6 +51,8 @@ class ARCoreGl {
   void ProduceFrame(const gfx::Size& frame_size,
                     display::Display::Rotation display_rotation,
                     mojom::VRMagicWindowProvider::GetFrameDataCallback);
+  void Pause();
+  void Resume();
 
   const scoped_refptr<base::SingleThreadTaskRunner>& GetGlThreadTaskRunner() {
     return gl_thread_task_runner_;

chrome/browser/android/vr/arcore_device/arcore_impl.cc

@@ -83,13 +83,16 @@ bool ARCoreImpl::Initialize() {
     return false;
   }
 
-  ArFrame_create(session.get(), arcore_frame_.receive());
-  if (!arcore_frame_.is_valid()) {
+  internal::ScopedArCoreObject<ArFrame*> frame;
+
+  ArFrame_create(session.get(), frame.receive());
+  if (!frame.is_valid()) {
     DLOG(ERROR) << "ArFrame_create failed";
     return false;
   }
 
-  // Success, we now have a valid session.
+  // Success, we now have a valid session and a valid frame.
+  arcore_frame_ = std::move(frame);
   arcore_session_ = std::move(session);
   return true;
 }
@@ -131,14 +134,6 @@ mojom::VRPosePtr ARCoreImpl::Update() {
   DCHECK(arcore_frame_.is_valid());
 
   ArStatus status;
-  if (!is_tracking_) {
-    status = ArSession_resume(arcore_session_.get());
-    if (status != AR_SUCCESS) {
-      DLOG(ERROR) << "ArSession_resume failed: " << status;
-      return nullptr;
-    }
-    is_tracking_ = true;
-  }
 
   status = ArSession_update(arcore_session_.get(), arcore_frame_.get());
   if (status != AR_SUCCESS) {
@@ -182,6 +177,22 @@ mojom::VRPosePtr ARCoreImpl::Update() {
   return pose;
 }
 
+void ARCoreImpl::Pause() {
+  DCHECK(IsOnGlThread());
+  DCHECK(arcore_session_.is_valid());
+  ArStatus status = ArSession_pause(arcore_session_.get());
+  DLOG_IF(ERROR, status != AR_SUCCESS)
+      << "ArSession_pause failed: status = " << status;
+}
+
+void ARCoreImpl::Resume() {
+  DCHECK(IsOnGlThread());
+  DCHECK(arcore_session_.is_valid());
+  ArStatus status = ArSession_resume(arcore_session_.get());
+  DLOG_IF(ERROR, status != AR_SUCCESS)
+      << "ArSession_resume failed: status = " << status;
+}
+
 gfx::Transform ARCoreImpl::GetProjectionMatrix(float near, float far) {
   DCHECK(IsOnGlThread());
   DCHECK(arcore_session_.is_valid());

chrome/browser/android/vr/arcore_device/arcore_impl.h

@@ -79,6 +79,8 @@ class ARCoreImpl : public ARCore {
       const base::span<const float> uvs) override;
   gfx::Transform GetProjectionMatrix(float near, float far) override;
   mojom::VRPosePtr Update() override;
+  void Pause() override;
+  void Resume() override;
 
   bool RequestHitTest(const mojom::XRRayPtr& ray,
                       const gfx::Size& image_size,
@@ -105,9 +107,6 @@ class ARCoreImpl : public ARCore {
   internal::ScopedArCoreObject<ArSession*> arcore_session_;
   internal::ScopedArCoreObject<ArFrame*> arcore_frame_;
 
-  // TODO(https://crbug.com/838513): replace this with more sophisticated logic.
-  bool is_tracking_ = false;
-
   // Must be last.
   base::WeakPtrFactory<ARCoreImpl> weak_ptr_factory_;
   DISALLOW_COPY_AND_ASSIGN(ARCoreImpl);

chrome/browser/android/vr/arcore_device/fake_arcore.cc

@@ -13,21 +13,26 @@
 
 namespace device {
 
-FakeARCore::FakeARCore() = default;
+FakeARCore::FakeARCore()
+    : gl_thread_task_runner_(base::ThreadTaskRunnerHandle::Get()) {}
+
 FakeARCore::~FakeARCore() = default;
 
 bool FakeARCore::Initialize() {
+  DCHECK(IsOnGlThread());
   return true;
 }
 
 void FakeARCore::SetDisplayGeometry(
     const gfx::Size& frame_size,
     display::Display::Rotation display_rotation) {
+  DCHECK(IsOnGlThread());
   display_rotation_ = display_rotation;
   frame_size_ = frame_size;
 }
 
 void FakeARCore::SetCameraTexture(GLuint texture) {
+  DCHECK(IsOnGlThread());
   // We need a GL_TEXTURE_EXTERNAL_OES to be compatible with the real ARCore.
   // The content doesn't really matter, just create an AHardwareBuffer-backed
   // GLImage and bind it to the texture.
@@ -217,6 +222,7 @@ std::vector<float> FakeARCore::TransformDisplayUvCoords(
 }
 
 gfx::Transform FakeARCore::GetProjectionMatrix(float near, float far) {
+  DCHECK(IsOnGlThread());
   // Get a projection matrix matching the current screen orientation and
   // aspect. Currently, this uses a hardcoded FOV angle for the smaller screen
   // dimension, and adjusts the other angle to preserve the aspect. A better
@@ -248,6 +254,7 @@ gfx::Transform FakeARCore::GetProjectionMatrix(float near, float far) {
 }
 
 mojom::VRPosePtr FakeARCore::Update() {
+  DCHECK(IsOnGlThread());
   // 1m up from the origin, neutral orientation facing forward.
   mojom::VRPosePtr pose = mojom::VRPose::New();
   pose->orientation.emplace(4);
@@ -271,4 +278,16 @@ bool FakeARCore::RequestHitTest(
   return false;
 }
 
+void FakeARCore::Pause() {
+  DCHECK(IsOnGlThread());
+}
+
+void FakeARCore::Resume() {
+  DCHECK(IsOnGlThread());
+}
+
+bool FakeARCore::IsOnGlThread() const {
+  return gl_thread_task_runner_->BelongsToCurrentThread();
+}
+
 }  // namespace device

chrome/browser/android/vr/arcore_device/fake_arcore.h

@@ -34,6 +34,8 @@ class FakeARCore : public ARCore {
       const base::span<const float> uvs) override;
   gfx::Transform GetProjectionMatrix(float near, float far) override;
   mojom::VRPosePtr Update() override;
+  void Pause() override;
+  void Resume() override;
 
   bool RequestHitTest(const mojom::XRRayPtr& ray,
                       const gfx::Size& image_size,
@@ -42,6 +44,10 @@ class FakeARCore : public ARCore {
   void SetCameraAspect(float aspect) { camera_aspect_ = aspect; }
 
  private:
+  bool IsOnGlThread() const;
+
+  scoped_refptr<base::SingleThreadTaskRunner> gl_thread_task_runner_;
+
   float camera_aspect_ = 1.0f;
   display::Display::Rotation display_rotation_ =
       display::Display::Rotation::ROTATE_0;

chrome/browser/vr/service/vr_display_host.cc

@@ -42,6 +42,8 @@ VRDisplayHost::VRDisplayHost(BrowserXrDevice* device,
                              device::mojom::VRServiceClient* service_client,
                              device::mojom::VRDisplayInfoPtr display_info)
     : browser_device_(device),
+      // TODO(https://crbug.com/846392): render_frame_host can be null because
+      // of a test, not because a VRDisplayHost can be created without it.
       in_focused_frame_(
           render_frame_host ? render_frame_host->GetView()->HasFocus() : false),
       render_frame_host_(render_frame_host),

device/vr/vr_device_base.cc

@@ -103,6 +103,9 @@ void VRDeviceBase::OnListeningForActivateChanged(VRDisplayImpl* display) {
 
 void VRDeviceBase::OnFrameFocusChanged(VRDisplayImpl* display) {
   UpdateListeningForActivate(display);
+  if (ShouldPauseAndResumeOnFocusChange()) {
+    display->InFocusedFrame() ? ResumeTracking() : PauseTracking();
+  }
 }
 
 void VRDeviceBase::SetVRDisplayInfo(mojom::VRDisplayInfoPtr display_info) {
@@ -125,6 +128,10 @@ void VRDeviceBase::OnActivate(mojom::VRDisplayEventReason reason,
     listener_->OnActivate(reason, std::move(on_handled));
 }
 
+bool VRDeviceBase::ShouldPauseAndResumeOnFocusChange() {
+  return false;
+}
+
 void VRDeviceBase::OnListeningForActivate(bool listening) {}
 
 void VRDeviceBase::OnMagicWindowPoseRequest(

device/vr/vr_device_base.h

@@ -64,7 +64,17 @@ class DEVICE_VR_EXPORT VRDeviceBase : public VRDevice {
                   base::Callback<void(bool)> on_handled);
 
  private:
+  // Subclasses should implement these methods if they need to perform
+  // device-specific operations.
   virtual void UpdateListeningForActivate(VRDisplayImpl* display);
+
+  // TODO(https://crbug.com/845283): This method is a temporary solution
+  // until a XR related refactor lands. It allows to keep using the
+  // existing PauseTracking/ResumeTracking while not changing the
+  // existing VR functionality.
+  virtual bool ShouldPauseAndResumeOnFocusChange();
+
+  // TODO(https://crbug.com/842227): Rename methods to HandleOnXXX
   virtual void OnListeningForActivate(bool listening);
   virtual void OnMagicWindowPoseRequest(
       mojom::VRMagicWindowProvider::GetPoseCallback callback);