WebXR: Handle AR tracking loss

When ARCore loses tracking, continue providing camera images, but send null
poses. This allows JS applications to handle tracking loss, i.e. by displaying
a pose-independent info message.

BUG: 837116
Change-Id: I92238f1bf158ca2714a4068ddba8ac5bbdb123fc
Reviewed-on: https://chromium-review.googlesource.com/1043207Reviewed-by: Brandon Jones <bajones@chromium.org>
Reviewed-by: Martin Barbella <mbarbella@chromium.org>
Commit-Queue: Klaus Weidner <klausw@chromium.org>
Cr-Commit-Position: refs/heads/master@{#565051}

chrome/browser/android/vr/arcore_device/arcore.h

@@ -34,7 +34,12 @@ class ARCore {
       const base::span<const float> uvs) = 0;
   virtual gfx::Transform GetProjectionMatrix(float near, float far) = 0;
 
-  virtual mojom::VRPosePtr Update() = 0;
+  // Update ARCore state. This call blocks for up to 1/30s while waiting for a
+  // new camera image. The output parameter |camera_updated| must be non-null,
+  // the stored value indicates if the camera image was updated successfully.
+  // The returned pose is nullptr if tracking was lost, this can happen even
+  // when the camera image was updated successfully.
+  virtual mojom::VRPosePtr Update(bool* camera_updated) = 0;
 
   virtual bool RequestHitTest(
       const mojom::XRRayPtr& ray,

chrome/browser/android/vr/arcore_device/arcore_gl.cc

@@ -165,10 +165,11 @@ void ARCoreGl::ProduceFrame(
   arcore_->SetDisplayGeometry(transfer_size, display_rotation);
 
   TRACE_EVENT_BEGIN0("gpu", "ARCore Update");
-  mojom::VRPosePtr ar_pose = arcore_->Update();
+  bool camera_updated = false;
+  mojom::VRPosePtr pose = arcore_->Update(&camera_updated);
   TRACE_EVENT_END0("gpu", "ARCore Update");
-  if (!ar_pose) {
-    DLOG(ERROR) << "Failed get pose from arcore_->Update()!";
+  if (!camera_updated) {
+    DVLOG(1) << "arcore_->Update() failed";
     std::move(callback).Run(nullptr);
     return;
   }
@@ -187,7 +188,7 @@ void ARCoreGl::ProduceFrame(
   // Create the frame data to return to the renderer.
   mojom::VRMagicWindowFrameDataPtr frame_data =
       mojom::VRMagicWindowFrameData::New();
-  frame_data->pose = std::move(ar_pose);
+  frame_data->pose = std::move(pose);
   frame_data->buffer_holder = buffer_holder;
   frame_data->buffer_size = transfer_size;
   frame_data->time_delta = base::TimeTicks::Now() - base::TimeTicks();

chrome/browser/android/vr/arcore_device/arcore_impl.cc

@@ -128,19 +128,24 @@ std::vector<float> ARCoreImpl::TransformDisplayUvCoords(
   return uvs_out;
 }
 
-mojom::VRPosePtr ARCoreImpl::Update() {
+mojom::VRPosePtr ARCoreImpl::Update(bool* camera_updated) {
   DCHECK(IsOnGlThread());
   DCHECK(arcore_session_.is_valid());
   DCHECK(arcore_frame_.is_valid());
+  DCHECK(camera_updated);
 
   ArStatus status;
 
   status = ArSession_update(arcore_session_.get(), arcore_frame_.get());
   if (status != AR_SUCCESS) {
     DLOG(ERROR) << "ArSession_update failed: " << status;
+    *camera_updated = false;
     return nullptr;
   }
 
+  // If we get here, assume we have a valid camera image, but we don't know yet
+  // if tracking is working.
+  *camera_updated = true;
   internal::ScopedArCoreObject<ArCamera*> arcore_camera;
   ArFrame_acquireCamera(arcore_session_.get(), arcore_frame_.get(),
                         arcore_camera.receive());
@@ -153,7 +158,7 @@ mojom::VRPosePtr ARCoreImpl::Update() {
   ArCamera_getTrackingState(arcore_session_.get(), arcore_camera.get(),
                             &tracking_state);
   if (tracking_state != AR_TRACKING_STATE_TRACKING) {
-    DLOG(ERROR) << "Tracking state is not AR_TRACKING_STATE_TRACKING: "
+    DVLOG(1) << "Tracking state is not AR_TRACKING_STATE_TRACKING: "
              << tracking_state;
     return nullptr;
   }

chrome/browser/android/vr/arcore_device/arcore_impl.h

@@ -78,7 +78,7 @@ class ARCoreImpl : public ARCore {
   std::vector<float> TransformDisplayUvCoords(
       const base::span<const float> uvs) override;
   gfx::Transform GetProjectionMatrix(float near, float far) override;
-  mojom::VRPosePtr Update() override;
+  mojom::VRPosePtr Update(bool* camera_updated) override;
   void Pause() override;
   void Resume() override;
 

chrome/browser/android/vr/arcore_device/fake_arcore.cc

@@ -253,8 +253,12 @@ gfx::Transform FakeARCore::GetProjectionMatrix(float near, float far) {
   return result;
 }
 
-mojom::VRPosePtr FakeARCore::Update() {
+mojom::VRPosePtr FakeARCore::Update(bool* camera_updated) {
   DCHECK(IsOnGlThread());
+  DCHECK(camera_updated);
+
+  *camera_updated = true;
+
   // 1m up from the origin, neutral orientation facing forward.
   mojom::VRPosePtr pose = mojom::VRPose::New();
   pose->orientation.emplace(4);

chrome/browser/android/vr/arcore_device/fake_arcore.h

@@ -33,7 +33,7 @@ class FakeARCore : public ARCore {
   std::vector<float> TransformDisplayUvCoords(
       const base::span<const float> uvs) override;
   gfx::Transform GetProjectionMatrix(float near, float far) override;
-  mojom::VRPosePtr Update() override;
+  mojom::VRPosePtr Update(bool* camera_updated) override;
   void Pause() override;
   void Resume() override;
 

device/vr/public/mojom/vr_service.mojom

@@ -195,15 +195,17 @@ struct VRDisplayFrameTransportOptions {
 // just a VRPose - ex: non-exclusive AR needs a camera image and
 // to get a projection matrix directly from the backend rather than
 // FOV values to support features like focus.
+//
+// The pose may be null if the device lost tracking. It can still
+// send a camera image in that case.
 struct VRMagicWindowFrameData {
-  VRPose pose;
+  VRPose? pose;
   gpu.mojom.MailboxHolder buffer_holder;
   gfx.mojom.Size buffer_size;
   // TODO(https://crbug.com/838515): Is this delta since the last
   // frame? OR an unspecified origin? Something else?
   mojo_base.mojom.TimeDelta time_delta;
   array<float, 16> projection_matrix;
-
 };
 
 enum VRDisplayEventReason {

third_party/blink/renderer/modules/xr/xr_frame_provider.cc

@@ -394,23 +394,6 @@ void XRFrameProvider::OnNonExclusiveFrameData(
                 WrapWeakPersistent(this), std::move(frame_data), timestamp));
 }
 
-// TODO(836349): revisit sending this data to blink at all.
-void XRFrameProvider::RenderBackgroundImage(
-    const device::mojom::blink::VRMagicWindowFrameDataPtr& frame_data,
-    XRSession* session) {
-  DCHECK(frame_data);
-  TRACE_EVENT0("gpu", __FUNCTION__);
-
-  XRLayer* layer = session->baseLayer();
-  if (!layer)
-    return;
-
-  // TODO(https://crbug.com/837509): Remove this static_cast.
-  XRWebGLLayer* webgl_layer = static_cast<XRWebGLLayer*>(layer);
-  webgl_layer->OverwriteColorBufferFromMailboxTexture(
-      frame_data->buffer_holder, IntSize(frame_data->buffer_size));
-}
-
 void XRFrameProvider::ProcessScheduledFrame(
     device::mojom::blink::VRMagicWindowFrameDataPtr frame_data,
     double timestamp) {
@@ -447,7 +430,8 @@ void XRFrameProvider::ProcessScheduledFrame(
     // holder must be present.
     DCHECK(!frame_transport_->DrawingIntoSharedBuffer() ||
            buffer_mailbox_holder_);
-    exclusive_session_->OnFrame(std::move(pose_matrix), buffer_mailbox_holder_);
+    exclusive_session_->OnFrame(std::move(pose_matrix), buffer_mailbox_holder_,
+                                base::nullopt, base::nullopt);
   } else {
     // In the process of fulfilling the frame requests for each session they are
     // extremely likely to request another frame. Work off of a separate list
@@ -464,11 +448,6 @@ void XRFrameProvider::ProcessScheduledFrame(
     // Inform sessions with a pending request of the new frame
     for (unsigned i = 0; i < processing_sessions.size(); ++i) {
       XRSession* session = processing_sessions.at(i).Get();
-      if (frame_data) {
-        // TODO(https://crbug.com/837883): only render background for
-        // sessions that are using AR.
-        RenderBackgroundImage(frame_data, session);
-      }
 
       if (frame_pose_ && frame_pose_->input_state.has_value()) {
         session->OnInputStateChange(frame_id_,
@@ -485,7 +464,19 @@ void XRFrameProvider::ProcessScheduledFrame(
 
       std::unique_ptr<TransformationMatrix> pose_matrix =
           getPoseMatrix(frame_pose_);
-      session->OnFrame(std::move(pose_matrix), base::nullopt);
+      // TODO(https://crbug.com/837883): only render background for
+      // sessions that are using AR.
+      if (frame_data) {
+        // buffer_holder and buffer_size are non-optional members of
+        // the mojo frame_data struct. We pass them to OnFrame as
+        // optional arguments.
+        session->OnFrame(std::move(pose_matrix), base::nullopt,
+                         frame_data->buffer_holder,
+                         IntSize(frame_data->buffer_size));
+      } else {
+        session->OnFrame(std::move(pose_matrix), base::nullopt, base::nullopt,
+                         base::nullopt);
+      }
     }
   }
 }

third_party/blink/renderer/modules/xr/xr_frame_provider.h

@@ -65,10 +65,6 @@ class XRFrameProvider final
       device::mojom::blink::VRMagicWindowFrameDataPtr frame_data,
       double timestamp);
 
-  void RenderBackgroundImage(
-      const device::mojom::blink::VRMagicWindowFrameDataPtr& frame_data,
-      XRSession* session);
-
   const Member<XRDevice> device_;
   Member<XRSession> exclusive_session_;
   Member<ScriptPromiseResolver> pending_exclusive_session_resolver_;

third_party/blink/renderer/modules/xr/xr_session.cc

@@ -29,6 +29,7 @@
 #include "third_party/blink/renderer/modules/xr/xr_presentation_frame.h"
 #include "third_party/blink/renderer/modules/xr/xr_session_event.h"
 #include "third_party/blink/renderer/modules/xr/xr_view.h"
+#include "third_party/blink/renderer/modules/xr/xr_webgl_layer.h"
 #include "third_party/blink/renderer/platform/transforms/transformation_matrix.h"
 
 namespace blink {
@@ -422,7 +423,9 @@ void XRSession::OnFocusChanged() {
 
 void XRSession::OnFrame(
     std::unique_ptr<TransformationMatrix> base_pose_matrix,
-    const base::Optional<gpu::MailboxHolder>& buffer_mailbox_holder) {
+    const base::Optional<gpu::MailboxHolder>& output_mailbox_holder,
+    const base::Optional<gpu::MailboxHolder>& background_mailbox_holder,
+    const base::Optional<IntSize>& background_size) {
   TRACE_EVENT0("gpu", __FUNCTION__);
   DVLOG(2) << __FUNCTION__;
   // Don't process any outstanding frames once the session is ended.
@@ -449,7 +452,18 @@ void XRSession::OnFrame(
 
     // Cache the base layer, since it could change during the frame callback.
     XRLayer* frame_base_layer = base_layer_;
-    frame_base_layer->OnFrameStart(buffer_mailbox_holder);
+    frame_base_layer->OnFrameStart(output_mailbox_holder);
+
+    // TODO(836349): revisit sending background image data to blink at all.
+    if (background_mailbox_holder) {
+      // If using a background image, the caller must provide its pixel size
+      // also. The source size can differ from the current drawing buffer size.
+      DCHECK(background_size);
+      // TODO(https://crbug.com/837509): Remove this static_cast.
+      XRWebGLLayer* webgl_layer = static_cast<XRWebGLLayer*>(frame_base_layer);
+      webgl_layer->OverwriteColorBufferFromMailboxTexture(
+          background_mailbox_holder.value(), background_size.value());
+    }
 
     // Resolve the queued requestAnimationFrame callbacks. All XR rendering will
     // happen within these calls. resolving_frame_ will be true for the duration

third_party/blink/renderer/modules/xr/xr_session.h

@@ -115,7 +115,9 @@ class XRSession final : public EventTargetWithInlineData {
 
   void OnFocusChanged();
   void OnFrame(std::unique_ptr<TransformationMatrix>,
-               const base::Optional<gpu::MailboxHolder>&);
+               const base::Optional<gpu::MailboxHolder>& output_mailbox_holder,
+               const base::Optional<gpu::MailboxHolder>& bg_mailbox_holder,
+               const base::Optional<IntSize>& background_size);
   void OnInputStateChange(
       int16_t frame_id,
       const WTF::Vector<device::mojom::blink::XRInputSourceStatePtr>&);

third_party/blink/renderer/modules/xr/xr_webgl_layer.cc

@@ -245,6 +245,7 @@ void XRWebGLLayer::OverwriteColorBufferFromMailboxTexture(
     const gpu::MailboxHolder& mailbox_holder,
     const IntSize& size) {
   drawing_buffer_->OverwriteColorBufferFromMailboxTexture(mailbox_holder, size);
+  framebuffer_->SetContentsChanged(true);
 }
 
 void XRWebGLLayer::OnFrameStart(