Support ARCore immersive-ar on Android N

Currently, WebXR's immersive-ar mode requires Android O due to using
SharedBuffer image transport which needs AHardwareBuffer support.

On Android N, use a Surface/SurfaceTexture pair instead. This is based
on the existing XR frame transport method SUBMIT_AS_MAILBOX_HOLDER that's
also used by the GVR immersive-vr backend.

Bug: 1012972
Change-Id: I87759daf7d317873dea0fb569fffaee379783ca1
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1854802Reviewed-by: Piotr Bialecki <bialpio@chromium.org>
Commit-Queue: Klaus Weidner <klausw@chromium.org>
Cr-Commit-Position: refs/heads/master@{#705629}

chrome/browser/android/vr/arcore_device/ar_image_transport.cc

@@ -14,6 +14,8 @@
 #include "gpu/command_buffer/common/shared_image_usage.h"
 #include "gpu/ipc/common/gpu_memory_buffer_impl_android_hardware_buffer.h"
 #include "ui/gfx/gpu_fence.h"
+#include "ui/gl/android/scoped_java_surface.h"
+#include "ui/gl/android/surface_texture.h"
 #include "ui/gl/gl_bindings.h"
 #include "ui/gl/gl_context.h"
 #include "ui/gl/gl_fence_egl.h"
@@ -36,7 +38,7 @@ void ArImageTransport::DestroySharedBuffers(vr::WebXrPresentationState* webxr) {
   DVLOG(2) << __func__;
   DCHECK(IsOnGlThread());
 
-  if (!webxr)
+  if (!webxr || !UseSharedBuffer())
     return;
 
   std::vector<std::unique_ptr<vr::WebXrSharedBuffer>> buffers =
@@ -65,6 +67,21 @@ void ArImageTransport::Initialize(vr::WebXrPresentationState* webxr,
 
   glGenFramebuffersEXT(1, &camera_fbo_);
 
+  shared_buffer_draw_ = base::AndroidHardwareBufferCompat::IsSupportAvailable();
+
+  if (shared_buffer_draw_) {
+    DVLOG(2) << __func__ << ": UseSharedBuffer()=true";
+  } else {
+    DVLOG(2) << __func__ << ": UseSharedBuffer()=false, setting up surface";
+    glGenTextures(1, &transport_texture_id_);
+    transport_surface_texture_ =
+        gl::SurfaceTexture::Create(transport_texture_id_);
+    surface_size_ = {0, 0};
+    mailbox_bridge_->CreateSurface(transport_surface_texture_.get());
+    transport_surface_texture_->SetFrameAvailableCallback(base::BindRepeating(
+        &ArImageTransport::OnFrameAvailable, weak_ptr_factory_.GetWeakPtr()));
+  }
+
   mailbox_bridge_->CreateAndBindContextProvider(
       base::BindOnce(&ArImageTransport::OnMailboxBridgeReady,
                      weak_ptr_factory_.GetWeakPtr(), std::move(callback)));
@@ -79,6 +96,39 @@ void ArImageTransport::OnMailboxBridgeReady(base::OnceClosure callback) {
   std::move(callback).Run();
 }
 
+void ArImageTransport::SetFrameAvailableCallback(
+    XrFrameCallback on_transport_frame_available) {
+  DVLOG(2) << __func__;
+  on_transport_frame_available_ = std::move(on_transport_frame_available);
+}
+
+void ArImageTransport::OnFrameAvailable() {
+  DVLOG(2) << __func__;
+  DCHECK(on_transport_frame_available_);
+
+  // This function assumes that there's only at most one frame in "processing"
+  // state at any given time, the webxr_ state handling ensures that. Drawing
+  // and swapping twice without an intervening UpdateTexImage call would lose
+  // an image, and that would lead to images and poses getting out of sync.
+  //
+  // It also assumes that the ArImageTransport and Surface only exist for the
+  // duration of a single session, and a new session will use fresh objects. For
+  // comparison, see GvrSchedulerDelegate::OnWebXrFrameAvailable which has more
+  // complex logic to support a lifetime across multiple sessions, including
+  // handling a possibly-unconsumed frame left over from a previous session.
+
+  transport_surface_texture_->UpdateTexImage();
+
+  // The SurfaceTexture needs to be drawn using the corresponding
+  // UV transform, that's usually a Y flip.
+  transport_surface_texture_->GetTransformMatrix(
+      &transport_surface_texture_uv_matrix_[0]);
+  transport_surface_texture_uv_transform_.matrix().setColMajorf(
+      transport_surface_texture_uv_matrix_);
+
+  on_transport_frame_available_.Run(transport_surface_texture_uv_transform_);
+}
+
 GLuint ArImageTransport::GetCameraTextureId() {
   return camera_texture_id_arcore_;
 }
@@ -190,8 +240,14 @@ void ArImageTransport::CopyDrawnImageToFramebuffer(
     const gfx::Transform& uv_transform) {
   DVLOG(2) << __func__;
 
-  vr::WebXrSharedBuffer* shared_buffer =
-      webxr->GetRenderingFrame()->shared_buffer.get();
+  GLuint source_texture;
+  if (UseSharedBuffer()) {
+    vr::WebXrSharedBuffer* shared_buffer =
+        webxr->GetRenderingFrame()->shared_buffer.get();
+    source_texture = shared_buffer->local_texture;
+  } else {
+    source_texture = transport_texture_id_;
+  }
 
   // Set the blend mode for combining the drawn image (source) with the camera
   // image (destination). WebXR assumes that the canvas has premultiplied alpha,
@@ -201,8 +257,7 @@ void ArImageTransport::CopyDrawnImageToFramebuffer(
   glEnable(GL_BLEND);
   glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
   glBindFramebufferEXT(GL_DRAW_FRAMEBUFFER, 0);
-  CopyTextureToFramebuffer(shared_buffer->local_texture, frame_size,
-                           uv_transform);
+  CopyTextureToFramebuffer(source_texture, frame_size, uv_transform);
 }
 
 void ArImageTransport::CopyTextureToFramebuffer(
@@ -225,6 +280,26 @@ void ArImageTransport::CopyTextureToFramebuffer(
   ar_renderer_->Draw(texture, uv_transform_floats, 0, 0);
 }
 
+void ArImageTransport::CopyMailboxToSurfaceAndSwap(
+    const gfx::Size& frame_size,
+    const gpu::MailboxHolder& mailbox) {
+  DVLOG(2) << __func__;
+  if (frame_size != surface_size_) {
+    DVLOG(2) << __func__ << " resize from " << surface_size_.ToString()
+             << " to " << frame_size.ToString();
+    transport_surface_texture_->SetDefaultBufferSize(frame_size.width(),
+                                                     frame_size.height());
+    mailbox_bridge_->ResizeSurface(frame_size.width(), frame_size.height());
+    surface_size_ = frame_size;
+  }
+
+  // Draw the image to the surface in the GPU process's command buffer context.
+  // This will trigger an OnFrameAvailable event once the corresponding
+  // SurfaceTexture in the local GL context is ready for updating.
+  bool swapped = mailbox_bridge_->CopyMailboxToSurfaceAndSwap(mailbox);
+  DCHECK(swapped);
+}
+
 bool ArImageTransport::IsOnGlThread() const {
   return gl_thread_task_runner_->BelongsToCurrentThread();
 }

chrome/browser/android/vr/arcore_device/ar_image_transport.h

@@ -13,6 +13,10 @@
 #include "device/vr/public/mojom/vr_service.mojom.h"
 #include "ui/gfx/geometry/size_f.h"
 
+namespace gl {
+class SurfaceTexture;
+}  // namespace gl
+
 namespace gfx {
 class GpuFence;
 }  // namespace gfx
@@ -30,6 +34,8 @@ struct WebXrSharedBuffer;
 
 namespace device {
 
+using XrFrameCallback = base::RepeatingCallback<void(const gfx::Transform&)>;
+
 // This class handles transporting WebGL rendered output from the GPU process's
 // command buffer GL context to the local GL context, and compositing WebGL
 // output onto the camera image using the local GL context.
@@ -69,14 +75,21 @@ class ArImageTransport {
                                         const gfx::Size& frame_size,
                                         const gfx::Transform& uv_transform);
   virtual void WaitSyncToken(const gpu::SyncToken& sync_token);
+  virtual void CopyMailboxToSurfaceAndSwap(const gfx::Size& frame_size,
+                                           const gpu::MailboxHolder& mailbox);
+
+  bool UseSharedBuffer() { return shared_buffer_draw_; }
+  void SetFrameAvailableCallback(XrFrameCallback on_frame_available);
 
  private:
   std::unique_ptr<vr::WebXrSharedBuffer> CreateBuffer();
   void ResizeSharedBuffer(vr::WebXrPresentationState* webxr,
                           const gfx::Size& size,
                           vr::WebXrSharedBuffer* buffer);
+  void ResizeSurface(const gfx::Size& size);
   bool IsOnGlThread() const;
   void OnMailboxBridgeReady(base::OnceClosure callback);
+  void OnFrameAvailable();
   std::unique_ptr<ArRenderer> ar_renderer_;
   // samplerExternalOES texture for the camera image.
   GLuint camera_texture_id_arcore_ = 0;
@@ -86,6 +99,20 @@ class ArImageTransport {
 
   std::unique_ptr<vr::MailboxToSurfaceBridge> mailbox_bridge_;
 
+  // If true, use shared buffer transport aka DRAW_INTO_TEXTURE_MAILBOX.
+  // If false, use Surface transport aka SUBMIT_AS_MAILBOX_HOLDER.
+  bool shared_buffer_draw_ = false;
+
+  // Used for Surface transport (Android N)
+  //
+  // samplerExternalOES texture data for WebXR content image.
+  GLuint transport_texture_id_ = 0;
+  gfx::Size surface_size_;
+  scoped_refptr<gl::SurfaceTexture> transport_surface_texture_;
+  gfx::Transform transport_surface_texture_uv_transform_;
+  float transport_surface_texture_uv_matrix_[16];
+  XrFrameCallback on_transport_frame_available_;
+
   // Must be last.
   base::WeakPtrFactory<ArImageTransport> weak_ptr_factory_{this};
   DISALLOW_COPY_AND_ASSIGN(ArImageTransport);

chrome/browser/android/vr/arcore_device/arcore_gl.cc

@@ -187,12 +187,20 @@ void ArCoreGl::CreateSession(mojom::VRDisplayInfoPtr display_info,
       device::mojom::XRPresentationTransportOptions::New();
   transport_options->wait_for_gpu_fence = true;
 
-  // Currently, AR mode only supports Android O+ due to requiring
-  // AHardwareBuffer-backed GpuMemoryBuffer shared images. This could be
-  // extended back to Android N by using the SUBMIT_AS_MAILBOX_HOLDER method
-  // that uses Surface/SurfaceTexture.
-  transport_options->transport_method =
-      device::mojom::XRPresentationTransportMethod::DRAW_INTO_TEXTURE_MAILBOX;
+  if (ar_image_transport_->UseSharedBuffer()) {
+    DVLOG(2) << __func__
+             << ": UseSharedBuffer()=true, DRAW_INTO_TEXTURE_MAILBOX";
+    transport_options->transport_method =
+        device::mojom::XRPresentationTransportMethod::DRAW_INTO_TEXTURE_MAILBOX;
+  } else {
+    DVLOG(2) << __func__
+             << ": UseSharedBuffer()=false, SUBMIT_AS_MAILBOX_HOLDER";
+    transport_options->transport_method =
+        device::mojom::XRPresentationTransportMethod::SUBMIT_AS_MAILBOX_HOLDER;
+    transport_options->wait_for_transfer_notification = true;
+    ar_image_transport_->SetFrameAvailableCallback(base::BindRepeating(
+        &ArCoreGl::OnTransportFrameAvailable, weak_ptr_factory_.GetWeakPtr()));
+  }
 
   auto submit_frame_sink = device::mojom::XRPresentationConnection::New();
   submit_frame_sink->client_receiver =
@@ -389,14 +397,17 @@ void ArCoreGl::GetFrameData(
     frame_data->left_eye = display_info_->left_eye.Clone();
     display_info_changed_ = false;
   }
-  // Set up a shared buffer for the renderer to draw into, it'll be sent
-  // alongside the frame pose.
-  gpu::MailboxHolder buffer_holder = ar_image_transport_->TransferFrame(
-      webxr_.get(), transfer_size_, uv_transform_);
+
+  if (ar_image_transport_->UseSharedBuffer()) {
+    // Set up a shared buffer for the renderer to draw into, it'll be sent
+    // alongside the frame pose.
+    gpu::MailboxHolder buffer_holder = ar_image_transport_->TransferFrame(
+        webxr_.get(), transfer_size_, uv_transform_);
+    frame_data->buffer_holder = buffer_holder;
+  }
 
   // Create the frame data to return to the renderer.
   frame_data->pose = std::move(pose);
-  frame_data->buffer_holder = buffer_holder;
   frame_data->time_delta = base::TimeTicks::Now() - base::TimeTicks();
 
   if (options && options->include_plane_data) {
@@ -481,7 +492,59 @@ void ArCoreGl::SubmitFrameMissing(int16_t frame_index,
 void ArCoreGl::SubmitFrame(int16_t frame_index,
                            const gpu::MailboxHolder& mailbox,
                            base::TimeDelta time_waited) {
-  NOTIMPLEMENTED();
+  DVLOG(2) << __func__ << ": frame=" << frame_index;
+  DCHECK(!ar_image_transport_->UseSharedBuffer());
+
+  if (!IsSubmitFrameExpected(frame_index))
+    return;
+
+  webxr_->ProcessOrDefer(base::BindOnce(&ArCoreGl::ProcessFrameFromMailbox,
+                                        weak_ptr_factory_.GetWeakPtr(),
+                                        frame_index, mailbox));
+}
+
+void ArCoreGl::ProcessFrameFromMailbox(int16_t frame_index,
+                                       const gpu::MailboxHolder& mailbox) {
+  DVLOG(2) << __func__ << ": frame=" << frame_index;
+  DCHECK(webxr_->HaveProcessingFrame());
+  DCHECK(!ar_image_transport_->UseSharedBuffer());
+
+  ar_image_transport_->CopyMailboxToSurfaceAndSwap(transfer_size_, mailbox);
+  // Notify the client that we're done with the mailbox so that the underlying
+  // image is eligible for destruction.
+  submit_client_->OnSubmitFrameTransferred(true);
+
+  CopyCameraImageToFramebuffer();
+
+  // Now wait for ar_image_transport_ to call OnTransportFrameAvailable
+  // indicating that the image drawn onto the Surface is ready for consumption
+  // from the SurfaceTexture.
+}
+
+void ArCoreGl::OnTransportFrameAvailable(const gfx::Transform& uv_transform) {
+  DVLOG(2) << __func__;
+  DCHECK(!ar_image_transport_->UseSharedBuffer());
+  DCHECK(webxr_->HaveProcessingFrame());
+  webxr_->TransitionFrameProcessingToRendering();
+
+  glBindFramebufferEXT(GL_DRAW_FRAMEBUFFER, 0);
+  ar_image_transport_->CopyDrawnImageToFramebuffer(
+      webxr_.get(), camera_image_size_, uv_transform);
+
+  surface_->SwapBuffers(base::DoNothing());
+  DVLOG(3) << __func__ << ": SwapBuffers";
+
+  webxr_->EndFrameRendering();
+
+  if (submit_client_) {
+    // Create a local GpuFence and pass it to the Renderer via IPC.
+    std::unique_ptr<gl::GLFence> gl_fence = gl::GLFence::CreateForGpuFence();
+    std::unique_ptr<gfx::GpuFence> gpu_fence2 = gl_fence->GetGpuFence();
+    submit_client_->OnSubmitFrameGpuFence(
+        gfx::CloneHandleForIPC(gpu_fence2->GetGpuFenceHandle()));
+  }
+  // We finished processing a frame, unblock a potentially waiting next frame.
+  webxr_->TryDeferredProcessing();
 }
 
 void ArCoreGl::SubmitFrameWithTextureHandle(int16_t frame_index,
@@ -493,6 +556,7 @@ void ArCoreGl::SubmitFrameDrawnIntoTexture(int16_t frame_index,
                                            const gpu::SyncToken& sync_token,
                                            base::TimeDelta time_waited) {
   DVLOG(2) << __func__ << ": frame=" << frame_index;
+  DCHECK(ar_image_transport_->UseSharedBuffer());
 
   if (!IsSubmitFrameExpected(frame_index))
     return;
@@ -509,6 +573,7 @@ void ArCoreGl::ProcessFrameDrawnIntoTexture(int16_t frame_index,
   TRACE_EVENT0("gpu", "ArCore SubmitFrame");
 
   DCHECK(webxr_->HaveProcessingFrame());
+  DCHECK(ar_image_transport_->UseSharedBuffer());
   CopyCameraImageToFramebuffer();
 
   ar_image_transport_->CreateGpuFenceForSyncToken(
@@ -521,6 +586,7 @@ void ArCoreGl::OnWebXrTokenSignaled(int16_t frame_index,
   DVLOG(3) << __func__ << ": frame=" << frame_index;
 
   DCHECK(webxr_->HaveProcessingFrame());
+  DCHECK(ar_image_transport_->UseSharedBuffer());
   webxr_->TransitionFrameProcessingToRendering();
 
   glBindFramebufferEXT(GL_DRAW_FRAMEBUFFER, 0);

chrome/browser/android/vr/arcore_device/arcore_gl.h

@@ -130,6 +130,8 @@ class ArCoreGl : public mojom::XRFrameDataProvider,
   // mojom::XRSessionController
   void SetFrameDataRestricted(bool restricted) override;
 
+  void ProcessFrameFromMailbox(int16_t frame_index,
+                               const gpu::MailboxHolder& mailbox);
   void ProcessFrameDrawnIntoTexture(int16_t frame_index,
                                     const gpu::SyncToken& sync_token);
   void OnWebXrTokenSignaled(int16_t frame_index,
@@ -152,6 +154,7 @@ class ArCoreGl : public mojom::XRFrameDataProvider,
   void OnArImageTransportReady(base::OnceCallback<void(bool)> callback);
   bool IsOnGlThread() const;
   void CopyCameraImageToFramebuffer();
+  void OnTransportFrameAvailable(const gfx::Transform& uv_transform);
 
   base::OnceClosure session_shutdown_callback_;
 

chrome/browser/android/vr/arcore_device/arcore_shim.cc

@@ -7,6 +7,7 @@
 #include <dlfcn.h>
 
 #include "base/android/android_hardware_buffer_compat.h"
+#include "base/android/build_info.h"
 #include "base/logging.h"
 
 namespace {
@@ -120,7 +121,8 @@ bool LoadArCoreSdk(const std::string& libraryPath) {
 }
 
 bool IsArCoreSupported() {
-  return base::AndroidHardwareBufferCompat::IsSupportAvailable();
+  return base::android::BuildInfo::GetInstance()->sdk_int() >=
+         base::android::SDK_VERSION_NOUGAT;
 }
 
 }  // namespace vr