Enables WebXR API to access camera image texture.

I2P: https://groups.google.com/a/chromium.org/d/msg/blink-dev/4kMGCyCUIao/5vjPk4tuAQAJ
Change-Id: Ib30e5d2ec4d833d2cc44246bf95827f4e2083f4a
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2230280
Commit-Queue: Luis Garcia <luisggarcia@google.com>
Reviewed-by: Brandon Jones <bajones@chromium.org>
Reviewed-by: Klaus Weidner <klausw@chromium.org>
Reviewed-by: Ken Buchanan <kenrb@chromium.org>
Reviewed-by: Piotr Bialecki <bialpio@chromium.org>
Cr-Commit-Position: refs/heads/master@{#786483}

chrome/browser/android/vr/arcore_device/ar_image_transport.cc

@@ -206,10 +206,12 @@ gpu::MailboxHolder ArImageTransport::TransferFrame(
     const gfx::Size& frame_size,
     const gfx::Transform& uv_transform) {
   DCHECK(IsOnGlThread());
+  DCHECK(UseSharedBuffer());
 
   if (!webxr->GetAnimatingFrame()->shared_buffer) {
     webxr->GetAnimatingFrame()->shared_buffer = CreateBuffer();
   }
+
   vr::WebXrSharedBuffer* shared_buffer =
       webxr->GetAnimatingFrame()->shared_buffer.get();
   ResizeSharedBuffer(webxr, frame_size, shared_buffer);
@@ -218,6 +220,53 @@ gpu::MailboxHolder ArImageTransport::TransferFrame(
   return shared_buffer->mailbox_holder;
 }
 
+gpu::MailboxHolder ArImageTransport::TransferCameraImageFrame(
+    vr::WebXrPresentationState* webxr,
+    const gfx::Size& frame_size,
+    const gfx::Transform& uv_transform) {
+  DCHECK(IsOnGlThread());
+  DCHECK(UseSharedBuffer());
+
+  if (!webxr->GetAnimatingFrame()->camera_image_shared_buffer) {
+    webxr->GetAnimatingFrame()->camera_image_shared_buffer = CreateBuffer();
+  }
+
+  vr::WebXrSharedBuffer* camera_image_shared_buffer =
+      webxr->GetAnimatingFrame()->camera_image_shared_buffer.get();
+  ResizeSharedBuffer(webxr, frame_size, camera_image_shared_buffer);
+
+  // Temporarily change drawing buffer to the camera image buffer.
+  if (!camera_image_fbo_) {
+    glGenFramebuffersEXT(1, &camera_image_fbo_);
+  }
+  glBindFramebufferEXT(GL_DRAW_FRAMEBUFFER, camera_image_fbo_);
+  glFramebufferTexture2DEXT(GL_DRAW_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
+                            GL_TEXTURE_EXTERNAL_OES,
+                            camera_image_shared_buffer->local_texture, 0);
+
+  CopyCameraImageToFramebuffer(frame_size, uv_transform);
+
+#if DCHECK_IS_ON()
+  if (!framebuffer_complete_checked_for_camera_buffer_) {
+    auto status = glCheckFramebufferStatusEXT(GL_DRAW_FRAMEBUFFER);
+    DVLOG(1) << __func__ << ": framebuffer status=" << std::hex << status;
+    DCHECK(status == GL_FRAMEBUFFER_COMPLETE);
+    framebuffer_complete_checked_for_camera_buffer_ = true;
+  }
+#endif
+
+  // Restore default drawing buffer.
+  glBindFramebufferEXT(GL_DRAW_FRAMEBUFFER, 0);
+
+  std::unique_ptr<gl::GLFence> gl_fence = gl::GLFence::CreateForGpuFence();
+  std::unique_ptr<gfx::GpuFence> gpu_fence = gl_fence->GetGpuFence();
+  mailbox_bridge_->WaitForClientGpuFence(gpu_fence.release());
+
+  mailbox_bridge_->GenSyncToken(
+      &camera_image_shared_buffer->mailbox_holder.sync_token);
+  return camera_image_shared_buffer->mailbox_holder;
+}
+
 void ArImageTransport::CreateGpuFenceForSyncToken(
     const gpu::SyncToken& sync_token,
     base::OnceCallback<void(std::unique_ptr<gfx::GpuFence>)> callback) {

chrome/browser/android/vr/arcore_device/ar_image_transport.h

@@ -57,12 +57,21 @@ class ArImageTransport {
 
   virtual GLuint GetCameraTextureId();
 
+  // This creates a shared buffer if one doesn't already exist, and populates it
+  // with the current animating frame's buffer data. It returns a
+  // gpu::Mailboxholder with this shared buffer data.
+  virtual gpu::MailboxHolder TransferFrame(vr::WebXrPresentationState* webxr,
+                                           const gfx::Size& frame_size,
+                                           const gfx::Transform& uv_transform);
+
   // This transfers whatever the contents of the texture specified
   // by GetCameraTextureId() is at the time it is called and returns
   // a gpu::MailboxHolder with that texture copied to a shared buffer.
-  virtual gpu::MailboxHolder TransferFrame(vr::WebXrPresentationState* webxr,
+  virtual gpu::MailboxHolder TransferCameraImageFrame(
+      vr::WebXrPresentationState* webxr,
       const gfx::Size& frame_size,
       const gfx::Transform& uv_transform);
+
   virtual void CreateGpuFenceForSyncToken(
       const gpu::SyncToken& sync_token,
       base::OnceCallback<void(std::unique_ptr<gfx::GpuFence>)>);
@@ -95,6 +104,7 @@ class ArImageTransport {
   // samplerExternalOES texture for the camera image.
   GLuint camera_texture_id_arcore_ = 0;
   GLuint camera_fbo_ = 0;
+  GLuint camera_image_fbo_ = 0;
 
   scoped_refptr<base::SingleThreadTaskRunner> gl_thread_task_runner_;
 
@@ -104,6 +114,10 @@ class ArImageTransport {
   // If false, use Surface transport aka SUBMIT_AS_MAILBOX_HOLDER.
   bool shared_buffer_draw_ = false;
 
+  // Used to limit framebuffer complete check to occurring once, due to it being
+  // expensive.
+  bool framebuffer_complete_checked_for_camera_buffer_ = false;
+
   // Used for Surface transport (Android N)
   //
   // samplerExternalOES texture data for WebXR content image.

chrome/browser/android/vr/arcore_device/arcore_gl.cc

@@ -434,6 +434,13 @@ void ArCoreGl::GetFrameData(
     gpu::MailboxHolder buffer_holder = ar_image_transport_->TransferFrame(
         webxr_.get(), transfer_size_, uv_transform_);
     frame_data->buffer_holder = buffer_holder;
+
+    if (IsFeatureEnabled(device::mojom::XRSessionFeature::CAMERA_ACCESS)) {
+      gpu::MailboxHolder camera_image_buffer_holder =
+          ar_image_transport_->TransferCameraImageFrame(
+              webxr_.get(), transfer_size_, uv_transform_);
+      frame_data->camera_image_buffer_holder = camera_image_buffer_holder;
+    }
   }
 
   // Create the frame data to return to the renderer.
@@ -1071,6 +1078,10 @@ std::vector<mojom::XRInputSourceStatePtr> ArCoreGl::GetInputSourceStates() {
   return result;
 }
 
+bool ArCoreGl::IsFeatureEnabled(mojom::XRSessionFeature feature) {
+  return base::Contains(enabled_features_, feature);
+}
+
 void ArCoreGl::Pause() {
   DCHECK(IsOnGlThread());
   DCHECK(is_initialized_);

chrome/browser/android/vr/arcore_device/arcore_gl.h

@@ -181,6 +181,8 @@ class ArCoreGl : public mojom::XRFrameDataProvider,
   // state on session shutdown. See https://crbug.com/1065572.
   void RunNextGetFrameData();
 
+  bool IsFeatureEnabled(mojom::XRSessionFeature feature);
+
   // List of features enabled on this session. Required to correctly configure
   // the session and only send out necessary data related to reference spaces to
   // blink. Valid after the call to |Initialize()| method.

chrome/browser/android/vr/web_xr_presentation_state.cc

@@ -121,6 +121,8 @@ WebXrPresentationState::TakeSharedBuffers() {
   for (auto& frame : frames_storage_) {
     if (frame->shared_buffer)
       shared_buffers.emplace_back(std::move(frame->shared_buffer));
+    if (frame->camera_image_shared_buffer)
+      shared_buffers.emplace_back(std::move(frame->camera_image_shared_buffer));
   }
   return shared_buffers;
 }

chrome/browser/android/vr/web_xr_presentation_state.h

@@ -125,6 +125,8 @@ struct WebXrFrame {
   // In SharedBuffer mode, keep a swap chain.
   std::unique_ptr<WebXrSharedBuffer> shared_buffer;
 
+  std::unique_ptr<WebXrSharedBuffer> camera_image_shared_buffer;
+
   DISALLOW_COPY_AND_ASSIGN(WebXrFrame);
 };
 

device/vr/public/mojom/vr_service.mojom

@@ -543,12 +543,12 @@ struct XRFrameData {
   VRPose? pose;
   // Time delta from an unspecified origin.
   mojo_base.mojom.TimeDelta time_delta;
-  // The buffer_holder is used for sending data imagery back and forth across
-  // the process boundary. For application with pass through camera, it holds
-  // the camera image to be passed to the renderer. For immersive sessions, it
-  // is the place for the renderer to draw into to pass imagery to the device
-  // for rendering.
+  // The buffer_holder is used for sending imagery data back and forth across
+  // the process boundary.
   gpu.mojom.MailboxHolder? buffer_holder;
+  // The camera_image_buffer_holder is used to send a copy of the camera image
+  // across the process boundary for immersive-ar sessions.
+  gpu.mojom.MailboxHolder? camera_image_buffer_holder;
 
   // Indicates that there has been a significant discontinuity in the mojo space
   // coordinate system, and that poses from this point forward with the same

third_party/blink/renderer/modules/webgl/webgl_texture.cc

@@ -37,6 +37,13 @@ WebGLTexture::WebGLTexture(WebGLRenderingContextBase* ctx)
   SetObject(texture);
 }
 
+WebGLTexture::WebGLTexture(WebGLRenderingContextBase* ctx,
+                           GLuint texture,
+                           GLenum target)
+    : WebGLSharedPlatform3DObject(ctx), target_(target) {
+  SetObject(texture);
+}
+
 WebGLTexture::~WebGLTexture() = default;
 
 void WebGLTexture::SetTarget(GLenum target) {

third_party/blink/renderer/modules/webgl/webgl_texture.h

@@ -36,6 +36,13 @@ class WebGLTexture final : public WebGLSharedPlatform3DObject {
 
  public:
   explicit WebGLTexture(WebGLRenderingContextBase*);
+
+  // The provided GLuint must have been created in the same
+  // WebGLRenderingContextBase that is provided.
+  explicit WebGLTexture(WebGLRenderingContextBase* ctx,
+                        GLuint texture,
+                        GLenum target);
+
   ~WebGLTexture() override;
 
   void SetTarget(GLenum);

third_party/blink/renderer/modules/xr/xr_frame.cc

@@ -172,6 +172,10 @@ void XRFrame::Deactivate() {
   is_animation_frame_ = false;
 }
 
+bool XRFrame::IsActive() const {
+  return is_active_;
+}
+
 HeapVector<Member<XRHitTestResult>> XRFrame::getHitTestResults(
     XRHitTestSource* hit_test_source,
     ExceptionState& exception_state) {

third_party/blink/renderer/modules/xr/xr_frame.h

@@ -52,6 +52,8 @@ class XRFrame final : public ScriptWrappable {
 
   void Deactivate();
 
+  bool IsActive() const;
+
   void SetAnimationFrame(bool is_animation_frame) {
     is_animation_frame_ = is_animation_frame;
   }

third_party/blink/renderer/modules/xr/xr_frame_provider.cc

@@ -303,6 +303,7 @@ void XRFrameProvider::OnImmersiveFrameData(
 
   frame_id_ = data->frame_id;
   buffer_mailbox_holder_ = data->buffer_holder;
+  camera_image_mailbox_holder_ = data->camera_image_buffer_holder;
 
   pending_immersive_vsync_ = false;
 
@@ -490,7 +491,8 @@ void XRFrameProvider::ProcessScheduledFrame(
         ->PostTask(FROM_HERE,
                    WTF::Bind(&XRSession::OnFrame,
                              WrapWeakPersistent(immersive_session_.Get()),
-                             high_res_now_ms, buffer_mailbox_holder_));
+                             high_res_now_ms, buffer_mailbox_holder_,
+                             camera_image_mailbox_holder_));
   } else {
     // In the process of fulfilling the frame requests for each session they are
     // extremely likely to request another frame. Work off of a separate list
@@ -539,7 +541,7 @@ void XRFrameProvider::ProcessScheduledFrame(
       frame->GetTaskRunner(blink::TaskType::kInternalMedia)
           ->PostTask(FROM_HERE,
                      WTF::Bind(&XRSession::OnFrame, WrapWeakPersistent(session),
-                               high_res_now_ms, base::nullopt));
+                               high_res_now_ms, base::nullopt, base::nullopt));
     }
   }
 }

third_party/blink/renderer/modules/xr/xr_frame_provider.h

@@ -112,6 +112,7 @@ class XRFrameProvider final : public GarbageCollected<XRFrameProvider> {
   bool pending_non_immersive_vsync_ = false;
 
   base::Optional<gpu::MailboxHolder> buffer_mailbox_holder_;
+  base::Optional<gpu::MailboxHolder> camera_image_mailbox_holder_;
   bool last_has_focus_ = false;
 };
 

third_party/blink/renderer/modules/xr/xr_session.cc

@@ -1548,7 +1548,8 @@ void XRSession::SetMetricsReporter(std::unique_ptr<MetricsReporter> reporter) {
 
 void XRSession::OnFrame(
     double timestamp,
-    const base::Optional<gpu::MailboxHolder>& output_mailbox_holder) {
+    const base::Optional<gpu::MailboxHolder>& output_mailbox_holder,
+    const base::Optional<gpu::MailboxHolder>& camera_image_mailbox_holder) {
   TRACE_EVENT0("gpu", __func__);
   DVLOG(2) << __func__ << ": ended_=" << ended_
            << ", pending_frame_=" << pending_frame_;
@@ -1575,7 +1576,8 @@ void XRSession::OnFrame(
       if (prev_base_layer_) {
         DVLOG(2) << __func__
                  << ": prev_base_layer_ is valid, submitting frame to it";
-        prev_base_layer_->OnFrameStart(output_mailbox_holder);
+        prev_base_layer_->OnFrameStart(output_mailbox_holder,
+                                       camera_image_mailbox_holder);
         prev_base_layer_->OnFrameEnd();
         prev_base_layer_ = nullptr;
       }
@@ -1591,7 +1593,8 @@ void XRSession::OnFrame(
       return;
     }
 
-    frame_base_layer->OnFrameStart(output_mailbox_holder);
+    frame_base_layer->OnFrameStart(output_mailbox_holder,
+                                   camera_image_mailbox_holder);
 
     // Don't allow frames to be processed if the session's visibility state is
     // "hidden".

third_party/blink/renderer/modules/xr/xr_session.h

@@ -238,8 +238,10 @@ class XRSession final
   const AtomicString& InterfaceName() const override;
 
   void OnFocusChanged();
-  void OnFrame(double timestamp,
-               const base::Optional<gpu::MailboxHolder>& output_mailbox_holder);
+  void OnFrame(
+      double timestamp,
+      const base::Optional<gpu::MailboxHolder>& output_mailbox_holder,
+      const base::Optional<gpu::MailboxHolder>& camera_image_mailbox_holder);
 
   // XRInputSourceButtonListener
   void OnButtonEvent(

third_party/blink/renderer/modules/xr/xr_webgl_binding.cc

@@ -7,9 +7,13 @@
 #include "third_party/blink/renderer/modules/webgl/webgl_rendering_context_base.h"
 #include "third_party/blink/renderer/modules/webgl/webgl_texture.h"
 #include "third_party/blink/renderer/modules/xr/xr_cube_map.h"
+#include "third_party/blink/renderer/modules/xr/xr_frame.h"
 #include "third_party/blink/renderer/modules/xr/xr_light_probe.h"
+#include "third_party/blink/renderer/modules/xr/xr_render_state.h"
 #include "third_party/blink/renderer/modules/xr/xr_session.h"
 #include "third_party/blink/renderer/modules/xr/xr_utils.h"
+#include "third_party/blink/renderer/modules/xr/xr_viewer_pose.h"
+#include "third_party/blink/renderer/modules/xr/xr_webgl_layer.h"
 #include "third_party/blink/renderer/platform/bindings/exception_state.h"
 #include "third_party/blink/renderer/platform/graphics/gpu/extensions_3d_util.h"
 
@@ -78,7 +82,28 @@ WebGLTexture* XRWebGLBinding::getReflectionCubeMap(
 }
 
 WebGLTexture* XRWebGLBinding::getCameraImage(XRFrame* frame, XRView* view) {
+  // Verify that frame is currently active.
+  if (!frame->IsActive()) {
     return nullptr;
+  }
+
+  // TODO(https://crbug.com/1100978): Verify view is in camera_views_.
+
+  XRWebGLLayer* base_layer = view->session()->renderState()->baseLayer();
+  DCHECK(base_layer);
+
+  base::Optional<gpu::MailboxHolder> camera_image_mailbox_holder =
+      base_layer->CameraImageMailboxHolder();
+
+  if (!camera_image_mailbox_holder) {
+    return nullptr;
+  }
+
+  GLuint texture_id = base_layer->CameraImageTextureId();
+
+  WebGLTexture* texture = MakeGarbageCollected<WebGLTexture>(
+      webgl_context_, texture_id, GL_TEXTURE_2D);
+  return texture;
 }
 
 void XRWebGLBinding::Trace(Visitor* visitor) const {

third_party/blink/renderer/modules/xr/xr_webgl_binding.h

@@ -15,9 +15,9 @@ namespace blink {
 class ExceptionState;
 class WebGLRenderingContextBase;
 class WebGLTexture;
+class XRFrame;
 class XRLightProbe;
 class XRSession;
-class XRFrame;
 class XRView;
 
 class XRWebGLBinding final : public ScriptWrappable {

third_party/blink/renderer/modules/xr/xr_webgl_layer.cc

@@ -250,17 +250,59 @@ HTMLCanvasElement* XRWebGLLayer::output_canvas() const {
   return nullptr;
 }
 
+uint32_t XRWebGLLayer::CameraImageTextureId() const {
+  return camera_image_texture_id_;
+}
+
+base::Optional<gpu::MailboxHolder> XRWebGLLayer::CameraImageMailboxHolder()
+    const {
+  return camera_image_mailbox_holder_;
+}
+
 void XRWebGLLayer::OnFrameStart(
-    const base::Optional<gpu::MailboxHolder>& buffer_mailbox_holder) {
+    const base::Optional<gpu::MailboxHolder>& buffer_mailbox_holder,
+    const base::Optional<gpu::MailboxHolder>& camera_image_mailbox_holder) {
   if (framebuffer_) {
     framebuffer_->MarkOpaqueBufferComplete(true);
     framebuffer_->SetContentsChanged(false);
     if (buffer_mailbox_holder) {
       drawing_buffer_->UseSharedBuffer(buffer_mailbox_holder.value());
+      DVLOG(3) << __func__ << ": buffer_mailbox_holder->mailbox="
+               << buffer_mailbox_holder->mailbox.ToDebugString();
       is_direct_draw_frame = true;
     } else {
       is_direct_draw_frame = false;
     }
+
+    if (camera_image_mailbox_holder) {
+      DVLOG(3) << __func__ << ":camera_image_mailbox_holder->mailbox="
+               << camera_image_mailbox_holder->mailbox.ToDebugString();
+      camera_image_mailbox_holder_ = camera_image_mailbox_holder;
+      camera_image_texture_id_ =
+          GetBufferTextureId(camera_image_mailbox_holder_);
+      BindBufferTexture(camera_image_mailbox_holder_);
+    }
+  }
+}
+
+uint32_t XRWebGLLayer::GetBufferTextureId(
+    const base::Optional<gpu::MailboxHolder>& buffer_mailbox_holder) {
+  gpu::gles2::GLES2Interface* gl = drawing_buffer_->ContextGL();
+  gl->WaitSyncTokenCHROMIUM(buffer_mailbox_holder->sync_token.GetConstData());
+  GLuint texture_id = gl->CreateAndTexStorage2DSharedImageCHROMIUM(
+      buffer_mailbox_holder->mailbox.name);
+  return texture_id;
+}
+
+void XRWebGLLayer::BindBufferTexture(
+    const base::Optional<gpu::MailboxHolder>& buffer_mailbox_holder) {
+  gpu::gles2::GLES2Interface* gl = drawing_buffer_->ContextGL();
+
+  if (buffer_mailbox_holder) {
+    uint32_t texture_target = buffer_mailbox_holder->texture_target;
+    gl->BindTexture(texture_target, camera_image_texture_id_);
+    gl->BeginSharedImageAccessDirectCHROMIUM(
+        camera_image_texture_id_, GL_SHARED_IMAGE_ACCESS_MODE_READ_CHROMIUM);
   }
 }
 
@@ -300,6 +342,14 @@ void XRWebGLLayer::OnFrameEnd() {
       // Always call submit, but notify if the contents were changed or not.
       session()->xr()->frameProvider()->SubmitWebGLLayer(this,
                                                          framebuffer_dirty);
+      if (camera_image_mailbox_holder_ && camera_image_texture_id_) {
+        DVLOG(3) << __func__ << "Deleting camera image texture";
+        gpu::gles2::GLES2Interface* gl = drawing_buffer_->ContextGL();
+        gl->EndSharedImageAccessDirectCHROMIUM(camera_image_texture_id_);
+        gl->DeleteTextures(1, &camera_image_texture_id_);
+        camera_image_texture_id_ = 0;
+        camera_image_mailbox_holder_ = base::nullopt;
+      }
     }
   }
 }

third_party/blink/renderer/modules/xr/xr_webgl_layer.h

@@ -64,8 +64,12 @@ class XRWebGLLayer final : public XRLayer {
   void UpdateViewports();
 
   HTMLCanvasElement* output_canvas() const;
+  uint32_t CameraImageTextureId() const;
+  base::Optional<gpu::MailboxHolder> CameraImageMailboxHolder() const;
 
-  void OnFrameStart(const base::Optional<gpu::MailboxHolder>&);
+  void OnFrameStart(
+      const base::Optional<gpu::MailboxHolder>& buffer_mailbox_holder,
+      const base::Optional<gpu::MailboxHolder>& camera_image_mailbox_holder);
   void OnFrameEnd();
   void OnResize();
 
@@ -78,6 +82,12 @@ class XRWebGLLayer final : public XRLayer {
   void Trace(Visitor*) const override;
 
  private:
+  uint32_t GetBufferTextureId(
+      const base::Optional<gpu::MailboxHolder>& buffer_mailbox_holder);
+
+  void BindBufferTexture(
+      const base::Optional<gpu::MailboxHolder>& buffer_mailbox_holder);
+
   Member<XRViewport> left_viewport_;
   Member<XRViewport> right_viewport_;
 
@@ -91,6 +101,9 @@ class XRWebGLLayer final : public XRLayer {
   bool ignore_depth_values_ = false;
 
   uint32_t clean_frame_count = 0;
+
+  uint32_t camera_image_texture_id_;
+  base::Optional<gpu::MailboxHolder> camera_image_mailbox_holder_;
 };
 
 }  // namespace blink