WebXR: implement viewport scaling for ARCore

This adds the needed changes to use dynamic viewports for the ARCore
WebXR device, and opts in to using that.

It also changes the texture MAG filter used when enlarging a texture to
use LINEAR interpolation. This should also be beneficial when using a
custom framebufferScaleFactor independent of dynamic viewport scaling
to avoid pixelation artifacts.

Bug: 1133381
Change-Id: I442124de1090f971edbc822dd3ba8550301c2e7a
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2436683
Commit-Queue: Klaus Weidner <klausw@chromium.org>
Reviewed-by: Piotr Bialecki <bialpio@chromium.org>
Cr-Commit-Position: refs/heads/master@{#813092}

components/webxr/mailbox_to_surface_bridge_impl.cc

@@ -28,6 +28,7 @@
 #include "gpu/ipc/common/gpu_surface_tracker.h"
 #include "services/viz/public/cpp/gpu/context_provider_command_buffer.h"
 #include "ui/gfx/color_space.h"
+#include "ui/gfx/transform.h"
 #include "ui/gl/android/surface_texture.h"
 
 #include <android/native_window_jni.h>
@@ -43,8 +44,10 @@ const char kQuadCopyVertex[] = SHADER(
     attribute vec4 a_Position;
     attribute vec2 a_TexCoordinate;
     varying highp vec2 v_TexCoordinate;
+    uniform mat4 u_UvTransform;
     void main() {
-      v_TexCoordinate = a_TexCoordinate;
+      highp vec4 uv_in = vec4(a_TexCoordinate.x, a_TexCoordinate.y, 0, 1);
+      v_TexCoordinate = (u_UvTransform * uv_in).xy;
       gl_Position = a_Position;
     }
 );
@@ -291,6 +294,12 @@ void MailboxToSurfaceBridgeImpl::ResizeSurface(int width, int height) {
 
 bool MailboxToSurfaceBridgeImpl::CopyMailboxToSurfaceAndSwap(
     const gpu::MailboxHolder& mailbox) {
+  return CopyMailboxToSurfaceAndSwap(mailbox, gfx::Transform());
+}
+
+bool MailboxToSurfaceBridgeImpl::CopyMailboxToSurfaceAndSwap(
+    const gpu::MailboxHolder& mailbox,
+    const gfx::Transform& uv_transform) {
   if (!IsConnected()) {
     // We may not have a context yet, i.e. due to surface initialization
     // being incomplete. This is not an error, but we obviously can't draw
@@ -316,7 +325,7 @@ bool MailboxToSurfaceBridgeImpl::CopyMailboxToSurfaceAndSwap(
   GLuint sourceTexture = ConsumeTexture(gl_, mailbox);
   gl_->BeginSharedImageAccessDirectCHROMIUM(
       sourceTexture, GL_SHARED_IMAGE_ACCESS_MODE_READ_CHROMIUM);
-  DrawQuad(sourceTexture);
+  DrawQuad(sourceTexture, uv_transform);
   gl_->EndSharedImageAccessDirectCHROMIUM(sourceTexture);
   gl_->DeleteTextures(1, &sourceTexture);
   gl_->SwapBuffers(swap_id_++);
@@ -423,6 +432,8 @@ void MailboxToSurfaceBridgeImpl::InitializeRenderer() {
       gl_->GetAttribLocation(program_handle, "a_TexCoordinate");
   GLuint texUniform_handle =
       gl_->GetUniformLocation(program_handle, "u_Texture");
+  uniform_uv_transform_handle_ =
+      gl_->GetUniformLocation(program_handle, "u_UvTransform");
 
   GLuint vertexBuffer = 0;
   gl_->GenBuffers(1, &vertexBuffer);
@@ -466,7 +477,8 @@ void MailboxToSurfaceBridgeImpl::InitializeRenderer() {
   gl_->Uniform1i(texUniform_handle, 0);
 }
 
-void MailboxToSurfaceBridgeImpl::DrawQuad(unsigned int texture_handle) {
+void MailboxToSurfaceBridgeImpl::DrawQuad(unsigned int texture_handle,
+                                          const gfx::Transform& uv_transform) {
   DCHECK(IsConnected());
 
   // We're redrawing over the entire viewport, but it's generally more
@@ -477,6 +489,11 @@ void MailboxToSurfaceBridgeImpl::DrawQuad(unsigned int texture_handle) {
   // it's not supported on older devices such as Nexus 5X.
   gl_->Clear(GL_COLOR_BUFFER_BIT);
 
+  float uv_transform_floats[16];
+  uv_transform.matrix().asColMajorf(uv_transform_floats);
+  gl_->UniformMatrix4fv(uniform_uv_transform_handle_, 1, GL_FALSE,
+                        &uv_transform_floats[0]);
+
   // Configure texture. This is a 1:1 pixel copy since the surface
   // size is resized to match the source canvas, so we can use
   // GL_NEAREST.
@@ -484,7 +501,11 @@ void MailboxToSurfaceBridgeImpl::DrawQuad(unsigned int texture_handle) {
   gl_->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
   gl_->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
   gl_->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
-  gl_->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+  if (uv_transform.IsIdentity()) {
+    gl_->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+  } else {
+    gl_->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+  }
   gl_->DrawArrays(GL_TRIANGLE_FAN, 0, 4);
 }
 

components/webxr/mailbox_to_surface_bridge_impl.h

@@ -54,6 +54,9 @@ class MailboxToSurfaceBridgeImpl : public device::MailboxToSurfaceBridge {
 
   bool CopyMailboxToSurfaceAndSwap(const gpu::MailboxHolder& mailbox) override;
 
+  bool CopyMailboxToSurfaceAndSwap(const gpu::MailboxHolder& mailbox,
+                                   const gfx::Transform& uv_transform) override;
+
   void GenSyncToken(gpu::SyncToken* out_sync_token) override;
 
   void WaitSyncToken(const gpu::SyncToken& sync_token) override;
@@ -77,7 +80,7 @@ class MailboxToSurfaceBridgeImpl : public device::MailboxToSurfaceBridge {
       scoped_refptr<viz::ContextProvider> provider);
   void InitializeRenderer();
   void DestroyContext();
-  void DrawQuad(unsigned int textureHandle);
+  void DrawQuad(unsigned int textureHandle, const gfx::Transform& uv_transform);
 
   scoped_refptr<viz::ContextProvider> context_provider_;
   std::unique_ptr<gl::ScopedJavaSurface> surface_;
@@ -98,6 +101,9 @@ class MailboxToSurfaceBridgeImpl : public device::MailboxToSurfaceBridge {
   // A swap ID which is passed to GL swap. Incremented each call.
   uint64_t swap_id_ = 0;
 
+  // Uniform handle for the UV transform used by DrawQuad.
+  uint32_t uniform_uv_transform_handle_ = 0;
+
   // A task runner for the GL thread
   scoped_refptr<base::SingleThreadTaskRunner> gl_thread_task_runner_;
 

device/vr/android/arcore/ar_image_transport.cc

@@ -378,7 +378,8 @@ void ArImageTransport::CopyTextureToFramebuffer(
 
 void ArImageTransport::CopyMailboxToSurfaceAndSwap(
     const gfx::Size& frame_size,
-    const gpu::MailboxHolder& mailbox) {
+    const gpu::MailboxHolder& mailbox,
+    const gfx::Transform& uv_transform) {
   DVLOG(2) << __func__;
   if (frame_size != surface_size_) {
     DVLOG(2) << __func__ << " resize from " << surface_size_.ToString()
@@ -392,7 +393,8 @@ void ArImageTransport::CopyMailboxToSurfaceAndSwap(
   // Draw the image to the surface in the GPU process's command buffer context.
   // This will trigger an OnFrameAvailable event once the corresponding
   // SurfaceTexture in the local GL context is ready for updating.
-  bool swapped = mailbox_bridge_->CopyMailboxToSurfaceAndSwap(mailbox);
+  bool swapped =
+      mailbox_bridge_->CopyMailboxToSurfaceAndSwap(mailbox, uv_transform);
   DCHECK(swapped);
 }
 

device/vr/android/arcore/ar_image_transport.h

@@ -86,7 +86,8 @@ class COMPONENT_EXPORT(VR_ARCORE) ArImageTransport {
                                         const gfx::Transform& uv_transform);
   virtual void WaitSyncToken(const gpu::SyncToken& sync_token);
   virtual void CopyMailboxToSurfaceAndSwap(const gfx::Size& frame_size,
-                                           const gpu::MailboxHolder& mailbox);
+                                           const gpu::MailboxHolder& mailbox,
+                                           const gfx::Transform& uv_transform);
 
   bool UseSharedBuffer() { return shared_buffer_draw_; }
   void SetFrameAvailableCallback(XrFrameCallback on_frame_available);

device/vr/android/arcore/ar_renderer.cc

@@ -100,12 +100,15 @@ void ArRenderer::Draw(int texture_handle, const float (&uv_transform)[16]) {
   glVertexAttribPointer(position_handle_, 2, GL_FLOAT, false, 0, 0);
   glEnableVertexAttribArray(position_handle_);
 
-  // Bind texture. This is a 1:1 pixel copy since the source surface
-  // and renderbuffer destination size are resized to match, so use
-  // GL_NEAREST.
+  // Bind texture. This is not necessarily a 1:1 pixel copy since the
+  // size is modified by framebufferScaleFactor and requestViewportScale,
+  // so use GL_LINEAR.
   glActiveTexture(GL_TEXTURE0);
   glBindTexture(GL_TEXTURE_EXTERNAL_OES, texture_handle);
-  vr::SetTexParameters(GL_TEXTURE_EXTERNAL_OES);
+  glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+  glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+  glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+  glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
   glUniform1i(texture_handle_, 0);
 
   glUniformMatrix4fv(uv_transform_, 1, GL_FALSE, &uv_transform[0]);

device/vr/android/arcore/arcore_device.cc

@@ -276,6 +276,7 @@ void ArCoreDevice::OnCreateSessionCallback(
   session->data_provider = std::move(frame_data_provider);
   session->display_info = std::move(display_info);
   session->submit_frame_sink = std::move(presentation_connection);
+  session->supports_viewport_scaling = true;
 
   std::move(deferred_callback)
       .Run(std::move(session), std::move(session_controller));

device/vr/android/arcore/arcore_gl.cc

@@ -62,6 +62,22 @@ const char kInputSourceProfileName[] = "generic-touchscreen";
 const gfx::Size kDefaultFrameSize = {1, 1};
 const display::Display::Rotation kDefaultRotation = display::Display::ROTATE_0;
 
+gfx::Transform GetContentTransform(const gfx::RectF& bounds) {
+  // Calculate the transform matrix from quad coordinates (range 0..1 with
+  // origin at bottom left of the quad) to texture lookup UV coordinates (also
+  // range 0..1 with origin at bottom left), where the active viewport uses a
+  // subset of the texture range that needs to be magnified to fill the quad.
+  // The bounds as used by the UpdateLayerBounds mojo messages appear to use an
+  // old WebVR convention with origin at top left, so the Y range needs to be
+  // mirrored.
+  gfx::Transform transform;
+  transform.matrix().set(0, 0, bounds.width());
+  transform.matrix().set(1, 1, bounds.height());
+  transform.matrix().set(0, 3, bounds.x());
+  transform.matrix().set(1, 3, 1.f - bounds.y() - bounds.height());
+  return transform;
+}
+
 }  // namespace
 
 namespace device {
@@ -400,6 +416,7 @@ void ArCoreGl::GetFrameData(
 
   vr::WebXrFrame* xrframe = webxr_->GetAnimatingFrame();
   xrframe->time_pose = now;
+  xrframe->bounds_left = viewport_bounds_;
 
   if (display_info_changed_) {
     frame_data->left_eye = display_info_->left_eye.Clone();
@@ -573,7 +590,13 @@ void ArCoreGl::ProcessFrameFromMailbox(int16_t frame_index,
   DCHECK(webxr_->HaveProcessingFrame());
   DCHECK(!ar_image_transport_->UseSharedBuffer());
 
-  ar_image_transport_->CopyMailboxToSurfaceAndSwap(transfer_size_, mailbox);
+  // Use only the active bounds of the viewport, converting the
+  // bounds UV boundaries to a transform. See also OnWebXrTokenSignaled().
+  gfx::Transform transform =
+      GetContentTransform(webxr_->GetProcessingFrame()->bounds_left);
+  ar_image_transport_->CopyMailboxToSurfaceAndSwap(transfer_size_, mailbox,
+                                                   transform);
+
   // Notify the client that we're done with the mailbox so that the underlying
   // image is eligible for destruction.
   submit_client_->OnSubmitFrameTransferred(true);
@@ -661,9 +684,13 @@ void ArCoreGl::OnWebXrTokenSignaled(int16_t frame_index,
   webxr_->GetProcessingFrame()->time_copied = base::TimeTicks::Now();
   webxr_->TransitionFrameProcessingToRendering();
 
+  // Use only the active bounds of the viewport, converting the
+  // bounds UV boundaries to a transform. See also ProcessFrameFromMailbox().
+  gfx::Transform transform =
+      GetContentTransform(webxr_->GetRenderingFrame()->bounds_left);
   glBindFramebufferEXT(GL_DRAW_FRAMEBUFFER, 0);
   ar_image_transport_->CopyDrawnImageToFramebuffer(
-      webxr_.get(), camera_image_size_, shared_buffer_transform_);
+      webxr_.get(), camera_image_size_, transform);
 
   FinishFrame(frame_index);
 
@@ -684,8 +711,22 @@ void ArCoreGl::UpdateLayerBounds(int16_t frame_index,
                                  const gfx::RectF& left_bounds,
                                  const gfx::RectF& right_bounds,
                                  const gfx::Size& source_size) {
-  DVLOG(2) << __func__ << " source_size=" << source_size.ToString();
+  DVLOG(2) << __func__ << " source_size=" << source_size.ToString()
+           << " left_bounds=" << left_bounds.ToString();
+
+  // The first UpdateLayerBounds may arrive early, when there's
+  // no animating frame yet. In that case, just save it in viewport_bounds_
+  // so that it's applied to the next animating frame.
+  if (webxr_->HaveAnimatingFrame()) {
+    // Handheld AR mode is monoscopic and only uses the left bounds.
+    webxr_->GetAnimatingFrame()->bounds_left = left_bounds;
+    (void)right_bounds;
+  }
+  viewport_bounds_ = left_bounds;
 
+  // Early setting of transfer_size_ is OK since that's only used by the
+  // animating frame. Processing/rendering frames use the bounds from
+  // WebXRPresentationState.
   transfer_size_ = source_size;
 }
 

device/vr/android/arcore/arcore_gl.h

@@ -225,6 +225,11 @@ class ArCoreGl : public mojom::XRFrameDataProvider,
   // smaller than the camera image if framebufferScaleFactor is < 1.0.
   gfx::Size transfer_size_ = gfx::Size(0, 0);
 
+  // Viewport size to use for new animating frames. Currently in-flight
+  // processing/rendering frames continue using the viewport size stored
+  // in their WebXrFrame state.
+  gfx::RectF viewport_bounds_ = gfx::RectF(0.f, 0.f, 1.f, 1.f);
+
   // The camera image size stays locked to the screen size even if
   // framebufferScaleFactor changes.
   gfx::Size camera_image_size_ = gfx::Size(0, 0);
@@ -235,10 +240,6 @@ class ArCoreGl : public mojom::XRFrameDataProvider,
   // and can include 90 degree rotations or other nontrivial transforms.
   gfx::Transform uv_transform_;
 
-  // UV transform for drawing received WebGL content from a shared buffer's
-  // texture, this is simply an identity.
-  gfx::Transform shared_buffer_transform_;
-
   gfx::Transform projection_;
   gfx::Transform inverse_projection_;
   // The first run of ProduceFrame should set uv_transform_ and projection_

device/vr/android/mailbox_to_surface_bridge.h

@@ -8,6 +8,7 @@
 namespace gfx {
 class ColorSpace;
 class GpuFence;
+class Transform;
 }  // namespace gfx
 
 namespace gl {
@@ -54,6 +55,9 @@ class MailboxToSurfaceBridge {
   // won't get a new frame on the SurfaceTexture.
   virtual bool CopyMailboxToSurfaceAndSwap(
       const gpu::MailboxHolder& mailbox) = 0;
+  virtual bool CopyMailboxToSurfaceAndSwap(
+      const gpu::MailboxHolder& mailbox,
+      const gfx::Transform& uv_transform) = 0;
 
   virtual void GenSyncToken(gpu::SyncToken* out_sync_token) = 0;
 

device/vr/android/web_xr_presentation_state.h

@@ -13,6 +13,7 @@
 #include "base/macros.h"
 #include "base/memory/ref_counted.h"
 #include "gpu/command_buffer/common/mailbox_holder.h"
+#include "ui/gfx/geometry/rect_f.h"
 #include "ui/gfx/geometry/size.h"
 #include "ui/gfx/transform.h"
 
@@ -127,6 +128,16 @@ struct WebXrFrame {
 
   std::unique_ptr<WebXrSharedBuffer> camera_image_shared_buffer;
 
+  // Viewport bounds used for rendering, in texture coordinates with uv=(0, 1)
+  // corresponding to viewport pixel (0, 0) as set by UpdateLayerBounds.
+  //
+  // Currently this is only used by the ARCore handheld AR mode which is
+  // monoscopic and uses the left viewport. TODO(https://crbug.com/1134203): The
+  // GVR device currently has its own separate bounds tracking implementation.
+  // That should be updated to use this implementation, at that time a matching
+  // bounds_right would need to be added.
+  gfx::RectF bounds_left;
+
   DISALLOW_COPY_AND_ASSIGN(WebXrFrame);
 };