Add hit-test API to blink for AR

This change adds the capability to request a hit-test
on an XRSession. The session queries the magic window
provider to get any hits based on the given ray and
then resolves a promise with an array of hit results.

This test is added by https://crrev.com/c/1040880, which
will land immediately after this one.

Bug: 833633
Test: third_party/WebKit/LayoutTests/xr/ar_hittest.html
Change-Id: I225297bcf052765f156790a588eafb817ed75e08
Reviewed-on: https://chromium-review.googlesource.com/1018524
Commit-Queue: Max Rebuschatis <lincolnfrog@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Brandon Jones <bajones@chromium.org>
Reviewed-by: David Dorwin <ddorwin@chromium.org>
Reviewed-by: Ian Vollick <vollick@chromium.org>
Reviewed-by: Bill Orr <billorr@chromium.org>
Cr-Commit-Position: refs/heads/master@{#562553}

chrome/browser/android/vr/arcore_device/arcore.h

@@ -31,7 +31,13 @@ class ARCore {
   virtual std::vector<float> TransformDisplayUvCoords(
       const base::span<const float> uvs) = 0;
   virtual gfx::Transform GetProjectionMatrix(float near, float far) = 0;
+
   virtual mojom::VRPosePtr Update() = 0;
+
+  virtual bool RequestHitTest(
+      const mojom::XRRayPtr& ray,
+      const gfx::Size& image_size,
+      std::vector<mojom::XRHitResultPtr>* hit_results) = 0;
 };
 
 }  // namespace device

chrome/browser/android/vr/arcore_device/arcore_device.cc

@@ -85,7 +85,7 @@ void ARCoreDevice::OnMailboxBridgeReady() {
   // TODO(https://crbug.com/836553): use same GL thread as GVR.
   arcore_gl_thread_ = std::make_unique<ARCoreGlThread>(
       std::move(mailbox_bridge_),
-      CreateMainThreadCallback<bool>(base::BindOnce(
+      CreateMainThreadCallback(base::BindOnce(
           &ARCoreDevice::OnARCoreGlThreadInitialized, GetWeakPtr())));
   arcore_gl_thread_->Start();
 }
@@ -132,8 +132,17 @@ void ARCoreDevice::OnMagicWindowFrameDataRequest(
   PostTaskToGlThread(base::BindOnce(
       &ARCoreGl::ProduceFrame, arcore_gl_thread_->GetARCoreGl()->GetWeakPtr(),
       frame_size, display_rotation,
-      CreateMainThreadCallback<mojom::VRMagicWindowFrameDataPtr>(
-          std::move(callback))));
+      CreateMainThreadCallback(std::move(callback))));
+}
+
+void ARCoreDevice::RequestHitTest(
+    mojom::XRRayPtr ray,
+    mojom::VRMagicWindowProvider::RequestHitTestCallback callback) {
+  DCHECK(IsOnMainThread());
+
+  PostTaskToGlThread(base::BindOnce(
+      &ARCoreGl::RequestHitTest, arcore_gl_thread_->GetARCoreGl()->GetWeakPtr(),
+      std::move(ray), CreateMainThreadCallback(std::move(callback))));
 }
 
 bool ARCoreDevice::IsOnMainThread() {

chrome/browser/android/vr/arcore_device/arcore_device.h

@@ -39,9 +39,9 @@ class ARCoreDevice : public VRDeviceBase {
       const gfx::Size& frame_size,
       display::Display::Rotation frame_rotation,
       mojom::VRMagicWindowProvider::GetFrameDataCallback callback) override;
-  void OnMagicWindowFrameDataRequestComplete(
-      mojom::VRMagicWindowProvider::GetFrameDataCallback callback,
-      mojom::VRMagicWindowFrameDataPtr frame_data);
+  void RequestHitTest(
+      mojom::XRRayPtr ray,
+      mojom::VRMagicWindowProvider::RequestHitTestCallback callback) override;
   void OnMailboxBridgeReady();
   void OnARCoreGlThreadInitialized(bool success);
 

chrome/browser/android/vr/arcore_device/arcore_gl.cc

@@ -34,8 +34,6 @@
 #include "ui/gl/gl_surface.h"
 #include "ui/gl/init/gl_factory.h"
 
-namespace device {
-
 namespace {
 // Input display coordinates (range 0..1) used with ARCore's
 // transformDisplayUvCoords to calculate the output matrix.
@@ -79,6 +77,18 @@ gfx::Transform ConvertUvsToTransformMatrix(const std::vector<float>& uvs) {
 
 }  // namespace
 
+namespace device {
+
+struct ARCoreHitTestRequest {
+  ARCoreHitTestRequest() = default;
+  ~ARCoreHitTestRequest() = default;
+  mojom::XRRayPtr ray;
+  mojom::VRMagicWindowProvider::RequestHitTestCallback callback;
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(ARCoreHitTestRequest);
+};
+
 ARCoreGl::ARCoreGl(std::unique_ptr<vr::MailboxToSurfaceBridge> mailbox_bridge)
     : gl_thread_task_runner_(base::ThreadTaskRunnerHandle::Get()),
       arcore_(std::make_unique<ARCoreImpl>()),
@@ -195,6 +205,62 @@ void ARCoreGl::ProduceFrame(
   fps_meter_.AddFrame(base::TimeTicks::Now());
   TRACE_COUNTER1("gpu", "WebXR FPS", fps_meter_.GetFPS());
 
+  // Post a task to finish processing the frame so any calls to
+  // RequestHitTest() that were made during this function, which can block
+  // on the arcore_->Update() call above, can be processed in this frame.
+  gl_thread_task_runner_->PostTask(
+      FROM_HERE,
+      base::BindOnce(&ARCoreGl::ProcessFrame, weak_ptr_factory_.GetWeakPtr(),
+                     base::Passed(&frame_data), frame_size,
+                     base::Passed(&callback)));
+}
+
+void ARCoreGl::RequestHitTest(
+    mojom::XRRayPtr ray,
+    mojom::VRMagicWindowProvider::RequestHitTestCallback callback) {
+  DCHECK(IsOnGlThread());
+  DCHECK(is_initialized_);
+
+  std::unique_ptr<ARCoreHitTestRequest> request =
+      std::make_unique<ARCoreHitTestRequest>();
+  request->ray = std::move(ray);
+  request->callback = std::move(callback);
+  hit_test_requests_.push_back(std::move(request));
+}
+
+void ARCoreGl::ProcessFrame(
+    mojom::VRMagicWindowFrameDataPtr frame_data,
+    const gfx::Size& frame_size,
+    mojom::VRMagicWindowProvider::GetFrameDataCallback callback) {
+  DCHECK(IsOnGlThread());
+  DCHECK(is_initialized_);
+
+  // The timing requirements for hit-test are documented here:
+  // https://github.com/immersive-web/hit-test/blob/master/explainer.md#timing
+  // The current implementation of frame generation on the renderer side is
+  // 1:1 with calls to this method, so it is safe to fire off the hit-test
+  // results here, one at a time, in the order they were enqueued prior to
+  // running the GetFrameDataCallback.
+  // Since mojo callbacks are processed in order, this will result in the
+  // correct sequence of hit-test callbacks / promise resolutions. If
+  // the implementation of the renderer processing were to change, this
+  // code is fragile and could break depending on the new implementation.
+  // TODO(https://crbug.com/844174): In order to be more correct by design,
+  // hit results should be bundled with the frame data - that way it would be
+  // obvious how the timing between the results and the frame should go.
+  for (auto& request : hit_test_requests_) {
+    std::vector<mojom::XRHitResultPtr> results;
+    if (!arcore_->RequestHitTest(request->ray, frame_size, &results)) {
+      std::move(request->callback).Run(base::nullopt);
+    }
+    std::move(request->callback).Run(std::move(results));
+  }
+  hit_test_requests_.clear();
+
+  // Running this callback after resolving all the hit-test requests ensures
+  // that we satisfy the guarantee of the WebXR hit-test spec - that the
+  // hit-test promise resolves immediately prior to the frame for which it is
+  // valid.
   std::move(callback).Run(std::move(frame_data));
 }
 

chrome/browser/android/vr/arcore_device/arcore_gl.h

@@ -36,6 +36,7 @@ class MailboxToSurfaceBridge;
 namespace device {
 
 class ARCore;
+struct ARCoreHitTestRequest;
 class ARImageTransport;
 
 // All of this class's methods must be called on the same valid GL thread with
@@ -55,9 +56,18 @@ class ARCoreGl {
     return gl_thread_task_runner_;
   }
 
+  void RequestHitTest(mojom::XRRayPtr,
+                      mojom::VRMagicWindowProvider::RequestHitTestCallback);
+
   base::WeakPtr<ARCoreGl> GetWeakPtr();
 
  private:
+  // TODO(https://crbug/835948): remove frame_size.
+  void ProcessFrame(
+      mojom::VRMagicWindowFrameDataPtr frame_data,
+      const gfx::Size& frame_size,
+      mojom::VRMagicWindowProvider::GetFrameDataCallback callback);
+
   bool IsOnGlThread() const;
 
   scoped_refptr<gl::GLSurface> surface_;
@@ -71,6 +81,9 @@ class ARCoreGl {
   bool is_initialized_ = false;
 
   vr::FPSMeter fps_meter_;
+
+  std::vector<std::unique_ptr<ARCoreHitTestRequest>> hit_test_requests_;
+
   // Must be last.
   base::WeakPtrFactory<ARCoreGl> weak_ptr_factory_;
   DISALLOW_COPY_AND_ASSIGN(ARCoreGl);

chrome/browser/android/vr/arcore_device/arcore_impl.cc

@@ -11,7 +11,11 @@
 #include "base/trace_event/trace_event.h"
 #include "chrome/browser/android/vr/arcore_device/arcore_java_utils.h"
 #include "device/vr/public/mojom/vr_service.mojom.h"
+#include "third_party/skia/include/core/SkMatrix44.h"
 #include "ui/display/display.h"
+#include "ui/gfx/geometry/point3_f.h"
+#include "ui/gfx/geometry/point_f.h"
+#include "ui/gfx/transform.h"
 
 using base::android::JavaRef;
 
@@ -198,6 +202,130 @@ gfx::Transform ARCoreImpl::GetProjectionMatrix(float near, float far) {
   return result;
 }
 
+// TODO(835948): remove image-size
+bool ARCoreImpl::RequestHitTest(
+    const mojom::XRRayPtr& ray,
+    const gfx::Size& image_size,
+    std::vector<mojom::XRHitResultPtr>* hit_results) {
+  DCHECK(IsOnGlThread());
+  DCHECK(arcore_session_.is_valid());
+  DCHECK(arcore_frame_.is_valid());
+
+  internal::ScopedArCoreObject<ArHitResultList*> arcore_hit_result_list;
+  ArHitResultList_create(arcore_session_.get(),
+                         arcore_hit_result_list.receive());
+  if (!arcore_hit_result_list.is_valid()) {
+    DLOG(ERROR) << "ArHitResultList_create failed!";
+    return false;
+  }
+
+  gfx::PointF screen_point;
+  if (!TransformRayToScreenSpace(ray, image_size, &screen_point)) {
+    return false;
+  }
+
+  // ARCore returns hit-results in sorted order, thus providing the guarantee
+  // of sorted results promised by the WebXR spec for requestHitTest().
+  ArFrame_hitTest(arcore_session_.get(), arcore_frame_.get(),
+                  screen_point.x() * image_size.width(),
+                  screen_point.y() * image_size.height(),
+                  arcore_hit_result_list.get());
+
+  int arcore_hit_result_list_size = 0;
+  ArHitResultList_getSize(arcore_session_.get(), arcore_hit_result_list.get(),
+                          &arcore_hit_result_list_size);
+
+  for (int i = 0; i < arcore_hit_result_list_size; i++) {
+    internal::ScopedArCoreObject<ArHitResult*> arcore_hit;
+    ArHitResult_create(arcore_session_.get(), arcore_hit.receive());
+    if (!arcore_hit.is_valid()) {
+      DLOG(ERROR) << "ArHitResult_create failed!";
+      return false;
+    }
+
+    ArHitResultList_getItem(arcore_session_.get(), arcore_hit_result_list.get(),
+                            i, arcore_hit.get());
+
+    mojom::XRHitResultPtr mojo_hit = mojom::XRHitResult::New();
+    if (!ArHitResultToXRHitResult(arcore_hit.get(), mojo_hit.get())) {
+      return false;
+    }
+    hit_results->push_back(std::move(mojo_hit));
+  }
+  return true;
+}
+
+// TODO(835948): remove this method.
+bool ARCoreImpl::TransformRayToScreenSpace(const mojom::XRRayPtr& ray,
+                                           const gfx::Size& image_size,
+                                           gfx::PointF* screen_point) {
+  DCHECK(IsOnGlThread());
+  DCHECK(arcore_session_.is_valid());
+  DCHECK(arcore_frame_.is_valid());
+
+  internal::ScopedArCoreObject<ArCamera*> arcore_camera;
+  ArFrame_acquireCamera(arcore_session_.get(), arcore_frame_.get(),
+                        arcore_camera.receive());
+  DCHECK(arcore_camera.is_valid())
+      << "ArFrame_acquireCamera failed despite documentation saying it cannot";
+
+  // Get the projection matrix.
+  float projection_matrix[16];
+  ArCamera_getProjectionMatrix(arcore_session_.get(), arcore_camera.get(), 0.1,
+                               1000, projection_matrix);
+  SkMatrix44 projection44;
+  projection44.setColMajorf(projection_matrix);
+  gfx::Transform projection_transform(projection44);
+
+  // Get the view matrix.
+  float view_matrix[16];
+  ArCamera_getViewMatrix(arcore_session_.get(), arcore_camera.get(),
+                         view_matrix);
+  SkMatrix44 view44;
+  view44.setColMajorf(view_matrix);
+  gfx::Transform view_transform(view44);
+
+  // Create the combined matrix.
+  gfx::Transform proj_view_transform = projection_transform * view_transform;
+
+  // Transform the ray into screen space.
+  gfx::Point3F screen_point_3d{ray->origin[0] + ray->direction[0],
+                               ray->origin[1] + ray->direction[1],
+                               ray->origin[2] + ray->direction[2]};
+  proj_view_transform.TransformPoint(&screen_point_3d);
+  if (screen_point_3d.x() < -1 || screen_point_3d.x() > 1 ||
+      screen_point_3d.y() < -1 || screen_point_3d.y() > 1) {
+    // The point does not project back into screen space, so this won't
+    // work with the screen-space-based hit-test API.
+    DLOG(ERROR) << "Invalid ray - does not originate from device screen.";
+    return false;
+  }
+
+  screen_point->set_x((screen_point_3d.x() + 1) / 2);
+  screen_point->set_y((screen_point_3d.y() + 1) / 2);
+  return true;
+}
+
+bool ARCoreImpl::ArHitResultToXRHitResult(ArHitResult* arcore_hit,
+                                          mojom::XRHitResult* hit_result) {
+  DCHECK(IsOnGlThread());
+  DCHECK(arcore_session_.is_valid());
+  DCHECK(arcore_frame_.is_valid());
+
+  internal::ScopedArCoreObject<ArPose*> arcore_pose;
+  ArPose_create(arcore_session_.get(), nullptr, arcore_pose.receive());
+  if (!arcore_pose.is_valid()) {
+    DLOG(ERROR) << "ArPose_create failed!";
+    return false;
+  }
+  ArHitResult_getHitPose(arcore_session_.get(), arcore_hit, arcore_pose.get());
+  hit_result->hit_matrix.resize(16);
+  ArPose_getMatrix(arcore_session_.get(), arcore_pose.get(),
+                   hit_result->hit_matrix.data());
+
+  return true;
+}
+
 bool ARCoreImpl::IsOnGlThread() {
   return gl_thread_task_runner_->BelongsToCurrentThread();
 }

chrome/browser/android/vr/arcore_device/arcore_impl.h

@@ -47,6 +47,18 @@ void inline ScopedGenericArObject<ArCamera*>::Free(ArCamera* ar_camera) {
   // Do nothing - ArCamera has no destroy method and is managed by ARCore.
 }
 
+template <>
+void inline ScopedGenericArObject<ArHitResultList*>::Free(
+    ArHitResultList* ar_hit_result_list) {
+  ArHitResultList_destroy(ar_hit_result_list);
+}
+
+template <>
+void inline ScopedGenericArObject<ArHitResult*>::Free(
+    ArHitResult* ar_hit_result) {
+  ArHitResult_destroy(ar_hit_result);
+}
+
 template <class T>
 using ScopedArCoreObject = base::ScopedGeneric<T, ScopedGenericArObject<T>>;
 
@@ -68,7 +80,18 @@ class ARCoreImpl : public ARCore {
   gfx::Transform GetProjectionMatrix(float near, float far) override;
   mojom::VRPosePtr Update() override;
 
+  bool RequestHitTest(const mojom::XRRayPtr& ray,
+                      const gfx::Size& image_size,
+                      std::vector<mojom::XRHitResultPtr>* hit_results) override;
+
  private:
+  bool TransformRayToScreenSpace(const mojom::XRRayPtr& ray,
+                                 const gfx::Size& image_size,
+                                 gfx::PointF* screen_point);
+
+  bool ArHitResultToXRHitResult(ArHitResult* ar_hit,
+                                mojom::XRHitResult* hit_result);
+
   bool IsOnGlThread();
   base::WeakPtr<ARCoreImpl> GetWeakPtr() {
     return weak_ptr_factory_.GetWeakPtr();

chrome/browser/android/vr/arcore_device/arcore_shim.cc

@@ -15,14 +15,24 @@ namespace {
   CALL(ArCamera_getDisplayOrientedPose)  \
   CALL(ArCamera_getProjectionMatrix)     \
   CALL(ArCamera_getTrackingState)        \
+  CALL(ArCamera_getViewMatrix)           \
   CALL(ArConfig_create)                  \
   CALL(ArConfig_destroy)                 \
   CALL(ArFrame_acquireCamera)            \
   CALL(ArFrame_create)                   \
   CALL(ArFrame_destroy)                  \
+  CALL(ArFrame_hitTest)                  \
   CALL(ArFrame_transformDisplayUvCoords) \
+  CALL(ArHitResult_create)               \
+  CALL(ArHitResult_destroy)              \
+  CALL(ArHitResult_getHitPose)           \
+  CALL(ArHitResultList_create)           \
+  CALL(ArHitResultList_destroy)          \
+  CALL(ArHitResultList_getItem)          \
+  CALL(ArHitResultList_getSize)          \
   CALL(ArPose_create)                    \
   CALL(ArPose_destroy)                   \
+  CALL(ArPose_getMatrix)                 \
   CALL(ArPose_getPoseRaw)                \
   CALL(ArSession_checkSupported)         \
   CALL(ArSession_configure)              \
@@ -101,6 +111,12 @@ void ArCamera_getTrackingState(const ArSession* session,
                                              out_tracking_state);
 }
 
+void ArCamera_getViewMatrix(const ArSession* session,
+                            const ArCamera* camera,
+                            float* out_matrix) {
+  arcore_api->impl_ArCamera_getViewMatrix(session, camera, out_matrix);
+}
+
 void ArConfig_create(const ArSession* session, ArConfig** out_config) {
   arcore_api->impl_ArConfig_create(session, out_config);
 }
@@ -123,6 +139,15 @@ void ArFrame_destroy(ArFrame* frame) {
   arcore_api->impl_ArFrame_destroy(frame);
 }
 
+void ArFrame_hitTest(const ArSession* session,
+                     const ArFrame* frame,
+                     float pixel_x,
+                     float pixel_y,
+                     ArHitResultList* out_hit_results) {
+  arcore_api->impl_ArFrame_hitTest(session, frame, pixel_x, pixel_y,
+                                   out_hit_results);
+}
+
 void ArFrame_transformDisplayUvCoords(const ArSession* session,
                                       const ArFrame* frame,
                                       int32_t num_elements,
@@ -132,6 +157,44 @@ void ArFrame_transformDisplayUvCoords(const ArSession* session,
       session, frame, num_elements, uvs_in, uvs_out);
 }
 
+void ArHitResult_create(const ArSession* session,
+                        ArHitResult** out_hit_result) {
+  arcore_api->impl_ArHitResult_create(session, out_hit_result);
+}
+
+void ArHitResult_destroy(ArHitResult* hit_result) {
+  arcore_api->impl_ArHitResult_destroy(hit_result);
+}
+
+void ArHitResult_getHitPose(const ArSession* session,
+                            const ArHitResult* hit_result,
+                            ArPose* out_pose) {
+  arcore_api->impl_ArHitResult_getHitPose(session, hit_result, out_pose);
+}
+
+void ArHitResultList_create(const ArSession* session,
+                            ArHitResultList** out_hit_result_list) {
+  arcore_api->impl_ArHitResultList_create(session, out_hit_result_list);
+}
+
+void ArHitResultList_destroy(ArHitResultList* hit_result_list) {
+  arcore_api->impl_ArHitResultList_destroy(hit_result_list);
+}
+
+void ArHitResultList_getItem(const ArSession* session,
+                             const ArHitResultList* hit_result_list,
+                             int index,
+                             ArHitResult* out_hit_result) {
+  arcore_api->impl_ArHitResultList_getItem(session, hit_result_list, index,
+                                           out_hit_result);
+}
+
+void ArHitResultList_getSize(const ArSession* session,
+                             const ArHitResultList* hit_result_list,
+                             int* out_size) {
+  arcore_api->impl_ArHitResultList_getSize(session, hit_result_list, out_size);
+}
+
 void ArPose_create(const ArSession* session,
                    const float* pose_raw,
                    ArPose** out_pose) {
@@ -142,6 +205,12 @@ void ArPose_destroy(ArPose* pose) {
   arcore_api->impl_ArPose_destroy(pose);
 }
 
+void ArPose_getMatrix(const ArSession* session,
+                      const ArPose* pose,
+                      float* out_matrix) {
+  arcore_api->impl_ArPose_getMatrix(session, pose, out_matrix);
+}
+
 void ArPose_getPoseRaw(const ArSession* session,
                        const ArPose* pose,
                        float* out_pose_raw) {

chrome/browser/android/vr/arcore_device/fake_arcore.cc

@@ -263,4 +263,12 @@ mojom::VRPosePtr FakeARCore::Update() {
   return pose;
 }
 
+bool FakeARCore::RequestHitTest(
+    const mojom::XRRayPtr& ray,
+    const gfx::Size& image_size,
+    std::vector<mojom::XRHitResultPtr>* hit_results) {
+  // TODO(https://crbug.com/837834): implement for testing.
+  return false;
+}
+
 }  // namespace device

chrome/browser/android/vr/arcore_device/fake_arcore.h

@@ -35,6 +35,10 @@ class FakeARCore : public ARCore {
   gfx::Transform GetProjectionMatrix(float near, float far) override;
   mojom::VRPosePtr Update() override;
 
+  bool RequestHitTest(const mojom::XRRayPtr& ray,
+                      const gfx::Size& image_size,
+                      std::vector<mojom::XRHitResultPtr>* hit_results) override;
+
   void SetCameraAspect(float aspect) { camera_aspect_ = aspect; }
 
  private:

device/vr/public/mojom/vr_service.mojom

@@ -89,6 +89,20 @@ struct VRPose {
   bool pose_reset;
 };
 
+struct XRRay {
+  // TODO(https://crbug.com/845293): use Point3F and Vector3F from
+  // ui/gfx/geometry and inline directly in requestHitTest().
+  array<float, 3> origin;
+  array<float, 3> direction;
+};
+
+struct XRHitResult {
+  // A 4x4 transformation matrix representing the position of the object hit
+  // and the orientation of the normal of the object at the hit location.
+  // TODO(https://crbug.com/845293): use gfx.mojom.Transform.
+  array<float, 16> hit_matrix;
+};
+
 struct VRDisplayCapabilities {
   bool hasPosition;
   bool hasExternalDisplay;
@@ -290,6 +304,17 @@ interface VRMagicWindowProvider {
   GetFrameData(gfx.mojom.Size frame_size,
       display.mojom.Rotation display_rotation) =>
       (VRMagicWindowFrameData? frame_data);
+
+  // Performs a raycast into the magic window scene and returns a list of
+  // XRHitResults sorted from closest to furthest hit from the ray. Each
+  // hit result contains a hit_matrix containing the transform of the hit
+  // where the rotation represents the normal of the surface hit.
+  // An empty result list means there were no hits. If a nullopt is returned,
+  // there was an error.
+  // TODO(https://crbug.com/842025): have one "session" type, merging
+  // VRMagicWindowProvider and VRPresentationProvider because RequestHitTest
+  // makes sense for both types of sessions.
+  RequestHitTest(XRRay ray) => (array<XRHitResult>? results);
 };
 
 // Provides the necessary functionality for a presenting WebVR page to draw

device/vr/vr_device_base.cc

@@ -147,6 +147,13 @@ void VRDeviceBase::SetListeningForActivate(bool is_listening) {
   OnListeningForActivate(is_listening);
 }
 
+void VRDeviceBase::RequestHitTest(
+    mojom::XRRayPtr ray,
+    mojom::VRMagicWindowProvider::RequestHitTestCallback callback) {
+  NOTREACHED() << "Unexpected call to a device without hit-test support";
+  std::move(callback).Run(base::nullopt);
+}
+
 void VRDeviceBase::UpdateListeningForActivate(VRDisplayImpl* display) {
   if (display->ListeningForActivate() && display->InFocusedFrame()) {
     bool was_listening = !!listening_for_activate_diplay_;

device/vr/vr_device_base.h

@@ -53,6 +53,9 @@ class DEVICE_VR_EXPORT VRDeviceBase : public VRDevice {
       const gfx::Size& frame_size,
       display::Display::Rotation display_rotation,
       mojom::VRMagicWindowProvider::GetFrameDataCallback callback);
+  virtual void RequestHitTest(
+      mojom::XRRayPtr ray,
+      mojom::VRMagicWindowProvider::RequestHitTestCallback callback);
 
  protected:
   void SetIsPresenting();

device/vr/vr_display_impl.cc

@@ -33,9 +33,13 @@ VRDisplayImpl::VRDisplayImpl(VRDevice* device,
 
 VRDisplayImpl::~VRDisplayImpl() = default;
 
+bool VRDisplayImpl::IsPrivilegedOperationAllowed() {
+  return device_->IsAccessAllowed(this) && InFocusedFrame();
+}
+
 void VRDisplayImpl::RequestSession(
     mojom::VRDisplayHost::RequestSessionCallback callback) {
-  if (!CanStartNewSession()) {
+  if (!IsPrivilegedOperationAllowed()) {
     std::move(callback).Run(false);
     return;
   }
@@ -74,8 +78,13 @@ void VRDisplayImpl::GetFrameData(const gfx::Size& frame_size,
   device_->GetMagicWindowFrameData(frame_size, rotation, std::move(callback));
 }
 
-bool VRDisplayImpl::CanStartNewSession() {
-  return device_->IsAccessAllowed(this) && InFocusedFrame();
+void VRDisplayImpl::RequestHitTest(mojom::XRRayPtr ray,
+                                   RequestHitTestCallback callback) {
+  if (!IsPrivilegedOperationAllowed()) {
+    std::move(callback).Run(base::nullopt);
+    return;
+  }
+  device_->RequestHitTest(std::move(ray), std::move(callback));
 }
 
 void VRDisplayImpl::SetListeningForActivate(bool listening) {

device/vr/vr_display_impl.h

@@ -43,13 +43,15 @@ class DEVICE_VR_EXPORT VRDisplayImpl : public mojom::VRMagicWindowProvider {
   void RequestSession(mojom::VRDisplayHost::RequestSessionCallback callback);
 
  private:
+  bool IsPrivilegedOperationAllowed();
+
   // mojom::VRMagicWindowProvider
   void GetPose(GetPoseCallback callback) override;
   void GetFrameData(const gfx::Size& frame_size,
                     display::Display::Rotation rotation,
                     GetFrameDataCallback callback) override;
-
-  bool CanStartNewSession();
+  void RequestHitTest(mojom::XRRayPtr ray,
+                      RequestHitTestCallback callback) override;
 
   mojo::Binding<mojom::VRMagicWindowProvider> binding_;
   device::VRDeviceBase* device_;

third_party/blink/renderer/modules/modules_idl_files.gni

@@ -425,6 +425,7 @@ modules_idl_files =
           "xr/xr_device.idl",
           "xr/xr_device_pose.idl",
           "xr/xr_frame_of_reference.idl",
+          "xr/xr_hit_result.idl",
           "xr/xr_input_pose.idl",
           "xr/xr_input_source.idl",
           "xr/xr_input_source_event.idl",

third_party/blink/renderer/modules/xr/BUILD.gn

@@ -22,6 +22,8 @@ blink_modules_sources("xr") {
     "xr_frame_provider.h",
     "xr_frame_request_callback_collection.cc",
     "xr_frame_request_callback_collection.h",
+    "xr_hit_result.cc",
+    "xr_hit_result.h",
     "xr_input_pose.cc",
     "xr_input_pose.h",
     "xr_input_source.cc",

third_party/blink/renderer/modules/xr/xr_hit_result.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "third_party/blink/renderer/modules/xr/xr_hit_result.h"
+
+#include "third_party/blink/renderer/modules/xr/xr_utils.h"
+#include "third_party/blink/renderer/platform/transforms/transformation_matrix.h"
+
+namespace blink {
+XRHitResult::XRHitResult(std::unique_ptr<TransformationMatrix> hit_transform)
+    : hit_transform_(std::move(hit_transform)) {}
+
+XRHitResult::~XRHitResult() {}
+
+DOMFloat32Array* XRHitResult::hitMatrix() const {
+  if (!hit_transform_)
+    return nullptr;
+  // TODO(https://crbug.com/845296): rename
+  // transformationMatrixToFloat32Array() to
+  // TransformationMatrixToDOMFloat32Array().
+  return transformationMatrixToFloat32Array(*hit_transform_);
+}
+
+}  // namespace blink

third_party/blink/renderer/modules/xr/xr_hit_result.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_XR_XR_HIT_RESULT_H_
+#define THIRD_PARTY_BLINK_RENDERER_MODULES_XR_XR_HIT_RESULT_H_
+
+#include "third_party/blink/renderer/core/typed_arrays/dom_typed_array.h"
+
+namespace blink {
+
+class TransformationMatrix;
+
+class XRHitResult final : public ScriptWrappable {
+  DEFINE_WRAPPERTYPEINFO();
+
+ public:
+  explicit XRHitResult(std::unique_ptr<TransformationMatrix>);
+  ~XRHitResult() override;
+
+  DOMFloat32Array* hitMatrix() const;
+
+ private:
+  const std::unique_ptr<TransformationMatrix> hit_transform_;
+};
+
+}  // namespace blink
+
+#endif  // THIRD_PARTY_BLINK_RENDERER_MODULES_XR_XR_HIT_RESULT_H_

third_party/blink/renderer/modules/xr/xr_hit_result.idl

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+[RuntimeEnabled=WebXRHitTest, SecureContext, Exposed=Window] interface XRHitResult {
+  readonly attribute Float32Array hitMatrix;
+};
\ No newline at end of file

third_party/blink/renderer/modules/xr/xr_session.cc

@@ -22,12 +22,14 @@
 #include "third_party/blink/renderer/modules/xr/xr_frame_of_reference.h"
 #include "third_party/blink/renderer/modules/xr/xr_frame_of_reference_options.h"
 #include "third_party/blink/renderer/modules/xr/xr_frame_provider.h"
+#include "third_party/blink/renderer/modules/xr/xr_hit_result.h"
 #include "third_party/blink/renderer/modules/xr/xr_input_source_event.h"
 #include "third_party/blink/renderer/modules/xr/xr_layer.h"
 #include "third_party/blink/renderer/modules/xr/xr_presentation_context.h"
 #include "third_party/blink/renderer/modules/xr/xr_presentation_frame.h"
 #include "third_party/blink/renderer/modules/xr/xr_session_event.h"
 #include "third_party/blink/renderer/modules/xr/xr_view.h"
+#include "third_party/blink/renderer/platform/transforms/transformation_matrix.h"
 
 namespace blink {
 
@@ -245,6 +247,79 @@ HeapVector<Member<XRInputSource>> XRSession::getInputSources() const {
   return source_array;
 }
 
+ScriptPromise XRSession::requestHitTest(ScriptState* script_state,
+                                        NotShared<DOMFloat32Array> origin,
+                                        NotShared<DOMFloat32Array> direction,
+                                        XRCoordinateSystem* coordinate_system) {
+  if (ended_) {
+    return ScriptPromise::RejectWithDOMException(
+        script_state, DOMException::Create(kInvalidStateError, kSessionEnded));
+  }
+
+  if (!coordinate_system) {
+    return ScriptPromise::Reject(
+        script_state, V8ThrowException::CreateTypeError(
+                          script_state->GetIsolate(),
+                          "The coordinateSystem parameter is empty."));
+  }
+
+  if (origin.View()->length() != 3 || direction.View()->length() != 3) {
+    return ScriptPromise::RejectWithDOMException(
+        script_state, DOMException::Create(kNotSupportedError, "Invalid ray!"));
+  }
+
+  // TODO(https://crbug.com/846411): use coordinate_system.
+
+  // TODO(https://crbug.com/843376): Reject the promise if device doesn't
+  // support the hit-test API.
+
+  device::mojom::blink::XRRayPtr ray = device::mojom::blink::XRRay::New();
+  ray->origin.resize(3);
+  ray->origin[0] = origin.View()->Data()[0];
+  ray->origin[1] = origin.View()->Data()[1];
+  ray->origin[2] = origin.View()->Data()[2];
+  ray->direction.resize(3);
+  ray->direction[0] = direction.View()->Data()[0];
+  ray->direction[1] = direction.View()->Data()[1];
+  ray->direction[2] = direction.View()->Data()[2];
+
+  ScriptPromiseResolver* resolver = ScriptPromiseResolver::Create(script_state);
+  ScriptPromise promise = resolver->Promise();
+
+  // TODO(https://crbug.com/845520): Promise should be rejected if session
+  // is deleted.
+  device_->xrMagicWindowProviderPtr()->RequestHitTest(
+      std::move(ray),
+      WTF::Bind(&XRSession::OnHitTestResults, WrapWeakPersistent(this),
+                WrapPersistent(resolver)));
+
+  return promise;
+}
+
+void XRSession::OnHitTestResults(
+    ScriptPromiseResolver* resolver,
+    base::Optional<WTF::Vector<device::mojom::blink::XRHitResultPtr>> results) {
+  if (!results) {
+    resolver->Reject();
+    return;
+  }
+
+  HeapVector<Member<XRHitResult>> hit_results;
+  for (const auto& mojom_result : results.value()) {
+    XRHitResult* hit_result = new XRHitResult(TransformationMatrix::Create(
+        mojom_result->hit_matrix[0], mojom_result->hit_matrix[1],
+        mojom_result->hit_matrix[2], mojom_result->hit_matrix[3],
+        mojom_result->hit_matrix[4], mojom_result->hit_matrix[5],
+        mojom_result->hit_matrix[6], mojom_result->hit_matrix[7],
+        mojom_result->hit_matrix[8], mojom_result->hit_matrix[9],
+        mojom_result->hit_matrix[10], mojom_result->hit_matrix[11],
+        mojom_result->hit_matrix[12], mojom_result->hit_matrix[13],
+        mojom_result->hit_matrix[14], mojom_result->hit_matrix[15]));
+    hit_results.push_back(hit_result);
+  }
+  resolver->Resolve(hit_results);
+}
+
 ScriptPromise XRSession::end(ScriptState* script_state) {
   // Don't allow a session to end twice.
   if (ended_) {

third_party/blink/renderer/modules/xr/xr_session.h

@@ -9,6 +9,8 @@
 #include "mojo/public/cpp/bindings/binding.h"
 #include "third_party/blink/renderer/bindings/core/v8/script_promise.h"
 #include "third_party/blink/renderer/core/dom/events/event_target.h"
+#include "third_party/blink/renderer/core/typed_arrays/array_buffer_view_helpers.h"
+#include "third_party/blink/renderer/core/typed_arrays/dom_typed_array.h"
 #include "third_party/blink/renderer/modules/xr/xr_frame_request_callback_collection.h"
 #include "third_party/blink/renderer/modules/xr/xr_input_source.h"
 #include "third_party/blink/renderer/platform/bindings/trace_wrapper_member.h"
@@ -21,8 +23,10 @@ namespace blink {
 
 class Element;
 class ResizeObserver;
+class ScriptPromiseResolver;
 class V8XRFrameRequestCallback;
 class XRCanvasInputProvider;
+class XRCoordinateSystem;
 class XRDevice;
 class XRFrameOfReferenceOptions;
 class XRInputSourceEvent;
@@ -73,6 +77,11 @@ class XRSession final : public EventTargetWithInlineData {
 
   HeapVector<Member<XRInputSource>> getInputSources() const;
 
+  ScriptPromise requestHitTest(ScriptState* script_state,
+                               NotShared<DOMFloat32Array> origin,
+                               NotShared<DOMFloat32Array> direction,
+                               XRCoordinateSystem* coordinate_system);
+
   // Called by JavaScript to manually end the session.
   ScriptPromise end(ScriptState*);
 
@@ -139,6 +148,11 @@ class XRSession final : public EventTargetWithInlineData {
   void OnBlur();
   bool HasAppropriateFocus();
 
+  void OnHitTestResults(
+      ScriptPromiseResolver* resolver,
+      base::Optional<WTF::Vector<device::mojom::blink::XRHitResultPtr>>
+          results);
+
   const Member<XRDevice> device_;
   const bool exclusive_;
   const Member<XRPresentationContext> output_context_;

third_party/blink/renderer/modules/xr/xr_session.idl

@@ -28,5 +28,7 @@
 
   [MeasureAs=XRSessionGetInputSources] FrozenArray<XRInputSource> getInputSources();
 
+  [RuntimeEnabled=WebXRHitTest, CallWith=ScriptState] Promise<FrozenArray<XRHitResult>> requestHitTest(Float32Array origin, Float32Array direction, XRCoordinateSystem coordinateSystem);
+
   [CallWith=ScriptState] Promise<void> end();
 };