Hook up AR hit-test to 3D ARCore hit-test API

Now that we can use ARCore SDK 1.5, we can start using 3D version of
hit test and remove previously used one. Since we no longer use
ArFrame_hitTest, it is getting removed from arcore_shim.cc.

Bug: 835948, 854823
Change-Id: Ibf8e953135b04925dbebd384e16f8f08ad52a869
Reviewed-on: https://chromium-review.googlesource.com/c/1357723Reviewed-by: Klaus Weidner <klausw@chromium.org>
Commit-Queue: Piotr Bialecki <bialpio@chromium.org>
Cr-Commit-Position: refs/heads/master@{#618649}

chrome/browser/android/vr/arcore_device/arcore.h

@@ -45,7 +45,6 @@ class ArCore {
 
   virtual bool RequestHitTest(
       const mojom::XRRayPtr& ray,
-      const gfx::Size& image_size,
       std::vector<mojom::XRHitResultPtr>* hit_results) = 0;
 
   virtual void Pause() = 0;

chrome/browser/android/vr/arcore_device/arcore_gl.cc

@@ -265,8 +265,7 @@ void ArCoreGl::ProduceFrame(
   gl_thread_task_runner_->PostTask(
       FROM_HERE,
       base::BindOnce(&ArCoreGl::ProcessFrame, weak_ptr_factory_.GetWeakPtr(),
-                     base::Passed(&frame_data), frame_size,
-                     base::Passed(&callback)));
+                     base::Passed(&frame_data), base::Passed(&callback)));
 }
 
 void ArCoreGl::RequestHitTest(
@@ -284,7 +283,6 @@ void ArCoreGl::RequestHitTest(
 
 void ArCoreGl::ProcessFrame(
     mojom::XRFrameDataPtr frame_data,
-    const gfx::Size& frame_size,
     mojom::XRFrameDataProvider::GetFrameDataCallback callback) {
   DCHECK(IsOnGlThread());
   DCHECK(is_initialized_);
@@ -304,7 +302,7 @@ void ArCoreGl::ProcessFrame(
   // obvious how the timing between the results and the frame should go.
   for (auto& request : hit_test_requests_) {
     std::vector<mojom::XRHitResultPtr> results;
-    if (arcore_->RequestHitTest(request->ray, frame_size, &results)) {
+    if (arcore_->RequestHitTest(request->ray, &results)) {
       std::move(request->callback).Run(std::move(results));
     } else {
       // Hit test failed, i.e. unprojected location was offscreen.

chrome/browser/android/vr/arcore_device/arcore_gl.h

@@ -67,9 +67,7 @@ class ArCoreGl {
   base::WeakPtr<ArCoreGl> GetWeakPtr();
 
  private:
-  // TODO(https://crbug/835948): remove frame_size.
   void ProcessFrame(mojom::XRFrameDataPtr frame_data,
-                    const gfx::Size& frame_size,
                     mojom::XRFrameDataProvider::GetFrameDataCallback callback);
 
   bool InitializeGl();

chrome/browser/android/vr/arcore_device/arcore_impl.cc

@@ -207,10 +207,8 @@ gfx::Transform ArCoreImpl::GetProjectionMatrix(float near, float far) {
   return result;
 }
 
-// TODO(835948): remove image-size
 bool ArCoreImpl::RequestHitTest(
     const mojom::XRRayPtr& ray,
-    const gfx::Size& image_size,
     std::vector<mojom::XRHitResultPtr>* hit_results) {
   DCHECK(IsOnGlThread());
   DCHECK(arcore_session_.is_valid());
@@ -227,17 +225,14 @@ bool ArCoreImpl::RequestHitTest(
     return false;
   }
 
-  gfx::PointF screen_point;
-  if (!TransformRayToScreenSpace(ray, image_size, &screen_point)) {
-    return false;
-  }
-
   // ArCore returns hit-results in sorted order, thus providing the guarantee
   // of sorted results promised by the WebXR spec for requestHitTest().
-  ArFrame_hitTest(arcore_session_.get(), arcore_frame_.get(),
-                  screen_point.x() * image_size.width(),
-                  screen_point.y() * image_size.height(),
-                  arcore_hit_result_list.get());
+  float origin[3] = {ray->origin.x(), ray->origin.y(), ray->origin.z()};
+  float direction[3] = {ray->direction.x(), ray->direction.y(),
+                        ray->direction.z()};
+
+  ArFrame_hitTestRay(arcore_session_.get(), arcore_frame_.get(), origin,
+                     direction, arcore_hit_result_list.get());
 
   int arcore_hit_result_list_size = 0;
   ArHitResultList_getSize(arcore_session_.get(), arcore_hit_result_list.get(),
@@ -315,61 +310,6 @@ bool ArCoreImpl::RequestHitTest(
   return true;
 }
 
-// TODO(835948): remove this method.
-bool ArCoreImpl::TransformRayToScreenSpace(const mojom::XRRayPtr& ray,
-                                           const gfx::Size& image_size,
-                                           gfx::PointF* screen_point) {
-  DCHECK(IsOnGlThread());
-  DCHECK(arcore_session_.is_valid());
-  DCHECK(arcore_frame_.is_valid());
-
-  internal::ScopedArCoreObject<ArCamera*> arcore_camera;
-  ArFrame_acquireCamera(
-      arcore_session_.get(), arcore_frame_.get(),
-      internal::ScopedArCoreObject<ArCamera*>::Receiver(arcore_camera).get());
-  DCHECK(arcore_camera.is_valid())
-      << "ArFrame_acquireCamera failed despite documentation saying it cannot";
-
-  // Get the projection matrix.
-  float projection_matrix[16];
-  ArCamera_getProjectionMatrix(arcore_session_.get(), arcore_camera.get(), 0.1,
-                               1000, projection_matrix);
-  SkMatrix44 projection44;
-  projection44.setColMajorf(projection_matrix);
-  gfx::Transform projection_transform(projection44);
-
-  // Get the view matrix.
-  float view_matrix[16];
-  ArCamera_getViewMatrix(arcore_session_.get(), arcore_camera.get(),
-                         view_matrix);
-  SkMatrix44 view44;
-  view44.setColMajorf(view_matrix);
-  gfx::Transform view_transform(view44);
-
-  // Create the combined matrix.
-  gfx::Transform proj_view_transform = projection_transform * view_transform;
-
-  // Transform the ray into screen space.
-  gfx::Point3F screen_point_3d = ray->origin + ray->direction;
-
-  proj_view_transform.TransformPoint(&screen_point_3d);
-  if (screen_point_3d.x() < -1 || screen_point_3d.x() > 1 ||
-      screen_point_3d.y() < -1 || screen_point_3d.y() > 1) {
-    // The point does not project back into screen space, so this won't
-    // work with the screen-space-based hit-test API.
-    DLOG(ERROR) << "Invalid ray - does not originate from device screen.";
-    return false;
-  }
-
-  screen_point->set_x((screen_point_3d.x() + 1) / 2);
-  // The calculated point in GL's normalized device coordinates (NDC) ranges
-  // from -1..1, with -1, -1 at the bottom left of the screen, +1 at the top.
-  // The output screen space coordinates range from 0..1, with 0, 0 at the
-  // top left.
-  screen_point->set_y((-screen_point_3d.y() + 1) / 2);
-  return true;
-}
-
 bool ArCoreImpl::IsOnGlThread() {
   return gl_thread_task_runner_->BelongsToCurrentThread();
 }

chrome/browser/android/vr/arcore_device/arcore_impl.h

@@ -82,14 +82,9 @@ class ArCoreImpl : public ArCore {
   void Pause() override;
   void Resume() override;
   bool RequestHitTest(const mojom::XRRayPtr& ray,
-                      const gfx::Size& image_size,
                       std::vector<mojom::XRHitResultPtr>* hit_results) override;
 
  private:
-  bool TransformRayToScreenSpace(const mojom::XRRayPtr& ray,
-                                 const gfx::Size& image_size,
-                                 gfx::PointF* screen_point);
-
   bool IsOnGlThread();
   base::WeakPtr<ArCoreImpl> GetWeakPtr() {
     return weak_ptr_factory_.GetWeakPtr();

chrome/browser/android/vr/arcore_device/arcore_shim.cc

@@ -22,7 +22,7 @@ namespace {
   CALL(ArFrame_acquireCamera)            \
   CALL(ArFrame_create)                   \
   CALL(ArFrame_destroy)                  \
-  CALL(ArFrame_hitTest)                  \
+  CALL(ArFrame_hitTestRay)               \
   CALL(ArFrame_transformDisplayUvCoords) \
   CALL(ArHitResult_create)               \
   CALL(ArHitResult_destroy)              \
@@ -152,13 +152,13 @@ void ArFrame_destroy(ArFrame* frame) {
   arcore_api->impl_ArFrame_destroy(frame);
 }
 
-void ArFrame_hitTest(const ArSession* session,
-                     const ArFrame* frame,
-                     float pixel_x,
-                     float pixel_y,
-                     ArHitResultList* out_hit_results) {
-  arcore_api->impl_ArFrame_hitTest(session, frame, pixel_x, pixel_y,
-                                   out_hit_results);
+void ArFrame_hitTestRay(const ArSession* session,
+                        const ArFrame* frame,
+                        const float* ray_origin_3,
+                        const float* ray_direction_3,
+                        ArHitResultList* out_hit_results) {
+  arcore_api->impl_ArFrame_hitTestRay(session, frame, ray_origin_3,
+                                      ray_direction_3, out_hit_results);
 }
 
 void ArFrame_transformDisplayUvCoords(const ArSession* session,

chrome/browser/android/vr/arcore_device/fake_arcore.cc

@@ -203,7 +203,6 @@ mojom::VRPosePtr FakeArCore::Update(bool* camera_updated) {
 
 bool FakeArCore::RequestHitTest(
     const mojom::XRRayPtr& ray,
-    const gfx::Size& image_size,
     std::vector<mojom::XRHitResultPtr>* hit_results) {
   mojom::XRHitResultPtr hit = mojom::XRHitResult::New();
   hit->hit_matrix.resize(16);

chrome/browser/android/vr/arcore_device/fake_arcore.h

@@ -35,7 +35,6 @@ class FakeArCore : public ArCore {
   void Resume() override;
 
   bool RequestHitTest(const mojom::XRRayPtr& ray,
-                      const gfx::Size& image_size,
                       std::vector<mojom::XRHitResultPtr>* hit_results) override;
 
   void SetCameraAspect(float aspect) { camera_aspect_ = aspect; }