Return anchor information retrieved from ARCore via mojo's XRFrameData

Ship anchor information from ARCore side to blink. The data will be
surfaced to the JavaScript side in subsequent CLs.

Bug: 992029
Change-Id: Ica8a0d424c64c96d9d1c3679be07e5569bc6bd8b
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1763876Reviewed-by: David Dorwin <ddorwin@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Commit-Queue: Piotr Bialecki <bialpio@chromium.org>
Cr-Commit-Position: refs/heads/master@{#689949}

chrome/browser/android/vr/arcore_device/arcore.h

@@ -47,6 +47,9 @@ class ArCore {
   // Returns information about all planes detected in the current frame.
   virtual mojom::XRPlaneDetectionDataPtr GetDetectedPlanesData() = 0;
 
+  // Returns information about all anchors tracked in the current frame.
+  virtual mojom::XRAnchorsDataPtr GetAnchorsData() = 0;
+
   virtual bool RequestHitTest(
       const mojom::XRRayPtr& ray,
       std::vector<mojom::XRHitResultPtr>* hit_results) = 0;

chrome/browser/android/vr/arcore_device/arcore_impl.cc

@@ -201,6 +201,16 @@ void ArCoreImpl::EnsureArCorePlanesList() {
   }
 }
 
+void ArCoreImpl::EnsureArCoreAnchorsList() {
+  if (!arcore_anchors_.is_valid()) {
+    ArAnchorList_create(
+        arcore_session_.get(),
+        internal::ScopedArCoreObject<ArAnchorList*>::Receiver(arcore_anchors_)
+            .get());
+    DCHECK(arcore_anchors_.is_valid());
+  }
+}
+
 template <typename FunctionType>
 void ArCoreImpl::ForEachArCorePlane(FunctionType fn) {
   DCHECK(arcore_planes_.is_valid());
@@ -253,6 +263,99 @@ void ArCoreImpl::ForEachArCorePlane(FunctionType fn) {
   }
 }  // namespace device
 
+std::vector<mojom::XRAnchorDataPtr> ArCoreImpl::GetUpdatedAnchorsData() {
+  EnsureArCoreAnchorsList();
+
+  std::vector<mojom::XRAnchorDataPtr> result;
+
+  ArFrame_getUpdatedAnchors(arcore_session_.get(), arcore_frame_.get(),
+                            arcore_anchors_.get());
+
+  ForEachArCoreAnchor([this, &result](ArAnchor* ar_anchor) {
+    // pose
+    internal::ScopedArCoreObject<ArPose*> anchor_pose;
+    ArPose_create(
+        arcore_session_.get(), nullptr,
+        internal::ScopedArCoreObject<ArPose*>::Receiver(anchor_pose).get());
+    ArAnchor_getPose(arcore_session_.get(), ar_anchor, anchor_pose.get());
+    mojom::VRPosePtr pose =
+        GetMojomPoseFromArPose(arcore_session_.get(), std::move(anchor_pose));
+
+    // ID
+    auto maybe_pair_with_anchor_id_and_created = CreateOrGetAnchorId(ar_anchor);
+    if (!maybe_pair_with_anchor_id_and_created) {
+      // Skip the anchor - we have run out of IDs and there is not much we can
+      // do about it.
+      return;
+    }
+
+    int32_t anchor_id;
+    bool created;
+    std::tie(anchor_id, created) = *maybe_pair_with_anchor_id_and_created;
+
+    result.push_back(mojom::XRAnchorData::New(anchor_id, std::move(pose)));
+  });
+
+  return result;
+}
+
+std::vector<int32_t> ArCoreImpl::GetAllAnchorIds() {
+  EnsureArCoreAnchorsList();
+
+  std::vector<int32_t> result;
+
+  ArSession_getAllAnchors(arcore_session_.get(), arcore_anchors_.get());
+
+  ForEachArCoreAnchor([this, &result](ArAnchor* ar_anchor) {
+    // ID
+    auto maybe_pair_with_anchor_id_and_created = CreateOrGetAnchorId(ar_anchor);
+    if (!maybe_pair_with_anchor_id_and_created) {
+      // Skip the anchor - we have run out of IDs and there is not much we can
+      // do about it.
+      return;
+    }
+
+    int32_t anchor_id;
+    bool created;
+    std::tie(anchor_id, created) = *maybe_pair_with_anchor_id_and_created;
+
+    // TODO(https://crbug.com/992033): Add explanation for the below DCHECK when
+    // implementing anchor creation.
+    DCHECK(!created);
+
+    result.emplace_back(anchor_id);
+  });
+
+  return result;
+}
+
+template <typename FunctionType>
+void ArCoreImpl::ForEachArCoreAnchor(FunctionType fn) {
+  DCHECK(arcore_anchors_.is_valid());
+
+  int32_t anchor_list_size;
+  ArAnchorList_getSize(arcore_session_.get(), arcore_anchors_.get(),
+                       &anchor_list_size);
+
+  for (int i = 0; i < anchor_list_size; i++) {
+    internal::ScopedArCoreObject<ArAnchor*> anchor;
+    ArAnchorList_acquireItem(
+        arcore_session_.get(), arcore_anchors_.get(), i,
+        internal::ScopedArCoreObject<ArAnchor*>::Receiver(anchor).get());
+
+    ArTrackingState tracking_state;
+    ArAnchor_getTrackingState(arcore_session_.get(), anchor.get(),
+                              &tracking_state);
+
+    if (tracking_state != ArTrackingState::AR_TRACKING_STATE_TRACKING) {
+      // Skip all anchors that are not currently tracked.
+      continue;
+    }
+
+    fn(anchor.get());
+  }
+}
+
 std::vector<mojom::XRPlaneDataPtr> ArCoreImpl::GetUpdatedPlanesData() {
   EnsureArCorePlanesList();
 
@@ -293,10 +396,17 @@ std::vector<mojom::XRPlaneDataPtr> ArCoreImpl::GetUpdatedPlanesData() {
           mojom::XRPlanePointData::New(vertices_raw[i], vertices_raw[i + 1]));
     }
 
-    // id
+    // ID
+    auto maybe_pair_with_plane_id_and_created = CreateOrGetPlaneId(ar_plane);
+    if (!maybe_pair_with_plane_id_and_created) {
+      // Skip the plane - we have run out of IDs and there is not much we can
+      // do about it.
+      return;
+    }
+
     int32_t plane_id;
     bool created;
-    std::tie(plane_id, created) = CreateOrGetPlaneId(ar_plane);
+    std::tie(plane_id, created) = *maybe_pair_with_plane_id_and_created;
 
     result.push_back(mojom::XRPlaneData::New(
         plane_id,
@@ -317,13 +427,20 @@ std::vector<int32_t> ArCoreImpl::GetAllPlaneIds() {
                              arcore_planes_.get());
 
   ForEachArCorePlane([this, &result](ArPlane* ar_plane) {
-    // id
+    // ID
+    auto maybe_pair_with_plane_id_and_created = CreateOrGetPlaneId(ar_plane);
+    if (!maybe_pair_with_plane_id_and_created) {
+      // Skip the plane - we have run out of IDs and there is not much we can
+      // do about it.
+      return;
+    }
+
     int32_t plane_id;
     bool created;
-    std::tie(plane_id, created) = CreateOrGetPlaneId(ar_plane);
+    std::tie(plane_id, created) = *maybe_pair_with_plane_id_and_created;
 
-    // Newly detected planes should be handled by GetUpdatedPlanesData().
-    DCHECK(!created);
+    DCHECK(!created)
+        << "Newly detected planes should be handled by GetUpdatedPlanesData().";
 
     result.emplace_back(plane_id);
   });
@@ -341,14 +458,25 @@ mojom::XRPlaneDetectionDataPtr ArCoreImpl::GetDetectedPlanesData() {
                                           std::move(updated_planes));
 }
 
-std::pair<int32_t, bool> ArCoreImpl::CreateOrGetPlaneId(void* plane_address) {
+mojom::XRAnchorsDataPtr ArCoreImpl::GetAnchorsData() {
+  TRACE_EVENT0("gpu", __FUNCTION__);
+
+  auto updated_anchors = GetUpdatedAnchorsData();
+  auto all_anchor_ids = GetAllAnchorIds();
+
+  return mojom::XRAnchorsData::New(all_anchor_ids, std::move(updated_anchors));
+}
+
+base::Optional<std::pair<int32_t, bool>> ArCoreImpl::CreateOrGetPlaneId(
+    void* plane_address) {
   auto it = ar_plane_address_to_id_.find(plane_address);
   if (it != ar_plane_address_to_id_.end()) {
     return std::make_pair(it->second, false);
   }
 
-  // Make sure that incrementing next_id_ won't cause an overflow.
-  CHECK(next_id_ != std::numeric_limits<int32_t>::max());
+  if (next_id_ == std::numeric_limits<int32_t>::max()) {
+    return base::nullopt;
+  }
 
   int32_t current_id = next_id_++;
   ar_plane_address_to_id_.emplace(plane_address, current_id);
@@ -356,6 +484,23 @@ std::pair<int32_t, bool> ArCoreImpl::CreateOrGetPlaneId(void* plane_address) {
   return std::make_pair(current_id, true);
 }
 
+base::Optional<std::pair<int32_t, bool>> ArCoreImpl::CreateOrGetAnchorId(
+    void* anchor_address) {
+  auto it = ar_anchor_address_to_id_.find(anchor_address);
+  if (it != ar_anchor_address_to_id_.end()) {
+    return std::make_pair(it->second, false);
+  }
+
+  if (next_id_ == std::numeric_limits<int32_t>::max()) {
+    return base::nullopt;
+  }
+
+  int32_t current_id = next_id_++;
+  ar_anchor_address_to_id_.emplace(anchor_address, current_id);
+
+  return std::make_pair(current_id, true);
+}
+
 void ArCoreImpl::Pause() {
   DVLOG(2) << __func__;
 

chrome/browser/android/vr/arcore_device/arcore_impl.h

@@ -98,9 +98,14 @@ class ArCoreImpl : public ArCore {
       const base::span<const float> uvs) override;
   gfx::Transform GetProjectionMatrix(float near, float far) override;
   mojom::VRPosePtr Update(bool* camera_updated) override;
+
   mojom::XRPlaneDetectionDataPtr GetDetectedPlanesData() override;
+
+  mojom::XRAnchorsDataPtr GetAnchorsData() override;
+
   void Pause() override;
   void Resume() override;
+
   bool RequestHitTest(const mojom::XRRayPtr& ray,
                       std::vector<mojom::XRHitResultPtr>* hit_results) override;
 
@@ -120,13 +125,23 @@ class ArCoreImpl : public ArCore {
 
   // List of trackables - used for retrieving planes detected by ARCore. The
   // list will initially be null - call EnsureArCorePlanesList() before using
-  // it. Instead of creating it in every call to GetUpdatedPlanesData(), the
-  // list can be reused as it will be cleaned by calls to ArCore SDK.
+  // it.
+  // Allows reuse of the list across updates; ARCore clears the list on each
+  // call to the ARCore SDK.
   internal::ScopedArCoreObject<ArTrackableList*> arcore_planes_;
 
+  // List of anchors - used for retrieving anchors tracked by ARCore. The list
+  // will initially be null - call EnsureArCoreAnchorsList() before using it.
+  // Allows reuse of the list across updates; ARCore clears the list on each
+  // call to the ARCore SDK.
+  internal::ScopedArCoreObject<ArAnchorList*> arcore_anchors_;
+
   // Initializes |arcore_planes_| list.
   void EnsureArCorePlanesList();
 
+  // Initializes |arcore_anchors_| list.
+  void EnsureArCoreAnchorsList();
+
   // Returns vector containing information about all planes updated in current
   // frame, assigning IDs for newly detected planes as needed.
   std::vector<mojom::XRPlaneDataPtr> GetUpdatedPlanesData();
@@ -137,18 +152,35 @@ class ArCoreImpl : public ArCore {
   // and unchanged planes. It excludes planes that are no longer being tracked.
   std::vector<int32_t> GetAllPlaneIds();
 
+  // Returns vector containing information about all anchors updated in the
+  // current frame.
+  std::vector<mojom::XRAnchorDataPtr> GetUpdatedAnchorsData();
+
+  // The result will contain IDs of all anchors still tracked in the current
+  // frame.
+  std::vector<int32_t> GetAllAnchorIds();
+
   int32_t next_id_ = 1;
   std::unordered_map<void*, int32_t> ar_plane_address_to_id_;
+  std::unordered_map<void*, int32_t> ar_anchor_address_to_id_;
 
   // Returns tuple containing plane id and a boolean signifying that the plane
-  // was created.
-  std::pair<int32_t, bool> CreateOrGetPlaneId(void* plane_address);
+  // was created. The result will be a nullopt in case the ID should be assigned
+  // but next ID would result in an integer overflow.
+  base::Optional<std::pair<int32_t, bool>> CreateOrGetPlaneId(
+      void* plane_address);
+  base::Optional<std::pair<int32_t, bool>> CreateOrGetAnchorId(
+      void* anchor_address);
 
   // Executes |fn| for each still tracked, non-subsumed plane present in
   // |arcore_planes_|.
   template <typename FunctionType>
   void ForEachArCorePlane(FunctionType fn);
 
+  // Executes |fn| for each still tracked anchor present in |arcore_anchors_|.
+  template <typename FunctionType>
+  void ForEachArCoreAnchor(FunctionType fn);
+
   // Must be last.
   base::WeakPtrFactory<ArCoreImpl> weak_ptr_factory_{this};
   DISALLOW_COPY_AND_ASSIGN(ArCoreImpl);

chrome/browser/android/vr/arcore_device/fake_arcore.cc

@@ -229,6 +229,19 @@ mojom::XRPlaneDetectionDataPtr FakeArCore::GetDetectedPlanesData() {
                                           std::move(result));
 }
 
+mojom::XRAnchorsDataPtr FakeArCore::GetAnchorsData() {
+  std::vector<mojom::XRAnchorDataPtr> result;
+
+  // 2m ahead of the origin, neutral orientation facing forward.
+  mojom::VRPosePtr pose = mojom::VRPose::New();
+  pose->position = gfx::Point3F(0.0, 0.0, -2.0);
+  pose->orientation = gfx::Quaternion();
+
+  result.push_back(mojom::XRAnchorData::New(2, std::move(pose)));
+
+  return mojom::XRAnchorsData::New(std::vector<int32_t>{2}, std::move(result));
+}
+
 void FakeArCore::Pause() {
   DCHECK(IsOnGlThread());
 }

chrome/browser/android/vr/arcore_device/fake_arcore.h

@@ -38,6 +38,7 @@ class FakeArCore : public ArCore {
                       std::vector<mojom::XRHitResultPtr>* hit_results) override;
 
   mojom::XRPlaneDetectionDataPtr GetDetectedPlanesData() override;
+  mojom::XRAnchorsDataPtr GetAnchorsData() override;
 
   void SetCameraAspect(float aspect) { camera_aspect_ = aspect; }
 

device/vr/public/mojom/vr_service.mojom

@@ -284,6 +284,8 @@ struct XRPlanePointData {
   float z;
 };
 
+// Struct containing plane-related information. A plane describes a flat surface
+// detected in the real world by the underlying system.
 struct XRPlaneData {
   // Unique (per session) identifier of the plane.
   int32 id;
@@ -300,17 +302,47 @@ struct XRPlaneData {
   array<XRPlanePointData> polygon;
 };
 
-// Struct containing information about existing & updated planes in frame N.
-// If the plane was removed between frame N-1 and N, its ID will not be present
-// in all_plane_ids.
+// Struct containing information about all tracked & updated planes in a given
+// frame. If a plane tracked in frame N-1 is no longer being tracked in frame N,
+// its ID will not be present in all_plane_ids and its XRPlaneData will not be
+// present in updated_planes_data.
 struct XRPlaneDetectionData {
-  // Array with ids of all planes.
+  // Array with ids of all tracked planes.
   array<int32> all_planes_ids;
 
-  // Array with plane data for updated planes.
+  // Array with plane data for all updated planes. Plane is considered updated
+  // if its position or polygon has changed. Updated planes are always a subset
+  // of all planes (i.e. for each plane found in updated_planes_data, its ID
+  // will be present in all_planes_ids).
   array<XRPlaneData> updated_planes_data;
 };
 
+// Struct containing anchor-related information. An anchor represents a pose
+// (position and orientation) in the real world. The anchor's pose will be
+// adjusted by the underlying system as its understanding of the real world
+// evolves.
+struct XRAnchorData {
+  // Unique (per session) identifier of the anchor.
+  int32 id;
+
+  // Pose of the anchor.
+  VRPose pose;
+};
+
+// Struct containing information about all tracked & updated anchors in a given
+// frame. If an anchor tracked in frame N-1 is no longer being tracked in
+// frame N, its ID will not be present in all_anchors_ids and its XRAnchorData
+// will not be present in updated_anchors_data.
+struct XRAnchorsData {
+  // Array with ids of all tracked anchors.
+  array<int32> all_anchors_ids;
+
+  // Array with anchor data for all updated anchors. Updated anchors are always
+  // a subset of all anchors (i.e. for each anchor found in
+  // updated_anchors_data, its ID will be present in all_anchors_ids).
+  array<XRAnchorData> updated_anchors_data;
+};
+
 // The data needed for each animation frame of an XRSession.
 struct XRFrameData {
   // General XRSession value
@@ -357,6 +389,9 @@ struct XRFrameData {
 
   // Detected plane information. Only present if plane detection is enabled.
   XRPlaneDetectionData? detected_planes_data;
+
+  // Tracked anchors information.
+  XRAnchorsData? anchors_data;
 };
 
 enum VRDisplayEventReason {