ShapeDetection: Parse landmarks of the detected face on Mac Vision

Update the mojom interface definition of landmark to match the Spec, and adjust
the usage in services/blink/LayoutTests.
Export eyes and mouth landmarks that detect with Mac Vision Framework, the points
are normalized to the bounding box of the detected face, with the origin at the
bounding box of the detected face's lower-left corner.

Link Mac 10.13 build bots [1] and Face Detection demo[2] here.

[1] https://ci.chromium.org/buildbot/chromium.fyi/Chromium%20Mac%2010.13/
[2] https://codepen.io/miguelao/pen/PmJWro

BUG=799319

Cq-Include-Trybots: luci.chromium.try:mac_optional_gpu_tests_rel;master.tryserver.chromium.win:win10_chromium_x64_rel_ng
Change-Id: Id034721a517e4ab2a6b9aa0e3eec9d10997b5b80
Reviewed-on: https://chromium-review.googlesource.com/982579Reviewed-by: Miguel Casas <mcasas@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Reilly Grant <reillyg@chromium.org>
Reviewed-by: Mark Mentovai <mark@chromium.org>
Commit-Queue: Junwei Fu <junwei.fu@intel.com>
Cr-Commit-Position: refs/heads/master@{#552217}

base/mac/sdk_forward_declarations.h

@@ -293,6 +293,23 @@ typedef NSString* VNImageOption NS_STRING_ENUM;
 - (BOOL)performRequests:(NSArray<VNRequest*>*)requests error:(NSError**)error;
 @end
 
+// VNFaceLandmarks2D forward declarations.
+@interface VNFaceLandmarkRegion : NSObject
+@property(readonly) NSUInteger pointCount;
+@end
+
+@interface VNFaceLandmarkRegion2D : VNFaceLandmarkRegion
+@property(readonly, assign)
+    const CGPoint* normalizedPoints NS_RETURNS_INNER_POINTER;
+@end
+
+@interface VNFaceLandmarks2D : NSObject
+@property(readonly) VNFaceLandmarkRegion2D* leftEye;
+@property(readonly) VNFaceLandmarkRegion2D* rightEye;
+@property(readonly) VNFaceLandmarkRegion2D* outerLips;
+@property(readonly) VNFaceLandmarkRegion2D* nose;
+@end
+
 // VNFaceObservation forward declarations.
 @interface VNObservation : NSObject<NSCopying, NSSecureCoding>
 @end
@@ -302,6 +319,7 @@ typedef NSString* VNImageOption NS_STRING_ENUM;
 @end
 
 @interface VNFaceObservation : VNDetectedObjectObservation
+@property(readonly, nonatomic, strong) VNFaceLandmarks2D* landmarks;
 @end
 
 #endif  // MAC_OS_X_VERSION_10_13

services/shape_detection/android/java/src/org/chromium/shape_detection/FaceDetectionImplGmsCore.java

@@ -103,9 +103,10 @@ public class FaceDetectionImplGmsCore implements FaceDetection {
                         || landmarkType == Landmark.BOTTOM_MOUTH) {
                     org.chromium.shape_detection.mojom.Landmark mojoLandmark =
                             new org.chromium.shape_detection.mojom.Landmark();
-                    mojoLandmark.location = new org.chromium.gfx.mojom.PointF();
-                    mojoLandmark.location.x = landmark.getPosition().x;
-                    mojoLandmark.location.y = landmark.getPosition().y;
+                    mojoLandmark.locations = new org.chromium.gfx.mojom.PointF[1];
+                    mojoLandmark.locations[0] = new org.chromium.gfx.mojom.PointF();
+                    mojoLandmark.locations[0].x = landmark.getPosition().x;
+                    mojoLandmark.locations[0].y = landmark.getPosition().y;
                     mojoLandmark.type = landmarkType == Landmark.BOTTOM_MOUTH ? LandmarkType.MOUTH
                                                                               : LandmarkType.EYE;
                     mojoLandmarks.add(mojoLandmark);

services/shape_detection/face_detection_impl_mac.mm

@@ -42,22 +42,22 @@ void FaceDetectionImplMac::Detect(const SkBitmap& bitmap,
     if (f.hasLeftEyePosition) {
       auto landmark = shape_detection::mojom::Landmark::New();
       landmark->type = shape_detection::mojom::LandmarkType::EYE;
-      landmark->location =
-          gfx::PointF(f.leftEyePosition.x, height - f.leftEyePosition.y);
+      landmark->locations.emplace_back(f.leftEyePosition.x,
+                                       height - f.leftEyePosition.y);
       face->landmarks.push_back(std::move(landmark));
     }
     if (f.hasRightEyePosition) {
       auto landmark = shape_detection::mojom::Landmark::New();
       landmark->type = shape_detection::mojom::LandmarkType::EYE;
-      landmark->location =
-          gfx::PointF(f.rightEyePosition.x, height - f.rightEyePosition.y);
+      landmark->locations.emplace_back(f.rightEyePosition.x,
+                                       height - f.rightEyePosition.y);
       face->landmarks.push_back(std::move(landmark));
     }
     if (f.hasMouthPosition) {
       auto landmark = shape_detection::mojom::Landmark::New();
       landmark->type = shape_detection::mojom::LandmarkType::MOUTH;
-      landmark->location =
-          gfx::PointF(f.mouthPosition.x, height - f.mouthPosition.y);
+      landmark->locations.emplace_back(f.mouthPosition.x,
+                                       height - f.mouthPosition.y);
       face->landmarks.push_back(std::move(landmark));
     }
 

services/shape_detection/face_detection_impl_mac_vision.mm

@@ -19,6 +19,26 @@
 
 namespace shape_detection {
 
+namespace {
+
+mojom::LandmarkPtr BuildLandmark(VNFaceLandmarkRegion2D* landmark_region,
+                                 mojom::LandmarkType landmark_type,
+                                 gfx::RectF bounding_box) {
+  auto landmark = mojom::Landmark::New();
+  landmark->type = landmark_type;
+  landmark->locations.reserve(landmark_region.pointCount);
+  for (NSUInteger i = 0; i < landmark_region.pointCount; ++i) {
+    // The points are normalized to the bounding box of the detected face.
+    landmark->locations.emplace_back(
+        landmark_region.normalizedPoints[i].x * bounding_box.width() +
+            bounding_box.x(),
+        (1 - landmark_region.normalizedPoints[i].y) * bounding_box.height() +
+            bounding_box.y());
+  }
+  return landmark;
+}
+}
+
 // The VisionAPIAsyncRequestMac class submits an image analysis request for
 // asynchronous execution on a dispatch queue with default priority.
 class API_AVAILABLE(macos(10.13))
@@ -147,7 +167,7 @@ void FaceDetectionImplMacVision::OnFacesDetected(VNRequest* request,
 
   std::vector<mojom::FaceDetectionResultPtr> results;
   for (VNFaceObservation* const observation in request.results) {
-    auto face = shape_detection::mojom::FaceDetectionResult::New();
+    auto face = mojom::FaceDetectionResult::New();
     // The coordinate are normalized to the dimensions of the processed image.
     face->bounding_box = ConvertCGToGfxCoordinates(
         CGRectMake(observation.boundingBox.origin.x * image_size_.width,
@@ -156,6 +176,19 @@ void FaceDetectionImplMacVision::OnFacesDetected(VNRequest* request,
                    observation.boundingBox.size.height * image_size_.height),
         image_size_.height);
 
+    if (VNFaceLandmarkRegion2D* leftEye = observation.landmarks.leftEye) {
+      face->landmarks.push_back(
+          BuildLandmark(leftEye, mojom::LandmarkType::EYE, face->bounding_box));
+    }
+    if (VNFaceLandmarkRegion2D* rightEye = observation.landmarks.rightEye) {
+      face->landmarks.push_back(BuildLandmark(
+          rightEye, mojom::LandmarkType::EYE, face->bounding_box));
+    }
+    if (VNFaceLandmarkRegion2D* outerLips = observation.landmarks.outerLips) {
+      face->landmarks.push_back(BuildLandmark(
+          outerLips, mojom::LandmarkType::MOUTH, face->bounding_box));
+    }
+
     results.push_back(std::move(face));
   }
   std::move(detected_callback_).Run(std::move(results));

services/shape_detection/public/mojom/facedetection.mojom

@@ -14,7 +14,7 @@ enum LandmarkType { MOUTH, EYE };
 
 // https://wicg.github.io/shape-detection-api/#dictdef-landmark
 struct Landmark {
-  gfx.mojom.PointF location;
+  array<gfx.mojom.PointF> locations;
   LandmarkType type;
 };
 

third_party/WebKit/LayoutTests/shapedetection/detection-HTMLImageElement.html

@@ -37,10 +37,12 @@ function FaceDetectorDetectionResultTest(detectionResult, mock) {
   const GREEN_PIXEL = 0xFF00FF00;
   assert_equals(imageReceivedByMock[0], GREEN_PIXEL, "Pixel color");
   assert_equals(detectionResult.length, 3, "Number of faces");
-  assert_equals(detectionResult[0].landmarks.length, 1, "Number of landmarks");
+  assert_equals(detectionResult[0].landmarks.length, 2, "Number of landmarks");
   assert_object_equals(detectionResult[0].landmarks[0],
                       {type : 'eye', locations : [{x : 4.0, y : 5.0}]},
                       "landmark #1");
+  assert_equals(detectionResult[0].landmarks[1].locations.length, 8,
+                "Number of locations along the landmark");
 }
 
 function BarcodeDetectorDetectionResultTest(detectionResult, mock) {

third_party/WebKit/LayoutTests/shapedetection/resources/mock-facedetection.js

@@ -46,7 +46,15 @@ class MockFaceDetection {
           boundingBox: {x: 1.0, y: 1.0, width: 100.0, height: 100.0},
           landmarks: [{
             type: shapeDetection.mojom.LandmarkType.EYE,
-            location: {x: 4.0, y: 5.0}
+            locations: [{x: 4.0, y: 5.0}]
+          },
+          {
+            type: shapeDetection.mojom.LandmarkType.EYE,
+            locations: [
+              {x: 4.0, y: 5.0}, {x: 5.0, y: 4.0}, {x: 6.0, y: 3.0},
+              {x: 7.0, y: 4.0}, {x: 8.0, y: 5.0}, {x: 7.0, y: 6.0},
+              {x: 6.0, y: 7.0}, {x: 5.0, y: 6.0}
+            ]
           }]
         },
         {

third_party/blink/renderer/modules/shapedetection/face_detector.cc

@@ -71,11 +71,13 @@ void FaceDetector::OnDetectFaces(
   for (const auto& face : face_detection_results) {
     HeapVector<Landmark> landmarks;
     for (const auto& landmark : face->landmarks) {
-      Point2D location;
-      location.setX(landmark->location.x);
-      location.setY(landmark->location.y);
       HeapVector<Point2D> locations;
-      locations.push_back(location);
+      for (const auto& location : landmark->locations) {
+        Point2D web_location;
+        web_location.setX(location.x);
+        web_location.setY(location.y);
+        locations.push_back(web_location);
+      }
 
       Landmark web_landmark;
       web_landmark.setLocations(locations);