Reland "ShapeDetection: Parse landmarks of the detected face on Mac Vision"

Original CL got reverted because the num of landmarks aren't updated in services
tests that aren't checked on GPU FYI.

This is a reland of 5e1aa79b

Original change's description:
> ShapeDetection: Parse landmarks of the detected face on Mac Vision
>
> Update the mojom interface definition of landmark to match the Spec, and adjust
> the usage in services/blink/LayoutTests.
> Export eyes and mouth landmarks that detect with Mac Vision Framework, the points
> are normalized to the bounding box of the detected face, with the origin at the
> bounding box of the detected face's lower-left corner.
>
> Link Mac 10.13 build bots [1] and Face Detection demo[2] here.
>
> [1] https://ci.chromium.org/buildbot/chromium.fyi/Chromium%20Mac%2010.13/
> [2] https://codepen.io/miguelao/pen/PmJWro
>
> BUG=799319
>
> Cq-Include-Trybots: luci.chromium.try:mac_optional_gpu_tests_rel;master.tryserver.chromium.win:win10_chromium_x64_rel_ng
> Change-Id: Id034721a517e4ab2a6b9aa0e3eec9d10997b5b80
> Reviewed-on: https://chromium-review.googlesource.com/982579
> Reviewed-by: Miguel Casas <mcasas@chromium.org>
> Reviewed-by: Daniel Cheng <dcheng@chromium.org>
> Reviewed-by: Reilly Grant <reillyg@chromium.org>
> Reviewed-by: Mark Mentovai <mark@chromium.org>
> Commit-Queue: Junwei Fu <junwei.fu@intel.com>
> Cr-Commit-Position: refs/heads/master@{#552217}

TBR=mark@chromium.org

Bug: 799319
Change-Id: Id521f2a92dd3e1c6d8ac29d571bc1c58cfdb020e
Cq-Include-Trybots: luci.chromium.try:mac_optional_gpu_tests_rel;master.tryserver.chromium.win:win10_chromium_x64_rel_ng
Reviewed-on: https://chromium-review.googlesource.com/1023181
Commit-Queue: Junwei Fu <junwei.fu@intel.com>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Reilly Grant <reillyg@chromium.org>
Reviewed-by: Miguel Casas <mcasas@chromium.org>
Cr-Commit-Position: refs/heads/master@{#552924}

base/mac/sdk_forward_declarations.h

@@ -293,6 +293,23 @@ typedef NSString* VNImageOption NS_STRING_ENUM;
 - (BOOL)performRequests:(NSArray<VNRequest*>*)requests error:(NSError**)error;
 @end
 
+// VNFaceLandmarks2D forward declarations.
+@interface VNFaceLandmarkRegion : NSObject
+@property(readonly) NSUInteger pointCount;
+@end
+
+@interface VNFaceLandmarkRegion2D : VNFaceLandmarkRegion
+@property(readonly, assign)
+    const CGPoint* normalizedPoints NS_RETURNS_INNER_POINTER;
+@end
+
+@interface VNFaceLandmarks2D : NSObject
+@property(readonly) VNFaceLandmarkRegion2D* leftEye;
+@property(readonly) VNFaceLandmarkRegion2D* rightEye;
+@property(readonly) VNFaceLandmarkRegion2D* outerLips;
+@property(readonly) VNFaceLandmarkRegion2D* nose;
+@end
+
 // VNFaceObservation forward declarations.
 @interface VNObservation : NSObject<NSCopying, NSSecureCoding>
 @end
@@ -302,6 +319,7 @@ typedef NSString* VNImageOption NS_STRING_ENUM;
 @end
 
 @interface VNFaceObservation : VNDetectedObjectObservation
+@property(readonly, nonatomic, strong) VNFaceLandmarks2D* landmarks;
 @end
 
 #endif  // MAC_OS_X_VERSION_10_13

services/shape_detection/android/java/src/org/chromium/shape_detection/FaceDetectionImplGmsCore.java

@@ -103,9 +103,10 @@ public class FaceDetectionImplGmsCore implements FaceDetection {
                         || landmarkType == Landmark.BOTTOM_MOUTH) {
                     org.chromium.shape_detection.mojom.Landmark mojoLandmark =
                             new org.chromium.shape_detection.mojom.Landmark();
-                    mojoLandmark.location = new org.chromium.gfx.mojom.PointF();
-                    mojoLandmark.location.x = landmark.getPosition().x;
-                    mojoLandmark.location.y = landmark.getPosition().y;
+                    mojoLandmark.locations = new org.chromium.gfx.mojom.PointF[1];
+                    mojoLandmark.locations[0] = new org.chromium.gfx.mojom.PointF();
+                    mojoLandmark.locations[0].x = landmark.getPosition().x;
+                    mojoLandmark.locations[0].y = landmark.getPosition().y;
                     mojoLandmark.type = landmarkType == Landmark.BOTTOM_MOUTH ? LandmarkType.MOUTH
                                                                               : LandmarkType.EYE;
                     mojoLandmarks.add(mojoLandmark);

services/shape_detection/face_detection_impl_mac.mm

@@ -42,22 +42,22 @@ void FaceDetectionImplMac::Detect(const SkBitmap& bitmap,
     if (f.hasLeftEyePosition) {
       auto landmark = shape_detection::mojom::Landmark::New();
       landmark->type = shape_detection::mojom::LandmarkType::EYE;
-      landmark->location =
-          gfx::PointF(f.leftEyePosition.x, height - f.leftEyePosition.y);
+      landmark->locations.emplace_back(f.leftEyePosition.x,
+                                       height - f.leftEyePosition.y);
       face->landmarks.push_back(std::move(landmark));
     }
     if (f.hasRightEyePosition) {
       auto landmark = shape_detection::mojom::Landmark::New();
       landmark->type = shape_detection::mojom::LandmarkType::EYE;
-      landmark->location =
-          gfx::PointF(f.rightEyePosition.x, height - f.rightEyePosition.y);
+      landmark->locations.emplace_back(f.rightEyePosition.x,
+                                       height - f.rightEyePosition.y);
       face->landmarks.push_back(std::move(landmark));
     }
     if (f.hasMouthPosition) {
       auto landmark = shape_detection::mojom::Landmark::New();
       landmark->type = shape_detection::mojom::LandmarkType::MOUTH;
-      landmark->location =
-          gfx::PointF(f.mouthPosition.x, height - f.mouthPosition.y);
+      landmark->locations.emplace_back(f.mouthPosition.x,
+                                       height - f.mouthPosition.y);
       face->landmarks.push_back(std::move(landmark));
     }
 

services/shape_detection/face_detection_impl_mac_unittest.mm

@@ -69,13 +69,13 @@ struct TestParams {
      base::Bind(&CreateFaceDetectorImplMac)},
     {true, 120, 120, "services/test/data/mona_lisa.jpg", 1, 3,
      base::Bind(&CreateFaceDetectorImplMac)},
-    {false, 120, 120, "services/test/data/mona_lisa.jpg", 1, 0,
+    {false, 120, 120, "services/test/data/mona_lisa.jpg", 1, 3,
      base::Bind(&CreateFaceDetectorImplMacVision)},
     {false, 240, 240, "services/test/data/the_beatles.jpg", 3, 3,
      base::Bind(&CreateFaceDetectorImplMac)},
     {true, 240, 240, "services/test/data/the_beatles.jpg", 3, 3,
      base::Bind(&CreateFaceDetectorImplMac)},
-    {false, 240, 240, "services/test/data/the_beatles.jpg", 4, 0,
+    {false, 240, 240, "services/test/data/the_beatles.jpg", 4, 3,
      base::Bind(&CreateFaceDetectorImplMacVision)},
 };
 

services/shape_detection/face_detection_impl_mac_vision.mm

@@ -19,6 +19,26 @@
 
 namespace shape_detection {
 
+namespace {
+
+mojom::LandmarkPtr BuildLandmark(VNFaceLandmarkRegion2D* landmark_region,
+                                 mojom::LandmarkType landmark_type,
+                                 gfx::RectF bounding_box) {
+  auto landmark = mojom::Landmark::New();
+  landmark->type = landmark_type;
+  landmark->locations.reserve(landmark_region.pointCount);
+  for (NSUInteger i = 0; i < landmark_region.pointCount; ++i) {
+    // The points are normalized to the bounding box of the detected face.
+    landmark->locations.emplace_back(
+        landmark_region.normalizedPoints[i].x * bounding_box.width() +
+            bounding_box.x(),
+        (1 - landmark_region.normalizedPoints[i].y) * bounding_box.height() +
+            bounding_box.y());
+  }
+  return landmark;
+}
+}
+
 // The VisionAPIAsyncRequestMac class submits an image analysis request for
 // asynchronous execution on a dispatch queue with default priority.
 class API_AVAILABLE(macos(10.13))
@@ -147,7 +167,7 @@ void FaceDetectionImplMacVision::OnFacesDetected(VNRequest* request,
 
   std::vector<mojom::FaceDetectionResultPtr> results;
   for (VNFaceObservation* const observation in request.results) {
-    auto face = shape_detection::mojom::FaceDetectionResult::New();
+    auto face = mojom::FaceDetectionResult::New();
     // The coordinate are normalized to the dimensions of the processed image.
     face->bounding_box = ConvertCGToGfxCoordinates(
         CGRectMake(observation.boundingBox.origin.x * image_size_.width,
@@ -156,6 +176,19 @@ void FaceDetectionImplMacVision::OnFacesDetected(VNRequest* request,
                    observation.boundingBox.size.height * image_size_.height),
         image_size_.height);
 
+    if (VNFaceLandmarkRegion2D* leftEye = observation.landmarks.leftEye) {
+      face->landmarks.push_back(
+          BuildLandmark(leftEye, mojom::LandmarkType::EYE, face->bounding_box));
+    }
+    if (VNFaceLandmarkRegion2D* rightEye = observation.landmarks.rightEye) {
+      face->landmarks.push_back(BuildLandmark(
+          rightEye, mojom::LandmarkType::EYE, face->bounding_box));
+    }
+    if (VNFaceLandmarkRegion2D* outerLips = observation.landmarks.outerLips) {
+      face->landmarks.push_back(BuildLandmark(
+          outerLips, mojom::LandmarkType::MOUTH, face->bounding_box));
+    }
+
     results.push_back(std::move(face));
   }
   std::move(detected_callback_).Run(std::move(results));

services/shape_detection/public/mojom/facedetection.mojom

@@ -14,7 +14,7 @@ enum LandmarkType { MOUTH, EYE };
 
 // https://wicg.github.io/shape-detection-api/#dictdef-landmark
 struct Landmark {
-  gfx.mojom.PointF location;
+  array<gfx.mojom.PointF> locations;
   LandmarkType type;
 };
 

third_party/WebKit/LayoutTests/shapedetection/detection-HTMLImageElement.html

@@ -37,10 +37,12 @@ function FaceDetectorDetectionResultTest(detectionResult, mock) {
   const GREEN_PIXEL = 0xFF00FF00;
   assert_equals(imageReceivedByMock[0], GREEN_PIXEL, "Pixel color");
   assert_equals(detectionResult.length, 3, "Number of faces");
-  assert_equals(detectionResult[0].landmarks.length, 1, "Number of landmarks");
+  assert_equals(detectionResult[0].landmarks.length, 2, "Number of landmarks");
   assert_object_equals(detectionResult[0].landmarks[0],
                       {type : 'eye', locations : [{x : 4.0, y : 5.0}]},
                       "landmark #1");
+  assert_equals(detectionResult[0].landmarks[1].locations.length, 8,
+                "Number of locations along the landmark");
 }
 
 function BarcodeDetectorDetectionResultTest(detectionResult, mock) {

third_party/WebKit/LayoutTests/shapedetection/resources/mock-facedetection.js

@@ -46,7 +46,15 @@ class MockFaceDetection {
           boundingBox: {x: 1.0, y: 1.0, width: 100.0, height: 100.0},
           landmarks: [{
             type: shapeDetection.mojom.LandmarkType.EYE,
-            location: {x: 4.0, y: 5.0}
+            locations: [{x: 4.0, y: 5.0}]
+          },
+          {
+            type: shapeDetection.mojom.LandmarkType.EYE,
+            locations: [
+              {x: 4.0, y: 5.0}, {x: 5.0, y: 4.0}, {x: 6.0, y: 3.0},
+              {x: 7.0, y: 4.0}, {x: 8.0, y: 5.0}, {x: 7.0, y: 6.0},
+              {x: 6.0, y: 7.0}, {x: 5.0, y: 6.0}
+            ]
           }]
         },
         {

third_party/blink/renderer/modules/shapedetection/face_detector.cc

@@ -71,11 +71,13 @@ void FaceDetector::OnDetectFaces(
   for (const auto& face : face_detection_results) {
     HeapVector<Landmark> landmarks;
     for (const auto& landmark : face->landmarks) {
-      Point2D location;
-      location.setX(landmark->location.x);
-      location.setY(landmark->location.y);
       HeapVector<Point2D> locations;
-      locations.push_back(location);
+      for (const auto& location : landmark->locations) {
+        Point2D web_location;
+        web_location.setX(location.x);
+        web_location.setY(location.y);
+        locations.push_back(web_location);
+      }
 
       Landmark web_landmark;
       web_landmark.setLocations(locations);