ShapeDetection: Detect faces asynchronously on windows 10

Create Windows bitmap from detected SkBitmap that is Rgba8/Bgra8 color type,
then convert the pixel format SoftwareBitmap to Gray8/Nv12 that is supported
in current version.
Detect faces asynchronously with the created Windows bitmap, and pause incoming
method call to prevent the Detect function from being called before the
AsyncOperation completes.
Add unittest to test the result of detecting faces.

Split original large CL[1] up in smaller subpatches including this CL.
Link Face Detection demo[2] here.

[1] https://chromium-review.googlesource.com/c/chromium/src/+/708336
[2] https://codepen.io/miguelao/pen/PmJWro

BUG=767021

Cq-Include-Trybots: master.tryserver.chromium.mac:mac_optional_gpu_tests_rel;master.tryserver.chromium.win:win10_chromium_x64_rel_ng
Change-Id: If2f93ede374bdebc6921cee4b0c9a09df879b693
Reviewed-on: https://chromium-review.googlesource.com/787790Reviewed-by: Miguel Casas <mcasas@chromium.org>
Reviewed-by: Reilly Grant <reillyg@chromium.org>
Reviewed-by: Greg Thompson <grt@chromium.org>
Commit-Queue: Junwei Fu <junwei.fu@intel.com>
Cr-Commit-Position: refs/heads/master@{#523694}

services/shape_detection/BUILD.gn

@@ -31,6 +31,7 @@ source_set("lib") {
     sources -= [ "face_detection_provider_impl.h" ]
     sources += [
       "barcode_detection_impl.cc",
+      "detection_utils_win.cc",
       "detection_utils_win.h",
       "face_detection_impl_win.cc",
       "face_detection_impl_win.h",

services/shape_detection/detection_utils_win.cc

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "services/shape_detection/detection_utils_win.h"
+
+#include "base/numerics/checked_math.h"
+#include "base/win/winrt_storage_util.h"
+#include "third_party/skia/include/core/SkBitmap.h"
+
+namespace shape_detection {
+
+Microsoft::WRL::ComPtr<ISoftwareBitmap> CreateWinBitmapFromSkBitmap(
+    ISoftwareBitmapStatics* bitmap_factory,
+    BitmapPixelFormat pixel_format,
+    const SkBitmap& bitmap) {
+  DCHECK(bitmap_factory);
+  DCHECK_EQ(bitmap.colorType(), kN32_SkColorType);
+  if (!base::CheckedNumeric<uint32_t>(bitmap.computeByteSize()).IsValid()) {
+    DLOG(ERROR) << "Data overflow.";
+    return nullptr;
+  }
+
+  // Create IBuffer from bitmap data.
+  Microsoft::WRL::ComPtr<ABI::Windows::Storage::Streams::IBuffer> buffer;
+  HRESULT hr = base::win::CreateIBufferFromData(
+      static_cast<uint8_t*>(bitmap.getPixels()),
+      static_cast<UINT32>(bitmap.computeByteSize()), &buffer);
+  if (FAILED(hr)) {
+    DLOG(ERROR) << "Create IBuffer from bitmap data failed: "
+                << logging::SystemErrorCodeToString(hr);
+    return nullptr;
+  }
+
+  Microsoft::WRL::ComPtr<ISoftwareBitmap> win_bitmap;
+  const BitmapPixelFormat pixelFormat =
+      (kN32_SkColorType == kRGBA_8888_SkColorType)
+          ? ABI::Windows::Graphics::Imaging::BitmapPixelFormat_Rgba8
+          : ABI::Windows::Graphics::Imaging::BitmapPixelFormat_Bgra8;
+  // Create ISoftwareBitmap from SKBitmap that is kN32_SkColorType and copy the
+  // IBuffer into it.
+  hr = bitmap_factory->CreateCopyFromBuffer(buffer.Get(), pixelFormat,
+                                            bitmap.width(), bitmap.height(),
+                                            win_bitmap.GetAddressOf());
+  if (FAILED(hr)) {
+    DLOG(ERROR) << "Create ISoftwareBitmap from buffer failed: "
+                << logging::SystemErrorCodeToString(hr);
+    return nullptr;
+  }
+
+  // Convert Rgba8/Bgra8 to Gray8/Nv12 SoftwareBitmap.
+  hr = bitmap_factory->Convert(win_bitmap.Get(), pixel_format,
+                               win_bitmap.GetAddressOf());
+  if (FAILED(hr)) {
+    DLOG(ERROR) << "Convert Rgba8/Bgra8 to Gray8/Nv12 failed: "
+                << logging::SystemErrorCodeToString(hr);
+    return nullptr;
+  }
+
+  return win_bitmap;
+}
+
+}  // namespace shape_detection

services/shape_detection/detection_utils_win.h

@@ -5,21 +5,30 @@
 #ifndef SERVICES_SHAPE_DETECTION_DETECTION_UTILS_WIN_H_
 #define SERVICES_SHAPE_DETECTION_DETECTION_UTILS_WIN_H_
 
-#include <wrl\event.h>
-#include <wrl\implements.h>
+#include <windows.storage.streams.h>
+#include <wrl/client.h>
+#include <wrl/event.h>
+#include <wrl/implements.h>
+#include <memory>
 #include <utility>
 
 #include "base/bind.h"
 #include "base/callback.h"
 #include "base/location.h"
+#include "base/logging.h"
 #include "base/macros.h"
 #include "base/memory/ptr_util.h"
 #include "base/memory/ref_counted.h"
 #include "base/memory/weak_ptr.h"
 #include "base/sequenced_task_runner.h"
 
+class SkBitmap;
+
 using ABI::Windows::Foundation::IAsyncOperation;
 using ABI::Windows::Foundation::IAsyncOperationCompletedHandler;
+using ABI::Windows::Graphics::Imaging::ISoftwareBitmapStatics;
+using ABI::Windows::Graphics::Imaging::ISoftwareBitmap;
+using ABI::Windows::Graphics::Imaging::BitmapPixelFormat;
 
 namespace shape_detection {
 
@@ -27,7 +36,9 @@ namespace WRL = Microsoft::WRL;
 
 // This template represents an asynchronous operation which returns a result
 // upon completion, internal async callback will be not called if the instance
-// is deleted. RuntimeType is Windows Runtime APIs that have a result.
+// is deleted. RuntimeType is Windows Runtime APIs that has a result.
+// TODO(junwei.fu): https://crbug.com/791371 consider moving the implementation
+// of AsyncOperation to .cc file.
 template <typename RuntimeType>
 class AsyncOperation {
  public:
@@ -40,8 +51,8 @@ class AsyncOperation {
 
   ~AsyncOperation() = default;
 
-  // Creates an AsyncOperation instance which set Callback to be called when the
-  // asynchronous action completes.
+  // Creates an AsyncOperation instance which sets |callback| to be called when
+  // the asynchronous action completes.
   static std::unique_ptr<AsyncOperation<RuntimeType>> Create(
       Callback callback,
       IAsyncOperationPtr async_op_ptr) {
@@ -63,8 +74,8 @@ class AsyncOperation {
           // A reference to |async_op| is kept in |async_op_ptr_|, safe to pass
           // outside.  This is happening on an OS thread.
           task_runner->PostTask(
-              FROM_HERE, base::Bind(&AsyncOperation::AsyncCallbackInternal,
-                                    std::move(weak_ptr), async_op, status));
+              FROM_HERE, base::BindOnce(&AsyncOperation::AsyncCallbackInternal,
+                                        std::move(weak_ptr), async_op, status));
 
           return S_OK;
         });
@@ -96,11 +107,20 @@ class AsyncOperation {
 
   IAsyncOperationPtr async_op_ptr_;
   Callback callback_;
+  // TODO(junwei.fu): https://crbug.com/790843 guarantee |callback_| will be
+  // called instead of canceling the callback if this object is freed.
   base::WeakPtrFactory<AsyncOperation<RuntimeType>> weak_factory_;
 
   DISALLOW_COPY_AND_ASSIGN(AsyncOperation);
 };
 
+// Creates a Gray8/Nv12 ISoftwareBitmap from a kN32_SkColorType |bitmap|, or
+// returns nullptr.
+Microsoft::WRL::ComPtr<ISoftwareBitmap> CreateWinBitmapFromSkBitmap(
+    ISoftwareBitmapStatics* bitmap_factory,
+    BitmapPixelFormat pixel_format,
+    const SkBitmap& bitmap);
+
 }  // namespace shape_detection
 
-#endif  // SERVICES_SHAPE_DETECTION_DETECTION_UTILS_WIN_H_
+#endif  // SERVICES_SHAPE_DETECTION_DETECTION_UTILS_WIN_H_
\ No newline at end of file

services/shape_detection/face_detection_impl_win.cc

@@ -6,25 +6,118 @@
 
 #include <windows.media.faceanalysis.h>
 
+#include "base/bind.h"
+#include "base/logging.h"
+
 namespace shape_detection {
 
-base::OnceCallback<void(bool)> g_callback_for_testing;
+namespace {
+
+using ABI::Windows::Media::FaceAnalysis::DetectedFace;
+
+}  // namespace
 
 FaceDetectionImplWin::FaceDetectionImplWin(
-    Microsoft::WRL::ComPtr<IFaceDetectorStatics> factory,
-    Microsoft::WRL::ComPtr<IFaceDetector> face_detector)
-    : face_detector_factory_(std::move(factory)),
-      face_detector_(std::move(face_detector)) {}
+    Microsoft::WRL::ComPtr<IFaceDetector> face_detector,
+    Microsoft::WRL::ComPtr<ISoftwareBitmapStatics> bitmap_factory,
+    BitmapPixelFormat pixel_format)
+    : face_detector_(std::move(face_detector)),
+      bitmap_factory_(std::move(bitmap_factory)),
+      pixel_format_(pixel_format) {
+  DCHECK(face_detector_);
+  DCHECK(bitmap_factory_);
+}
 FaceDetectionImplWin::~FaceDetectionImplWin() = default;
 
 void FaceDetectionImplWin::Detect(const SkBitmap& bitmap,
                                   DetectCallback callback) {
-  DCHECK_EQ(kN32_SkColorType, bitmap.colorType());
-
-  if (g_callback_for_testing) {
-    std::move(g_callback_for_testing).Run(face_detector_ ? true : false);
+  if ((async_detect_face_ops_ = BeginDetect(bitmap))) {
+    // Hold on the callback until AsyncOperation completes.
+    detected_face_callback_ = std::move(callback);
+    // This prevents the Detect function from being called before the
+    // AsyncOperation completes.
+    binding_->PauseIncomingMethodCallProcessing();
+  } else {
+    // No detection taking place; run |callback| with an empty array of results.
     std::move(callback).Run(std::vector<mojom::FaceDetectionResultPtr>());
   }
 }
 
-}  // namespace shape_detection
+std::unique_ptr<AsyncOperation<IVector<DetectedFace*>>>
\ No newline at end of file
+FaceDetectionImplWin::BeginDetect(const SkBitmap& bitmap) {
+  Microsoft::WRL::ComPtr<ISoftwareBitmap> win_bitmap =
+      CreateWinBitmapFromSkBitmap(bitmap_factory_.Get(), pixel_format_, bitmap);
+  if (!win_bitmap)
+    return nullptr;
+
+  // Detect faces asynchronously.
+  AsyncOperation<IVector<DetectedFace*>>::IAsyncOperationPtr async_op;
+  const HRESULT hr =
+      face_detector_->DetectFacesAsync(win_bitmap.Get(), &async_op);
+  if (FAILED(hr)) {
+    DLOG(ERROR) << "Detect faces asynchronously failed: "
+                << logging::SystemErrorCodeToString(hr);
+    return nullptr;
+  }
+
+  // The once callback will not be called if this object is deleted, so it's
+  // fine to use Unretained to bind the callback.
+  // |win_bitmap| needs to be kept alive until OnFaceDetected().
+  return AsyncOperation<IVector<DetectedFace*>>::Create(
+      base::BindOnce(&FaceDetectionImplWin::OnFaceDetected,
+                     base::Unretained(this), std::move(win_bitmap)),
+      std::move(async_op));
+}
+
+std::vector<mojom::FaceDetectionResultPtr>
+FaceDetectionImplWin::BuildFaceDetectionResult(
+    AsyncOperation<IVector<DetectedFace*>>::IAsyncOperationPtr async_op) {
+  std::vector<mojom::FaceDetectionResultPtr> results;
+  Microsoft::WRL::ComPtr<IVector<DetectedFace*>> detected_face;
+  HRESULT hr =
+      async_op ? async_op->GetResults(detected_face.GetAddressOf()) : E_FAIL;
+  if (FAILED(hr)) {
+    DLOG(ERROR) << "GetResults failed: "
+                << logging::SystemErrorCodeToString(hr);
+    return results;
+  }
+
+  uint32_t count;
+  hr = detected_face->get_Size(&count);
+  if (FAILED(hr)) {
+    DLOG(ERROR) << "get_Size failed: " << logging::SystemErrorCodeToString(hr);
+    return results;
+  }
+
+  results.reserve(count);
+  for (uint32_t i = 0; i < count; i++) {
+    Microsoft::WRL::ComPtr<IDetectedFace> face;
+    hr = detected_face->GetAt(i, &face);
+    if (FAILED(hr))
+      break;
+
+    ABI::Windows::Graphics::Imaging::BitmapBounds bounds;
+    hr = face->get_FaceBox(&bounds);
+    if (FAILED(hr))
+      break;
+
+    auto result = shape_detection::mojom::FaceDetectionResult::New();
+    result->bounding_box =
+        gfx::RectF(bounds.X, bounds.Y, bounds.Width, bounds.Height);
+    results.push_back(std::move(result));
+  }
+  return results;
+}
+
+// The extra passing of |win_bitmap| need to be kept until AsyncOperation
+// completes, otherwise random crashes will happen.
+void FaceDetectionImplWin::OnFaceDetected(
+    Microsoft::WRL::ComPtr<ISoftwareBitmap> /* win_bitmap */,
+    AsyncOperation<IVector<DetectedFace*>>::IAsyncOperationPtr async_op) {
+  std::move(detected_face_callback_)
+      .Run(BuildFaceDetectionResult(std::move(async_op)));
+  async_detect_face_ops_.reset();
+  binding_->ResumeIncomingMethodCallProcessing();
+}
+
+}  // namespace shape_detection

services/shape_detection/face_detection_impl_win.h

@@ -6,51 +6,70 @@
 #define SERVICES_SHAPE_DETECTION_FACE_DETECTION_IMPL_WIN_H_
 
 #include <windows.foundation.h>
+#include <windows.graphics.imaging.h>
+#include <windows.media.faceanalysis.h>
 #include <wrl/client.h>
+#include <memory>
+#include <utility>
+#include <vector>
 
-#include "detection_utils_win.h"
+#include "base/macros.h"
+#include "mojo/public/cpp/bindings/strong_binding.h"
+#include "services/shape_detection/detection_utils_win.h"
 #include "services/shape_detection/public/interfaces/facedetection.mojom.h"
 
 class SkBitmap;
 
-namespace ABI {
-namespace Windows {
-namespace Media {
-namespace FaceAnalysis {
-interface IFaceDetectorStatics;
-interface IFaceDetector;
-class FaceDetector;
-}  // namespace FaceAnalysis
-}  // namespace Media
-}  // namespace Windows
-}  // namespace ABI
-
 namespace shape_detection {
 
-extern base::OnceCallback<void(bool)> g_callback_for_testing;
+using ABI::Windows::Foundation::Collections::IVector;
 
 class FaceDetectionImplWin : public mojom::FaceDetection {
  public:
-  using IFaceDetectorStatics =
-      ABI::Windows::Media::FaceAnalysis::IFaceDetectorStatics;
   using FaceDetector = ABI::Windows::Media::FaceAnalysis::FaceDetector;
   using IFaceDetector = ABI::Windows::Media::FaceAnalysis::IFaceDetector;
+  using DetectedFace = ABI::Windows::Media::FaceAnalysis::DetectedFace;
+  using IDetectedFace = ABI::Windows::Media::FaceAnalysis::IDetectedFace;
+  using BitmapPixelFormat = ABI::Windows::Graphics::Imaging::BitmapPixelFormat;
+  using ISoftwareBitmapStatics =
+      ABI::Windows::Graphics::Imaging::ISoftwareBitmapStatics;
 
-  FaceDetectionImplWin(Microsoft::WRL::ComPtr<IFaceDetectorStatics> factory,
-                       Microsoft::WRL::ComPtr<IFaceDetector> face_detector);
+  FaceDetectionImplWin(
+      Microsoft::WRL::ComPtr<IFaceDetector> face_detector,
+      Microsoft::WRL::ComPtr<ISoftwareBitmapStatics> bitmap_factory,
+      BitmapPixelFormat pixel_format);
   ~FaceDetectionImplWin() override;
 
+  void SetBinding(mojo::StrongBindingPtr<mojom::FaceDetection> binding) {
+    binding_ = std::move(binding);
+  }
+
   // mojom::FaceDetection implementation.
   void Detect(const SkBitmap& bitmap,
               mojom::FaceDetection::DetectCallback callback) override;
 
  private:
-  Microsoft::WRL::ComPtr<IFaceDetectorStatics> face_detector_factory_;
+  std::unique_ptr<AsyncOperation<IVector<DetectedFace*>>> BeginDetect(
+      const SkBitmap& bitmap);
+  std::vector<mojom::FaceDetectionResultPtr> BuildFaceDetectionResult(
+      AsyncOperation<IVector<DetectedFace*>>::IAsyncOperationPtr async_op);
+  void OnFaceDetected(
+      Microsoft::WRL::ComPtr<ISoftwareBitmap> win_bitmap,
+      AsyncOperation<IVector<DetectedFace*>>::IAsyncOperationPtr async_op);
+
   Microsoft::WRL::ComPtr<IFaceDetector> face_detector_;
 
+  Microsoft::WRL::ComPtr<ISoftwareBitmapStatics> bitmap_factory_;
+  BitmapPixelFormat pixel_format_;
+
+  std::unique_ptr<AsyncOperation<IVector<DetectedFace*>>>
+      async_detect_face_ops_;
+  DetectCallback detected_face_callback_;
+  mojo::StrongBindingPtr<mojom::FaceDetection> binding_;
+
   DISALLOW_COPY_AND_ASSIGN(FaceDetectionImplWin);
 };
 
 }  // namespace shape_detection
 
-#endif  // SERVICES_SHAPE_DETECTION_FACE_DETECTION_IMPL_WIN_H_
+#endif  // SERVICES_SHAPE_DETECTION_FACE_DETECTION_IMPL_WIN_H_
\ No newline at end of file

services/shape_detection/face_detection_impl_win_unittest.cc

@@ -2,72 +2,100 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-#include "services/shape_detection/face_detection_impl_win.h"
-
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "base/bind.h"
+#include "base/callback.h"
+#include "base/files/file_util.h"
+#include "base/macros.h"
+#include "base/path_service.h"
 #include "base/run_loop.h"
 #include "base/test/scoped_task_environment.h"
 #include "base/win/scoped_com_initializer.h"
-#include "face_detection_impl_win.h"
-#include "face_detection_provider_win.h"
+#include "base/win/windows_version.h"
+#include "services/shape_detection/face_detection_provider_win.h"
 #include "services/shape_detection/public/interfaces/facedetection_provider.mojom.h"
 #include "testing/gtest/include/gtest/gtest.h"
+#include "ui/gfx/codec/jpeg_codec.h"
 
 namespace shape_detection {
 
+namespace {
+void DetectCallback(base::Closure quit_closure,
+                    uint32_t* num_faces,
+                    std::vector<mojom::FaceDetectionResultPtr> results) {
+  *num_faces = results.size();
+  quit_closure.Run();
+}
+}  // namespace
+
 class FaceDetectionImplWinTest : public testing::Test {
  protected:
+  FaceDetectionImplWinTest() = default;
+  ~FaceDetectionImplWinTest() override = default;
+
   void SetUp() override {
     scoped_com_initializer_ = std::make_unique<base::win::ScopedCOMInitializer>(
         base::win::ScopedCOMInitializer::kMTA);
     ASSERT_TRUE(scoped_com_initializer_->Succeeded());
   }
 
- public:
-  void CreateFaceDetectorCallback(bool succeeded) { EXPECT_TRUE(succeeded); }
-
-  void DetectCallback(base::Closure quit_closure,
-                      std::vector<mojom::FaceDetectionResultPtr> results) {
-    CloseConnection(quit_closure);
-  }
-
-  void CloseConnection(base::Closure quit_closure) { quit_closure.Run(); }
-
  private:
   std::unique_ptr<base::win::ScopedCOMInitializer> scoped_com_initializer_;
 
   base::test::ScopedTaskEnvironment scoped_task_environment_;
+
+  DISALLOW_COPY_AND_ASSIGN(FaceDetectionImplWinTest);
 };
 
-TEST_F(FaceDetectionImplWinTest, CreateFaceDetector) {
+TEST_F(FaceDetectionImplWinTest, ScanOneFace) {
+  // FaceDetector not supported before Windows 10
+  if (base::win::GetVersion() < base::win::VERSION_WIN10)
+    return;
+
   mojom::FaceDetectionProviderPtr provider;
   mojom::FaceDetectionPtr face_service;
 
   auto request = mojo::MakeRequest(&provider);
-  auto providerWin = base::MakeUnique<FaceDetectionProviderWin>();
-  auto* provider_ptr = providerWin.get();
+  auto provider_win = std::make_unique<FaceDetectionProviderWin>();
+  auto* provider_ptr = provider_win.get();
+  // A raw pointer is obtained because ownership is passed to MakeStrongBinding.
   provider_ptr->binding_ =
-      mojo::MakeStrongBinding(std::move(providerWin), std::move(request));
+      mojo::MakeStrongBinding(std::move(provider_win), std::move(request));
 
   auto options = shape_detection::mojom::FaceDetectorOptions::New();
   provider->CreateFaceDetection(mojo::MakeRequest(&face_service),
                                 std::move(options));
 
-  SkBitmap bitmap;
-  bitmap.allocN32Pixels(100, 100);
-  bitmap.eraseColor(SK_ColorBLUE);
+  // Load image data from test directory.
+  base::FilePath image_path;
+  ASSERT_TRUE(PathService::Get(base::DIR_SOURCE_ROOT, &image_path));
+  image_path = image_path.Append(FILE_PATH_LITERAL("services"))
+                   .Append(FILE_PATH_LITERAL("test"))
+                   .Append(FILE_PATH_LITERAL("data"))
+                   .Append(FILE_PATH_LITERAL("mona_lisa.jpg"));
+  ASSERT_TRUE(base::PathExists(image_path));
+  std::string image_data;
+  ASSERT_TRUE(base::ReadFileToString(image_path, &image_data));
+
+  std::unique_ptr<SkBitmap> image = gfx::JPEGCodec::Decode(
+      reinterpret_cast<const uint8_t*>(image_data.data()), image_data.size());
+  ASSERT_TRUE(image);
+
+  const gfx::Size size(image->width(), image->height());
+  const uint32_t num_bytes = size.GetArea() * 4 /* bytes per pixel */;
+  ASSERT_EQ(num_bytes, image->computeByteSize());
 
   base::RunLoop run_loop;
-  g_callback_for_testing =
-      base::BindOnce(&FaceDetectionImplWinTest::CreateFaceDetectorCallback,
-                     base::Unretained(this));
-
-  face_service.set_connection_error_handler(
-      base::Bind(&FaceDetectionImplWinTest::CloseConnection,
-                 base::Unretained(this), run_loop.QuitClosure()));
+  uint32_t num_faces = 0;
   face_service->Detect(
-      bitmap, base::BindOnce(&FaceDetectionImplWinTest::DetectCallback,
-                             base::Unretained(this), run_loop.QuitClosure()));
+      *image,
+      base::BindOnce(&DetectCallback, run_loop.QuitClosure(), &num_faces));
   run_loop.Run();
+  EXPECT_EQ(1u, num_faces);
 }
 
 }  // namespace shape_detection

services/shape_detection/face_detection_provider_win.cc

@@ -6,6 +6,8 @@
 
 #include <windows.media.faceanalysis.h>
 
+#include "base/bind.h"
+#include "base/logging.h"
 #include "base/scoped_generic.h"
 #include "base/win/core_winrt_util.h"
 #include "base/win/scoped_hstring.h"
@@ -15,10 +17,28 @@
 
 namespace shape_detection {
 
-using ABI::Windows::Media::FaceAnalysis::FaceDetector;
+namespace {
+
+using ABI::Windows::Media::FaceAnalysis::IFaceDetectorStatics;
 using base::win::ScopedHString;
 using base::win::GetActivationFactory;
 
+BitmapPixelFormat GetPreferredPixelFormat(IFaceDetectorStatics* factory) {
+  static constexpr BitmapPixelFormat kFormats[] = {
+      ABI::Windows::Graphics::Imaging::BitmapPixelFormat_Gray8,
+      ABI::Windows::Graphics::Imaging::BitmapPixelFormat_Nv12};
+
+  for (const auto& format : kFormats) {
+    boolean is_supported = false;
+    factory->IsBitmapPixelFormatSupported(format, &is_supported);
+    if (is_supported)
+      return format;
+  }
+  return ABI::Windows::Graphics::Imaging::BitmapPixelFormat_Unknown;
+}
+
+}  // namespace
+
 void FaceDetectionProviderWin::CreateFaceDetection(
     shape_detection::mojom::FaceDetectionRequest request,
     shape_detection::mojom::FaceDetectorOptionsPtr options) {
@@ -51,10 +71,19 @@ void FaceDetectionProviderWin::CreateFaceDetection(
     return;
   }
 
-  boolean is_supported = FALSE;
+  boolean is_supported = false;
   factory->get_IsSupported(&is_supported);
-  if (is_supported == FALSE)
+  if (!is_supported)
+    return;
+
+  // In the current version, the FaceDetector class only supports images in
+  // Gray8 or Nv12. Gray8 should be a good type but verify it against
+  // FaceDetector’s supported formats.
+  BitmapPixelFormat pixel_format = GetPreferredPixelFormat(factory.Get());
+  if (pixel_format ==
+      ABI::Windows::Graphics::Imaging::BitmapPixelFormat_Unknown) {
     return;
+  }
 
   // Create an instance of FaceDetector asynchronously.
   AsyncOperation<FaceDetector>::IAsyncOperationPtr async_op;
@@ -69,8 +98,7 @@ void FaceDetectionProviderWin::CreateFaceDetection(
   // fine to use Unretained to bind the callback.
   auto async_operation = AsyncOperation<FaceDetector>::Create(
       base::BindOnce(&FaceDetectionProviderWin::OnFaceDetectorCreated,
-                     base::Unretained(this), std::move(request),
-                     std::move(factory)),
+                     base::Unretained(this), std::move(request), pixel_format),
       std::move(async_op));
   if (!async_operation)
     return;
@@ -88,13 +116,13 @@ FaceDetectionProviderWin::~FaceDetectionProviderWin() = default;
 
 void FaceDetectionProviderWin::OnFaceDetectorCreated(
     shape_detection::mojom::FaceDetectionRequest request,
-    Microsoft::WRL::ComPtr<IFaceDetectorStatics> factory,
+    BitmapPixelFormat pixel_format,
     AsyncOperation<FaceDetector>::IAsyncOperationPtr async_op) {
   binding_->ResumeIncomingMethodCallProcessing();
   async_create_detector_ops_.reset();
 
   Microsoft::WRL::ComPtr<IFaceDetector> face_detector;
-  const HRESULT hr =
+  HRESULT hr =
       async_op ? async_op->GetResults(face_detector.GetAddressOf()) : E_FAIL;
   if (FAILED(hr)) {
     DLOG(ERROR) << "GetResults failed: "
@@ -102,9 +130,21 @@ void FaceDetectionProviderWin::OnFaceDetectorCreated(
     return;
   }
 
-  mojo::MakeStrongBinding(base::MakeUnique<FaceDetectionImplWin>(
-                              std::move(factory), std::move(face_detector)),
-                          std::move(request));
+  Microsoft::WRL::ComPtr<ISoftwareBitmapStatics> bitmap_factory;
+  hr = GetActivationFactory<
+      ISoftwareBitmapStatics,
+      RuntimeClass_Windows_Graphics_Imaging_SoftwareBitmap>(&bitmap_factory);
+  if (FAILED(hr)) {
+    DLOG(ERROR) << "ISoftwareBitmapStatics factory failed: "
+                << logging::SystemErrorCodeToString(hr);
+    return;
+  }
+
+  auto impl = std::make_unique<FaceDetectionImplWin>(
+      std::move(face_detector), std::move(bitmap_factory), pixel_format);
+  auto* impl_ptr = impl.get();
+  impl_ptr->SetBinding(
+      mojo::MakeStrongBinding(std::move(impl), std::move(request)));
 }
 
 }  // namespace shape_detection

services/shape_detection/face_detection_provider_win.h

@@ -2,14 +2,18 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-#ifndef SERVICES_SHAPE_DETECTION_FACE_DETECTION_WIN_PROVIDER_H_
-#define SERVICES_SHAPE_DETECTION_FACE_DETECTION_WIN_PROVIDER_H_
+#ifndef SERVICES_SHAPE_DETECTION_FACE_DETECTION_PROVIDER_WIN_H_
+#define SERVICES_SHAPE_DETECTION_FACE_DETECTION_PROVIDER_WIN_H_
 
 #include <windows.foundation.h>
+#include <windows.graphics.imaging.h>
+#include <memory>
+#include <utility>
 
-#include "detection_utils_win.h"
-#include "face_detection_impl_win.h"
+#include "base/macros.h"
 #include "mojo/public/cpp/bindings/strong_binding.h"
+#include "services/shape_detection/detection_utils_win.h"
+#include "services/shape_detection/face_detection_impl_win.h"
 #include "services/shape_detection/public/interfaces/facedetection_provider.mojom.h"
 
 namespace shape_detection {
@@ -17,17 +21,16 @@ namespace shape_detection {
 class FaceDetectionProviderWin
     : public shape_detection::mojom::FaceDetectionProvider {
  public:
-  using IFaceDetectorStatics =
-      ABI::Windows::Media::FaceAnalysis::IFaceDetectorStatics;
   using FaceDetector = ABI::Windows::Media::FaceAnalysis::FaceDetector;
   using IFaceDetector = ABI::Windows::Media::FaceAnalysis::IFaceDetector;
+  using BitmapPixelFormat = ABI::Windows::Graphics::Imaging::BitmapPixelFormat;
 
   FaceDetectionProviderWin();
   ~FaceDetectionProviderWin() override;
 
   static void Create(
       shape_detection::mojom::FaceDetectionProviderRequest request) {
-    auto provider = base::MakeUnique<FaceDetectionProviderWin>();
+    auto provider = std::make_unique<FaceDetectionProviderWin>();
     auto* provider_ptr = provider.get();
     provider_ptr->binding_ =
         mojo::MakeStrongBinding(std::move(provider), std::move(request));
@@ -40,10 +43,10 @@ class FaceDetectionProviderWin
  private:
   void OnFaceDetectorCreated(
       shape_detection::mojom::FaceDetectionRequest request,
-      Microsoft::WRL::ComPtr<IFaceDetectorStatics> factory,
+      BitmapPixelFormat pixel_format,
       AsyncOperation<FaceDetector>::IAsyncOperationPtr async_op);
 
-  FRIEND_TEST_ALL_PREFIXES(FaceDetectionImplWinTest, CreateFaceDetector);
+  FRIEND_TEST_ALL_PREFIXES(FaceDetectionImplWinTest, ScanOneFace);
   mojo::StrongBindingPtr<mojom::FaceDetectionProvider> binding_;
   std::unique_ptr<AsyncOperation<FaceDetector>> async_create_detector_ops_;
 
@@ -52,4 +55,4 @@ class FaceDetectionProviderWin
 
 }  // namespace shape_detection
 
-#endif  // SERVICES_SHAPE_DETECTION_FACE_DETECTION_WIN_PROVIDER_H_
+#endif  // SERVICES_SHAPE_DETECTION_FACE_DETECTION_PROVIDER_WIN_H_