VideoCaptureDeviceMac: Match output and capture format

This relates to the following two classes used when capturing
content on macOS:
- AVCaptureDevice which does the actual capturing from the device
  into IOSurfaces
- AVCaptureVideoDataOutput takes those IOSurfaces and wrangles
  them into CMSampleBuffers that are handed to us

Bug 1: Not telling AVCaptureDevice our preferences
  When Chrome wants a specific (width, height, framerate)
  combination, we request that, along with a pixel format,
  to AVCaptureVideoDataOutput.

  While we indicate our preferences to our
  AVCaptureVideoDataOutput, we do not indicate them to our
  AVCaptureDevice.

  As a result, the AVCaptureDevice will often have a completely
  different format that it's using, and the aforementioned
  "wrangling" will involve a format conversion and maybe even
  some downsampling(!). In bad cases, this can burn 10% of a
  CPU core on its own (fortunately bad cases are rare).

Bug 2: Using an unsupported pixel format
  We currently prefer "422YpCbCr8" aka "UYVY" aka "2vuy", and
  request it regardless of whether or not it is supported by
  the capture. Most devices don't actually support it, and so
  the wrangling starts up again.

The fix for these bugs is to trawl through the list of supported
pixel formats to find the one that best matches the requested
format, and then specify this to our AVCaptureDevice when we
start capture.

Do this by adding a FindBestCaptureFormat to do the trawling.

Also don't prefer the "422YpCbCr8" aka "UYVY" aka "2vuy" format
above supported formats.

The consequence of this is that many Facetime cameras will now
be outputting "422YpCbCr8_yuvs" aka "YUY2" aka "yuvs", and will
hopefully use less CPU.

When we do get an exact match between input and output, we
also gain access to the IOSurface that the camera's data
is being thrown into. This could allow a zero copy path all
the way through to the compositor. Eventually.

Bug: 1124884
Change-Id: I33b4b62e152ba9f3b2be04dc3a5f3084d6ee9438
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2391767
Commit-Queue: ccameron <ccameron@chromium.org>
Reviewed-by: Miguel Casas <mcasas@chromium.org>
Cr-Commit-Position: refs/heads/master@{#804584}

media/capture/BUILD.gn

@@ -370,6 +370,7 @@ test("capture_unittests") {
     "video/linux/v4l2_capture_delegate_unittest.cc",
     "video/linux/video_capture_device_factory_linux_unittest.cc",
     "video/mac/video_capture_device_factory_mac_unittest.mm",
+    "video/mac/video_capture_device_mac_unittest.mm",
     "video/mock_gpu_memory_buffer_manager.cc",
     "video/mock_gpu_memory_buffer_manager.h",
     "video/video_capture_device_client_unittest.cc",
@@ -413,6 +414,13 @@ test("capture_unittests") {
     ]
   }
 
+  if (is_mac) {
+    frameworks = [
+      "AVFoundation.framework",
+      "CoreMedia.framework",
+    ]
+  }
+
   if (is_win) {
     sources += [
       "video/win/video_capture_device_factory_win_unittest.cc",

media/capture/video/mac/video_capture_device_avfoundation_mac.h

@@ -16,7 +16,15 @@
 
 namespace media {
 class VideoCaptureDeviceMac;
-}
+
+// Find the best capture format from |formats| for the specified dimensions and
+// frame rate. Returns an element of |formats|, or nil.
+AVCaptureDeviceFormat* CAPTURE_EXPORT
+FindBestCaptureFormat(NSArray<AVCaptureDeviceFormat*>* formats,
+                      int width,
+                      int height,
+                      float frame_rate);
+}  // namespace media
 
 // Class used by VideoCaptureDeviceMac (VCDM) for video and image capture using
 // AVFoundation API. This class lives inside the thread created by its owner
@@ -58,6 +66,9 @@ class VideoCaptureDeviceMac;
   int _frameHeight;
   float _frameRate;
 
+  // The capture format that best matches the above attributes.
+  base::scoped_nsobject<AVCaptureDeviceFormat> _bestCaptureFormat;
+
   base::Lock _lock;  // Protects concurrent setting and using |frameReceiver_|.
   media::VideoCaptureDeviceMac* _frameReceiver;  // weak.
 

media/capture/video/mac/video_capture_device_avfoundation_mac.mm

@@ -18,6 +18,7 @@
 #include "base/strings/string_util.h"
 #include "base/strings/sys_string_conversions.h"
 #include "media/base/timestamp_constants.h"
+#include "media/base/video_types.h"
 #include "media/capture/video/mac/video_capture_device_factory_mac.h"
 #include "media/capture/video/mac/video_capture_device_mac.h"
 #include "media/capture/video_capture_types.h"
@@ -177,6 +178,83 @@ void ExtractBaseAddressAndLength(char** base_address,
 
 }  // anonymous namespace
 
+namespace media {
+
+// Find the best capture format from |formats| for the specified dimensions and
+// frame rate. Returns an element of |formats|, or nil.
+AVCaptureDeviceFormat* FindBestCaptureFormat(
+    NSArray<AVCaptureDeviceFormat*>* formats,
+    int width,
+    int height,
+    float frame_rate) {
+  AVCaptureDeviceFormat* bestCaptureFormat = nil;
+  VideoPixelFormat bestPixelFormat = VideoPixelFormat::PIXEL_FORMAT_UNKNOWN;
+  bool bestMatchesFrameRate = false;
+  Float64 bestMaxFrameRate = 0;
+
+  for (AVCaptureDeviceFormat* captureFormat in formats) {
+    const FourCharCode fourcc =
+        CMFormatDescriptionGetMediaSubType([captureFormat formatDescription]);
+    VideoPixelFormat pixelFormat = FourCCToChromiumPixelFormat(fourcc);
+    CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(
+        [captureFormat formatDescription]);
+    Float64 maxFrameRate = 0;
+    bool matchesFrameRate = false;
+    for (AVFrameRateRange* frameRateRange in
+         [captureFormat videoSupportedFrameRateRanges]) {
+      maxFrameRate = std::max(maxFrameRate, [frameRateRange maxFrameRate]);
+      matchesFrameRate |= [frameRateRange minFrameRate] <= frame_rate &&
+                          frame_rate <= [frameRateRange maxFrameRate];
+    }
+
+    // If the pixel format is unsupported by our code, then it is not useful.
+    if (pixelFormat == VideoPixelFormat::PIXEL_FORMAT_UNKNOWN)
+      continue;
+
+    // If our CMSampleBuffers will have a different size than the native
+    // capture, then we will not be the fast path.
+    if (dimensions.width != width || dimensions.height != height)
+      continue;
+
+    // Prefer a capture format that handles the requested framerate to one
+    // that doesn't.
+    if (bestCaptureFormat) {
+      if (bestMatchesFrameRate && !matchesFrameRate)
+        continue;
+      if (matchesFrameRate && !bestMatchesFrameRate)
+        bestCaptureFormat = nil;
+    }
+
+    // Prefer a capture format with a lower maximum framerate, under the
+    // assumption that that may have lower power consumption.
+    if (bestCaptureFormat) {
+      if (bestMaxFrameRate < maxFrameRate)
+        continue;
+      if (maxFrameRate < bestMaxFrameRate)
+        bestCaptureFormat = nil;
+    }
+
+    // Finally, compare according to Chromium preference.
+    if (bestCaptureFormat) {
+      if (VideoCaptureFormat::ComparePixelFormatPreference(bestPixelFormat,
+                                                           pixelFormat)) {
+        continue;
+      }
+    }
+
+    bestCaptureFormat = captureFormat;
+    bestPixelFormat = pixelFormat;
+    bestMaxFrameRate = maxFrameRate;
+    bestMatchesFrameRate = matchesFrameRate;
+  }
+
+  VLOG(1) << "Selecting AVCaptureDevice format "
+          << VideoPixelFormatToString(bestPixelFormat);
+  return bestCaptureFormat;
+}
+
+}  // namespace media
+
 @implementation VideoCaptureDeviceAVFoundation
 
 #pragma mark Class methods
@@ -345,17 +423,15 @@ void ExtractBaseAddressAndLength(char** base_address,
   _frameWidth = width;
   _frameHeight = height;
   _frameRate = frameRate;
+  _bestCaptureFormat.reset(
+      media::FindBestCaptureFormat([_captureDevice formats], width, height,
+                                   frameRate),
+      base::scoped_policy::RETAIN);
 
   FourCharCode best_fourcc = kCMPixelFormat_422YpCbCr8;
-  for (AVCaptureDeviceFormat* format in [_captureDevice formats]) {
-    const FourCharCode fourcc =
-        CMFormatDescriptionGetMediaSubType([format formatDescription]);
-    // Compare according to Chromium preference.
-    if (media::VideoCaptureFormat::ComparePixelFormatPreference(
-            FourCCToChromiumPixelFormat(fourcc),
-            FourCCToChromiumPixelFormat(best_fourcc))) {
-      best_fourcc = fourcc;
-    }
+  if (_bestCaptureFormat) {
+    best_fourcc = CMFormatDescriptionGetMediaSubType(
+        [_bestCaptureFormat formatDescription]);
   }
 
   if (best_fourcc == kCMVideoCodecType_JPEG_OpenDML) {
@@ -414,6 +490,16 @@ void ExtractBaseAddressAndLength(char** base_address,
              name:AVCaptureSessionRuntimeErrorNotification
            object:_captureSession];
   [_captureSession startRunning];
+
+  // Update the active capture format once the capture session is running.
+  // Setting it before the capture session is running has no effect.
+  if (_bestCaptureFormat) {
+    if ([_captureDevice lockForConfiguration:nil]) {
+      [_captureDevice setActiveFormat:_bestCaptureFormat];
+      [_captureDevice unlockForConfiguration];
+    }
+  }
+
   return YES;
 }
 

media/capture/video/mac/video_capture_device_mac_unittest.mm

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/capture/video/mac/video_capture_device_avfoundation_mac.h"
+
+#include "base/mac/scoped_cftyperef.h"
+#include "base/mac/scoped_nsobject.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+// Create a subclass of AVFrameRateRange because there is no API to initialize
+// a custom AVFrameRateRange.
+@interface FakeAVFrameRateRange : AVFrameRateRange {
+  Float64 _minFrameRate;
+  Float64 _maxFrameRate;
+}
+- (id)initWithMinFrameRate:(Float64)minFrameRate
+              maxFrameRate:(Float64)maxFrameRate;
+@end
+
+@implementation FakeAVFrameRateRange
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wobjc-designated-initializers"
+- (id)initWithMinFrameRate:(Float64)minFrameRate
+              maxFrameRate:(Float64)maxFrameRate {
+  _minFrameRate = minFrameRate;
+  _maxFrameRate = maxFrameRate;
+  return self;
+}
+- (void)dealloc {
+  [super dealloc];
+}
+- (Float64)minFrameRate {
+  return _minFrameRate;
+}
+- (Float64)maxFrameRate {
+  return _maxFrameRate;
+}
+@end
+
+// Create a subclass of AVCaptureDeviceFormat because there is no API to
+// initialize a custom AVCaptureDeviceFormat.
+@interface FakeAVCaptureDeviceFormat : AVCaptureDeviceFormat {
+  base::ScopedCFTypeRef<CMVideoFormatDescriptionRef> _formatDescription;
+  base::scoped_nsobject<FakeAVFrameRateRange> _frameRateRange1;
+  base::scoped_nsobject<FakeAVFrameRateRange> _frameRateRange2;
+};
+- (id)initWithWidth:(int)width
+             height:(int)height
+             fourCC:(FourCharCode)fourCC
+          frameRate:(Float64)frameRate;
+- (void)setSecondFrameRate:(Float64)frameRate;
+@end
+
+@implementation FakeAVCaptureDeviceFormat
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wobjc-designated-initializers"
+- (id)initWithWidth:(int)width
+             height:(int)height
+             fourCC:(FourCharCode)fourCC
+          frameRate:(Float64)frameRate {
+  CMVideoFormatDescriptionCreate(nullptr, fourCC, width, height, nullptr,
+                                 _formatDescription.InitializeInto());
+  _frameRateRange1.reset([[FakeAVFrameRateRange alloc]
+      initWithMinFrameRate:frameRate
+              maxFrameRate:frameRate]);
+  return self;
+}
+#pragma clang diagnostic pop
+
+- (void)setSecondFrameRate:(Float64)frameRate {
+  _frameRateRange2.reset([[FakeAVFrameRateRange alloc]
+      initWithMinFrameRate:frameRate
+              maxFrameRate:frameRate]);
+}
+
+- (CMFormatDescriptionRef)formatDescription {
+  return _formatDescription;
+}
+
+- (NSArray<AVFrameRateRange*>*)videoSupportedFrameRateRanges {
+  return _frameRateRange2 ? @[ _frameRateRange1, _frameRateRange2 ]
+                          : @[ _frameRateRange1 ];
+}
+@end
+
+namespace media {
+
+// Test the behavior of the function FindBestCaptureFormat which is used to
+// determine the capture format.
+TEST(VideoCaptureDeviceMacTest, FindBestCaptureFormat) {
+  base::scoped_nsobject<FakeAVCaptureDeviceFormat> fmt_320_240_xyzw_30(
+      [[FakeAVCaptureDeviceFormat alloc] initWithWidth:320
+                                                height:240
+                                                fourCC:'xyzw'
+                                             frameRate:30]);
+
+  base::scoped_nsobject<FakeAVCaptureDeviceFormat> fmt_320_240_yuvs_30(
+      [[FakeAVCaptureDeviceFormat alloc] initWithWidth:320
+                                                height:240
+                                                fourCC:'yuvs'
+                                             frameRate:30]);
+  base::scoped_nsobject<FakeAVCaptureDeviceFormat> fmt_640_480_yuvs_30(
+      [[FakeAVCaptureDeviceFormat alloc] initWithWidth:640
+                                                height:480
+                                                fourCC:'yuvs'
+                                             frameRate:30]);
+
+  base::scoped_nsobject<FakeAVCaptureDeviceFormat> fmt_320_240_2vuy_30(
+      [[FakeAVCaptureDeviceFormat alloc] initWithWidth:320
+                                                height:240
+                                                fourCC:'2vuy'
+                                             frameRate:30]);
+  base::scoped_nsobject<FakeAVCaptureDeviceFormat> fmt_640_480_2vuy_30(
+      [[FakeAVCaptureDeviceFormat alloc] initWithWidth:640
+                                                height:480
+                                                fourCC:'2vuy'
+                                             frameRate:30]);
+  base::scoped_nsobject<FakeAVCaptureDeviceFormat> fmt_640_480_2vuy_60(
+      [[FakeAVCaptureDeviceFormat alloc] initWithWidth:640
+                                                height:480
+                                                fourCC:'2vuy'
+                                             frameRate:30]);
+  base::scoped_nsobject<FakeAVCaptureDeviceFormat> fmt_640_480_2vuy_30_60(
+      [[FakeAVCaptureDeviceFormat alloc] initWithWidth:640
+                                                height:480
+                                                fourCC:'2vuy'
+                                             frameRate:30]);
+  [fmt_640_480_2vuy_30_60 setSecondFrameRate:60];
+
+  // We'll be using this for the result in all of the below tests. Note that
+  // in all of the tests, the test is run with the candidate capture functions
+  // in two orders (forward and reversed). This is to avoid having the traversal
+  // order of FindBestCaptureFormat affect the result.
+  AVCaptureDeviceFormat* result = nil;
+
+  // If we can't find a valid format, we should return nil;
+  result = FindBestCaptureFormat(@[ fmt_320_240_xyzw_30 ], 320, 240, 30);
+  EXPECT_EQ(result, nil);
+
+  // Can't find a matching resolution
+  result = FindBestCaptureFormat(@[ fmt_320_240_yuvs_30, fmt_320_240_2vuy_30 ],
+                                 640, 480, 30);
+  EXPECT_EQ(result, nil);
+  result = FindBestCaptureFormat(@[ fmt_320_240_2vuy_30, fmt_320_240_yuvs_30 ],
+                                 640, 480, 30);
+  EXPECT_EQ(result, nil);
+
+  // Simple exact match.
+  result = FindBestCaptureFormat(@[ fmt_640_480_yuvs_30, fmt_320_240_yuvs_30 ],
+                                 320, 240, 30);
+  EXPECT_EQ(result, fmt_320_240_yuvs_30.get());
+  result = FindBestCaptureFormat(@[ fmt_320_240_yuvs_30, fmt_640_480_yuvs_30 ],
+                                 320, 240, 30);
+  EXPECT_EQ(result, fmt_320_240_yuvs_30.get());
+
+  // Different frame rate.
+  result = FindBestCaptureFormat(@[ fmt_640_480_2vuy_30 ], 640, 480, 60);
+  EXPECT_EQ(result, fmt_640_480_2vuy_30.get());
+
+  // Prefer the same frame rate.
+  result = FindBestCaptureFormat(@[ fmt_640_480_yuvs_30, fmt_640_480_2vuy_60 ],
+                                 640, 480, 60);
+  EXPECT_EQ(result, fmt_640_480_2vuy_60.get());
+  result = FindBestCaptureFormat(@[ fmt_640_480_2vuy_60, fmt_640_480_yuvs_30 ],
+                                 640, 480, 60);
+  EXPECT_EQ(result, fmt_640_480_2vuy_60.get());
+
+  // Prefer version with matching frame rate.
+  result = FindBestCaptureFormat(@[ fmt_640_480_yuvs_30, fmt_640_480_2vuy_60 ],
+                                 640, 480, 60);
+  EXPECT_EQ(result, fmt_640_480_2vuy_60.get());
+  result = FindBestCaptureFormat(@[ fmt_640_480_2vuy_60, fmt_640_480_yuvs_30 ],
+                                 640, 480, 60);
+  EXPECT_EQ(result, fmt_640_480_2vuy_60.get());
+
+  // Prefer version with matching frame rate when there are multiple framerates.
+  result = FindBestCaptureFormat(
+      @[ fmt_640_480_yuvs_30, fmt_640_480_2vuy_30_60 ], 640, 480, 60);
+  EXPECT_EQ(result, fmt_640_480_2vuy_30_60.get());
+  result = FindBestCaptureFormat(
+      @[ fmt_640_480_2vuy_30_60, fmt_640_480_yuvs_30 ], 640, 480, 60);
+  EXPECT_EQ(result, fmt_640_480_2vuy_30_60.get());
+
+  // Prefer version with the lower maximum framerate when there are multiple
+  // framerates.
+  result = FindBestCaptureFormat(
+      @[ fmt_640_480_2vuy_30, fmt_640_480_2vuy_30_60 ], 640, 480, 30);
+  EXPECT_EQ(result, fmt_640_480_2vuy_30.get());
+  result = FindBestCaptureFormat(
+      @[ fmt_640_480_2vuy_30_60, fmt_640_480_2vuy_30 ], 640, 480, 30);
+  EXPECT_EQ(result, fmt_640_480_2vuy_30.get());
+
+  // Prefer the Chromium format order.
+  result = FindBestCaptureFormat(@[ fmt_640_480_yuvs_30, fmt_640_480_2vuy_30 ],
+                                 640, 480, 30);
+  EXPECT_EQ(result, fmt_640_480_2vuy_30.get());
+  result = FindBestCaptureFormat(@[ fmt_640_480_2vuy_30, fmt_640_480_yuvs_30 ],
+                                 640, 480, 30);
+  EXPECT_EQ(result, fmt_640_480_2vuy_30.get());
+}
+
+}  // namespace media