[macOS] In-capturer convert/rescale class: SampleBufferTransformer.

This class supports pixel conversions and rescaling, backed up by both
libyuv functions for SW conversions and PixelBufferTransferer for HW
conversions. It supports conversions from all supported capture formats
(NV12, YUY2, UYVY, MJPEG) to NV12 and I420. (The pixel transfer path
supports even more destination pixel formats, but libyuv's destination
must be NV12 or I420, and these are the only two of interest to us.)

Whether HW or SW conversion is used, the resulting buffer is put in an
IOSurface-backed pixel buffer.

Code paths:
- When we have a pixel buffer, i.e. this is NOT MJPEG:
  - PixelBufferTransferer: X -> I420/NV12 in a single step.
  - libyuv: X -> I420 in a single step. X -> NV12 may involve an
    intermediate step X -> I420 -> NV12. Conversion and rescaling are
    done in separate steps.
- MJPEG (when we only have a sample buffer):
  - PixelBufferTransferer: Not supported.
  - libyuv: MJPEG -> I420 in a single step. MJPEG -> NV12 in two steps
    MJPEG -> I420 -> NV12. Conversion and rescaling are done in separate
    steps.

In a follow-up, we will do MJPEG -> NV12 in a single step based on the
libyuv CL that eshr@ recently landed.

Measurements show that which conversion method is most efficient is
dependent on input and output pixel formats and may even be affected by
cache hits/misses. In future CLs, we should wire up and measure the
optimal SampleBufferTransformer configurations (in which formats to use
libyuv and in which formats to use pixel transfer).

Bug: chromium:1132299
Change-Id: I4f6053c65c0c3b02483d130e25bebd1f656abd89
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2498563
Commit-Queue: Henrik Boström <hbos@chromium.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@chromium.org>
Cr-Commit-Position: refs/heads/master@{#823502}

media/capture/BUILD.gn

@@ -173,6 +173,8 @@ component("capture_lib") {
       "video/mac/pixel_buffer_pool_mac.h",
       "video/mac/pixel_buffer_transferer_mac.cc",
       "video/mac/pixel_buffer_transferer_mac.h",
+      "video/mac/sample_buffer_transformer_mac.cc",
+      "video/mac/sample_buffer_transformer_mac.h",
       "video/mac/video_capture_device_avfoundation_legacy_mac.h",
       "video/mac/video_capture_device_avfoundation_legacy_mac.mm",
       "video/mac/video_capture_device_avfoundation_mac.h",
@@ -473,6 +475,7 @@ test("capture_unittests") {
     sources += [
       "video/mac/pixel_buffer_pool_mac_unittest.mm",
       "video/mac/pixel_buffer_transferer_mac_unittest.mm",
+      "video/mac/sample_buffer_transformer_mac_unittest.mm",
       "video/mac/test/mock_video_capture_device_avfoundation_frame_receiver_mac.h",
       "video/mac/test/mock_video_capture_device_avfoundation_frame_receiver_mac.mm",
       "video/mac/test/pixel_buffer_test_utils_mac.cc",

media/capture/video/mac/pixel_buffer_pool_mac.h

@@ -32,11 +32,9 @@ class CAPTURE_EXPORT PixelBufferPool {
   // Creates a new buffer from the pool, or returns null if |max_buffers_| would
   // be exceeded. The underlying buffers may be recycled.
   //
-  // The caller owns the returned buffer and is responsible for calling
-  // CFRelease() after they are done using it. This returns the underlying
-  // buffer to the pool. In order to free memory, you must both release all
-  // buffers and call Flush() or delete the pool. It is safe for a buffer to
-  // outlive its pool.
+  // Freeing all buffer references returns the underlying buffer to the pool. In
+  // order to free memory, you must both release all buffers and call Flush() or
+  // delete the pool. It is safe for a buffer to outlive its pool.
   base::ScopedCFTypeRef<CVPixelBufferRef> CreateBuffer();
 
   // Frees the memory of any released buffers returned to the pool.

media/capture/video/mac/sample_buffer_transformer_mac.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef MEDIA_CAPTURE_VIDEO_MAC_SAMPLE_BUFFER_TRANSFORMER_MAC_H_
+#define MEDIA_CAPTURE_VIDEO_MAC_SAMPLE_BUFFER_TRANSFORMER_MAC_H_
+
+#import <CoreMedia/CoreMedia.h>
+#import <CoreVideo/CoreVideo.h>
+#include <vector>
+
+#include "base/mac/scoped_cftyperef.h"
+#include "media/capture/capture_export.h"
+#include "media/capture/video/mac/pixel_buffer_pool_mac.h"
+#include "media/capture/video/mac/pixel_buffer_transferer_mac.h"
+
+namespace media {
+
+// Capable of converting from any supported capture format (NV12, YUY2, UYVY and
+// MJPEG) to NV12 or I420 and doing rescaling. This class can be configured to
+// use VTPixelTransferSession (sometimes HW-accelerated) or third_party/libyuv
+// (SW-only). The output is always an IOSurface-backed pixel buffer that comes
+// from an internal pixel buffer pool.
+class CAPTURE_EXPORT SampleBufferTransformer {
+ public:
+  enum class Transformer {
+    kNotConfigured,
+    // Supports (Any except MJPEG) -> (NV12, I420, ...)
+    kPixelBufferTransfer,
+    // Supports (Any) -> (NV12 or I420)
+    kLibyuv,
+  };
+
+  SampleBufferTransformer();
+  ~SampleBufferTransformer();
+
+  Transformer transformer() const;
+  OSType destination_pixel_format() const;
+
+  // Future calls to Transform() will output pixel buffers according to this
+  // configuration.
+  void Reconfigure(Transformer transformer,
+                   OSType destination_pixel_format,
+                   size_t destination_width,
+                   size_t destination_height,
+                   base::Optional<size_t> buffer_pool_size);
+
+  // Converts the sample buffer to an IOSurface-backed pixel buffer according to
+  // current configurations. If no transformation is needed (input format is the
+  // same as the configured output format), the sample buffer's pixel buffer is
+  // returned.
+  base::ScopedCFTypeRef<CVPixelBufferRef> Transform(
+      CMSampleBufferRef sample_buffer);
+
+ private:
+  // Sample buffers from the camera contain pixel buffers when an uncompressed
+  // pixel format is used (i.e. it's not MJPEG).
+  void TransformPixelBuffer(CVPixelBufferRef source_pixel_buffer,
+                            CVPixelBufferRef destination_pixel_buffer);
+  // (Any uncompressed -> Any uncompressed)
+  void TransformPixelBufferWithPixelTransfer(
+      CVPixelBufferRef source_pixel_buffer,
+      CVPixelBufferRef destination_pixel_buffer);
+  // (Any uncompressed -> NV12 or I420)
+  void TransformPixelBufferWithLibyuv(
+      CVPixelBufferRef source_pixel_buffer,
+      CVPixelBufferRef destination_pixel_buffer);
+  void TransformPixelBufferWithLibyuvFromAnyToI420(
+      CVPixelBufferRef source_pixel_buffer,
+      CVPixelBufferRef destination_pixel_buffer);
+  void TransformPixelBufferWithLibyuvFromAnyToNV12(
+      CVPixelBufferRef source_pixel_buffer,
+      CVPixelBufferRef destination_pixel_buffer);
+  // Sample buffers from the camera contain byte buffers when MJPEG is used.
+  void TransformSampleBuffer(CMSampleBufferRef source_sample_buffer,
+                             CVPixelBufferRef destination_pixel_buffer);
+
+  Transformer transformer_;
+  OSType destination_pixel_format_;
+  size_t destination_width_;
+  size_t destination_height_;
+  std::unique_ptr<PixelBufferPool> destination_pixel_buffer_pool_;
+  // For kPixelBufferTransfer.
+  std::unique_ptr<PixelBufferTransferer> pixel_buffer_transferer_;
+  // For kLibyuv in cases where an intermediate buffer is needed.
+  std::vector<uint8_t> intermediate_i420_buffer_;
+  std::vector<uint8_t> intermediate_nv12_buffer_;
+};
+
+}  // namespace media
+
+#endif  // MEDIA_CAPTURE_VIDEO_MAC_SAMPLE_BUFFER_TRANSFORMER_MAC_H_

media/capture/video/mac/test/pixel_buffer_test_utils_mac.cc

@@ -4,6 +4,8 @@
 
 #include "media/capture/video/mac/test/pixel_buffer_test_utils_mac.h"
 
+#include "media/capture/video/mac/pixel_buffer_pool_mac.h"
+#include "media/capture/video/mac/pixel_buffer_transferer_mac.h"
 #include "third_party/libyuv/include/libyuv/convert_argb.h"
 #include "third_party/libyuv/include/libyuv/convert_from_argb.h"
 
@@ -130,6 +132,11 @@ std::unique_ptr<ByteArrayPixelBuffer> CreateYuvsPixelBufferFromArgbBuffer(
     int width,
     int height,
     const std::vector<uint8_t>& argb_buffer) {
+  // These utility methods don't work well with widths that aren't multiples of
+  // 16. There could be assumptions about memory alignment, or there could
+  // simply be a loss of information in the YUVS <-> ARGB conversions since YUVS
+  // is packed. Either way, the pixels may change, so we avoid these widths.
+  DCHECK(width % 16 == 0);
   std::unique_ptr<ByteArrayPixelBuffer> result =
       std::make_unique<ByteArrayPixelBuffer>();
   size_t yuvs_stride = GetYuvsStride(width);
@@ -162,6 +169,11 @@ std::vector<uint8_t> CreateArgbBufferFromYuvsIOSurface(
   DCHECK(io_surface);
   size_t width = IOSurfaceGetWidth(io_surface);
   size_t height = IOSurfaceGetHeight(io_surface);
+  // These utility methods don't work well with widths that aren't multiples of
+  // 16. There could be assumptions about memory alignment, or there could
+  // simply be a loss of information in the YUVS <-> ARGB conversions since YUVS
+  // is packed. Either way, the pixels may change, so we avoid these widths.
+  DCHECK(width % 16 == 0);
   size_t argb_stride = GetArgbStride(width);
   size_t yuvs_stride = GetYuvsStride(width);
   uint8_t* pixels = static_cast<uint8_t*>(IOSurfaceGetBaseAddress(io_surface));
@@ -181,4 +193,30 @@ bool YuvsIOSurfaceIsSingleColor(IOSurfaceRef io_surface,
                                  r, g, b);
 }
 
+bool PixelBufferIsSingleColor(CVPixelBufferRef pixel_buffer,
+                              uint8_t r,
+                              uint8_t g,
+                              uint8_t b) {
+  OSType pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer);
+  base::ScopedCFTypeRef<CVPixelBufferRef> yuvs_pixel_buffer;
+  if (pixel_format == kPixelFormatYuvs) {
+    // The pixel buffer is already YUVS, so we know how to check its color.
+    yuvs_pixel_buffer.reset(pixel_buffer, base::scoped_policy::RETAIN);
+  } else {
+    // Convert to YUVS. We only know how to check the color of YUVS.
+    yuvs_pixel_buffer =
+        PixelBufferPool::Create(kPixelFormatYuvs,
+                                CVPixelBufferGetWidth(pixel_buffer),
+                                CVPixelBufferGetHeight(pixel_buffer), 1)
+            ->CreateBuffer();
+    PixelBufferTransferer transferer;
+    bool transfer_success =
+        transferer.TransferImage(pixel_buffer, yuvs_pixel_buffer);
+    DCHECK(transfer_success);
+  }
+  IOSurfaceRef io_surface = CVPixelBufferGetIOSurface(yuvs_pixel_buffer);
+  DCHECK(io_surface);
+  return YuvsIOSurfaceIsSingleColor(io_surface, r, g, b);
+}
+
 }  // namespace media

media/capture/video/mac/test/pixel_buffer_test_utils_mac.h

@@ -76,6 +76,13 @@ bool YuvsIOSurfaceIsSingleColor(IOSurfaceRef io_surface,
                                 uint8_t g,
                                 uint8_t b);
 
+// True if all pixels of the pixel buffer are the specified RGB color, within
+// some margin of error.
+bool PixelBufferIsSingleColor(CVPixelBufferRef pixel_buffer,
+                              uint8_t r,
+                              uint8_t g,
+                              uint8_t b);
+
 }  // namespace media
 
 #endif  // MEDIA_CAPTURE_VIDEO_MAC_TEST_PIXEL_BUFFER_TEST_UTILS_MAC_H_