Implement VP9 support for VideoToolbox on macOS.

Verified to work on Beta4. In overlay mode video looks great for both
HDR and 8-bit content. When we drop out of overlays and we're using
the validating cmd decoder HDR gets copied to non-HDR and looks wrong.
When using the passthrough cmd decoder, we just get black frames.

Remaining TODOs:
- Figure out how to get HDR displaying correctly out of overlay mode.
- Figure out difference between validating and passthrough cmd decoder.
- Skip reinitialize for config changes if possible.
- Still needs --no-sandbox until https://crrev.com/c/2346852 lands.
- Jank at playback startup for HDR content. https://crbug.com/1115333
- Playback hangs when starting with a seek. https://crbug.com/1115334

Bug: 1103432
Change-Id: I7d5860a49eee3856fec380eac8e78fdac798c117
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2290923Reviewed-by: Avi Drissman <avi@chromium.org>
Reviewed-by: Dan Sanders <sandersd@chromium.org>
Reviewed-by: ccameron <ccameron@chromium.org>
Commit-Queue: Dale Curtis <dalecurtis@chromium.org>
Auto-Submit: Dale Curtis <dalecurtis@chromium.org>
Cr-Commit-Position: refs/heads/master@{#797459}

base/mac/sdk_forward_declarations.h

@@ -67,6 +67,11 @@
 #if !defined(MAC_OS_VERSION_11_0)
 #include <CoreMedia/CoreMedia.h>
 enum : CMVideoCodecType { kCMVideoCodecType_VP9 = 'vp09' };
+
+extern "C" {
+void VTRegisterSupplementalVideoDecoderIfAvailable(CMVideoCodecType codecType);
+}
+
 #endif  // MAC_OS_VERSION_11_0
 
 #endif  // BASE_MAC_SDK_FORWARD_DECLARATIONS_H_

media/base/media_switches.cc

@@ -636,6 +636,11 @@ const base::Feature MEDIA_EXPORT kMediaFoundationVP8Decoding{
 
 #endif  // defined(OS_WIN)
 
+#if defined(OS_MAC)
+const base::Feature MEDIA_EXPORT kVideoToolboxVp9Decoding{
+    "VideoToolboxVp9Decoding", base::FEATURE_DISABLED_BY_DEFAULT};
+#endif  // defined(OS_MAC)
+
 std::string GetEffectiveAutoplayPolicy(const base::CommandLine& command_line) {
   // Return the autoplay policy set in the command line, if any.
   if (command_line.HasSwitch(switches::kAutoplayPolicy))

media/base/media_switches.h

@@ -214,6 +214,10 @@ MEDIA_EXPORT extern const base::Feature kMediaFoundationVideoCapture;
 MEDIA_EXPORT extern const base::Feature kMediaFoundationVP8Decoding;
 #endif  // defined(OS_WIN)
 
+#if defined(OS_MAC)
+MEDIA_EXPORT extern const base::Feature kVideoToolboxVp9Decoding;
+#endif
+
 // Based on a |command_line| and the current platform, returns the effective
 // autoplay policy. In other words, it will take into account the default policy
 // if none is specified via the command line and options passed for testing.

media/gpu/mac/BUILD.gn

@@ -13,6 +13,12 @@ assert(is_mac)
 
 import("//build/config/mac/mac_sdk.gni")
 
+generate_stubs("vt_beta_stubs") {
+  extra_header = "vt_beta_stubs_header.fragment"
+  sigs = [ "vt_beta.sig" ]
+  output_name = "vt_beta_stubs"
+}
+
 source_set("mac") {
   defines = [ "MEDIA_GPU_IMPLEMENTATION" ]
   visibility = [ "//media/gpu" ]
@@ -35,6 +41,7 @@ source_set("mac") {
     "VideoToolbox.framework",
   ]
   deps = [
+    ":vt_beta_stubs",
     "//base",
     "//components/crash/core/common:crash_key",
     "//gpu/ipc/service",

media/gpu/mac/vt_beta.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef MEDIA_GPU_MAC_VT_BETA_H_
+#define MEDIA_GPU_MAC_VT_BETA_H_
+
+// Dynamic library loader.
+#include "media/gpu/mac/vt_beta_stubs.h"
+
+// CoreMedia and VideoToolbox types.
+#include "media/gpu/mac/vt_beta_stubs_header.fragment"
+
+// CoreMedia and VideoToolbox functions.
+extern "C" {
+#include "media/gpu/mac/vt_beta.sig"
+}  // extern "C"
+
+#endif  // MEDIA_GPU_MAC_VT_BETA_H_

media/gpu/mac/vt_beta.sig

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This function isn't available until we're using the macOS 11.0 SDK.
+void VTRegisterSupplementalVideoDecoderIfAvailable(CMVideoCodecType codecType);

media/gpu/mac/vt_beta_stubs_header.fragment

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <CoreMedia/CoreMedia.h>

media/gpu/mac/vt_config_util.mm

@@ -40,6 +40,7 @@ CFStringRef GetPrimaries(media::VideoColorSpace::PrimaryID primary_id) {
   switch (primary_id) {
     case media::VideoColorSpace::PrimaryID::BT709:
     case media::VideoColorSpace::PrimaryID::UNSPECIFIED:  // Assume BT.709.
+    case media::VideoColorSpace::PrimaryID::INVALID:      // Assume BT.709.
       return kCMFormatDescriptionColorPrimaries_ITU_R_709_2;
 
     case media::VideoColorSpace::PrimaryID::BT2020:
@@ -100,6 +101,7 @@ CFStringRef GetTransferFunction(
     case media::VideoColorSpace::TransferID::SMPTE170M:
     case media::VideoColorSpace::TransferID::BT709:
     case media::VideoColorSpace::TransferID::UNSPECIFIED:  // Assume BT.709.
+    case media::VideoColorSpace::TransferID::INVALID:      // Assume BT.709.
       return kCMFormatDescriptionTransferFunction_ITU_R_709_2;
 
     case media::VideoColorSpace::TransferID::BT2020_10:
@@ -145,6 +147,7 @@ CFStringRef GetMatrix(media::VideoColorSpace::MatrixID matrix_id) {
   switch (matrix_id) {
     case media::VideoColorSpace::MatrixID::BT709:
     case media::VideoColorSpace::MatrixID::UNSPECIFIED:  // Assume BT.709.
+    case media::VideoColorSpace::MatrixID::INVALID:      // Assume BT.709.
       return kCMFormatDescriptionYCbCrMatrix_ITU_R_709_2;
 
     case media::VideoColorSpace::MatrixID::BT2020_NCL:

media/gpu/mac/vt_video_decode_accelerator_mac.h

@@ -29,6 +29,7 @@
 #include "ui/gl/gl_image_io_surface.h"
 
 namespace media {
+class VP9ConfigChangeDetector;
 
 // Preload VideoToolbox libraries, needed for sandbox warmup.
 MEDIA_GPU_EXPORT bool InitializeVideoToolbox();
@@ -162,6 +163,7 @@ class VTVideoDecodeAccelerator : public VideoDecodeAccelerator,
 
   // |frame| is owned by |pending_frames_|.
   void DecodeTask(scoped_refptr<DecoderBuffer> buffer, Frame* frame);
+  void DecodeTaskVp9(scoped_refptr<DecoderBuffer> buffer, Frame* frame);
   void DecodeDone(Frame* frame);
 
   //
@@ -188,6 +190,7 @@ class VTVideoDecodeAccelerator : public VideoDecodeAccelerator,
   // These methods returns true if a task was completed, false otherwise.
   bool ProcessTaskQueue();
   bool ProcessReorderQueue();
+  bool ProcessOutputQueue();
   bool ProcessFrame(const Frame& frame);
   bool SendFrame(const Frame& frame);
 
@@ -213,6 +216,12 @@ class VTVideoDecodeAccelerator : public VideoDecodeAccelerator,
                       FrameOrder>
       reorder_queue_;
 
+  // Queue of decoded frames in presentation order. Used by codecs which don't
+  // require reordering (VP9 only at the moment).
+  std::deque<std::unique_ptr<Frame>> output_queue_;
+
+  std::unique_ptr<VP9ConfigChangeDetector> cc_detector_;
+
   // Size of assigned picture buffers.
   gfx::Size picture_size_;
 
@@ -258,6 +267,9 @@ class VTVideoDecodeAccelerator : public VideoDecodeAccelerator,
   std::vector<uint8_t> configured_spsext_;
   std::vector<uint8_t> configured_pps_;
 
+  Config config_;
+  VideoCodec codec_;
+
   // Visible rect the decoder is configured to use.
   gfx::Size configured_size_;
 

ui/accelerated_widget_mac/ca_renderer_layer_tree.mm

@@ -447,8 +447,8 @@ CARendererLayerTree::ContentLayer::ContentLayer(
         if (has_hdr_color_space)
           type = CALayerType::kHDRCopier;
         break;
-      // TODO(crbug.com/1103432): We'll likely need YpCbCr10 here for HDR.
       case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
+      case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange:
         // Only allow 4:2:0 frames which fill the layer's contents to be
         // promoted to AV layers.
         if (tree->allow_av_sample_buffer_display_layer_ &&

ui/gfx/mac/io_surface.cc

@@ -58,10 +58,16 @@ int32_t BytesPerElement(gfx::BufferFormat format, int plane) {
     case gfx::BufferFormat::RGBA_F16:
       DCHECK_EQ(plane, 0);
       return 8;
-    case gfx::BufferFormat::YUV_420_BIPLANAR:
-      static int32_t bytes_per_element[] = {1, 2};
+    case gfx::BufferFormat::YUV_420_BIPLANAR: {
+      constexpr int32_t bytes_per_element[] = {1, 2};
+      DCHECK_LT(static_cast<size_t>(plane), base::size(bytes_per_element));
+      return bytes_per_element[plane];
+    }
+    case gfx::BufferFormat::P010: {
+      constexpr int32_t bytes_per_element[] = {2, 4};
       DCHECK_LT(static_cast<size_t>(plane), base::size(bytes_per_element));
       return bytes_per_element[plane];
+    }
     case gfx::BufferFormat::R_16:
     case gfx::BufferFormat::RG_88:
     case gfx::BufferFormat::BGR_565:
@@ -69,7 +75,6 @@ int32_t BytesPerElement(gfx::BufferFormat format, int plane) {
     case gfx::BufferFormat::RGBX_8888:
     case gfx::BufferFormat::RGBA_1010102:
     case gfx::BufferFormat::YVU_420:
-    case gfx::BufferFormat::P010:
       NOTREACHED();
       return 0;
   }
@@ -92,6 +97,8 @@ int32_t PixelFormat(gfx::BufferFormat format) {
       return 'RGhA';
     case gfx::BufferFormat::YUV_420_BIPLANAR:
       return '420v';
+    case gfx::BufferFormat::P010:
+      return 'x420';
     case gfx::BufferFormat::R_16:
     case gfx::BufferFormat::RG_88:
     case gfx::BufferFormat::BGR_565:
@@ -101,7 +108,6 @@ int32_t PixelFormat(gfx::BufferFormat format) {
     // Technically RGBA_1010102 should be accepted as 'R10k', but then it won't
     // be supported by CGLTexImageIOSurface2D(), so it's best to reject it here.
     case gfx::BufferFormat::YVU_420:
-    case gfx::BufferFormat::P010:
       NOTREACHED();
       return 0;
   }

ui/gl/gl_image_io_surface.mm

@@ -50,6 +50,7 @@ bool ValidInternalFormat(unsigned internalformat) {
     case GL_RGB10_A2_EXT:
     case GL_RGB_YCBCR_420V_CHROMIUM:
     case GL_RGB_YCBCR_422_CHROMIUM:
+    case GL_RGB_YCBCR_P010_CHROMIUM:
     case GL_RGBA:
       return true;
     default:
@@ -73,12 +74,13 @@ GLenum TextureFormat(gfx::BufferFormat format) {
       return GL_RGBA;
     case gfx::BufferFormat::YUV_420_BIPLANAR:
       return GL_RGB_YCBCR_420V_CHROMIUM;
+    case gfx::BufferFormat::P010:
+      return GL_RGB_YCBCR_P010_CHROMIUM;
     case gfx::BufferFormat::BGR_565:
     case gfx::BufferFormat::RGBA_4444:
     case gfx::BufferFormat::RGBX_8888:
     case gfx::BufferFormat::RGBA_1010102:
     case gfx::BufferFormat::YVU_420:
-    case gfx::BufferFormat::P010:
       NOTREACHED() << gfx::BufferFormatToString(format);
       return 0;
   }
@@ -246,10 +248,13 @@ unsigned GLImageIOSurface::GetDataType() {
 }
 
 GLImageIOSurface::BindOrCopy GLImageIOSurface::ShouldBindOrCopy() {
-  // YUV_420_BIPLANAR is not supported by BindTexImage.
-  // CopyTexImage is supported by this format as that performs conversion to RGB
-  // as part of the copy operation.
-  return format_ == gfx::BufferFormat::YUV_420_BIPLANAR ? COPY : BIND;
+  // YUV_420_BIPLANAR and P010 are not supported by BindTexImage. CopyTexImage
+  // is supported by these formats as that performs conversion to RGB as part of
+  // the copy operation.
+  return (format_ == gfx::BufferFormat::YUV_420_BIPLANAR ||
+          format_ == gfx::BufferFormat::P010)
+             ? COPY
+             : BIND;
 }
 
 bool GLImageIOSurface::BindTexImage(unsigned target) {
@@ -339,12 +344,15 @@ bool GLImageIOSurface::CopyTexImage(unsigned target) {
         glBindTexture(target, rgb_texture);
       })));
 
+  const auto src_type =
+      format_ == gfx::BufferFormat::P010 ? GL_UNSIGNED_SHORT : GL_UNSIGNED_BYTE;
+
   CGLContextObj cgl_context = CGLGetCurrentContext();
   {
     glBindTexture(GL_TEXTURE_RECTANGLE_ARB, yuv_to_rgb_converter->y_texture());
     CGLError cgl_error = CGLTexImageIOSurface2D(
         cgl_context, GL_TEXTURE_RECTANGLE_ARB, GL_RED, size_.width(),
-        size_.height(), GL_RED, GL_UNSIGNED_BYTE, io_surface_, 0);
+        size_.height(), GL_RED, src_type, io_surface_, 0);
     if (cgl_error != kCGLNoError) {
       LOG(ERROR) << "Error in CGLTexImageIOSurface2D for the Y plane. "
                  << cgl_error;
@@ -355,7 +363,7 @@ bool GLImageIOSurface::CopyTexImage(unsigned target) {
     glBindTexture(GL_TEXTURE_RECTANGLE_ARB, yuv_to_rgb_converter->uv_texture());
     CGLError cgl_error = CGLTexImageIOSurface2D(
         cgl_context, GL_TEXTURE_RECTANGLE_ARB, GL_RG, size_.width() / 2,
-        size_.height() / 2, GL_RG, GL_UNSIGNED_BYTE, io_surface_, 1);
+        size_.height() / 2, GL_RG, src_type, io_surface_, 1);
     if (cgl_error != kCGLNoError) {
       LOG(ERROR) << "Error in CGLTexImageIOSurface2D for the UV plane. "
                  << cgl_error;
@@ -363,7 +371,7 @@ bool GLImageIOSurface::CopyTexImage(unsigned target) {
     }
   }
 
-  yuv_to_rgb_converter->CopyYUV420ToRGB(target, size_, rgb_texture);
+  yuv_to_rgb_converter->CopyYUV420ToRGB(target, size_, rgb_texture, src_type);
   return true;
 }
 
@@ -506,6 +514,7 @@ bool GLImageIOSurface::ValidFormat(gfx::BufferFormat format) {
     case gfx::BufferFormat::RGBA_F16:
     case gfx::BufferFormat::BGRA_1010102:
     case gfx::BufferFormat::YUV_420_BIPLANAR:
+    case gfx::BufferFormat::P010:
       return true;
     case gfx::BufferFormat::R_16:
     case gfx::BufferFormat::RG_88:
@@ -514,7 +523,6 @@ bool GLImageIOSurface::ValidFormat(gfx::BufferFormat format) {
     case gfx::BufferFormat::RGBX_8888:
     case gfx::BufferFormat::RGBA_1010102:
     case gfx::BufferFormat::YVU_420:
-    case gfx::BufferFormat::P010:
       return false;
   }
 

ui/gl/gl_image_io_surface_egl.mm

@@ -250,8 +250,11 @@ bool GLImageIOSurfaceEGL::CopyTexImage(unsigned target) {
     return false;
   }
 
-  if (format_ != gfx::BufferFormat::YUV_420_BIPLANAR)
+  // TODO(crbug.com/1115621): We should support gfx::BufferFormat::P010 here,
+  // but EGL doesn't seem to be able to sample from the P010 Y and UV textures.
+  if (format_ != gfx::BufferFormat::YUV_420_BIPLANAR) {
     return false;
+  }
 
   GLContext* gl_context = GLContext::GetCurrent();
   DCHECK(gl_context);
@@ -361,7 +364,8 @@ bool GLImageIOSurfaceEGL::CopyTexImage(unsigned target) {
     return false;
   }
 
-  yuv_to_rgb_converter->CopyYUV420ToRGB(target, size_, rgb_texture);
+  yuv_to_rgb_converter->CopyYUV420ToRGB(target, size_, rgb_texture,
+                                        GL_UNSIGNED_BYTE);
   if (glGetError() != GL_NO_ERROR) {
     LOG(ERROR) << "Failed converting from YUV to RGB";
     return false;

ui/gl/yuv_to_rgb_converter.cc

@@ -206,7 +206,8 @@ YUVToRGBConverter::~YUVToRGBConverter() {
 
 void YUVToRGBConverter::CopyYUV420ToRGB(unsigned target,
                                         const gfx::Size& size,
-                                        unsigned rgb_texture) {
+                                        unsigned rgb_texture,
+                                        unsigned rgb_texture_type) {
   GLenum source_target_getter = 0;
   switch (source_texture_target_) {
     case GL_TEXTURE_2D:
@@ -233,8 +234,8 @@ void YUVToRGBConverter::CopyYUV420ToRGB(unsigned target,
   // Allocate the rgb texture.
   glActiveTexture(old_active_texture);
   glBindTexture(target, rgb_texture);
-  glTexImage2D(target, 0, GL_RGB, size.width(), size.height(),
-               0, GL_RGB, GL_UNSIGNED_BYTE, nullptr);
+  glTexImage2D(target, 0, GL_RGB, size.width(), size.height(), 0, GL_RGB,
+               rgb_texture_type, nullptr);
 
   // Set up and issue the draw call.
   glActiveTexture(GL_TEXTURE0);

ui/gl/yuv_to_rgb_converter.h

@@ -28,7 +28,8 @@ class YUVToRGBConverter {
 
   void CopyYUV420ToRGB(unsigned target,
                        const gfx::Size& size,
-                       unsigned rgb_texture);
+                       unsigned rgb_texture,
+                       unsigned rgb_texture_type);
 
  private:
   unsigned framebuffer_ = 0;