Bind RGB images to KHR streams in D3D11VideoDecoder

This patch adds support for 8-bit and fp16 RGB textures via KHR
streams, since ANGLE almost supports it.  8-bit textures work
as-is, but fp16 requires a small patch to ANGLE.

However, since fp16 support is off by default in D3D11VideoDecoder
anyway, it's okay to land this without ordering with respect to the
ANGLE changes.

Change-Id: I484760e462341bc6cad699d35f2c9733a7404c5b
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2079451Reviewed-by: Ted Meyer <tmathmeyer@chromium.org>
Commit-Queue: Frank Liberato <liberato@chromium.org>
Cr-Commit-Position: refs/heads/master@{#745267}

media/gpu/windows/d3d11_texture_selector.cc

@@ -16,10 +16,11 @@
 namespace media {
 
 TextureSelector::TextureSelector(VideoPixelFormat pixfmt,
+                                 DXGI_FORMAT output_dxgifmt,
                                  bool supports_swap_chain)
     : pixel_format_(pixfmt),
-      supports_swap_chain_(supports_swap_chain) {
-}
+      output_dxgifmt_(output_dxgifmt),
+      supports_swap_chain_(supports_swap_chain) {}
 
 bool SupportsZeroCopy(const gpu::GpuPreferences& preferences,
                       const gpu::GpuDriverBugWorkarounds& workarounds) {
@@ -55,12 +56,11 @@ std::unique_ptr<TextureSelector> TextureSelector::Create(
       break;
     case DXGI_FORMAT_P010:
       MEDIA_LOG(INFO, media_log) << "D3D11VideoDecoder producing FP16";
-      // Note: this combination isn't actually supported, since we don't support
-      // pbuffer textures right now.
       output_pixel_format = PIXEL_FORMAT_ARGB;
       output_dxgi_format = DXGI_FORMAT_R16G16B16A16_FLOAT;
-      // B8G8R8A8 is also an okay choice, if we don't have fp16 support.
-      needs_texture_copy = true;
+      // DXGI_FORMAT_B8G8R8A8_UNORM is also an okay choice, if we don't want to
+      // use fp16.
+      // TODO(liberato): Pick this better.
       break;
     default:
       // TODO(tmathmeyer) support other profiles in the future.
@@ -69,6 +69,10 @@ std::unique_ptr<TextureSelector> TextureSelector::Create(
       return nullptr;
   }
 
+  // If we're trying to produce an output texture that's different from what
+  // the decoder is providing, then we need to copy it.
+  needs_texture_copy = (decoder_output_format != output_dxgi_format);
+
   // Force texture copy on if requested for debugging.
   if (base::FeatureList::IsEnabled(kD3D11VideoDecoderAlwaysCopy))
     needs_texture_copy = true;
@@ -79,10 +83,9 @@ std::unique_ptr<TextureSelector> TextureSelector::Create(
         output_pixel_format, decoder_output_format, output_dxgi_format,
         supports_nv12_decode_swap_chain);  // TODO(tmathmeyer) false always?
   } else {
-    // We don't support anything except NV12 for binding right now.  With
-    // pbuffer textures, we could support rgb8 and / or fp16.
-    DCHECK_EQ(output_pixel_format, PIXEL_FORMAT_NV12);
+    MEDIA_LOG(INFO, media_log) << "D3D11VideoDecoder is binding textures";
     return std::make_unique<TextureSelector>(output_pixel_format,
+                                             output_dxgi_format,
                                              supports_nv12_decode_swap_chain);
   }
 }
@@ -93,7 +96,7 @@ std::unique_ptr<Texture2DWrapper> TextureSelector::CreateTextureWrapper(
     ComD3D11DeviceContext device_context,
     gfx::Size size) {
   // TODO(liberato): If the output format is rgb, then create a pbuffer wrapper.
-  return std::make_unique<DefaultTexture2DWrapper>(size);
+  return std::make_unique<DefaultTexture2DWrapper>(size, OutputDXGIFormat());
 }
 
 std::unique_ptr<Texture2DWrapper> CopyTextureSelector::CreateTextureWrapper(
@@ -125,7 +128,7 @@ std::unique_ptr<Texture2DWrapper> CopyTextureSelector::CreateTextureWrapper(
     return nullptr;
 
   return std::make_unique<CopyingTexture2DWrapper>(
-      size, std::make_unique<DefaultTexture2DWrapper>(size),
+      size, std::make_unique<DefaultTexture2DWrapper>(size, OutputDXGIFormat()),
       std::make_unique<VideoProcessorProxy>(video_device, device_context),
       out_texture);
 }

media/gpu/windows/d3d11_texture_selector.h

@@ -24,6 +24,7 @@ class MediaLog;
 class MEDIA_GPU_EXPORT TextureSelector {
  public:
   TextureSelector(VideoPixelFormat pixfmt,
+                  DXGI_FORMAT output_dxgifmt,
                   bool supports_swap_chain);
   virtual ~TextureSelector() = default;
 
@@ -39,12 +40,14 @@ class MEDIA_GPU_EXPORT TextureSelector {
       ComD3D11DeviceContext,
       gfx::Size size);
 
-  VideoPixelFormat PixelFormat() { return pixel_format_; }
+  VideoPixelFormat PixelFormat() const { return pixel_format_; }
+  DXGI_FORMAT OutputDXGIFormat() const { return output_dxgifmt_; }
 
  private:
   friend class CopyTextureSelector;
 
   const VideoPixelFormat pixel_format_;
+  const DXGI_FORMAT output_dxgifmt_;
   const bool supports_swap_chain_;
 };
 
@@ -55,18 +58,13 @@ class MEDIA_GPU_EXPORT CopyTextureSelector : public TextureSelector {
                       DXGI_FORMAT input_dxgifmt,
                       DXGI_FORMAT output_dxgifmt,
                       bool supports_swap_chain)
-      : TextureSelector(pixfmt,
-                        supports_swap_chain),
-        output_dxgifmt_(output_dxgifmt) {}
+      : TextureSelector(pixfmt, output_dxgifmt, supports_swap_chain) {}
 
   std::unique_ptr<Texture2DWrapper> CreateTextureWrapper(
       ComD3D11Device device,
       ComD3D11VideoDevice video_device,
       ComD3D11DeviceContext,
       gfx::Size size) override;
-
- private:
-  DXGI_FORMAT output_dxgifmt_;
 };
 
 }  // namespace media

media/gpu/windows/d3d11_texture_selector_unittest.cc

@@ -49,8 +49,9 @@ class D3D11TextureSelectorUnittest : public ::testing::Test {
 TEST_F(D3D11TextureSelectorUnittest, NV12BindsToNV12) {
   auto tex_sel = CreateWithDefaultGPUInfo(DXGI_FORMAT_NV12);
 
-  // TODO(liberato): checl "binds", somehow.
+  // TODO(liberato): check "binds", somehow.
   EXPECT_EQ(tex_sel->PixelFormat(), PIXEL_FORMAT_NV12);
+  EXPECT_EQ(tex_sel->OutputDXGIFormat(), DXGI_FORMAT_NV12);
 }
 
 TEST_F(D3D11TextureSelectorUnittest, P010CopiesToARGB) {
@@ -58,6 +59,9 @@ TEST_F(D3D11TextureSelectorUnittest, P010CopiesToARGB) {
 
   // TODO(liberato): check "copies", somehow.
   EXPECT_EQ(tex_sel->PixelFormat(), PIXEL_FORMAT_ARGB);
+  // Note that this might also produce 8 bit rgb, but for now always
+  // tries for fp16.
+  EXPECT_EQ(tex_sel->OutputDXGIFormat(), DXGI_FORMAT_R16G16B16A16_FLOAT);
 }
 
 }  // namespace media

media/gpu/windows/d3d11_texture_wrapper.cc

@@ -4,7 +4,10 @@
 
 #include "media/gpu/windows/d3d11_texture_wrapper.h"
 
+#include <list>
 #include <memory>
+#include <utility>
+#include <vector>
 
 #include "gpu/command_buffer/service/mailbox_manager.h"
 #include "media/base/win/mf_helpers.h"
@@ -12,12 +15,49 @@
 
 namespace media {
 
+// Handy structure so that we can activate / bind one or two textures.
+struct ScopedTextureEverything {
+  ScopedTextureEverything(GLenum unit, GLuint service_id)
+      : active_(unit), binder_(GL_TEXTURE_EXTERNAL_OES, service_id) {}
+  ~ScopedTextureEverything() = default;
+
+  // Order is important; we need |active_| to be constructed first
+  // and destructed last.
+  gl::ScopedActiveTexture active_;
+  gl::ScopedTextureBinder binder_;
+
+  DISALLOW_COPY_AND_ASSIGN(ScopedTextureEverything);
+};
+
+// Another handy helper class to guarantee that ScopedTextureEverythings
+// are deleted in reverse order.  This is required so that the scoped
+// active texture unit doesn't change.  Surprisingly, none of the stl
+// containers, or the chromium ones, seem to guarantee anything about
+// the order of destruction.
+struct OrderedDestructionList {
+  OrderedDestructionList() = default;
+  ~OrderedDestructionList() {
+    // Erase last-to-first.
+    while (!list_.empty())
+      list_.pop_back();
+  }
+
+  template <typename... Args>
+  void emplace_back(Args&&... args) {
+    list_.emplace_back(std::forward<Args>(args)...);
+  }
+
+  std::list<ScopedTextureEverything> list_;
+  DISALLOW_COPY_AND_ASSIGN(OrderedDestructionList);
+};
+
 Texture2DWrapper::Texture2DWrapper() = default;
 
 Texture2DWrapper::~Texture2DWrapper() = default;
 
-DefaultTexture2DWrapper::DefaultTexture2DWrapper(const gfx::Size& size)
-    : size_(size) {}
+DefaultTexture2DWrapper::DefaultTexture2DWrapper(const gfx::Size& size,
+                                                 DXGI_FORMAT dxgi_format)
+    : size_(size), dxgi_format_(dxgi_format) {}
 
 DefaultTexture2DWrapper::~DefaultTexture2DWrapper() = default;
 
@@ -33,6 +73,8 @@ bool DefaultTexture2DWrapper::ProcessTexture(ComD3D11Texture2D texture,
   if (!result.IsOk())
     return false;
 
+  // TODO(liberato): make sure that |mailbox_holders_| is zero-initialized in
+  // case we don't use all the planes.
   for (size_t i = 0; i < VideoFrame::kMaxPlanes; i++)
     (*mailbox_dest)[i] = mailbox_holders_[i];
 
@@ -44,8 +86,19 @@ bool DefaultTexture2DWrapper::Init(GetCommandBufferHelperCB get_helper_cb) {
   if (!gpu_resources_)
     return false;
 
-  // We currently only bind NV12, which requires two GL textures.
-  const int textures_per_picture = 2;
+  // YUV textures are mapped onto two GL textures, while RGB use one.
+  int textures_per_picture = 0;
+  switch (dxgi_format_) {
+    case DXGI_FORMAT_NV12:
+      textures_per_picture = 2;
+      break;
+    case DXGI_FORMAT_B8G8R8A8_UNORM:
+    case DXGI_FORMAT_R16G16B16A16_FLOAT:
+      textures_per_picture = 1;
+      break;
+    default:
+      return false;
+  }
 
   // Generate mailboxes and holders.
   std::vector<gpu::Mailbox> mailboxes;
@@ -89,6 +142,8 @@ bool DefaultTexture2DWrapper::GpuResources::Init(
     return false;
 
   // Create the textures and attach them to the mailboxes.
+  // TODO(liberato): Should we use GL_FLOAT for an fp16 texture?  It doesn't
+  // really seem to matter so far as I can tell.
   for (int texture_idx = 0; texture_idx < textures_per_picture; texture_idx++) {
     uint32_t service_id =
         helper_->CreateTexture(target, GL_RGBA, size.width(), size.height(),
@@ -100,11 +155,11 @@ bool DefaultTexture2DWrapper::GpuResources::Init(
   // Create the stream for zero-copy use by gl.
   EGLDisplay egl_display = gl::GLSurfaceEGL::GetHardwareDisplay();
   const EGLint stream_attributes[] = {
-      EGL_CONSUMER_LATENCY_USEC_KHR,
-      0,
-      EGL_CONSUMER_ACQUIRE_TIMEOUT_USEC_KHR,
-      0,
+      // clang-format off
+      EGL_CONSUMER_LATENCY_USEC_KHR,         0,
+      EGL_CONSUMER_ACQUIRE_TIMEOUT_USEC_KHR, 0,
       EGL_NONE,
+      // clang-format on
   };
   EGLStreamKHR stream = eglCreateStreamKHR(egl_display, stream_attributes);
   RETURN_ON_FAILURE(!!stream, "Could not create stream", false);
@@ -114,26 +169,35 @@ bool DefaultTexture2DWrapper::GpuResources::Init(
   // this function unless |helper_| retains it.  Also, this won't work if we
   // have a FakeCommandBufferHelper since the service IDs aren't meaningful.
   gl_image_ = base::MakeRefCounted<gl::GLImageDXGI>(size, stream);
-  gl::ScopedActiveTexture texture0(GL_TEXTURE0);
-  gl::ScopedTextureBinder texture0_binder(GL_TEXTURE_EXTERNAL_OES,
-                                          service_ids_[0]);
-  gl::ScopedActiveTexture texture1(GL_TEXTURE1);
-  gl::ScopedTextureBinder texture1_binder(GL_TEXTURE_EXTERNAL_OES,
-                                          service_ids_[1]);
-
-  EGLAttrib consumer_attributes[] = {
-      EGL_COLOR_BUFFER_TYPE,
-      EGL_YUV_BUFFER_EXT,
-      EGL_YUV_NUMBER_OF_PLANES_EXT,
-      2,
-      EGL_YUV_PLANE0_TEXTURE_UNIT_NV,
-      0,
-      EGL_YUV_PLANE1_TEXTURE_UNIT_NV,
-      1,
-      EGL_NONE,
-  };
+
+  // Bind all the textures so that the stream can find them.
+  OrderedDestructionList texture_everythings;
+  for (int i = 0; i < textures_per_picture; i++)
+    texture_everythings.emplace_back(GL_TEXTURE0 + i, service_ids_[i]);
+
+  std::vector<EGLAttrib> consumer_attributes;
+  if (textures_per_picture == 2) {
+    // Assume NV12.
+    consumer_attributes = {
+        // clang-format off
+        EGL_COLOR_BUFFER_TYPE,               EGL_YUV_BUFFER_EXT,
+        EGL_YUV_NUMBER_OF_PLANES_EXT,        2,
+        EGL_YUV_PLANE0_TEXTURE_UNIT_NV,      0,
+        EGL_YUV_PLANE1_TEXTURE_UNIT_NV,      1,
+        EGL_NONE,
+        // clang-format on
+    };
+  } else {
+    // Assume some rgb format.
+    consumer_attributes = {
+        // clang-format off
+        EGL_COLOR_BUFFER_TYPE,               EGL_RGB_BUFFER,
+        EGL_NONE,
+        // clang-format on
+    };
+  }
   EGLBoolean result = eglStreamConsumerGLTextureExternalAttribsNV(
-      egl_display, stream, consumer_attributes);
+      egl_display, stream, consumer_attributes.data());
   RETURN_ON_FAILURE(result, "Could not set stream consumer", false);
 
   EGLAttrib producer_attributes[] = {

media/gpu/windows/d3d11_texture_wrapper.h

@@ -55,7 +55,9 @@ class MEDIA_GPU_EXPORT Texture2DWrapper {
 // instance for each concurrently outstanding texture.
 class MEDIA_GPU_EXPORT DefaultTexture2DWrapper : public Texture2DWrapper {
  public:
-  DefaultTexture2DWrapper(const gfx::Size& size);
+  // While the specific texture instance can change on every call to
+  // ProcessTexture, the dxgi format must be the same for all of them.
+  DefaultTexture2DWrapper(const gfx::Size& size, DXGI_FORMAT dxgi_format);
   ~DefaultTexture2DWrapper() override;
 
   bool Init(GetCommandBufferHelperCB get_helper_cb) override;
@@ -96,6 +98,7 @@ class MEDIA_GPU_EXPORT DefaultTexture2DWrapper : public Texture2DWrapper {
   gfx::Size size_;
   std::unique_ptr<GpuResources> gpu_resources_;
   MailboxHolderArray mailbox_holders_;
+  DXGI_FORMAT dxgi_format_;
 };
 
 }  // namespace media

media/gpu/windows/d3d11_video_decoder.cc

@@ -743,10 +743,15 @@ bool D3D11VideoDecoder::OutputResult(const CodecPicture* picture,
   // For NV12, overlay is allowed by default. If the decoder is going to support
   // non-NV12 textures, then this may have to be conditionally set. Also note
   // that ALLOW_OVERLAY is required for encrypted video path.
+  //
   // Since all of our picture buffers allow overlay, we just use the finch
   // feature.  However, we may choose to set ALLOW_OVERLAY to false even if
   // the finch flag is enabled.  We may not choose to set ALLOW_OVERLAY if the
   // flag is off, however.
+  //
+  // Also note that, since we end up binding textures with GLImageDXGI, it's
+  // probably okay just to allow overlay always, and let the swap chain
+  // presenter decide if it wants to.
   const bool allow_overlay =
       base::FeatureList::IsEnabled(kD3D11VideoDecoderAllowOverlay);
   frame->metadata()->SetBoolean(VideoFrameMetadata::ALLOW_OVERLAY,