Fix processor formatting checks for DXVA

Note: this will also be used by D3D11VideoDecoder in the future.

The old way of just checking D3D11Device->CheckFormatSupport lies about
whether the processor / enumerator really can support those profiles.
This cl makes a new checker which can create an enumerator and then
check the formats that way.

Change-Id: I9f25e633c161095410909afa3ba88ca4f2e09000
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1864349
Commit-Queue: Ted Meyer <tmathmeyer@chromium.org>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Cr-Commit-Position: refs/heads/master@{#707537}

media/gpu/BUILD.gn

@@ -181,6 +181,8 @@ component("gpu") {
       "windows/d3d11_video_decoder_client.h",
       "windows/d3d11_video_decoder_impl.cc",
       "windows/d3d11_video_decoder_impl.h",
+      "windows/d3d11_video_device_format_support.cc",
+      "windows/d3d11_video_device_format_support.h",
       "windows/d3d11_video_processor_proxy.cc",
       "windows/d3d11_video_processor_proxy.h",
       "windows/d3d11_vp9_accelerator.cc",
@@ -599,6 +601,7 @@ source_set("unit_tests") {
       "windows/d3d11_decryptor_unittest.cc",
       "windows/d3d11_texture_selector_unittest.cc",
       "windows/d3d11_video_decoder_unittest.cc",
+      "windows/d3d11_video_device_format_support_unittest.cc",
       "windows/d3d11_video_processor_proxy_unittest.cc",
       "windows/supported_profile_helpers_unittest.cc",
     ]

media/gpu/windows/d3d11_video_device_format_support.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/gpu/windows/d3d11_video_device_format_support.h"
+
+namespace media {
+
+FormatSupportChecker::FormatSupportChecker(ComD3D11Device device)
+    : device_(std::move(device)) {
+  DCHECK(device_);
+}
+
+FormatSupportChecker::~FormatSupportChecker() {
+  if (initialized_)
+    enumerator_.Reset();
+  device_.Reset();
+}
+
+bool FormatSupportChecker::Initialize() {
+  ComD3D11VideoDevice v_device;
+  if (!SUCCEEDED(device_.As(&v_device)))
+    return false;
+
+  // The values here should have _no_ effect on supported profiles, but they
+  // are needed anyway for initialization.
+  D3D11_VIDEO_PROCESSOR_CONTENT_DESC desc;
+  desc.InputFrameFormat = D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE;
+  desc.InputFrameRate.Numerator = 60;
+  desc.InputFrameRate.Denominator = 1;
+  desc.InputWidth = 1920;
+  desc.InputHeight = 1080;
+  desc.OutputFrameRate.Numerator = 60;
+  desc.OutputFrameRate.Denominator = 1;
+  desc.OutputWidth = 1920;
+  desc.OutputHeight = 1080;
+  desc.Usage = D3D11_VIDEO_USAGE_PLAYBACK_NORMAL;
+
+  if (!SUCCEEDED(v_device->CreateVideoProcessorEnumerator(&desc, &enumerator_)))
+    return false;
+
+  // Check that the |CheckFormatSupport| and |CheckVideoProcessorFormat| calls
+  // won't be failing
+  UINT unneeded = 0;
+  DXGI_FORMAT example = DXGI_FORMAT_NV12;
+  if (!SUCCEEDED(device_->CheckFormatSupport(example, &unneeded)))
+    return false;
+
+  if (!SUCCEEDED(enumerator_->CheckVideoProcessorFormat(example, &unneeded)))
+    return false;
+
+  initialized_ = true;
+  return true;
+}
+
+bool FormatSupportChecker::CheckOutputFormatSupport(DXGI_FORMAT format) {
+  DCHECK(initialized_);
+  UINT device = 0, enumerator = 0;
+  if (!SUCCEEDED(device_->CheckFormatSupport(format, &device)))
+    return false;
+  if (!SUCCEEDED(enumerator_->CheckVideoProcessorFormat(format, &enumerator)))
+    return false;
+
+  return (enumerator & D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_OUTPUT) &&
+         (device & D3D11_FORMAT_SUPPORT_VIDEO_PROCESSOR_OUTPUT);
+}
+
+}  // namespace media

media/gpu/windows/d3d11_video_device_format_support.h

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef MEDIA_GPU_WINDOWS_D3D11_VIDEO_DEVICE_FORMAT_SUPPORT_H_
+#define MEDIA_GPU_WINDOWS_D3D11_VIDEO_DEVICE_FORMAT_SUPPORT_H_
+
+#include <d3d11_1.h>
+#include <vector>
+
+#include "base/optional.h"
+#include "media/base/media_log.h"
+#include "media/gpu/media_gpu_export.h"
+#include "media/gpu/windows/d3d11_com_defs.h"
+
+namespace media {
+
+// Helper class for Checking whether a video can be processed in any given
+// DXVI_FORMAT.
+class MEDIA_GPU_EXPORT FormatSupportChecker {
+ public:
+  explicit FormatSupportChecker(ComD3D11Device device);
+  ~FormatSupportChecker();
+
+  // Set up the device to be able to check format support.
+  // Returns false if there is a failure.
+  bool Initialize();
+
+  // Checks if the device's texture processing pipeline supports output textures
+  bool CheckOutputFormatSupport(DXGI_FORMAT format);
+
+ private:
+  ComD3D11Device device_;
+  ComD3D11VideoProcessorEnumerator enumerator_;
+  bool initialized_ = false;
+
+  DISALLOW_COPY_AND_ASSIGN(FormatSupportChecker);
+};
+
+}  // namespace media
+
+#endif  // MEDIA_GPU_WINDOWS_D3D11_VIDEO_DEVICE_FORMAT_SUPPORT_H_

media/gpu/windows/d3d11_video_device_format_support_unittest.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <utility>
+
+#include "media/base/media_util.h"
+#include "media/base/win/d3d11_mocks.h"
+#include "media/gpu/windows/d3d11_video_device_format_support.h"
+#include "testing/gmock/include/gmock/gmock.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+using ::testing::_;
+using ::testing::Combine;
+using ::testing::DoAll;
+using ::testing::NiceMock;
+using ::testing::Return;
+using ::testing::SetArgPointee;
+using ::testing::Values;
+
+namespace media {
+
+class FormatSupportCheckerUnittest : public ::testing::Test {};
+
+TEST_F(FormatSupportCheckerUnittest, CheckInitializationCantCast) {
+  auto device = CreateD3D11Mock<NiceMock<D3D11DeviceMock>>();
+  auto vdevice = CreateD3D11Mock<NiceMock<D3D11VideoDeviceMock>>();
+  auto enumerator =
+      CreateD3D11Mock<NiceMock<D3D11VideoProcessorEnumeratorMock>>();
+
+  ON_CALL(*device.Get(), QueryInterface(IID_ID3D11VideoDevice, _))
+      .WillByDefault(SetComPointeeAndReturnOk<1>(vdevice.Get()));
+
+  EXPECT_CALL(*vdevice.Get(), CreateVideoProcessorEnumerator(_, _))
+      .WillOnce(SetComPointeeAndReturnOk<1>(enumerator.Get()));
+
+  EXPECT_CALL(*device.Get(), CheckFormatSupport(_, _)).WillOnce(Return(S_OK));
+
+  EXPECT_CALL(*enumerator.Get(), CheckVideoProcessorFormat(_, _))
+      .WillOnce(Return(S_OK));
+
+  FormatSupportChecker checker(device);
+  EXPECT_TRUE(checker.Initialize());
+}
+
+TEST_F(FormatSupportCheckerUnittest, CheckFormatSupportWorks) {
+  auto device = CreateD3D11Mock<NiceMock<D3D11DeviceMock>>();
+  auto vdevice = CreateD3D11Mock<NiceMock<D3D11VideoDeviceMock>>();
+  auto enumerator =
+      CreateD3D11Mock<NiceMock<D3D11VideoProcessorEnumeratorMock>>();
+
+  ON_CALL(*device.Get(), QueryInterface(IID_ID3D11VideoDevice, _))
+      .WillByDefault(SetComPointeeAndReturnOk<1>(vdevice.Get()));
+
+  EXPECT_CALL(*vdevice.Get(), CreateVideoProcessorEnumerator(_, _))
+      .WillOnce(SetComPointeeAndReturnOk<1>(enumerator.Get()));
+
+  EXPECT_CALL(*device.Get(), CheckFormatSupport(_, _)).WillOnce(Return(S_OK));
+
+  EXPECT_CALL(*enumerator.Get(), CheckVideoProcessorFormat(_, _))
+      .WillOnce(Return(S_OK));
+
+  FormatSupportChecker checker(device);
+  EXPECT_TRUE(checker.Initialize());
+
+  UINT enumerator_outcome = D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_OUTPUT;
+  UINT device_outcome = D3D11_FORMAT_SUPPORT_VIDEO_PROCESSOR_OUTPUT;
+  EXPECT_CALL(*enumerator.Get(), CheckVideoProcessorFormat(_, _))
+      .WillOnce(DoAll(SetArgPointee<1>(enumerator_outcome), Return(S_OK)));
+  EXPECT_CALL(*device.Get(), CheckFormatSupport(_, _))
+      .WillOnce(DoAll(SetArgPointee<1>(device_outcome), Return(S_OK)));
+
+  EXPECT_TRUE(checker.CheckOutputFormatSupport(DXGI_FORMAT_NV12));
+}
+
+TEST_F(FormatSupportCheckerUnittest, CheckFormatSupportRequiresBoth) {
+  auto device = CreateD3D11Mock<NiceMock<D3D11DeviceMock>>();
+  auto vdevice = CreateD3D11Mock<NiceMock<D3D11VideoDeviceMock>>();
+  auto enumerator =
+      CreateD3D11Mock<NiceMock<D3D11VideoProcessorEnumeratorMock>>();
+
+  ON_CALL(*device.Get(), QueryInterface(IID_ID3D11VideoDevice, _))
+      .WillByDefault(SetComPointeeAndReturnOk<1>(vdevice.Get()));
+
+  EXPECT_CALL(*vdevice.Get(), CreateVideoProcessorEnumerator(_, _))
+      .WillOnce(SetComPointeeAndReturnOk<1>(enumerator.Get()));
+
+  EXPECT_CALL(*device.Get(), CheckFormatSupport(_, _)).WillOnce(Return(S_OK));
+
+  EXPECT_CALL(*enumerator.Get(), CheckVideoProcessorFormat(_, _))
+      .WillOnce(Return(S_OK));
+
+  FormatSupportChecker checker(device);
+  EXPECT_TRUE(checker.Initialize());
+
+  UINT enumerator_outcome = D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_OUTPUT;
+  UINT device_outcome = 0;
+  EXPECT_CALL(*enumerator.Get(), CheckVideoProcessorFormat(_, _))
+      .WillOnce(DoAll(SetArgPointee<1>(enumerator_outcome), Return(S_OK)));
+  EXPECT_CALL(*device.Get(), CheckFormatSupport(_, _))
+      .WillOnce(DoAll(SetArgPointee<1>(device_outcome), Return(S_OK)));
+
+  EXPECT_FALSE(checker.CheckOutputFormatSupport(DXGI_FORMAT_NV12));
+}
+
+}  // namespace media

media/gpu/windows/dxva_video_decode_accelerator_win.cc

@@ -50,6 +50,7 @@
 #include "media/base/win/mf_helpers.h"
 #include "media/base/win/mf_initializer.h"
 #include "media/filters/vp9_parser.h"
+#include "media/gpu/windows/d3d11_video_device_format_support.h"
 #include "media/gpu/windows/dxva_picture_buffer_win.h"
 #include "media/gpu/windows/supported_profile_helpers.h"
 #include "media/video/h264_parser.h"
@@ -881,19 +882,15 @@ bool DXVAVideoDecodeAccelerator::CreateDX11DevManager() {
   if (!options.ExtendedResourceSharing)
     support_copy_nv12_textures_ = false;
 
-  UINT nv12_format_support = 0;
-  hr =
-      D3D11Device()->CheckFormatSupport(DXGI_FORMAT_NV12, &nv12_format_support);
-  RETURN_ON_HR_FAILURE(hr, "Failed to check NV12 format support", false);
+  FormatSupportChecker checker(ShouldUseANGLEDevice() ? angle_device_
+                                                      : d3d11_device_);
+  RETURN_ON_FAILURE(checker.Initialize(), "Failed to check format supports!",
+                    false);
 
-  if (!(nv12_format_support & D3D11_FORMAT_SUPPORT_VIDEO_PROCESSOR_OUTPUT))
+  if (!checker.CheckOutputFormatSupport(DXGI_FORMAT_NV12))
     support_copy_nv12_textures_ = false;
 
-  UINT fp16_format_support = 0;
-  hr = D3D11Device()->CheckFormatSupport(DXGI_FORMAT_R16G16B16A16_FLOAT,
-                                         &fp16_format_support);
-  if (FAILED(hr) ||
-      !(fp16_format_support & D3D11_FORMAT_SUPPORT_VIDEO_PROCESSOR_OUTPUT))
+  if (!checker.CheckOutputFormatSupport(DXGI_FORMAT_R16G16B16A16_FLOAT))
     use_fp16_ = false;
 
   // Enable multithreaded mode on the device. This ensures that accesses to

media/gpu/windows/supported_profile_helpers.h

@@ -16,7 +16,6 @@
 #include "media/gpu/windows/d3d11_com_defs.h"
 #include "ui/gfx/geometry/rect.h"
 
-#include "ui/gfx/geometry/rect.h"
 
 namespace media {