Disable Direct Composition or overlay when the function fails to prevent a blank screen.

Returning a failure in the Direct Composition functions will trigger a
context lost in GL Renderer and Direct Composition will be reinitialized.
If the same Direct Composition function keeps failing, it will cause an
infinity loop of restart and the result is a blank screen or a blank
video.

If a DXGI or D3D11 call fails due to out of memory or a bad graphics
driver, most likely the same call will fail next time. We should disable
Direct Composition or Overlay to prevent the same failure.

Bug: 1091958
Change-Id: I3ac7d9bac0fdf5938302583b072e7bda5839d2c2
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2247390
Commit-Queue: Maggie Chen <magchen@chromium.org>
Reviewed-by: Sunny Sachanandani <sunnyps@chromium.org>
Cr-Commit-Position: refs/heads/master@{#781408}

ui/gl/dc_layer_tree.cc

@@ -6,6 +6,7 @@
 
 #include "base/trace_event/trace_event.h"
 #include "ui/gl/direct_composition_child_surface_win.h"
+#include "ui/gl/direct_composition_surface_win.h"
 #include "ui/gl/swap_chain_presenter.h"
 
 namespace gl {
@@ -63,6 +64,8 @@ bool DCLayerTree::InitializeVideoProcessor(const gfx::Size& input_size,
     // This can fail if the D3D device is "Microsoft Basic Display Adapter".
     if (FAILED(d3d11_device_.As(&video_device_))) {
       DLOG(ERROR) << "Failed to retrieve video device from D3D11 device";
+      DCHECK(false);
+      DirectCompositionSurfaceWin::DisableOverlays();
       return false;
     }
     DCHECK(video_device_);
@@ -100,6 +103,9 @@ bool DCLayerTree::InitializeVideoProcessor(const gfx::Size& input_size,
   if (FAILED(hr)) {
     DLOG(ERROR) << "CreateVideoProcessorEnumerator failed with error 0x"
                 << std::hex << hr;
+    // It might fail again next time. Disable overlay support so
+    // overlay processor will stop sending down overlay frames.
+    DirectCompositionSurfaceWin::DisableOverlays();
     return false;
   }
 
@@ -108,6 +114,9 @@ bool DCLayerTree::InitializeVideoProcessor(const gfx::Size& input_size,
   if (FAILED(hr)) {
     DLOG(ERROR) << "CreateVideoProcessor failed with error 0x" << std::hex
                 << hr;
+    // It might fail again next time. Disable overlay support so
+    // overlay processor will stop sending down overlay frames.
+    DirectCompositionSurfaceWin::DisableOverlays();
     return false;
   }
 

ui/gl/direct_composition_child_surface_win.cc

@@ -286,6 +286,9 @@ bool DirectCompositionChildSurfaceWin::SetDrawRectangle(
         DXGI_ALPHA_MODE_PREMULTIPLIED, &dcomp_surface_);
     if (FAILED(hr)) {
       DLOG(ERROR) << "CreateSurface failed with error " << std::hex << hr;
+      // Disable direct composition because CreateSurface might fail again next
+      // time.
+      g_direct_composition_swap_chain_failed = true;
       return false;
     }
   } else if (!swap_chain_ && !enable_dc_layers_) {
@@ -331,6 +334,8 @@ bool DirectCompositionChildSurfaceWin::SetDrawRectangle(
     if (FAILED(hr)) {
       DLOG(ERROR) << "CreateSwapChainForComposition failed with error "
                   << std::hex << hr;
+      // Disable direct composition because SwapChain creation might fail again
+      // next time.
       g_direct_composition_swap_chain_failed = true;
       return false;
     }

ui/gl/swap_chain_presenter.cc

@@ -262,6 +262,7 @@ Microsoft::WRL::ComPtr<ID3D11Texture2D> SwapChainPresenter::UploadVideoImages(
     if (FAILED(hr)) {
       DLOG(ERROR) << "Creating D3D11 video staging texture failed: " << std::hex
                   << hr;
+      DirectCompositionSurfaceWin::DisableOverlays();
       return nullptr;
     }
     DCHECK(staging_texture_);
@@ -314,6 +315,7 @@ Microsoft::WRL::ComPtr<ID3D11Texture2D> SwapChainPresenter::UploadVideoImages(
     if (FAILED(hr)) {
       DLOG(ERROR) << "Creating D3D11 video upload texture failed: " << std::hex
                   << hr;
+      DirectCompositionSurfaceWin::DisableOverlays();
       return nullptr;
     }
     DCHECK(copy_texture_);
@@ -602,6 +604,7 @@ bool SwapChainPresenter::PresentToDecodeSwapChain(
       DLOG(ERROR) << "CreateDecodeSwapChainForCompositionSurfaceHandle failed "
                      "with error 0x"
                   << std::hex << hr;
+      DirectCompositionSurfaceWin::DisableOverlays();
       return false;
     }
     DCHECK(decode_swap_chain_);
@@ -615,6 +618,7 @@ bool SwapChainPresenter::PresentToDecodeSwapChain(
     if (FAILED(hr)) {
       DLOG(ERROR) << "CreateSurfaceFromHandle failed with error 0x" << std::hex
                   << hr;
+      DirectCompositionSurfaceWin::DisableOverlays();
       return false;
     }
     DCHECK(decode_surface_);