Fix VANGLE initialize failure for remote desktop environment.

For remote desktop environment, when Vulkan is used, VANGLE is only used
for WebGL. It is not responsible for rendering to the native windows (we
will not create GLSurfaceEGL for native windows). We will use Vulkan
swapchain to render on native windows. However some Vulkan drivers
(NVIDIA) cannot be initialized successfully, if the $DISPLAY env
variable is set, but the Xserver(remote desktop Xserver) doesn't have
necessary extensions (NV-CONTROL, NV-GLX). To workaround this issue,
this CL unsets the $DISPLAY env variable before initializing VANGLE and
sets it back after it, if the Xserver doesn't support vulkan swapchain.

Change-Id: I528a6f7d2f618123ea519f0ab4f15e788c4dd607
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2424519Reviewed-by: Thomas Anderson <thomasanderson@chromium.org>
Reviewed-by: Jonathan Backer <backer@chromium.org>
Commit-Queue: Peng Huang <penghuang@chromium.org>
Cr-Commit-Position: refs/heads/master@{#809872}

gpu/vulkan/x/vulkan_implementation_x11.cc

@@ -16,6 +16,7 @@
 #include "gpu/vulkan/vulkan_surface.h"
 #include "gpu/vulkan/vulkan_util.h"
 #include "gpu/vulkan/x/vulkan_surface_x11.h"
+#include "ui/base/x/x11_util.h"
 #include "ui/gfx/gpu_fence.h"
 #include "ui/gfx/gpu_memory_buffer.h"
 #include "ui/gfx/x/x11.h"
@@ -25,43 +26,6 @@ namespace gpu {
 
 namespace {
 
-bool IsVulkanSurfaceSupported() {
-  static const char* extensions[] = {
-      "DRI3",         // open source driver.
-      "ATIFGLRXDRI",  // AMD proprietary driver.
-      "NV-CONTROL",   // NVidia proprietary driver.
-  };
-  auto* display = gfx::GetXDisplay();
-  int ext_code, first_event, first_error;
-  for (const auto* extension : extensions) {
-    if (XQueryExtension(display, extension, &ext_code, &first_event,
-                        &first_error)) {
-      return true;
-    }
-  }
-  return false;
-}
-
-class ScopedUnsetDisplay {
- public:
-  ScopedUnsetDisplay() {
-    const char* display = getenv("DISPLAY");
-    if (display) {
-      display_.emplace(display);
-      unsetenv("DISPLAY");
-    }
-  }
-  ~ScopedUnsetDisplay() {
-    if (display_) {
-      setenv("DISPLAY", display_->c_str(), 1);
-    }
-  }
-
- private:
-  base::Optional<std::string> display_;
-  DISALLOW_COPY_AND_ASSIGN(ScopedUnsetDisplay);
-};
-
 bool InitializeVulkanFunctionPointers(
     const base::FilePath& path,
     VulkanFunctionPointers* vulkan_function_pointers) {
@@ -81,12 +45,12 @@ VulkanImplementationX11::VulkanImplementationX11(bool use_swiftshader)
 VulkanImplementationX11::~VulkanImplementationX11() = default;
 
 bool VulkanImplementationX11::InitializeVulkanInstance(bool using_surface) {
-  if (using_surface && !use_swiftshader() && !IsVulkanSurfaceSupported())
+  if (using_surface && !use_swiftshader() && !ui::IsVulkanSurfaceSupported())
     using_surface = false;
   using_surface_ = using_surface;
   // Unset DISPLAY env, so the vulkan can be initialized successfully, if the X
   // server doesn't support Vulkan surface.
-  base::Optional<ScopedUnsetDisplay> unset_display;
+  base::Optional<ui::ScopedUnsetDisplay> unset_display;
   if (!using_surface_)
     unset_display.emplace();
 

ui/base/x/x11_util.cc

@@ -1128,6 +1128,23 @@ void SetX11ErrorHandlers(XErrorHandler error_handler,
                                       : DefaultX11IOErrorHandler);
 }
 
+bool IsVulkanSurfaceSupported() {
+  static const char* extensions[] = {
+      "DRI3",         // open source driver.
+      "ATIFGLRXDRI",  // AMD proprietary driver.
+      "NV-CONTROL",   // NVidia proprietary driver.
+  };
+  auto* display = gfx::GetXDisplay();
+  int ext_code, first_event, first_error;
+  for (const auto* extension : extensions) {
+    if (XQueryExtension(display, extension, &ext_code, &first_event,
+                        &first_error)) {
+      return true;
+    }
+  }
+  return false;
+}
+
 // static
 XVisualManager* XVisualManager::GetInstance() {
   return base::Singleton<XVisualManager>::get();
@@ -1265,4 +1282,18 @@ x11::ColorMap XVisualManager::XVisualData::GetColormap() {
   return colormap_;
 }
 
+ScopedUnsetDisplay::ScopedUnsetDisplay() {
+  const char* display = getenv("DISPLAY");
+  if (display) {
+    display_.emplace(display);
+    unsetenv("DISPLAY");
+  }
+}
+
+ScopedUnsetDisplay::~ScopedUnsetDisplay() {
+  if (display_) {
+    setenv("DISPLAY", display_->c_str(), 1);
+  }
+}
+
 }  // namespace ui

ui/base/x/x11_util.h

@@ -555,12 +555,8 @@ COMPONENT_EXPORT(UI_BASE_X)
 void SetX11ErrorHandlers(XErrorHandler error_handler,
                          XIOErrorHandler io_error_handler);
 
-// NOTE: This function should not be called directly from the
-// X11 Error handler because it queries the server to decode the
-// error message, which may trigger other errors. A suitable workaround
-// is to post a task in the error handler to call this function.
-COMPONENT_EXPORT(UI_BASE_X)
-void LogErrorEventDescription(Display* dpy, const XErrorEvent& error_event);
+// Return true if VulkanSurface is supported.
+COMPONENT_EXPORT(UI_BASE_X) bool IsVulkanSurfaceSupported();
 
 // --------------------------------------------------------------------------
 // Selects a visual with a preference for alpha support on compositing window
@@ -640,6 +636,16 @@ class COMPONENT_EXPORT(UI_BASE_X) XVisualManager {
   DISALLOW_COPY_AND_ASSIGN(XVisualManager);
 };
 
+class COMPONENT_EXPORT(UI_BASE_X) ScopedUnsetDisplay {
+ public:
+  ScopedUnsetDisplay();
+  ~ScopedUnsetDisplay();
+
+ private:
+  base::Optional<std::string> display_;
+  DISALLOW_COPY_AND_ASSIGN(ScopedUnsetDisplay);
+};
+
 }  // namespace ui
 
 #endif  // UI_BASE_X_X11_UTIL_H_

ui/gl/gl_surface_egl.cc

@@ -52,6 +52,10 @@
 #define EGL_OPENGL_ES3_BIT 0x00000040
 #endif
 
+#if defined(USE_X11)
+#include "ui/base/x/x11_util.h"
+#endif
+
 // Not present egl/eglext.h yet.
 
 #ifndef EGL_EXT_gl_colorspace_display_p3
@@ -347,6 +351,7 @@ EGLDisplay GetPlatformANGLEDisplay(
   // TODO(dbehr) Add an attrib to Angle to pass EGL platform.
 
   display_attribs.push_back(EGL_NONE);
+
   // This is an EGL 1.5 function that we know ANGLE supports. It's used to pass
   // EGLAttribs (pointers) instead of EGLints into the display
   return eglGetPlatformDisplay(
@@ -1289,6 +1294,14 @@ EGLDisplay GLSurfaceEGL::InitializeDisplay(EGLDisplayPlatform native_display) {
           GetANGLEImplementationFromDisplayType(display_type));
     }
 
+#if defined(USE_X11)
+    // Unset DISPLAY env, so the vulkan can be initialized successfully, if the
+    // X server doesn't support Vulkan surface.
+    base::Optional<ui::ScopedUnsetDisplay> unset_display;
+    if (display_type == ANGLE_VULKAN && !ui::IsVulkanSurfaceSupported())
+      unset_display.emplace();
+#endif  // defined(USE_X11)
+
     if (!eglInitialize(display, nullptr, nullptr)) {
       bool is_last = disp_index == init_displays.size() - 1;