WebGL: Add a direct uploading path for texImage2D with video frame backed by GPU

Current fast path for webgl uploading video frame is to render YUV planars to a RGBA8
intermediate gl texture and than do gpu to gpu copy to update the dst webgl texture.

It is possible to further optimize this progress by drawing the content from YUV planars
to dst webgl texture directly. This will save the extra gpu-gpu texture copy which may
hurt performance when uploading large resolution video (e.g. 4k/8k video).

BUG=1108154

Change-Id: I0abb269944f9eac9fb77e13b01860152a918ad95
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2319543Reviewed-by: Brian Salomon <bsalomon@google.com>
Reviewed-by: Dan Sanders <sandersd@chromium.org>
Reviewed-by: Kenneth Russell <kbr@chromium.org>
Commit-Queue: Shaobo Yan <shaobo.yan@intel.com>
Cr-Commit-Position: refs/heads/master@{#800811}

media/renderers/paint_canvas_video_renderer.cc

@@ -17,6 +17,7 @@
 #include "base/system/sys_info.h"
 #include "base/task/thread_pool.h"
 #include "base/threading/thread_restrictions.h"
+#include "build/build_config.h"
 #include "cc/paint/paint_canvas.h"
 #include "cc/paint/paint_flags.h"
 #include "cc/paint/paint_image_builder.h"
@@ -591,6 +592,27 @@ void ConvertVideoFrameToRGBPixelsTask(const VideoFrame* video_frame,
   done->Run();
 }
 
+// Valid gl texture internal format that can try to use direct uploading path.
+bool ValidFormatForDirectUploading(GrGLenum format, unsigned int type) {
+  switch (format) {
+    case GL_RGBA:
+      return type == GL_UNSIGNED_BYTE || type == GL_UNSIGNED_SHORT_4_4_4_4;
+    case GL_RGB:
+      return type == GL_UNSIGNED_BYTE || type == GL_UNSIGNED_SHORT_5_6_5;
+    // WebGL2 supported sized formats
+    case GL_RGBA8:
+    case GL_RGB565:
+    case GL_RGBA16F:
+    case GL_RGB8:
+    case GL_RGB10_A2:
+    case GL_RGBA4:
+    case GL_SRGB8_ALPHA8:
+      return true;
+    default:
+      return false;
+  }
+}
+
 }  // anonymous namespace
 
 // Generates an RGB image from a VideoFrame. Convert YUV to RGB plain on GPU.
@@ -1240,6 +1262,21 @@ bool PaintCanvasVideoRenderer::CopyVideoFrameTexturesToGLTexture(
     GrContext* gr_context = raster_context_provider->GrContext();
     if (!gr_context)
       return false;
+
+// TODO(crbug.com/1108154): This uploading path failed on mac with error
+// invalid mailbox name. Suspect it is related to texture target. Disable
+// direct uploading path on mac platform for now.
+#if !defined(OS_MAC)
+    // Try direct uploading path
+    if (!premultiply_alpha && level == 0) {
+      if (UploadVideoFrameToGLTexture(raster_context_provider, destination_gl,
+                                      video_frame, target, texture,
+                                      internal_format, format, type, flip_y)) {
+        return true;
+      }
+    }
+#endif  // !defined(OS_MAC)
+
     if (!UpdateLastImage(video_frame, raster_context_provider,
                          true /* allow_wrap_texture */)) {
       return false;
@@ -1297,6 +1334,75 @@ bool PaintCanvasVideoRenderer::CopyVideoFrameTexturesToGLTexture(
   return true;
 }
 
+bool PaintCanvasVideoRenderer::UploadVideoFrameToGLTexture(
+    viz::RasterContextProvider* raster_context_provider,
+    gpu::gles2::GLES2Interface* destination_gl,
+    scoped_refptr<VideoFrame> video_frame,
+    unsigned int target,
+    unsigned int texture,
+    unsigned int internal_format,
+    unsigned int format,
+    unsigned int type,
+    bool flip_y) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(video_frame);
+  DCHECK(video_frame->HasTextures());
+  // Support uploading for NV12 and I420 video frame only.
+  if (video_frame->format() != PIXEL_FORMAT_I420 &&
+      video_frame->format() != PIXEL_FORMAT_NV12) {
+    return false;
+  }
+
+  // TODO(crbug.com/1108154): Investigate how to support other texture target
+  if (target != GL_TEXTURE_2D) {
+    return false;
+  }
+
+  if (!ValidFormatForDirectUploading(static_cast<GLenum>(internal_format),
+                                     type)) {
+    return false;
+  }
+
+  if (!raster_context_provider || !raster_context_provider->GrContext())
+    return false;
+
+  // Trigger resource allocation for dst texture to back SkSurface.
+  // Dst texture size should equal to video frame visible rect.
+  destination_gl->BindTexture(target, texture);
+  destination_gl->TexImage2D(
+      target, 0, internal_format, video_frame->visible_rect().width(),
+      video_frame->visible_rect().height(), 0, format, type, nullptr);
+
+  gpu::MailboxHolder mailbox_holder;
+  mailbox_holder.texture_target = target;
+  destination_gl->ProduceTextureDirectCHROMIUM(texture,
+                                               mailbox_holder.mailbox.name);
+
+  destination_gl->GenUnverifiedSyncTokenCHROMIUM(
+      mailbox_holder.sync_token.GetData());
+
+  ConvertFromVideoFrameYUV(video_frame.get(), raster_context_provider,
+                           mailbox_holder, internal_format, type, flip_y,
+                           true  // use visible_rect
+  );
+
+  gpu::raster::RasterInterface* source_ri =
+      raster_context_provider->RasterInterface();
+  // Wait for mailbox creation on canvas context before consuming it and
+  // copying from it on the consumer context.
+  source_ri->GenUnverifiedSyncTokenCHROMIUM(
+      mailbox_holder.sync_token.GetData());
+
+  destination_gl->WaitSyncTokenCHROMIUM(
+      mailbox_holder.sync_token.GetConstData());
+
+  SyncTokenClientImpl client(source_ri);
+  video_frame->UpdateReleaseSyncToken(&client);
+
+  DCHECK(!cache_ || !cache_->wraps_video_frame_texture);
+  return true;
+}
+
 bool PaintCanvasVideoRenderer::PrepareVideoFrameForWebGL(
     viz::RasterContextProvider* raster_context_provider,
     gpu::gles2::GLES2Interface* destination_gl,

media/renderers/paint_canvas_video_renderer.h

@@ -257,6 +257,17 @@ class MEDIA_EXPORT PaintCanvasVideoRenderer {
                          viz::RasterContextProvider* raster_context_provider,
                          const gpu::MailboxHolder& dest_holder);
 
+  bool UploadVideoFrameToGLTexture(
+      viz::RasterContextProvider* raster_context_provider,
+      gpu::gles2::GLES2Interface* destination_gl,
+      scoped_refptr<VideoFrame> video_frame,
+      unsigned int target,
+      unsigned int texture,
+      unsigned int internal_format,
+      unsigned int format,
+      unsigned int type,
+      bool flip_y);
+
   base::Optional<Cache> cache_;
 
   // If |cache_| is not used for a while, it's deleted to save memory.

media/renderers/yuv_util.cc

@@ -7,14 +7,19 @@
 #include <GLES3/gl3.h>
 
 #include "components/viz/common/gpu/raster_context_provider.h"
-#include "gpu/GLES2/gl2extchromium.h"
 #include "gpu/command_buffer/client/raster_interface.h"
 #include "gpu/command_buffer/client/shared_image_interface.h"
 #include "gpu/command_buffer/common/mailbox_holder.h"
 #include "gpu/command_buffer/common/shared_image_usage.h"
 #include "media/base/video_frame.h"
+#include "third_party/skia/include/core/SkCanvas.h"
 #include "third_party/skia/include/core/SkImage.h"
+#include "third_party/skia/include/core/SkRefCnt.h"
+#include "third_party/skia/include/core/SkSurface.h"
+#include "third_party/skia/include/core/SkYUVAIndex.h"
+#include "third_party/skia/include/gpu/GrContext.h"
 #include "third_party/skia/include/gpu/GrDirectContext.h"
+#include "third_party/skia/include/gpu/gl/GrGLTypes.h"
 
 namespace media {
 
@@ -181,25 +186,18 @@ class VideoFrameYUVMailboxesHolder {
   YUVTexturesInfo textures_;
 };
 
-void ConvertFromVideoFrameYUVWithGrContext(
-    const VideoFrame* video_frame,
-    viz::RasterContextProvider* raster_context_provider,
-    const gpu::MailboxHolder& dest_mailbox_holder) {
-  gpu::raster::RasterInterface* ri = raster_context_provider->RasterInterface();
-  DCHECK(ri);
-  ri->WaitSyncTokenCHROMIUM(dest_mailbox_holder.sync_token.GetConstData());
-  GLuint dest_tex_id =
-      ri->CreateAndConsumeForGpuRaster(dest_mailbox_holder.mailbox);
-  if (dest_mailbox_holder.mailbox.IsSharedImage()) {
-    ri->BeginSharedImageAccessDirectCHROMIUM(
-        dest_tex_id, GL_SHARED_IMAGE_ACCESS_MODE_READWRITE_CHROMIUM);
+// Some YUVA factories infer the YUVAIndices. This helper identifies the channel
+// to use for single channel textures.
+static SkColorChannel GetSingleChannel(const GrBackendTexture& tex) {
+  switch (tex.getBackendFormat().channelMask()) {
+    case kGray_SkColorChannelFlag:  // Gray can be read as any of kR, kG, kB.
+    case kRed_SkColorChannelFlag:
+      return SkColorChannel::kR;
+    case kAlpha_SkColorChannelFlag:
+      return SkColorChannel::kA;
+    default:  // multiple channels in the texture. Guess kR.
+      return SkColorChannel::kR;
   }
-  // Let the SkImage fall out of scope and track the result using dest_tex_id
-  NewSkImageFromVideoFrameYUV(video_frame, raster_context_provider,
-                              dest_mailbox_holder.texture_target, dest_tex_id);
-  if (dest_mailbox_holder.mailbox.IsSharedImage())
-    ri->EndSharedImageAccessDirectCHROMIUM(dest_tex_id);
-  ri->DeleteGpuRasterTexture(dest_tex_id);
 }
 
 SkYUVColorSpace ColorSpaceToSkYUVColorSpace(
@@ -211,46 +209,113 @@ SkYUVColorSpace ColorSpaceToSkYUVColorSpace(
   return sk_color_space;
 }
 
-}  // namespace
+SkColorType GetCompatibleSurfaceColorType(GrGLenum format) {
+  switch (format) {
+    case GL_RGBA8:
+      return kRGBA_8888_SkColorType;
+    case GL_RGB565:
+      return kRGB_565_SkColorType;
+    case GL_RGBA16F:
+      return kRGBA_F16_SkColorType;
+    case GL_RGB8:
+      return kRGB_888x_SkColorType;
+    case GL_RGB10_A2:
+      return kRGBA_1010102_SkColorType;
+    case GL_RGBA4:
+      return kARGB_4444_SkColorType;
+    case GL_SRGB8_ALPHA8:
+      return kRGBA_8888_SkColorType;
+    default:
+      NOTREACHED();
+      return kUnknown_SkColorType;
+  }
+}
 
-void ConvertFromVideoFrameYUV(
+GrGLenum GetSurfaceColorFormat(GrGLenum format, GrGLenum type) {
+  if (format == GL_RGBA) {
+    if (type == GL_UNSIGNED_BYTE)
+      return GL_RGBA8;
+    if (type == GL_UNSIGNED_SHORT_4_4_4_4)
+      return GL_RGBA4;
+  }
+  if (format == GL_RGB) {
+    if (type == GL_UNSIGNED_BYTE)
+      return GL_RGB8;
+    if (type == GL_UNSIGNED_SHORT_5_6_5)
+      return GL_RGB565;
+  }
+  return format;
+}
+
+bool YUVGrBackendTexturesToSkSurface(GrDirectContext* gr_context,
                                      const VideoFrame* video_frame,
-    viz::RasterContextProvider* raster_context_provider,
-    const gpu::MailboxHolder& dest_mailbox_holder) {
-  DCHECK(raster_context_provider);
-  if (raster_context_provider->GrContext()) {
-    ConvertFromVideoFrameYUVWithGrContext(video_frame, raster_context_provider,
-                                          dest_mailbox_holder);
-    return;
+                                     GrBackendTexture* yuv_textures,
+                                     sk_sp<SkSurface> surface,
+                                     bool flip_y,
+                                     bool use_visible_rect) {
+  SkYUVAIndex indices[4];
+
+  switch (video_frame->format()) {
+    case PIXEL_FORMAT_NV12:
+      indices[SkYUVAIndex::kY_Index] = {
+          0, GetSingleChannel(yuv_textures[0])};  // the first backend texture
+      indices[SkYUVAIndex::kU_Index] = {
+          1, SkColorChannel::kR};  // the second backend texture
+      indices[SkYUVAIndex::kV_Index] = {1, SkColorChannel::kG};
+      indices[SkYUVAIndex::kA_Index] = {-1,
+                                        SkColorChannel::kA};  // no alpha plane
+      break;
+    case PIXEL_FORMAT_I420:
+      indices[SkYUVAIndex::kY_Index] = {
+          0, GetSingleChannel(yuv_textures[0])};  // the first backend texture
+      indices[SkYUVAIndex::kU_Index] = {
+          1, GetSingleChannel(yuv_textures[1])};  // the second backend texture
+      indices[SkYUVAIndex::kV_Index] = {2, GetSingleChannel(yuv_textures[2])};
+      indices[SkYUVAIndex::kA_Index] = {-1,
+                                        SkColorChannel::kA};  // no alpha plane
+      break;
+    default:
+      NOTREACHED();
+      return false;
   }
 
-  auto* ri = raster_context_provider->RasterInterface();
-  DCHECK(ri);
-  ri->WaitSyncTokenCHROMIUM(dest_mailbox_holder.sync_token.GetConstData());
-  SkYUVColorSpace color_space =
-      ColorSpaceToSkYUVColorSpace(video_frame->ColorSpace());
+  auto image = SkImage::MakeFromYUVATextures(
+      gr_context, ColorSpaceToSkYUVColorSpace(video_frame->ColorSpace()),
+      yuv_textures, indices,
+      {video_frame->coded_size().width(), video_frame->coded_size().height()},
+      kTopLeft_GrSurfaceOrigin, SkColorSpace::MakeSRGB());
 
-  VideoFrameYUVMailboxesHolder yuv_mailboxes(video_frame,
-                                             raster_context_provider, false);
+  if (!image) {
+    return false;
+  }
 
-  if (yuv_mailboxes.is_nv12()) {
-    ri->ConvertNV12MailboxesToRGB(dest_mailbox_holder.mailbox, color_space,
-                                  yuv_mailboxes.mailbox(kYIndex),
-                                  yuv_mailboxes.mailbox(kUIndex));
+  if (!use_visible_rect) {
+    surface->getCanvas()->drawImage(image, 0, 0);
   } else {
-    DCHECK_EQ(video_frame->NumTextures(), kNumYUVPlanes);
-    ri->ConvertYUVMailboxesToRGB(dest_mailbox_holder.mailbox, color_space,
-                                 yuv_mailboxes.mailbox(kYIndex),
-                                 yuv_mailboxes.mailbox(kUIndex),
-                                 yuv_mailboxes.mailbox(kVIndex));
+    // Draw the planar SkImage to the SkSurface wrapping the WebGL texture.
+    // Using drawImageRect to draw visible rect from video frame to dst texture.
+    const gfx::Rect& visible_rect = video_frame->visible_rect();
+    const SkRect src_rect =
+        SkRect::MakeXYWH(visible_rect.x(), visible_rect.y(),
+                         visible_rect.width(), visible_rect.height());
+    const SkRect dst_rect =
+        SkRect::MakeWH(visible_rect.width(), visible_rect.height());
+    surface->getCanvas()->drawImageRect(image, src_rect, dst_rect, nullptr);
   }
+
+  surface->flushAndSubmit();
+  return true;
 }
 
-sk_sp<SkImage> NewSkImageFromVideoFrameYUV(
+bool ConvertFromVideoFrameYUVToSkSurface(
     const VideoFrame* video_frame,
     viz::RasterContextProvider* raster_context_provider,
     unsigned int texture_target,
-    unsigned int texture_id) {
+    unsigned int texture_id,
+    unsigned int internal_format,
+    unsigned int type,
+    bool flip_y,
+    bool use_visible_rect) {
   DCHECK(video_frame->HasTextures() ||
          (video_frame->IsMappable() &&
           video_frame->format() == PIXEL_FORMAT_I420));
@@ -278,41 +343,103 @@ sk_sp<SkImage> NewSkImageFromVideoFrameYUV(
       GrBackendTexture(uv_tex_size.width(), uv_tex_size.height(),
                        GrMipMapped::kNo, yuv_textures_info.texture(kVIndex)),
   };
+
   backend_texture.fID = texture_id;
   backend_texture.fTarget = texture_target;
-  backend_texture.fFormat = GL_RGBA8;
-  GrBackendTexture result_texture(video_frame->coded_size().width(),
-                                  video_frame->coded_size().height(),
+  backend_texture.fFormat = GetSurfaceColorFormat(internal_format, type);
+
+  // For method like texImage2D, dst texture is Ok to use visible_rect
+  // but for method like create imageBitmap from video frame, it is better
+  // to use coded_size to store more pixels.
+  int backend_texture_width = use_visible_rect
+                                  ? video_frame->visible_rect().width()
+                                  : video_frame->coded_size().width();
+  int backend_texture_height = use_visible_rect
+                                   ? video_frame->visible_rect().height()
+                                   : video_frame->coded_size().height();
+
+  GrBackendTexture result_texture(backend_texture_width, backend_texture_height,
                                   GrMipMapped::kNo, backend_texture);
 
-  sk_sp<SkImage> img = YUVGrBackendTexturesToSkImage(
-      gr_context, video_frame->ColorSpace(), video_frame->format(),
-      yuv_textures, result_texture);
-  gr_context->flushAndSubmit();
+  // Wraps the result texture in a SkSurface which allows Skia to render to it
+  // (creates FBO, etc)
+  auto surface = SkSurface::MakeFromBackendTexture(
+      gr_context, result_texture,
+      flip_y ? kBottomLeft_GrSurfaceOrigin : kTopLeft_GrSurfaceOrigin, 1,
+      GetCompatibleSurfaceColorType(backend_texture.fFormat),
+      SkColorSpace::MakeSRGB(), nullptr);
 
-  return img;
+  if (!surface) {
+    return false;
+  }
+
+  return YUVGrBackendTexturesToSkSurface(gr_context, video_frame, yuv_textures,
+                                         surface, flip_y, use_visible_rect);
 }
 
-sk_sp<SkImage> YUVGrBackendTexturesToSkImage(
-    GrDirectContext* gr_context,
-    gfx::ColorSpace video_color_space,
-    VideoPixelFormat video_format,
-    GrBackendTexture* yuv_textures,
-    const GrBackendTexture& result_texture) {
-  SkYUVColorSpace color_space = ColorSpaceToSkYUVColorSpace(video_color_space);
+void ConvertFromVideoFrameYUVWithGrContext(
+    const VideoFrame* video_frame,
+    viz::RasterContextProvider* raster_context_provider,
+    const gpu::MailboxHolder& dest_mailbox_holder,
+    unsigned int internal_format,
+    unsigned int type,
+    bool flip_y,
+    bool use_visible_rect) {
+  gpu::raster::RasterInterface* ri = raster_context_provider->RasterInterface();
+  DCHECK(ri);
+  ri->WaitSyncTokenCHROMIUM(dest_mailbox_holder.sync_token.GetConstData());
+  GLuint dest_tex_id =
+      ri->CreateAndConsumeForGpuRaster(dest_mailbox_holder.mailbox);
+  if (dest_mailbox_holder.mailbox.IsSharedImage()) {
+    ri->BeginSharedImageAccessDirectCHROMIUM(
+        dest_tex_id, GL_SHARED_IMAGE_ACCESS_MODE_READWRITE_CHROMIUM);
+  }
+  // Let the SkImage fall out of scope and track the result using dest_tex_id
+  ConvertFromVideoFrameYUVToSkSurface(
+      video_frame, raster_context_provider, dest_mailbox_holder.texture_target,
+      dest_tex_id, internal_format, type, flip_y, use_visible_rect);
+  if (dest_mailbox_holder.mailbox.IsSharedImage())
+    ri->EndSharedImageAccessDirectCHROMIUM(dest_tex_id);
+  ri->DeleteGpuRasterTexture(dest_tex_id);
+}
 
-  switch (video_format) {
-    case PIXEL_FORMAT_NV12:
-      return SkImage::MakeFromNV12TexturesCopyWithExternalBackend(
-          gr_context, color_space, yuv_textures, kTopLeft_GrSurfaceOrigin,
-          result_texture);
-    case PIXEL_FORMAT_I420:
-      return SkImage::MakeFromYUVTexturesCopyWithExternalBackend(
-          gr_context, color_space, yuv_textures, kTopLeft_GrSurfaceOrigin,
-          result_texture);
-    default:
-      NOTREACHED();
-      return nullptr;
+}  // namespace
+
+void ConvertFromVideoFrameYUV(
+    const VideoFrame* video_frame,
+    viz::RasterContextProvider* raster_context_provider,
+    const gpu::MailboxHolder& dest_mailbox_holder,
+    unsigned int internal_format,
+    unsigned int type,
+    bool flip_y,
+    bool use_visible_rect) {
+  DCHECK(raster_context_provider);
+  if (raster_context_provider->GrContext()) {
+    ConvertFromVideoFrameYUVWithGrContext(video_frame, raster_context_provider,
+                                          dest_mailbox_holder, internal_format,
+                                          type, flip_y, use_visible_rect);
+    return;
+  }
+
+  auto* ri = raster_context_provider->RasterInterface();
+  DCHECK(ri);
+  ri->WaitSyncTokenCHROMIUM(dest_mailbox_holder.sync_token.GetConstData());
+  SkYUVColorSpace color_space =
+      ColorSpaceToSkYUVColorSpace(video_frame->ColorSpace());
+
+  VideoFrameYUVMailboxesHolder yuv_mailboxes(video_frame,
+                                             raster_context_provider, false);
+
+  if (yuv_mailboxes.is_nv12()) {
+    ri->ConvertNV12MailboxesToRGB(dest_mailbox_holder.mailbox, color_space,
+                                  yuv_mailboxes.mailbox(kYIndex),
+                                  yuv_mailboxes.mailbox(kUIndex));
+  } else {
+    DCHECK_EQ(video_frame->NumTextures(), kNumYUVPlanes);
+    ri->ConvertYUVMailboxesToRGB(dest_mailbox_holder.mailbox, color_space,
+                                 yuv_mailboxes.mailbox(kYIndex),
+                                 yuv_mailboxes.mailbox(kUIndex),
+                                 yuv_mailboxes.mailbox(kVIndex));
   }
 }
 

media/renderers/yuv_util.h

@@ -5,16 +5,12 @@
 #ifndef MEDIA_RENDERERS_YUV_UTIL_H_
 #define MEDIA_RENDERERS_YUV_UTIL_H_
 
+#include "gpu/GLES2/gl2extchromium.h"
 #include "media/base/media_export.h"
 #include "media/base/video_types.h"
 #include "third_party/skia/include/core/SkRefCnt.h"
 #include "ui/gfx/color_space.h"
 
-// Skia forward declarations
-class GrBackendTexture;
-class GrDirectContext;
-class SkImage;
-
 namespace gpu {
 struct MailboxHolder;
 }
@@ -35,20 +31,11 @@ class VideoFrame;
 MEDIA_EXPORT void ConvertFromVideoFrameYUV(
     const VideoFrame* video_frame,
     viz::RasterContextProvider* raster_context_provider,
-    const gpu::MailboxHolder& dest_mailbox_holder);
-
-MEDIA_EXPORT sk_sp<SkImage> NewSkImageFromVideoFrameYUV(
-    const VideoFrame* video_frame,
-    viz::RasterContextProvider* raster_context_provider,
-    unsigned int texture_target,
-    unsigned int texture_id);
-
-MEDIA_EXPORT sk_sp<SkImage> YUVGrBackendTexturesToSkImage(
-    GrDirectContext* gr_context,
-    gfx::ColorSpace video_color_space,
-    VideoPixelFormat video_format,
-    GrBackendTexture* yuv_textures,
-    const GrBackendTexture& result_texture);
+    const gpu::MailboxHolder& dest_mailbox_holder,
+    unsigned int internal_format = GL_RGBA,
+    unsigned int type = GL_UNSIGNED_BYTE,
+    bool flip_y = false,
+    bool use_visible_rect = false);
 
 }  // namespace media