webcodecs: Handling texture based frames in video encoder

1. Readback for textured frames in blink::VideoEncoder
2. Media foundation encoder can now handle NV12 frames

Test: https://kstreeter.github.io/wctranscode/
Bug: 1158665
Change-Id: Ie5865736eeee22f91bcf161f82a687c03679f912
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2598058
Commit-Queue: Eugene Zemtsov <eugene@chromium.org>
Reviewed-by: Sunny Sachanandani <sunnyps@chromium.org>
Reviewed-by: Dan Sanders <sandersd@chromium.org>
Reviewed-by: Jeremy Roman <jbroman@chromium.org>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Cr-Commit-Position: refs/heads/master@{#841018}

media/base/video_frame.cc

@@ -1265,6 +1265,8 @@ std::string VideoFrame::AsHumanReadableString() const {
   s << ConfigToString(format(), storage_type_, coded_size(), visible_rect_,
                       natural_size_)
     << " timestamp:" << timestamp_.InMicroseconds();
+  if (HasTextures())
+    s << " textures: " << NumTextures();
   return s.str();
 }
 

media/base/win/mf_helpers.h

@@ -62,6 +62,7 @@ class MF_INITIALIZER_EXPORT MediaBufferScopedPointer {
 
   uint8_t* get() { return buffer_; }
   DWORD current_length() const { return current_length_; }
+  DWORD max_length() const { return max_length_; }
 
  private:
   Microsoft::WRL::ComPtr<IMFMediaBuffer> media_buffer_;

media/gpu/windows/media_foundation_video_encode_accelerator_win.cc

@@ -45,7 +45,8 @@ const size_t kOneMicrosecondInMFSampleTimeUnits = 10;
 const size_t kOutputSampleBufferSizeRatio = 4;
 
 constexpr const wchar_t* const kMediaFoundationVideoEncoderDLLs[] = {
-    L"mf.dll", L"mfplat.dll",
+    L"mf.dll",
+    L"mfplat.dll",
 };
 
 eAVEncH264VProfile GetH264VProfile(VideoCodecProfile profile,
@@ -67,7 +68,6 @@ eAVEncH264VProfile GetH264VProfile(VideoCodecProfile profile,
       return eAVEncH264VProfile_unknown;
   }
 }
-
 }  // namespace
 
 class MediaFoundationVideoEncodeAccelerator::EncodeOutput {
@@ -181,7 +181,8 @@ bool MediaFoundationVideoEncodeAccelerator::Initialize(const Config& config,
   DVLOG(3) << __func__ << ": " << config.AsHumanReadableString();
   DCHECK(main_client_task_runner_->BelongsToCurrentThread());
 
-  if (PIXEL_FORMAT_I420 != config.input_format) {
+  if (PIXEL_FORMAT_I420 != config.input_format &&
+      PIXEL_FORMAT_NV12 != config.input_format) {
     DLOG(ERROR) << "Input format not supported= "
                 << VideoPixelFormatToString(config.input_format);
     return false;
@@ -682,9 +683,9 @@ void MediaFoundationVideoEncodeAccelerator::EncodeTask(
   DCHECK(encoder_thread_task_runner_->BelongsToCurrentThread());
 
   if (is_async_mft_) {
-    AsyncEncodeTask(frame, force_keyframe);
+    AsyncEncodeTask(std::move(frame), force_keyframe);
   } else {
-    SyncEncodeTask(frame, force_keyframe);
+    SyncEncodeTask(std::move(frame), force_keyframe);
   }
 }
 
@@ -695,7 +696,7 @@ void MediaFoundationVideoEncodeAccelerator::AsyncEncodeTask(
   HRESULT hr = E_FAIL;
   if (input_required_) {
     // Hardware MFT is waiting for this coming input.
-    hr = ProcessInput(frame, force_keyframe);
+    hr = ProcessInput(std::move(frame), force_keyframe);
     if (FAILED(hr)) {
       NotifyError(kPlatformFailureError);
       RETURN_ON_HR_FAILURE(hr, "Couldn't encode", );
@@ -721,7 +722,7 @@ void MediaFoundationVideoEncodeAccelerator::AsyncEncodeTask(
 
     // Always deliver the current input into HMFT.
     if (event_type == METransformNeedInput) {
-      hr = ProcessInput(frame, force_keyframe);
+      hr = ProcessInput(std::move(frame), force_keyframe);
       if (FAILED(hr)) {
         NotifyError(kPlatformFailureError);
         RETURN_ON_HR_FAILURE(hr, "Couldn't encode", );
@@ -748,7 +749,7 @@ void MediaFoundationVideoEncodeAccelerator::SyncEncodeTask(
     scoped_refptr<VideoFrame> frame,
     bool force_keyframe) {
   HRESULT hr = E_FAIL;
-  hr = ProcessInput(frame, force_keyframe);
+  hr = ProcessInput(std::move(frame), force_keyframe);
 
   // According to MSDN, if encoder returns MF_E_NOTACCEPTING, we need to try
   // processing the output. This error indicates that encoder does not accept
@@ -775,29 +776,50 @@ HRESULT MediaFoundationVideoEncodeAccelerator::ProcessInput(
     bool force_keyframe) {
   DVLOG(3) << __func__;
   DCHECK(encoder_thread_task_runner_->BelongsToCurrentThread());
-  DCHECK_EQ(frame->format(), PIXEL_FORMAT_I420);
-
-  // Convert I420 to NV12 as input.
-  Microsoft::WRL::ComPtr<IMFMediaBuffer> input_buffer;
-  input_sample_->GetBufferByIndex(0, &input_buffer);
 
   {
+    Microsoft::WRL::ComPtr<IMFMediaBuffer> input_buffer;
+    input_sample_->GetBufferByIndex(0, &input_buffer);
     MediaBufferScopedPointer scoped_buffer(input_buffer.Get());
     DCHECK(scoped_buffer.get());
-
+    uint8_t* dst_y = scoped_buffer.get();
     uint8_t* dst_uv =
         scoped_buffer.get() + frame->row_bytes(VideoFrame::kYPlane) *
                                   frame->rows(VideoFrame::kYPlane);
-    libyuv::I420ToNV12(frame->visible_data(VideoFrame::kYPlane),
-                       frame->stride(VideoFrame::kYPlane),
-                       frame->visible_data(VideoFrame::kUPlane),
-                       frame->stride(VideoFrame::kUPlane),
-                       frame->visible_data(VideoFrame::kVPlane),
-                       frame->stride(VideoFrame::kVPlane), scoped_buffer.get(),
-                       frame->row_bytes(VideoFrame::kYPlane), dst_uv,
-                       frame->row_bytes(VideoFrame::kUPlane) * 2,
-                       input_visible_size_.width(),
-                       input_visible_size_.height());
+    uint8_t* end = dst_uv + frame->row_bytes(VideoFrame::kUVPlane) *
+                                frame->rows(VideoFrame::kUVPlane);
+    DCHECK_GE(std::ptrdiff_t{scoped_buffer.max_length()},
+              end - scoped_buffer.get());
+
+    if (frame->format() == PIXEL_FORMAT_NV12) {
+      // Copy NV12 pixel data from |frame| to |input_buffer|.
+      int error = libyuv::NV12Copy(
+          frame->visible_data(VideoFrame::kYPlane),
+          frame->stride(VideoFrame::kYPlane),
+          frame->visible_data(VideoFrame::kUVPlane),
+          frame->stride(VideoFrame::kUVPlane), dst_y,
+          frame->row_bytes(VideoFrame::kYPlane), dst_uv,
+          frame->row_bytes(VideoFrame::kUPlane), input_visible_size_.width(),
+          input_visible_size_.height());
+      if (error)
+        return E_FAIL;
+    } else if (frame->format() == PIXEL_FORMAT_I420) {
+      // Convert I420 to NV12 as input.
+      int error = libyuv::I420ToNV12(
+          frame->visible_data(VideoFrame::kYPlane),
+          frame->stride(VideoFrame::kYPlane),
+          frame->visible_data(VideoFrame::kUPlane),
+          frame->stride(VideoFrame::kUPlane),
+          frame->visible_data(VideoFrame::kVPlane),
+          frame->stride(VideoFrame::kVPlane), dst_y,
+          frame->row_bytes(VideoFrame::kYPlane), dst_uv,
+          frame->row_bytes(VideoFrame::kUPlane) * 2,
+          input_visible_size_.width(), input_visible_size_.height());
+      if (error)
+        return E_FAIL;
+    } else {
+      NOTREACHED();
+    }
   }
 
   input_sample_->SetSampleTime(frame->timestamp().InMicroseconds() *
@@ -808,9 +830,6 @@ HRESULT MediaFoundationVideoEncodeAccelerator::ProcessInput(
   RETURN_ON_HR_FAILURE(hr, "Couldn't calculate sample duration", E_FAIL);
   input_sample_->SetSampleDuration(sample_duration);
 
-  // Release frame after input is copied.
-  frame = nullptr;
-
   if (force_keyframe) {
     VARIANT var;
     var.vt = VT_UI4;
@@ -818,7 +837,8 @@ HRESULT MediaFoundationVideoEncodeAccelerator::ProcessInput(
     hr = codec_api_->SetValue(&CODECAPI_AVEncVideoForceKeyFrame, &var);
     if (!compatible_with_win7_ && FAILED(hr)) {
       LOG(WARNING) << "Failed to set CODECAPI_AVEncVideoForceKeyFrame, "
-                      "HRESULT: 0x" << std::hex << hr;
+                      "HRESULT: 0x"
+                   << std::hex << hr;
     }
   }
 
@@ -1014,7 +1034,7 @@ bool MediaFoundationVideoEncodeAccelerator::TryToDeliverInputFrame(
         continue;
       }
       case METransformNeedInput: {
-        hr = ProcessInput(frame, force_keyframe);
+        hr = ProcessInput(std::move(frame), force_keyframe);
         if (FAILED(hr)) {
           NotifyError(kPlatformFailureError);
           RETURN_ON_HR_FAILURE(hr, "Couldn't encode", false);

media/video/video_encode_accelerator_adapter.cc

@@ -329,7 +329,9 @@ void VideoEncodeAcceleratorAdapter::EncodeOnAcceleratorThread(
     result = PrepareCpuFrame(options_.frame_size, frame);
 
   if (result.has_error()) {
-    std::move(done_cb).Run(std::move(result.error()).AddHere());
+    auto status = result.error();
+    status.WithData("frame", frame->AsHumanReadableString());
+    std::move(done_cb).Run(std::move(status).AddHere());
     return;
   }
 

third_party/blink/renderer/modules/webcodecs/DEPS

@@ -5,6 +5,8 @@ include_rules = [
     "+components/viz/common/resources/single_release_callback.h",
 
     "+gpu/command_buffer/client/shared_image_interface.h",
+    "+gpu/command_buffer/client/raster_interface.h",
+    "+gpu/GLES2/gl2extchromium.h",
 
     "+media/base",
     "+media/filters",

third_party/blink/renderer/modules/webcodecs/video_encoder.cc

@@ -8,21 +8,19 @@
 
 #include "base/bind.h"
 #include "base/callback.h"
+#include "base/callback_helpers.h"
 #include "base/logging.h"
 #include "base/macros.h"
 #include "build/build_config.h"
+#include "components/viz/common/gpu/raster_context_provider.h"
+#include "gpu/GLES2/gl2extchromium.h"
+#include "gpu/command_buffer/client/raster_interface.h"
 #include "media/base/async_destroy_video_encoder.h"
 #include "media/base/mime_util.h"
 #include "media/base/offloading_video_encoder.h"
 #include "media/base/video_codecs.h"
 #include "media/base/video_color_space.h"
 #include "media/base/video_encoder.h"
-#if BUILDFLAG(ENABLE_OPENH264)
-#include "media/video/openh264_video_encoder.h"
-#endif
-#if BUILDFLAG(ENABLE_LIBVPX)
-#include "media/video/vpx_video_encoder.h"
-#endif
 #include "media/video/gpu_video_accelerator_factories.h"
 #include "media/video/video_encode_accelerator_adapter.h"
 #include "third_party/blink/public/mojom/web_feature/web_feature.mojom-blink.h"
@@ -45,10 +43,25 @@
 #include "third_party/blink/renderer/platform/bindings/enumeration_base.h"
 #include "third_party/blink/renderer/platform/bindings/exception_state.h"
 #include "third_party/blink/renderer/platform/bindings/script_state.h"
+#include "third_party/blink/renderer/platform/graphics/gpu/shared_gpu_context.h"
 #include "third_party/blink/renderer/platform/instrumentation/use_counter.h"
 #include "third_party/blink/renderer/platform/scheduler/public/post_cross_thread_task.h"
 #include "third_party/blink/renderer/platform/scheduler/public/thread.h"
 #include "third_party/blink/renderer/platform/wtf/cross_thread_functional.h"
+#include "third_party/skia/include/core/SkImage.h"
+#include "third_party/skia/include/core/SkRefCnt.h"
+#include "third_party/skia/include/core/SkSurface.h"
+#include "third_party/skia/include/core/SkYUVAPixmaps.h"
+#include "third_party/skia/include/gpu/GrDirectContext.h"
+#include "third_party/skia/include/gpu/gl/GrGLTypes.h"
+
+#if BUILDFLAG(ENABLE_OPENH264)
+#include "media/video/openh264_video_encoder.h"
+#endif
+
+#if BUILDFLAG(ENABLE_LIBVPX)
+#include "media/video/vpx_video_encoder.h"
+#endif
 
 namespace blink {
 
@@ -125,6 +138,42 @@ std::unique_ptr<media::VideoEncoder> CreateOpenH264VideoEncoder() {
 #endif  // BUILDFLAG(ENABLE_OPENH264)
 }
 
+std::pair<SkColorType, GrGLenum> GetSkiaAndGlColorTypesForPlane(
+    media::VideoPixelFormat format,
+    size_t plane) {
+  // TODO(eugene): There is some strange channel switch during RGB readback.
+  // When frame's pixel format matches GL and Skia color types we get reversed
+  // channels. But why?
+  switch (format) {
+    case media::PIXEL_FORMAT_NV12:
+      if (plane == media::VideoFrame::kUVPlane)
+        return {kR8G8_unorm_SkColorType, GL_RG8_EXT};
+      if (plane == media::VideoFrame::kYPlane)
+        return {kAlpha_8_SkColorType, GL_R8_EXT};
+      break;
+    case media::PIXEL_FORMAT_XBGR:
+      if (plane == media::VideoFrame::kARGBPlane)
+        return {kRGBA_8888_SkColorType, GL_RGBA8_OES};
+      break;
+    case media::PIXEL_FORMAT_ABGR:
+      if (plane == media::VideoFrame::kARGBPlane)
+        return {kRGBA_8888_SkColorType, GL_RGBA8_OES};
+      break;
+    case media::PIXEL_FORMAT_XRGB:
+      if (plane == media::VideoFrame::kARGBPlane)
+        return {kBGRA_8888_SkColorType, GL_BGRA8_EXT};
+      break;
+    case media::PIXEL_FORMAT_ARGB:
+      if (plane == media::VideoFrame::kARGBPlane)
+        return {kBGRA_8888_SkColorType, GL_BGRA8_EXT};
+      break;
+    default:
+      break;
+  }
+  NOTREACHED();
+  return {kUnknown_SkColorType, 0};
+}
+
 }  // namespace
 
 // static
@@ -606,6 +655,20 @@ void VideoEncoder::ProcessEncode(Request* request) {
   };
 
   scoped_refptr<media::VideoFrame> frame = request->frame->frame();
+
+  if (frame->HasTextures()) {
+    frame = ReadbackTextureBackedFrameToMemory(std::move(frame));
+    if (!frame) {
+      auto status = media::Status(media::StatusCode::kEncoderFailedEncode,
+                                  "Can't readback frame textures.");
+      auto task_runner = Thread::Current()->GetTaskRunner();
+      task_runner->PostTask(
+          FROM_HERE, WTF::Bind(done_callback, WrapWeakPersistent(this),
+                               WrapPersistent(request), std::move(status)));
+      return;
+    }
+  }
+
   bool keyframe = request->encodeOpts->hasKeyFrameNonNull() &&
                   request->encodeOpts->keyFrameNonNull();
   --requested_encodes_;
@@ -804,6 +867,112 @@ bool VideoEncoder::HasPendingActivity() const {
   return stall_request_processing_ || !requests_.empty();
 }
 
+// This function reads pixel data from textures associated with |txt_frame|
+// and creates a new CPU memory backed frame. It's needed because
+// existing video encoders can't handle texture backed frames.
+//
+// TODO(crbug.com/1162530): Remove this code from blink::VideoEncoder, combine
+// with media::ConvertAndScaleFrame and put into a new class
+// media:FrameSizeAndFormatConverter.
+scoped_refptr<media::VideoFrame>
+VideoEncoder::ReadbackTextureBackedFrameToMemory(
+    scoped_refptr<media::VideoFrame> txt_frame) {
+  DCHECK(txt_frame);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+
+  if (txt_frame->NumTextures() > 2 || txt_frame->NumTextures() < 1) {
+    DLOG(ERROR) << "Readback is not possible for this frame: "
+                << txt_frame->AsHumanReadableString();
+    return nullptr;
+  }
+
+  media::VideoPixelFormat result_format = txt_frame->format();
+  if (txt_frame->NumTextures() == 1 &&
+      result_format == media::PIXEL_FORMAT_NV12) {
+    // Even though |txt_frame| format is NV12 and it is NV12 in GPU memory,
+    // the texture is a RGB view that is produced by a shader on the fly.
+    // So we currently we currently can only read it back as RGB.
+    result_format = media::PIXEL_FORMAT_ARGB;
+  }
+
+  scoped_refptr<viz::RasterContextProvider> raster_provider;
+  auto wrapper = SharedGpuContext::ContextProviderWrapper();
+  if (wrapper && wrapper->ContextProvider())
+    raster_provider = wrapper->ContextProvider()->RasterContextProvider();
+  if (!raster_provider)
+    return nullptr;
+
+  auto* ri = raster_provider->RasterInterface();
+  auto* gr_context = raster_provider->GrContext();
+
+  scoped_refptr<media::VideoFrame> result = readback_frame_pool_.CreateFrame(
+      result_format, txt_frame->coded_size(), txt_frame->visible_rect(),
+      txt_frame->natural_size(), txt_frame->timestamp());
+  result->set_color_space(txt_frame->ColorSpace());
+  result->metadata()->MergeMetadataFrom(txt_frame->metadata());
+
+  size_t planes = media::VideoFrame::NumPlanes(result->format());
+  for (size_t plane = 0; plane < planes; plane++) {
+    const gpu::MailboxHolder& holder = txt_frame->mailbox_holder(plane);
+    if (holder.mailbox.IsZero())
+      return nullptr;
+    ri->WaitSyncTokenCHROMIUM(holder.sync_token.GetConstData());
+
+    int width = media::VideoFrame::Columns(plane, result->format(),
+                                           result->coded_size().width());
+    int height = result->rows(plane);
+
+    auto texture_id = ri->CreateAndConsumeForGpuRaster(holder.mailbox);
+    if (holder.mailbox.IsSharedImage()) {
+      ri->BeginSharedImageAccessDirectCHROMIUM(
+          texture_id, GL_SHARED_IMAGE_ACCESS_MODE_READ_CHROMIUM);
+    }
+
+    auto cleanup_fn = [](GLuint texture_id, bool shared,
+                         gpu::raster::RasterInterface* ri) {
+      if (shared)
+        ri->EndSharedImageAccessDirectCHROMIUM(texture_id);
+      ri->DeleteGpuRasterTexture(texture_id);
+    };
+    base::ScopedClosureRunner cleanup(base::BindOnce(
+        cleanup_fn, texture_id, holder.mailbox.IsSharedImage(), ri));
+
+    GrGLenum texture_format;
+    SkColorType sk_color_type;
+    std::tie(sk_color_type, texture_format) =
+        GetSkiaAndGlColorTypesForPlane(result->format(), plane);
+    GrGLTextureInfo gl_texture_info;
+    gl_texture_info.fID = texture_id;
+    gl_texture_info.fTarget = holder.texture_target;
+    gl_texture_info.fFormat = texture_format;
+
+    GrBackendTexture texture(width, height, GrMipMapped::kNo, gl_texture_info);
+    auto image = SkImage::MakeFromTexture(
+        gr_context, texture, kTopLeft_GrSurfaceOrigin, sk_color_type,
+        kOpaque_SkAlphaType, nullptr /* colorSpace */);
+
+    if (!image) {
+      DLOG(ERROR) << "Can't create SkImage from texture!"
+                  << " plane:" << plane;
+      return nullptr;
+    }
+
+    SkImageInfo info =
+        SkImageInfo::Make(width, height, sk_color_type, kOpaque_SkAlphaType);
+    SkPixmap pixmap(info, result->data(plane), result->row_bytes(plane));
+    if (!image->readPixels(gr_context, pixmap, 0, 0,
+                           SkImage::kDisallow_CachingHint)) {
+      DLOG(ERROR) << "Plane readback failed."
+                  << " plane:" << plane << " width: " << width
+                  << " height: " << height
+                  << " minRowBytes: " << info.minRowBytes();
+      return nullptr;
+    }
+  }
+
+  return result;
+}
+
 void VideoEncoder::Trace(Visitor* visitor) const {
   visitor->Trace(active_config_);
   visitor->Trace(script_state_);

third_party/blink/renderer/modules/webcodecs/video_encoder.h

@@ -13,6 +13,7 @@
 #include "media/base/video_codecs.h"
 #include "media/base/video_color_space.h"
 #include "media/base/video_encoder.h"
+#include "media/base/video_frame_pool.h"
 #include "third_party/blink/renderer/bindings/core/v8/active_script_wrappable.h"
 #include "third_party/blink/renderer/bindings/core/v8/script_promise_resolver.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_codec_state.h"
@@ -146,6 +147,8 @@ class MODULES_EXPORT VideoEncoder final
       const ParsedConfig& config,
       media::GpuVideoAcceleratorFactories* gpu_factories);
   bool CanReconfigure(ParsedConfig& original_config, ParsedConfig& new_config);
+  scoped_refptr<media::VideoFrame> ReadbackTextureBackedFrameToMemory(
+      scoped_refptr<media::VideoFrame> txt_frame);
 
   std::unique_ptr<CodecLogger> logger_;
 
@@ -168,6 +171,7 @@ class MODULES_EXPORT VideoEncoder final
   // kEncode. This flag stops processing of new requests in the requests_ queue
   // till the current requests are finished.
   bool stall_request_processing_ = false;
+  media::VideoFramePool readback_frame_pool_;
 
   SEQUENCE_CHECKER(sequence_checker_);
 };

third_party/blink/tools/blinkpy/presubmit/audit_non_blink_usage.py

@@ -1004,6 +1004,10 @@ _CONFIG = [
         'allowed': [
             'gpu::kNullSurfaceHandle',
             'gpu::SHARED_IMAGE_.+',
+            'gpu::raster::RasterInterface',
+            'gpu::Mailbox',
+            'gpu::MailboxHolder',
+            "viz::RasterContextProvider",
             'media::.+',
             'libyuv::.+',
         ]