[WebCodecs] Add P3/Rec2020 uint8 support

This takes into account the color spaces of pixel data from
ImageBitmap being encoded into VideoFrame.

Bug: 897297
Change-Id: I5ee5b77eb067ce8f5cec91dbd584a1553f6e91f0
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2272134
Commit-Queue: Jie A Chen <jie.a.chen@intel.com>
Reviewed-by: Dan Sanders <sandersd@chromium.org>
Cr-Commit-Position: refs/heads/master@{#786156}

third_party/blink/renderer/modules/webcodecs/video_frame.cc

@@ -27,6 +27,41 @@
 
 namespace blink {
 
+namespace {
+
+bool IsValidSkColorSpace(sk_sp<SkColorSpace> sk_color_space) {
+  // Refer to CanvasColorSpaceToGfxColorSpace in CanvasColorParams.
+  sk_sp<SkColorSpace> valid_sk_color_spaces[] = {
+      gfx::ColorSpace::CreateSRGB().ToSkColorSpace(),
+      gfx::ColorSpace::CreateDisplayP3D65().ToSkColorSpace(),
+      gfx::ColorSpace(gfx::ColorSpace::PrimaryID::BT2020,
+                      gfx::ColorSpace::TransferID::GAMMA24)
+          .ToSkColorSpace()};
+  for (auto& valid_sk_color_space : valid_sk_color_spaces) {
+    if (SkColorSpace::Equals(sk_color_space.get(),
+                             valid_sk_color_space.get())) {
+      return true;
+    }
+  }
+  return false;
+}
+
+bool IsValidSkColorType(SkColorType sk_color_type) {
+  SkColorType valid_sk_color_types[] = {
+      kBGRA_8888_SkColorType, kRGBA_8888_SkColorType,
+      // TODO(jie.a.chen@intel.com): Add F16 support.
+      // kRGBA_F16_SkColorType
+  };
+  for (auto& valid_sk_color_type : valid_sk_color_types) {
+    if (sk_color_type == valid_sk_color_type) {
+      return true;
+    }
+  }
+  return false;
+}
+
+}  // namespace
+
 VideoFrame::VideoFrame(scoped_refptr<media::VideoFrame> frame)
     : frame_(std::move(frame)) {
   DCHECK(frame_);
@@ -101,6 +136,25 @@ VideoFrame* VideoFrame::Create(VideoFrameInit* init,
     return nullptr;
   }
 
+  auto sk_image =
+      source->BitmapImage()->PaintImageForCurrentFrame().GetSkImage();
+  auto sk_color_space = sk_image->refColorSpace();
+  if (!sk_color_space) {
+    sk_color_space = SkColorSpace::MakeSRGB();
+  }
+  if (!IsValidSkColorSpace(sk_color_space)) {
+    exception_state.ThrowDOMException(DOMExceptionCode::kInvalidStateError,
+                                      "Invalid color space");
+    return nullptr;
+  }
+  auto sk_color_type = sk_image->colorType();
+  if (!IsValidSkColorType(sk_color_type)) {
+    exception_state.ThrowDOMException(DOMExceptionCode::kInvalidStateError,
+                                      "Invalid pixel format");
+    return nullptr;
+  }
+
+  // TODO(jie.a.chen@intel.com): Handle data of float type.
   // Full copy #1
   WTF::Vector<uint8_t> pixel_data = source->CopyBitmapData();
   if (pixel_data.size() <
@@ -118,25 +172,41 @@ VideoFrame* VideoFrame::Create(VideoFrameInit* init,
     return nullptr;
   }
 
+#if SK_PMCOLOR_BYTE_ORDER(B, G, R, A)
+  auto libyuv_convert_to_i420 = libyuv::ARGBToI420;
+#else
+  auto libyuv_convert_to_i420 = libyuv::ABGRToI420;
+#endif
+  if (sk_color_type != kN32_SkColorType) {
+    // Swap ARGB and ABGR if not using the native pixel format.
+    libyuv_convert_to_i420 =
+        (libyuv_convert_to_i420 == libyuv::ARGBToI420 ? libyuv::ABGRToI420
+                                                      : libyuv::ARGBToI420);
+  }
+
+  // TODO(jie.a.chen@intel.com): Use GPU to do the conversion.
   // Full copy #2
   int error =
-      libyuv::ABGRToI420(pixel_data.data(), source->width() * 4,
-                         frame->visible_data(media::VideoFrame::kYPlane),
-                         frame->stride(media::VideoFrame::kYPlane),
-                         frame->visible_data(media::VideoFrame::kUPlane),
-                         frame->stride(media::VideoFrame::kUPlane),
-                         frame->visible_data(media::VideoFrame::kVPlane),
-                         frame->stride(media::VideoFrame::kVPlane),
-                         source->width(), source->height());
+      libyuv_convert_to_i420(pixel_data.data(), source->width() * 4,
+                             frame->visible_data(media::VideoFrame::kYPlane),
+                             frame->stride(media::VideoFrame::kYPlane),
+                             frame->visible_data(media::VideoFrame::kUPlane),
+                             frame->stride(media::VideoFrame::kUPlane),
+                             frame->visible_data(media::VideoFrame::kVPlane),
+                             frame->stride(media::VideoFrame::kVPlane),
+                             source->width(), source->height());
   if (error) {
     exception_state.ThrowDOMException(DOMExceptionCode::kOperationError,
                                       "ARGB to YUV420 conversion error");
     return nullptr;
   }
-  // TODO(jie.a.chen@intel.com): Figure out the right colorspace and conversion
-  // according to source ImageBitmap.
-  // libyuv::ABGRToI420 seems to assume Bt.601.
-  frame->set_color_space(gfx::ColorSpace::CreateREC601());
+  gfx::ColorSpace gfx_color_space(*sk_color_space);
+  // 'libyuv_convert_to_i420' assumes SMPTE170M.
+  // Refer to the func below to check the actual conversion:
+  // third_party/libyuv/source/row_common.cc -- RGBToY(...)
+  gfx_color_space = gfx_color_space.GetWithMatrixAndRange(
+      gfx::ColorSpace::MatrixID::SMPTE170M, gfx::ColorSpace::RangeID::LIMITED);
+  frame->set_color_space(gfx_color_space);
   auto* result = MakeGarbageCollected<VideoFrame>(std::move(frame));
   return result;
 }
@@ -166,6 +236,14 @@ ScriptPromise VideoFrame::CreateImageBitmap(ScriptState* script_state,
        (frame_->format() == media::PIXEL_FORMAT_NV12 &&
         frame_->HasTextures()))) {
     scoped_refptr<StaticBitmapImage> image;
+    gfx::ColorSpace gfx_color_space = frame_->ColorSpace();
+    gfx_color_space = gfx_color_space.GetWithMatrixAndRange(
+        gfx::ColorSpace::MatrixID::RGB, gfx::ColorSpace::RangeID::FULL);
+    auto sk_color_space = gfx_color_space.ToSkColorSpace();
+    if (!sk_color_space) {
+      sk_color_space = SkColorSpace::MakeSRGB();
+    }
+
     if (!preferAcceleratedImageBitmap()) {
       size_t bytes_per_row = sizeof(SkColor) * visibleWidth();
       size_t image_pixels_size = bytes_per_row * visibleHeight();
@@ -179,11 +257,9 @@ ScriptPromise VideoFrame::CreateImageBitmap(ScriptState* script_state,
       media::PaintCanvasVideoRenderer::ConvertVideoFrameToRGBPixels(
           frame_.get(), image_pixels->writable_data(), bytes_per_row);
 
-      // TODO(jie.a.chen@intel.com): Figure out the correct SkColorSpace.
-      sk_sp<SkColorSpace> skColorSpace = SkColorSpace::MakeSRGB();
       SkImageInfo info =
           SkImageInfo::Make(visibleWidth(), visibleHeight(), kN32_SkColorType,
-                            kUnpremul_SkAlphaType, std::move(skColorSpace));
+                            kUnpremul_SkAlphaType, std::move(sk_color_space));
       sk_sp<SkImage> skImage =
           SkImage::MakeRasterData(info, image_pixels, bytes_per_row);
       image = UnacceleratedStaticBitmapImage::Create(std::move(skImage));
@@ -224,7 +300,8 @@ ScriptPromise VideoFrame::CreateImageBitmap(ScriptState* script_state,
           base::Unretained(shared_image_interface), dest_holder.mailbox));
 
       const SkImageInfo sk_image_info =
-          SkImageInfo::MakeN32Premul(codedWidth(), codedHeight());
+          SkImageInfo::Make(codedWidth(), codedHeight(), kN32_SkColorType,
+                            kUnpremul_SkAlphaType, std::move(sk_color_space));
 
       image = AcceleratedStaticBitmapImage::CreateFromCanvasMailbox(
           dest_holder.mailbox, sync_token, 0u, sk_image_info,

third_party/blink/web_tests/webcodecs/videoframe-imagebitmap.html

@@ -3,49 +3,80 @@
 <script src="../resources/testharnessreport.js"></script>
 <script>
   'use strict';
-  function testCanvas_0f0(ctx, width, height, assert_compares) {
+  // Reference values generated by:
+  // https://fiddle.skia.org/c/f100d4d5f085a9e09896aabcbc463868
+
+  const kSRGBPixel = [50, 100, 150, 255];
+  const kP3Pixel = [62, 99, 146, 255];
+  const kRec2020Pixel = [87, 106, 151, 255];
+
+  const kCanvasOptionsP3Uint8 = {colorSpace: 'p3', pixelFormat:'uint8'};
+  const kCanvasOptionsRec2020Uint8 =
+      {colorSpace: 'rec2020', pixelFormat:'uint8'};
+
+  function testCanvas(ctx, width, height, expected_pixel, assert_compares) {
+    // The dup getImageData is to workaournd crbug.com/1100233
+    let imageData = ctx.getImageData(0, 0, width, height);
     let colorData = ctx.getImageData(0, 0, width, height).data;
-    for (let i = 0; i < width * height; i += 4) {
-      assert_compares(colorData[0], 0);
-      assert_compares(colorData[1], 255);
-      assert_compares(colorData[2], 0);
-      assert_compares(colorData[3], 255);
+    for (let i = 0; i < 4 * width * height; i += 4) {
+      assert_compares(colorData[i], expected_pixel[0]);
+      assert_compares(colorData[i + 1], expected_pixel[1]);
+      assert_compares(colorData[i + 2], expected_pixel[2]);
+      assert_compares(colorData[i + 3], expected_pixel[3]);
     }
   }
 
-  function testImageBitmapToAndFromVideoFrame(width, height) {
+  function testImageBitmapToAndFromVideoFrame(width, height, expectedPixel,
+      canvasOptions, imageBitmapOptions) {
     let canvas = document.createElement('canvas');
     canvas.width = width;
     canvas.height = height;
-    let ctx = canvas.getContext('2d');
-    ctx.fillStyle = '#0f0';
+    let ctx = canvas.getContext('2d', canvasOptions);
+    ctx.fillStyle = 'rgb(50, 100, 150)';
     ctx.fillRect(0, 0, width, height);
-    testCanvas_0f0(ctx, width, height, assert_equals);
+    testCanvas(ctx, width, height, expectedPixel, assert_equals);
 
-    return createImageBitmap(canvas)
+    return createImageBitmap(canvas, imageBitmapOptions)
     .then((fromImageBitmap) => {
       let videoFrame = new VideoFrame({ timestamp: 0 }, fromImageBitmap);
-      return videoFrame.createImageBitmap();
+      return videoFrame.createImageBitmap(imageBitmapOptions);
     })
     .then((toImageBitmap) => {
         let myCanvas = document.createElement('canvas');
         myCanvas.width = width;
         myCanvas.height = height;
-        let myCtx = myCanvas.getContext('2d');
+        let myCtx = myCanvas.getContext('2d', canvasOptions);
         myCtx.drawImage(toImageBitmap, 0, 0);
         let tolerance = 2;
-        testCanvas_0f0(myCtx, width, height, (actual, expected) => {
+        testCanvas(myCtx, width, height, expectedPixel, (actual, expected) => {
           assert_approx_equals(actual, expected, tolerance);
         });
     });
   }
 
   promise_test(() => {
-    return testImageBitmapToAndFromVideoFrame(320, 240);
-  }, 'ImageBitmap<->VideoFrame with canvas size 320x240.');
+    return testImageBitmapToAndFromVideoFrame(320, 240, kSRGBPixel);
+  }, 'ImageBitmap<->VideoFrame with canvas(320x240 srgb uint8).');
+  promise_test(() => {
+    return testImageBitmapToAndFromVideoFrame(640, 480, kSRGBPixel);
+  }, 'ImageBitmap<->VideoFrame with canvas(640x480 srgb uint8).');
+
+  promise_test(() => {
+    return testImageBitmapToAndFromVideoFrame(320, 240, kP3Pixel,
+      kCanvasOptionsP3Uint8, {colorSpaceConversion: "none"});
+  }, 'ImageBitmap<->VideoFrame with canvas(320x240 p3 uint8).');
+  promise_test(() => {
+    return testImageBitmapToAndFromVideoFrame(640, 480, kP3Pixel,
+      kCanvasOptionsP3Uint8, {colorSpaceConversion: "none"});
+  }, 'ImageBitmap<->VideoFrame with canvas(640x480 p3 uint8).');
 
   promise_test(() => {
-    return testImageBitmapToAndFromVideoFrame(640, 480);
-  }, 'ImageBitmap<->VideoFrame with canvas size 640x480.');
+    return testImageBitmapToAndFromVideoFrame(320, 240, kRec2020Pixel,
+      kCanvasOptionsRec2020Uint8, {colorSpaceConversion: "none"});
+  }, 'ImageBitmap<->VideoFrame with canvas(320x240 rec2020 uint8).');
+  promise_test(() => {
+    return testImageBitmapToAndFromVideoFrame(640, 480, kRec2020Pixel,
+      kCanvasOptionsRec2020Uint8, {colorSpaceConversion: "none"});
+  }, 'ImageBitmap<->VideoFrame with canvas(640x480 rec2020 uint8).');
 
 </script>