[WebCodecs] Parse codec for VideoDecoder

This parses the codec information in EncodedVideoConfig. Size has to
be explicitly specified in the config as well. With these changes,
a basic VP9 encoding and decoding test can run correctly. This test
can also cover the path of creating ImageBitmap from hardware decoder
VideoFrame.

Bug: 897297
Change-Id: Ifbac15db0305bcd781786707916b94be5a151215
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2301590
Commit-Queue: Jie A Chen <jie.a.chen@intel.com>
Reviewed-by: Dan Sanders <sandersd@chromium.org>
Cr-Commit-Position: refs/heads/master@{#790744}

media/base/video_codecs.cc

@@ -822,44 +822,71 @@ bool ParseDolbyVisionCodecId(const std::string& codec_id,
 #endif
 
 VideoCodec StringToVideoCodec(const std::string& codec_id) {
-  std::vector<std::string> elem = base::SplitString(
-      codec_id, ".", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
-  if (elem.empty())
-    return kUnknownVideoCodec;
-
+  VideoCodec codec = kUnknownVideoCodec;
   VideoCodecProfile profile = VIDEO_CODEC_PROFILE_UNKNOWN;
   uint8_t level = 0;
   VideoColorSpace color_space;
+  ParseCodec(codec_id, codec, profile, level, color_space);
+  return codec;
+}
+
+void ParseCodec(const std::string& codec_id,
+                VideoCodec& codec,
+                VideoCodecProfile& profile,
+                uint8_t& level,
+                VideoColorSpace& color_space) {
+  std::vector<std::string> elem = base::SplitString(
+      codec_id, ".", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
+  if (elem.empty()) {
+    codec = kUnknownVideoCodec;
+    return;
+  }
 
-  if (codec_id == "vp8" || codec_id == "vp8.0")
-    return kCodecVP8;
+  if (codec_id == "vp8" || codec_id == "vp8.0") {
+    codec = kCodecVP8;
+    return;
+  }
   if (ParseNewStyleVp9CodecID(codec_id, &profile, &level, &color_space) ||
       ParseLegacyVp9CodecID(codec_id, &profile, &level)) {
-    return kCodecVP9;
+    codec = kCodecVP9;
+    return;
   }
 
 #if BUILDFLAG(ENABLE_AV1_DECODER)
-  if (ParseAv1CodecId(codec_id, &profile, &level, &color_space))
-    return kCodecAV1;
+  if (ParseAv1CodecId(codec_id, &profile, &level, &color_space)) {
+    codec = kCodecAV1;
+    return;
+  }
 #endif
 
-  if (codec_id == "theora")
-    return kCodecTheora;
-  if (ParseAVCCodecId(codec_id, &profile, &level))
-    return kCodecH264;
+  if (codec_id == "theora") {
+    codec = kCodecTheora;
+    return;
+  }
+  if (ParseAVCCodecId(codec_id, &profile, &level)) {
+    codec = kCodecH264;
+    return;
+  }
 #if BUILDFLAG(ENABLE_MSE_MPEG2TS_STREAM_PARSER)
-  if (ParseAVCCodecId(TranslateLegacyAvc1CodecIds(codec_id), &profile, &level))
-    return kCodecH264;
+  if (ParseAVCCodecId(TranslateLegacyAvc1CodecIds(codec_id), &profile,
+                      &level)) {
+    codec = kCodecH264;
+    return;
+  }
 #endif
 #if BUILDFLAG(ENABLE_PLATFORM_HEVC)
-  if (ParseHEVCCodecId(codec_id, &profile, &level))
-    return kCodecHEVC;
+  if (ParseHEVCCodecId(codec_id, &profile, &level)) {
+    codec = kCodecHEVC;
+    return;
+  }
 #endif
 #if BUILDFLAG(ENABLE_PLATFORM_DOLBY_VISION)
-  if (ParseDolbyVisionCodecId(codec_id, &profile, &level))
-    return kCodecDolbyVision;
+  if (ParseDolbyVisionCodecId(codec_id, &profile, &level)) {
+    codec = kCodecDolbyVision;
+    return;
+  }
 #endif
-  return kUnknownVideoCodec;
+  codec = kUnknownVideoCodec;
 }
 
 }  // namespace media

media/base/video_codecs.h

@@ -147,6 +147,11 @@ MEDIA_EXPORT bool ParseDolbyVisionCodecId(const std::string& codec_id,
                                           uint8_t* level_id);
 #endif
 
+MEDIA_EXPORT void ParseCodec(const std::string& codec_id,
+                             VideoCodec& codec,
+                             VideoCodecProfile& profile,
+                             uint8_t& level,
+                             VideoColorSpace& color_space);
 MEDIA_EXPORT VideoCodec StringToVideoCodec(const std::string& codec_id);
 
 #if BUILDFLAG(ENABLE_MSE_MPEG2TS_STREAM_PARSER)

third_party/blink/renderer/modules/webcodecs/video_decoder.cc

@@ -58,15 +58,18 @@ void VideoDecoderTraits::InitializeDecoder(
     extra_data.assign(start, start + size);
   }
 
-  // TODO(sandersd): Parse |codec| to produce a VideoCodecProfile.
-  media::VideoCodec codec = media::kCodecH264;
-  media::VideoCodecProfile profile = media::H264PROFILE_BASELINE;
+  media::VideoCodec codec = media::kUnknownVideoCodec;
+  media::VideoCodecProfile profile = media::VIDEO_CODEC_PROFILE_UNKNOWN;
+  media::VideoColorSpace color_space = media::VideoColorSpace::REC709();
+  uint8_t level = 0;
+  media::ParseCodec(config.codec().Utf8(), codec, profile, level, color_space);
   // TODO(sandersd): Either remove sizes from VideoDecoderConfig (replace with
   // sample aspect) or parse the AvcC here to get the actual size.
+  // For the moment, hard-code 720p to prefer hardware decoders.
   gfx::Size size = gfx::Size(1280, 720);
   media::VideoDecoderConfig video_decoder_config(
       codec, profile, media::VideoDecoderConfig::AlphaMode::kIsOpaque,
-      media::VideoColorSpace::REC709(), media::kNoTransformation, size,
+      color_space, media::kNoTransformation, size,
       gfx::Rect(gfx::Point(), size), size, extra_data,
       media::EncryptionScheme::kUnencrypted);
 

third_party/blink/web_tests/webcodecs/videoframe-imagebitmap.html

@@ -10,9 +10,14 @@
   const kP3Pixel = [62, 99, 146, 255];
   const kRec2020Pixel = [87, 106, 151, 255];
 
-  const kCanvasOptionsP3Uint8 = { colorSpace: 'p3', pixelFormat: 'uint8' };
-  const kCanvasOptionsRec2020Uint8 =
-    { colorSpace: 'rec2020', pixelFormat: 'uint8' };
+  const kCanvasOptionsP3Uint8 = {
+    colorSpace: 'p3',
+    pixelFormat: 'uint8'
+  };
+  const kCanvasOptionsRec2020Uint8 = {
+    colorSpace: 'rec2020',
+    pixelFormat: 'uint8'
+  };
 
   function testCanvas(ctx, width, height, expected_pixel, assert_compares) {
     // The dup getImageData is to workaournd crbug.com/1100233
@@ -42,7 +47,9 @@
 
     return createImageBitmap(canvas, imageBitmapOptions)
       .then((fromImageBitmap) => {
-        let videoFrame = new VideoFrame({ timestamp: 0 }, fromImageBitmap);
+        let videoFrame = new VideoFrame({
+          timestamp: 0
+        }, fromImageBitmap);
         return videoFrame.createImageBitmap(imageBitmapOptions);
       })
       .then((toImageBitmap) => {
@@ -67,20 +74,115 @@
 
   promise_test(() => {
     return testImageBitmapToAndFromVideoFrame(48, 36, kP3Pixel,
-      kCanvasOptionsP3Uint8, { colorSpaceConversion: "none" });
+      kCanvasOptionsP3Uint8, {
+        colorSpaceConversion: "none"
+      });
   }, 'ImageBitmap<->VideoFrame with canvas(48x36 p3 uint8).');
   promise_test(() => {
     return testImageBitmapToAndFromVideoFrame(480, 360, kP3Pixel,
-      kCanvasOptionsP3Uint8, { colorSpaceConversion: "none" });
+      kCanvasOptionsP3Uint8, {
+        colorSpaceConversion: "none"
+      });
   }, 'ImageBitmap<->VideoFrame with canvas(480x360 p3 uint8).');
 
   promise_test(() => {
     return testImageBitmapToAndFromVideoFrame(48, 36, kRec2020Pixel,
-      kCanvasOptionsRec2020Uint8, { colorSpaceConversion: "none" });
+      kCanvasOptionsRec2020Uint8, {
+        colorSpaceConversion: "none"
+      });
   }, 'ImageBitmap<->VideoFrame with canvas(48x36 rec2020 uint8).');
   promise_test(() => {
     return testImageBitmapToAndFromVideoFrame(480, 360, kRec2020Pixel,
-      kCanvasOptionsRec2020Uint8, { colorSpaceConversion: "none" });
+      kCanvasOptionsRec2020Uint8, {
+        colorSpaceConversion: "none"
+      });
   }, 'ImageBitmap<->VideoFrame with canvas(480x360 rec2020 uint8).');
 
+
+  function testCreateImageBitmapFromVideoFrameVP9Decoder() {
+    // Prefers hardware decoders by setting video size as large as 720p.
+    const width = 1280;
+    const height = 720;
+
+    let canvas = document.createElement('canvas');
+    canvas.width = width;
+    canvas.height = height;
+    let ctx = canvas.getContext('2d');
+    ctx.fillStyle = 'rgb(50, 100, 150)';
+    ctx.fillRect(0, 0, width, height);
+
+    return createImageBitmap(canvas).then((fromImageBitmap) => {
+      let videoFrame = new VideoFrame({
+        timestamp: 0
+      }, fromImageBitmap);
+      return new Promise((resolve, reject) => {
+        let processVideoFrame = (frame) => {
+          frame.createImageBitmap().then((toImageBitmap) => {
+            let myCanvas = document.createElement('canvas');
+            myCanvas.width = width;
+            myCanvas.height = height;
+            let myCtx = myCanvas.getContext('2d');
+            myCtx.drawImage(toImageBitmap, 0, 0);
+            let tolerance = 6;
+            try {
+              testCanvas(myCtx, width, height, kSRGBPixel,
+                (actual, expected) => {
+                  assert_approx_equals(actual, expected, tolerance);
+                }
+              );
+            } catch (error) {
+              reject(error);
+            }
+            resolve('Done.');
+          });
+        };
+
+        const decoderInit = {
+          output: processVideoFrame,
+          error: (e) => {
+            console.log(e.message);
+          }
+        };
+
+        const encodedVideoConfig = {
+          codec: "vp09.00.10.08",
+        };
+
+        let decoder = new VideoDecoder(decoderInit);
+        decoder.configure(encodedVideoConfig);
+
+        let processVideoChunk = (chunk) => {
+          decoder.decode(chunk);
+          decoder.flush();
+        };
+
+        const encoderInit = {
+          output: processVideoChunk,
+          error: (e) => {
+            console.log(e.message);
+          }
+        };
+
+        const videoEncoderConfig = {
+          codec: "vp9",
+          profile: "vp09.00.10.08",
+          width: width,
+          height: height,
+          bitrate: 10e6,
+          framerate: 30,
+        };
+
+        let encoder = new VideoEncoder(encoderInit);
+        encoder.configure(videoEncoderConfig);
+        encoder.encode(videoFrame, {
+          keyFrame: true
+        });
+        encoder.flush();
+      });
+    });
+  }
+
+  promise_test(() => {
+    return testCreateImageBitmapFromVideoFrameVP9Decoder();
+  }, 'Create ImageBitmap for a VideoFrame from VP9 decoder.');
 </script>