Add h264 encoder format selection

This CL adds avcOptions, which allows user to choose between H264 stream
formats. By default, the encoded chunks will follow the AVC format, but
users can now choose to chunks converted into "Annex-B".

Bug: 1132050
Change-Id: I38ecfe9d2acec986050cc856a93d49d751152f75
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2587596
Commit-Queue: Thomas Guilbert <tguilbert@chromium.org>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Reviewed-by: Chrome Cunningham <chcunningham@chromium.org>
Reviewed-by: Eugene Zemtsov <eugene@chromium.org>
Cr-Commit-Position: refs/heads/master@{#840927}

media/base/video_encoder.h

@@ -34,6 +34,11 @@ struct MEDIA_EXPORT VideoEncoderOutput {
 
 class MEDIA_EXPORT VideoEncoder {
  public:
+  // TODO: Move this to a new file if there are more codec specific options.
+  struct MEDIA_EXPORT AvcOptions {
+    bool produce_annexb = false;
+  };
+
   struct MEDIA_EXPORT Options {
     Options();
     Options(const Options&);
@@ -44,6 +49,9 @@ class MEDIA_EXPORT VideoEncoder {
     gfx::Size frame_size;
 
     base::Optional<int> keyframe_interval = 10000;
+
+    // Only used for H264 encoding.
+    AvcOptions avc;
   };
 
   // A sequence of codec specific bytes, commonly known as extradata.

media/video/openh264_video_encoder.cc

@@ -132,10 +132,13 @@ void OpenH264VideoEncoder::Initialize(VideoCodecProfile profile,
     return;
   }
 
+  if (!options.avc.produce_annexb)
+    h264_converter_ = std::make_unique<H264AnnexBToAvcBitstreamConverter>();
+
   options_ = options;
   output_cb_ = BindToCurrentLoop(std::move(output_cb));
   codec_ = std::move(codec);
-  std::move(done_cb).Run(Status());
+  std::move(done_cb).Run(OkStatus());
 }
 
 void OpenH264VideoEncoder::Encode(scoped_refptr<VideoFrame> frame,
@@ -233,7 +236,16 @@ void OpenH264VideoEncoder::Encode(scoped_refptr<VideoFrame> frame,
 
   DCHECK_GT(frame_info.iFrameSizeInBytes, 0);
   size_t total_chunk_size = frame_info.iFrameSizeInBytes;
-  conversion_buffer_.resize(total_chunk_size);
+
+  result.data.reset(new uint8_t[total_chunk_size]);
+
+  auto* gather_buffer = result.data.get();
+
+  if (h264_converter_) {
+    // Copy data to a temporary buffer instead.
+    conversion_buffer_.resize(total_chunk_size);
+    gather_buffer = conversion_buffer_.data();
+  }
 
   size_t written_size = 0;
   for (int layer_idx = 0; layer_idx < frame_info.iLayerNum; ++layer_idx) {
@@ -247,16 +259,22 @@ void OpenH264VideoEncoder::Encode(scoped_refptr<VideoFrame> frame,
       return;
     }
 
-    memcpy(conversion_buffer_.data() + written_size, layer_info.pBsBuf,
-           layer_len);
+    memcpy(gather_buffer + written_size, layer_info.pBsBuf, layer_len);
     written_size += layer_len;
   }
   DCHECK_EQ(written_size, total_chunk_size);
 
+  if (!h264_converter_) {
+    result.size = total_chunk_size;
+
+    output_cb_.Run(std::move(result), base::Optional<CodecDescription>());
+    std::move(done_cb).Run(OkStatus());
+    return;
+  }
+
   size_t converted_output_size = 0;
   bool config_changed = false;
-  result.data.reset(new uint8_t[total_chunk_size]);
-  status = h264_converter_.ConvertChunk(
+  status = h264_converter_->ConvertChunk(
       conversion_buffer_,
       base::span<uint8_t>(result.data.get(), total_chunk_size), &config_changed,
       &converted_output_size);
@@ -265,11 +283,12 @@ void OpenH264VideoEncoder::Encode(scoped_refptr<VideoFrame> frame,
     std::move(done_cb).Run(std::move(status).AddHere(FROM_HERE));
     return;
   }
+
   result.size = converted_output_size;
 
   base::Optional<CodecDescription> desc;
   if (config_changed) {
-    const auto& config = h264_converter_.GetCurrentConfig();
+    const auto& config = h264_converter_->GetCurrentConfig();
     desc = CodecDescription();
     if (!config.Serialize(desc.value())) {
       std::move(done_cb).Run(Status(StatusCode::kEncoderFailedEncode,
@@ -279,7 +298,7 @@ void OpenH264VideoEncoder::Encode(scoped_refptr<VideoFrame> frame,
   }
 
   output_cb_.Run(std::move(result), std::move(desc));
-  std::move(done_cb).Run(Status());
+  std::move(done_cb).Run(OkStatus());
 }
 
 void OpenH264VideoEncoder::ChangeOptions(const Options& options,
@@ -316,9 +335,15 @@ void OpenH264VideoEncoder::ChangeOptions(const Options& options,
     return;
   }
 
+  if (options.avc.produce_annexb) {
+    h264_converter_.reset();
+  } else if (!h264_converter_) {
+    h264_converter_ = std::make_unique<H264AnnexBToAvcBitstreamConverter>();
+  }
+
   if (!output_cb.is_null())
     output_cb_ = BindToCurrentLoop(std::move(output_cb));
-  std::move(done_cb).Run(Status());
+  std::move(done_cb).Run(OkStatus());
 }
 
 void OpenH264VideoEncoder::Flush(StatusCB done_cb) {
@@ -329,7 +354,7 @@ void OpenH264VideoEncoder::Flush(StatusCB done_cb) {
   }
 
   // Nothing to do really.
-  std::move(done_cb).Run(Status());
+  std::move(done_cb).Run(OkStatus());
 }
 
 }  // namespace media

media/video/openh264_video_encoder.h

@@ -59,8 +59,11 @@ class MEDIA_EXPORT OpenH264VideoEncoder : public VideoEncoder {
   OutputCB output_cb_;
   std::vector<uint8_t> conversion_buffer_;
   VideoFramePool frame_pool_;
-  H264AnnexBToAvcBitstreamConverter h264_converter_;
+
+  // If |h264_converter_| is null, we output in annexb format. Otherwise, we
+  // output in avc format.
+  std::unique_ptr<H264AnnexBToAvcBitstreamConverter> h264_converter_;
 };
 
 }  // namespace media
-#endif  // MEDIA_VIDEO_OPENH264_VIDEO_ENCODER_H_
+#endif  // MEDIA_VIDEO_OPENH264_VIDEO_ENCODER_H_
\ No newline at end of file

media/video/video_encode_accelerator_adapter.cc

@@ -217,8 +217,10 @@ void VideoEncodeAcceleratorAdapter::InitializeOnAcceleratorThread(
   state_ = State::kWaitingForFirstFrame;
 
 #if BUILDFLAG(USE_PROPRIETARY_CODECS)
-  if (profile_ >= H264PROFILE_MIN && profile_ <= H264PROFILE_MAX)
+  if (profile_ >= H264PROFILE_MIN && profile_ <= H264PROFILE_MAX &&
+      !options_.avc.produce_annexb) {
     h264_converter_ = std::make_unique<H264AnnexBToAvcBitstreamConverter>();
+  }
 #endif  // BUILDFLAG(USE_PROPRIETARY_CODECS)
 
   std::move(done_cb).Run(Status());
@@ -378,9 +380,20 @@ void VideoEncodeAcceleratorAdapter::ChangeOptionsOnAcceleratorThread(
       options.framerate.value_or(VideoEncodeAccelerator::kDefaultFramerate))};
 
   accelerator_->RequestEncodingParametersChange(bitrate, framerate);
+
+#if BUILDFLAG(USE_PROPRIETARY_CODECS)
+  if (profile_ >= H264PROFILE_MIN && profile_ <= H264PROFILE_MAX) {
+    if (options.avc.produce_annexb) {
+      h264_converter_.reset();
+    } else if (!h264_converter_) {
+      h264_converter_ = std::make_unique<H264AnnexBToAvcBitstreamConverter>();
+    }
+  }
+#endif  // BUILDFLAG(USE_PROPRIETARY_CODECS)
+
   options_ = options;
   if (!output_cb.is_null())
-    output_cb_ = BindToCurrentLoop(std::move(output_cb));
+    output_cb_ = std::move(output_cb);
   std::move(done_cb).Run(Status());
 }
 

media/video/video_encode_accelerator_adapter.h

@@ -119,6 +119,8 @@ class MEDIA_EXPORT VideoEncodeAcceleratorAdapter
   GpuVideoAcceleratorFactories* gpu_factories_;
 
 #if BUILDFLAG(USE_PROPRIETARY_CODECS)
+  // If |h264_converter_| is null, we output in annexb format. Otherwise, we
+  // output in avc format.
   std::unique_ptr<H264AnnexBToAvcBitstreamConverter> h264_converter_;
 #endif  // BUILDFLAG(USE_PROPRIETARY_CODECS)
 
@@ -153,10 +155,10 @@ class MEDIA_EXPORT VideoEncodeAcceleratorAdapter
   InputBufferKind input_buffer_preference_ = InputBufferKind::Any;
   std::vector<uint8_t> resize_buf_;
 
-  VideoCodecProfile profile_;
+  VideoCodecProfile profile_ = VIDEO_CODEC_PROFILE_UNKNOWN;
   Options options_;
   OutputCB output_cb_;
 };
 
 }  // namespace media
-#endif  // MEDIA_VIDEO_VIDEO_ENCODE_ACCELERATOR_ADAPTER_H_
+#endif  // MEDIA_VIDEO_VIDEO_ENCODE_ACCELERATOR_ADAPTER_H_
\ No newline at end of file

third_party/blink/renderer/bindings/idl_in_modules.gni

@@ -746,6 +746,7 @@ static_idl_files_in_modules = get_path_info(
           "//third_party/blink/renderer/modules/webcodecs/audio_frame.idl",
           "//third_party/blink/renderer/modules/webcodecs/audio_frame_init.idl",
           "//third_party/blink/renderer/modules/webcodecs/audio_frame_output_callback.idl",
+          "//third_party/blink/renderer/modules/webcodecs/avc_encoder_config.idl",
           "//third_party/blink/renderer/modules/webcodecs/codec_state.idl",
           "//third_party/blink/renderer/modules/webcodecs/encoded_audio_chunk.idl",
           "//third_party/blink/renderer/modules/webcodecs/color_space_matrix_id.idl",

third_party/blink/renderer/modules/webcodecs/avc_encoder_config.idl

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// https://github.com/WICG/web-codecs
+
+enum AvcBitstreamFormat {
+  "annexb",
+  "avc",
+};
+
+dictionary AvcEncoderConfig {
+  AvcBitstreamFormat format = "avc";
+};

third_party/blink/renderer/modules/webcodecs/idls.gni

@@ -25,6 +25,7 @@ modules_callback_function_idl_files = [
 modules_dictionary_idl_files = [
   "audio_decoder_config.idl",
   "audio_decoder_init.idl",
+  "avc_encoder_config.idl",
   "encoded_video_chunk_init.idl",
   "encoded_audio_chunk_init.idl",
   "image_decoder_init.idl",

third_party/blink/renderer/modules/webcodecs/video_encoder.cc

@@ -31,6 +31,7 @@
 #include "third_party/blink/renderer/bindings/core/v8/script_function.h"
 #include "third_party/blink/renderer/bindings/core/v8/script_promise.h"
 #include "third_party/blink/renderer/bindings/core/v8/v8_dom_exception.h"
+#include "third_party/blink/renderer/bindings/modules/v8/v8_avc_encoder_config.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_video_decoder_config.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_video_encoder_config.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_video_encoder_encode_options.h"
@@ -233,6 +234,26 @@ VideoEncoder::ParsedConfig* VideoEncoder::ParseConfig(
     return nullptr;
   }
 
+  // We are done with the parsing.
+  if (!config->hasAvc())
+    return parsed;
+
+  // We should only get here with H264 codecs.
+  if (parsed->codec != media::VideoCodec::kCodecH264) {
+    exception_state.ThrowTypeError(
+        "'avcOptions' can only be used with AVC codecs");
+    return nullptr;
+  }
+
+  std::string avc_format = IDLEnumAsString(config->avc()->format()).Utf8();
+  if (avc_format == "avc") {
+    parsed->options.avc.produce_annexb = false;
+  } else if (avc_format == "annexb") {
+    parsed->options.avc.produce_annexb = true;
+  } else {
+    NOTREACHED();
+  }
+
   return parsed;
 }
 
@@ -375,7 +396,7 @@ std::unique_ptr<media::VideoEncoder> VideoEncoder::CreateMediaVideoEncoder(
 bool VideoEncoder::CanReconfigure(ParsedConfig& original_config,
                                   ParsedConfig& new_config) {
   // Reconfigure is intended for things that don't require changing underlying
-  // codec implementatio and can be changed on the fly.
+  // codec implementation and can be changed on the fly.
   return original_config.codec == new_config.codec &&
          original_config.profile == new_config.profile &&
          original_config.level == new_config.level &&

third_party/blink/renderer/modules/webcodecs/video_encoder_config.idl

@@ -4,7 +4,6 @@
 
 // https://github.com/WICG/web-codecs
 
-
 enum VideoEncoderAccelerationPreference {
   "allow",
   "deny",
@@ -21,4 +20,6 @@ dictionary VideoEncoderConfig {
 
   required unsigned long width;
   required unsigned long height;
-};
+
\ No newline at end of file
+  AvcEncoderConfig avc;
+};

third_party/blink/web_tests/wpt_internal/webcodecs/avc_encoder_config.any.js

+// META: global=window,dedicatedworker
+// META: script=/wpt_internal/webcodecs/encoder_utils.js
+
+const defaultWidth = 640;
+const defaultHeight = 360;
+
+let frameNumber = 0;
+
+async function configureAndEncode(encoder, config) {
+  encoder.configure(config);
+
+  let frame = await createFrame(defaultWidth, defaultHeight, ++frameNumber);
+  encoder.encode(frame, { keyFrame : true });
+  return encoder.flush()
+}
+
+function cycleAvcOutputFormats(acc, desc) {
+  promise_test(async t => {
+    var output = undefined;
+
+    let encoderInit = {
+      error: () => t.unreached_func("Unexpected error"),
+      output: (chunk, config) => {
+        assert_equals(output, undefined, "output undefined sanity");
+        output = {
+          chunk: chunk,
+          config: config,
+        };
+      },
+    };
+
+    let encoder = new VideoEncoder(encoderInit);
+
+    let encoderConfig = {
+      codec: "avc1.42001E",
+      acceleration: acc,
+      width: defaultWidth,
+      height: defaultHeight,
+    };
+
+    // Configure an encoder with no avcOptions (should default to avc format).
+    await configureAndEncode(encoder, encoderConfig);
+
+    // avc chunks should output a config with an avcC description.
+    assert_not_equals(output, undefined, "output default");
+    assert_not_equals(output.chunk, null, "chunk default");
+    assert_not_equals(output.config, null, "config default");
+    assert_not_equals(output.config.description, null, "desc default");
+
+    output = undefined;
+
+    // Configure with annex-b.
+    encoderConfig.avc = { format: "annexb" };
+    await configureAndEncode(encoder, encoderConfig);
+
+    // annexb chunks should start with a start code.
+    assert_not_equals(output, undefined, "output annexb");
+    assert_not_equals(output.chunk, null, "chunk annexb");
+    assert_not_equals(output.chunk.data, null, "chunk.data annexb");
+
+    let startCode = new Int8Array(output.chunk.data.slice(0,4));
+
+    assert_equals(startCode[0], 0x00, "startCode [0]");
+    assert_equals(startCode[1], 0x00, "startCode [1]");
+    assert_equals(startCode[2], 0x00, "startCode [2]");
+    assert_equals(startCode[3], 0x01, "startCode [3]");
+
+    // No config needs to be emitted for annexb, but we always emit one for now.
+    // There should not be an avcC 'description' with annexb.
+    // TODO: Update this if we only output configs when they change.
+    assert_not_equals(output.config, null, "config annexb");
+    assert_equals(output.config.description, undefined, "desc annexb");
+
+    output = undefined;
+
+    // Configure with avc.
+    encoderConfig.avc = { format: "avc" };
+    await configureAndEncode(encoder, encoderConfig);
+
+    // avc should output a config with an avcC description.
+    assert_not_equals(output, undefined, "output avc");
+    assert_not_equals(output.chunk, null, "chunk avc");
+    assert_not_equals(output.config, null, "config avc");
+    assert_not_equals(output.config.description, null, "desc avc");
+
+    encoder.close();
+  }, desc);
+}
+
+cycleAvcOutputFormats("deny",
+  "Test AvcConfig supports 'avc' and 'annexb' (acceleration = 'deny')");
+
+cycleAvcOutputFormats("allow",
+  "Test AvcConfig supports 'avc' and 'annexb' (acceleration = 'allow')");
+
+/* Uncomment this for manual testing, before we have GPU tests for that */
+// cycleAvcOutputFormats("require",
+//   "Test AvcConfig supports 'avc' and 'annexb' (acceleration = 'require')");
+
+promise_test(async t => {
+  let encoder = new VideoEncoder({
+    error: () => t.unreached_func("Unexpected error"),
+    output: () => t.unreached_func("Unexpected output"),
+  });
+
+  const vp8Config = {
+    codec: 'vp8',
+    acceleration: "allow",
+    width: defaultWidth,
+    height: defaultHeight,
+    avc: { outputFormat: "avc" },
+  };
+
+  assert_throws_js(TypeError,
+    () => { encoder.configure(vp8Config); },
+    "Only H264 should support avcOptions");
+
+  encoder.close();
+}, "Make sure non-H264 configurations reject avcOptions");

third_party/blink/web_tests/wpt_internal/webcodecs/basic_video_encoding.any.js

 // META: global=window,dedicatedworker
+// META: script=/wpt_internal/webcodecs/encoder_utils.js
 
-async function getImageAsBitmap(width, height) {
-  const src = "pattern.png";
-
-  var size = {
-    resizeWidth: width,
-    resizeHeight: height
-  };
-
-  return fetch(src)
-      .then(response => response.blob())
-      .then(blob => createImageBitmap(blob, size));
-}
-
-async function generateBitmap(width, height, text) {
-  let img = await getImageAsBitmap(width, height);
-  let cnv = new OffscreenCanvas(width, height);
-  var ctx = cnv.getContext('2d');
-  ctx.drawImage(img, 0, 0, width, height);
-  img.close();
-  ctx.font = '30px fantasy';
-  ctx.fillText(text, 5, 40);
-  return createImageBitmap(cnv);
-}
-
-async function createFrame(width, height, ts) {
-  let imageBitmap = await generateBitmap(width, height, ts.toString());
-  return new VideoFrame(imageBitmap, { timestamp: ts });
-}
-
-function delay(time_ms) {
-  return new Promise((resolve, reject) => {
-    setTimeout(resolve, time_ms);
-  });
-};
-
-async function encode_decode_test(codec, acc) {
+async function encode_decode_test(codec, acc, avc_options) {
   const w = 640;
   const h = 360;
   let next_ts = 0
@@ -73,7 +39,7 @@ async function encode_decode_test(codec, acc) {
     }
   };
 
-  const encoder_config = {
+  let encoder_config = {
     codec: codec,
     acceleration: acc,
     width: w,
@@ -81,6 +47,10 @@ async function encode_decode_test(codec, acc) {
     bitrate: 5000000,
   };
 
+  if (avc_options !== null) {
+    encoder_config.avcOptions = avc_options;
+  }
+
   let encoder = new VideoEncoder(encoder_init);
   encoder.configure(encoder_config);
 
@@ -88,6 +58,10 @@ async function encode_decode_test(codec, acc) {
     let frame = await createFrame(w, h, i);
     let keyframe = (i % 5 == 0);
     encoder.encode(frame, { keyFrame: keyframe });
+
+    // Wait to prevent queueing all frames before encoder.configure() completes.
+    // Queuing them all at once should still work, but would not be as
+    // repesentative of a real world scenario.
     await delay(1);
   }
   await encoder.flush();
@@ -158,17 +132,22 @@ promise_test(encode_test.bind(null, "vp09.00.10.08", "allow"),
 promise_test(encode_test.bind(null, "vp09.02.10.10", "allow"),
   "encoding vp9 profile2");
 
-promise_test(encode_decode_test.bind(null, "vp09.02.10.10", "allow"),
+promise_test(encode_decode_test.bind(null, "vp09.02.10.10", "allow", null),
   "encoding and decoding vp9 profile2");
 
 promise_test(encode_test.bind(null, "vp8", "allow"),
   "encoding vp8");
 
-promise_test(encode_decode_test.bind(null, "vp8", "allow"),
+promise_test(encode_decode_test.bind(null, "vp8", "allow", null),
   "encoding and decoding vp8");
 
-promise_test(encode_decode_test.bind(null, "avc1.42001E", "allow"),
-  "encoding and decoding avc1.42001E");
+promise_test(
+  encode_decode_test.bind(null, "avc1.42001E", "allow", { outputFormat: "annexb"}),
+  "encoding and decoding avc1.42001E (annexb)");
+
+promise_test(
+  encode_decode_test.bind(null, "avc1.42001E", "allow", { outputFormat: "avc"}),
+  "encoding and decoding avc1.42001E (avc)");
 
 /* Uncomment this for manual testing, before we have GPU tests for that */
 // promise_test(encode_test.bind(null, "avc1.42001E", "require"),

third_party/blink/web_tests/wpt_internal/webcodecs/encoder_utils.js

+async function getImageAsBitmap(width, height) {
+  const src = "pattern.png";
+
+  var size = {
+    resizeWidth: width,
+    resizeHeight: height
+  };
+
+  return fetch(src)
+      .then(response => response.blob())
+      .then(blob => createImageBitmap(blob, size));
+}
+
+async function generateBitmap(width, height, text) {
+  let img = await getImageAsBitmap(width, height);
+  let cnv = new OffscreenCanvas(width, height);
+  var ctx = cnv.getContext('2d');
+  ctx.drawImage(img, 0, 0, width, height);
+  img.close();
+  ctx.font = '30px fantasy';
+  ctx.fillText(text, 5, 40);
+  return createImageBitmap(cnv);
+}
+
+async function createFrame(width, height, ts) {
+  let imageBitmap = await generateBitmap(width, height, ts.toString());
+  return new VideoFrame(imageBitmap, { timestamp: ts });
+}
+
+function delay(time_ms) {
+  return new Promise((resolve, reject) => {
+    setTimeout(resolve, time_ms);
+  });
+};

third_party/blink/web_tests/wpt_internal/webcodecs/reconfiguring_encoder.any.js

 // META: global=window,dedicatedworker
-
-async function getImageAsBitmap(width, height) {
-  const src = "pattern.png";
-
-  var size = {
-    resizeWidth: width,
-    resizeHeight: height
-  };
-
-  return fetch(src)
-      .then(response => response.blob())
-      .then(blob => createImageBitmap(blob, size));
-}
-
-async function generateBitmap(width, height, text) {
-  let img = await getImageAsBitmap(width, height);
-  let cnv = new OffscreenCanvas(width, height);
-  var ctx = cnv.getContext('2d');
-  ctx.drawImage(img, 0, 0, width, height);
-  img.close();
-  ctx.font = '30px fantasy';
-  ctx.fillText(text, 5, 40);
-  return createImageBitmap(cnv);
-}
-
-
-async function createFrame(width, height, ts) {
-  let imageBitmap = await generateBitmap(width, height, ts.toString());
-  let frame = new VideoFrame(imageBitmap, { timestamp: ts });
-  imageBitmap.close();
-  return frame;
-}
-
-function delay(time_ms) {
-  return new Promise((resolve, reject) => {
-    setTimeout(resolve, time_ms);
-  });
-};
+// META: script=/wpt_internal/webcodecs/encoder_utils.js
 
 async function change_encoding_params_test(codec, acc) {
   let original_w = 800;