[webcodecs] Implement H.264 bitstream conversion.

WebCodecs expects AVC-formatted H.264 streams, but our decoders expect
Annex B.

Bug: 1119947
Change-Id: I8a0d92450890a9d625d1b2a67b1ea20377bdfd1e
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2368162
Commit-Queue: Dan Sanders <sandersd@chromium.org>
Reviewed-by: Chrome Cunningham <chcunningham@chromium.org>
Cr-Commit-Position: refs/heads/master@{#800704}

third_party/blink/renderer/modules/webcodecs/DEPS

@@ -8,6 +8,7 @@ include_rules = [
 
     "+media/base",
     "+media/filters",
+    "+media/formats/mp4/box_definitions.h",
     "+media/media_buildflags.h",
     "+media/mojo",
     "+media/renderers",

third_party/blink/renderer/modules/webcodecs/audio_decoder.cc

@@ -32,10 +32,37 @@ AudioDecoderTraits::CreateDecoder(ExecutionContext& execution_context,
 }
 
 // static
-CodecConfigEval AudioDecoderTraits::CreateMediaConfig(
-    const ConfigType& config,
-    MediaConfigType* out_media_config,
-    String* out_console_message) {
+void AudioDecoderTraits::InitializeDecoder(
+    MediaDecoderType& decoder,
+    const MediaConfigType& media_config,
+    MediaDecoderType::InitCB init_cb,
+    MediaDecoderType::OutputCB output_cb) {
+  decoder.Initialize(media_config, nullptr /* cdm_context */,
+                     std::move(init_cb), output_cb, media::WaitingCB());
+}
+
+// static
+int AudioDecoderTraits::GetMaxDecodeRequests(const MediaDecoderType& decoder) {
+  return 1;
+}
+
+// static
+AudioDecoder* AudioDecoder::Create(ScriptState* script_state,
+                                   const AudioDecoderInit* init,
+                                   ExceptionState& exception_state) {
+  return MakeGarbageCollected<AudioDecoder>(script_state, init,
+                                            exception_state);
+}
+
+AudioDecoder::AudioDecoder(ScriptState* script_state,
+                           const AudioDecoderInit* init,
+                           ExceptionState& exception_state)
+    : DecoderTemplate<AudioDecoderTraits>(script_state, init, exception_state) {
+}
+
+CodecConfigEval AudioDecoder::MakeMediaConfig(const ConfigType& config,
+                                              MediaConfigType* out_media_config,
+                                              String* out_console_message) {
   media::AudioCodec codec = media::kUnknownAudioCodec;
   bool is_codec_ambiguous = true;
   bool parse_succeeded = ParseAudioCodecString("", config.codec().Utf8(),
@@ -88,23 +115,7 @@ CodecConfigEval AudioDecoderTraits::CreateMediaConfig(
   return CodecConfigEval::kSupported;
 }
 
-// static
-void AudioDecoderTraits::InitializeDecoder(
-    MediaDecoderType& decoder,
-    const MediaConfigType& media_config,
-    MediaDecoderType::InitCB init_cb,
-    MediaDecoderType::OutputCB output_cb) {
-  decoder.Initialize(media_config, nullptr /* cdm_context */,
-                     std::move(init_cb), output_cb, media::WaitingCB());
-}
-
-// static
-int AudioDecoderTraits::GetMaxDecodeRequests(const MediaDecoderType& decoder) {
-  return 1;
-}
-
-// static
-scoped_refptr<media::DecoderBuffer> AudioDecoderTraits::MakeDecoderBuffer(
+scoped_refptr<media::DecoderBuffer> AudioDecoder::MakeDecoderBuffer(
     const InputType& chunk) {
   auto decoder_buffer = media::DecoderBuffer::CopyFrom(
       static_cast<uint8_t*>(chunk.data()->Data()),
@@ -115,18 +126,4 @@ scoped_refptr<media::DecoderBuffer> AudioDecoderTraits::MakeDecoderBuffer(
   return decoder_buffer;
 }
 
-// static
-AudioDecoder* AudioDecoder::Create(ScriptState* script_state,
-                                   const AudioDecoderInit* init,
-                                   ExceptionState& exception_state) {
-  return MakeGarbageCollected<AudioDecoder>(script_state, init,
-                                            exception_state);
-}
-
-AudioDecoder::AudioDecoder(ScriptState* script_state,
-                           const AudioDecoderInit* init,
-                           ExceptionState& exception_state)
-    : DecoderTemplate<AudioDecoderTraits>(script_state, init, exception_state) {
-}
-
 }  // namespace blink

third_party/blink/renderer/modules/webcodecs/audio_decoder.h

@@ -55,16 +55,11 @@ class MODULES_EXPORT AudioDecoderTraits {
   static std::unique_ptr<MediaDecoderType> CreateDecoder(
       ExecutionContext& execution_context,
       media::MediaLog* media_log);
-  static CodecConfigEval CreateMediaConfig(const ConfigType& config,
-                                           MediaConfigType* out_media_config,
-                                           String* out_console_message);
   static void InitializeDecoder(MediaDecoderType& decoder,
                                 const MediaConfigType& media_config,
                                 MediaDecoderType::InitCB init_cb,
                                 MediaDecoderType::OutputCB output_cb);
   static int GetMaxDecodeRequests(const MediaDecoderType& decoder);
-  static scoped_refptr<media::DecoderBuffer> MakeDecoderBuffer(
-      const InputType& input);
 };
 
 class MODULES_EXPORT AudioDecoder : public DecoderTemplate<AudioDecoderTraits> {
@@ -77,6 +72,13 @@ class MODULES_EXPORT AudioDecoder : public DecoderTemplate<AudioDecoderTraits> {
 
   AudioDecoder(ScriptState*, const AudioDecoderInit*, ExceptionState&);
   ~AudioDecoder() override = default;
+
+ protected:
+  CodecConfigEval MakeMediaConfig(const ConfigType& config,
+                                  MediaConfigType* out_media_config,
+                                  String* out_console_message) override;
+  scoped_refptr<media::DecoderBuffer> MakeDecoderBuffer(
+      const InputType& chunk) override;
 };
 
 }  // namespace blink

third_party/blink/renderer/modules/webcodecs/decoder_template.cc

@@ -73,7 +73,7 @@ void DecoderTemplate<Traits>::configure(const ConfigType* config,
   String console_message;
 
   CodecConfigEval eval =
-      Traits::CreateMediaConfig(*config, media_config.get(), &console_message);
+      MakeMediaConfig(*config, media_config.get(), &console_message);
   switch (eval) {
     case CodecConfigEval::kInvalid:
       exception_state.ThrowTypeError(console_message);
@@ -106,7 +106,7 @@ void DecoderTemplate<Traits>::decode(const InputType* chunk,
 
   Request* request = MakeGarbageCollected<Request>();
   request->type = Request::Type::kDecode;
-  request->chunk = chunk;
+  request->decoder_buffer = MakeDecoderBuffer(*chunk);
   requests_.push_back(request);
   ++requested_decodes_;
   ProcessRequests();
@@ -244,7 +244,6 @@ bool DecoderTemplate<Traits>::ProcessDecodeRequest(Request* request) {
   DCHECK(!is_closed_);
   DCHECK(!pending_request_);
   DCHECK_EQ(request->type, Request::Type::kDecode);
-  DCHECK(request->chunk);
   DCHECK_GT(requested_decodes_, 0);
 
   // TODO(sandersd): If a reset has been requested, complete immediately.
@@ -260,9 +259,8 @@ bool DecoderTemplate<Traits>::ProcessDecodeRequest(Request* request) {
     return false;
   }
 
-  scoped_refptr<media::DecoderBuffer> decoder_buffer =
-      Traits::MakeDecoderBuffer(*request->chunk);
-  if (decoder_buffer->data_size() == 0) {
+  // The request may be invalid, if so report that now.
+  if (!request->decoder_buffer || request->decoder_buffer->data_size() == 0) {
     HandleError();
     return false;
   }
@@ -276,7 +274,7 @@ bool DecoderTemplate<Traits>::ProcessDecodeRequest(Request* request) {
     ;
   pending_decodes_.Set(pending_decode_id_, request);
   --requested_decodes_;
-  decoder_->Decode(std::move(decoder_buffer),
+  decoder_->Decode(std::move(request->decoder_buffer),
                    WTF::Bind(&DecoderTemplate::OnDecodeDone,
                              WrapWeakPersistent(this), pending_decode_id_));
   return true;
@@ -491,7 +489,6 @@ void DecoderTemplate<Traits>::Trace(Visitor* visitor) const {
 
 template <typename Traits>
 void DecoderTemplate<Traits>::Request::Trace(Visitor* visitor) const {
-  visitor->Trace(chunk);
   visitor->Trace(resolver);
 }
 

third_party/blink/renderer/modules/webcodecs/decoder_template.h

@@ -15,6 +15,7 @@
 #include "third_party/blink/renderer/bindings/core/v8/script_promise_resolver.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_web_codecs_error_callback.h"
 #include "third_party/blink/renderer/modules/modules_export.h"
+#include "third_party/blink/renderer/modules/webcodecs/codec_config_eval.h"
 #include "third_party/blink/renderer/platform/bindings/script_wrappable.h"
 #include "third_party/blink/renderer/platform/heap/garbage_collected.h"
 #include "third_party/blink/renderer/platform/heap/heap.h"
@@ -48,6 +49,21 @@ class MODULES_EXPORT DecoderTemplate : public ScriptWrappable {
   // GarbageCollected override.
   void Trace(Visitor*) const override;
 
+ protected:
+  // TODO(sandersd): Consider moving these to the Traits class, and creating an
+  // instance of the traits.
+
+  // Convert a configuration to a DecoderConfig.
+  virtual CodecConfigEval MakeMediaConfig(const ConfigType& config,
+                                          MediaConfigType* out_media_config,
+                                          String* out_console_message) = 0;
+
+  // Convert a chunk to a DecoderBuffer. You can assume that the last
+  // configuration sent to MakeMediaConfig() is the active configuration for
+  // |chunk|.
+  virtual scoped_refptr<media::DecoderBuffer> MakeDecoderBuffer(
+      const InputType& chunk) = 0;
+
  private:
   struct Request final : public GarbageCollected<Request> {
     enum class Type {
@@ -65,7 +81,7 @@ class MODULES_EXPORT DecoderTemplate : public ScriptWrappable {
     std::unique_ptr<MediaConfigType> media_config;
 
     // For kDecode Requests.
-    Member<const InputType> chunk;
+    scoped_refptr<media::DecoderBuffer> decoder_buffer;
 
     // For kFlush Requests.
     Member<ScriptPromiseResolver> resolver;

third_party/blink/renderer/modules/webcodecs/video_decoder.cc

@@ -13,6 +13,7 @@
 #include "media/base/mime_util.h"
 #include "media/base/supported_types.h"
 #include "media/base/video_decoder.h"
+#include "media/media_buildflags.h"
 #include "third_party/blink/public/platform/platform.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_encoded_video_chunk.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_encoded_video_config.h"
@@ -29,6 +30,11 @@
 #include "ui/gfx/geometry/rect.h"
 #include "ui/gfx/geometry/size.h"
 
+#if BUILDFLAG(USE_PROPRIETARY_CODECS)
+#include "media/filters/h264_to_annex_b_bitstream_converter.h"
+#include "media/formats/mp4/box_definitions.h"
+#endif  // BUILDFLAG(USE_PROPRIETARY_CODECS)
+
 namespace blink {
 
 // static
@@ -40,10 +46,38 @@ VideoDecoderTraits::CreateDecoder(ExecutionContext& execution_context,
 }
 
 // static
-CodecConfigEval VideoDecoderTraits::CreateMediaConfig(
-    const ConfigType& config,
-    MediaConfigType* out_media_config,
-    String* out_console_message) {
+void VideoDecoderTraits::InitializeDecoder(
+    MediaDecoderType& decoder,
+    const MediaConfigType& media_config,
+    MediaDecoderType::InitCB init_cb,
+    MediaDecoderType::OutputCB output_cb) {
+  decoder.Initialize(media_config, false /* low_delay */,
+                     nullptr /* cdm_context */, std::move(init_cb), output_cb,
+                     media::WaitingCB());
+}
+
+// static
+int VideoDecoderTraits::GetMaxDecodeRequests(const MediaDecoderType& decoder) {
+  return decoder.GetMaxDecodeRequests();
+}
+
+// static
+VideoDecoder* VideoDecoder::Create(ScriptState* script_state,
+                                   const VideoDecoderInit* init,
+                                   ExceptionState& exception_state) {
+  return MakeGarbageCollected<VideoDecoder>(script_state, init,
+                                            exception_state);
+}
+
+VideoDecoder::VideoDecoder(ScriptState* script_state,
+                           const VideoDecoderInit* init,
+                           ExceptionState& exception_state)
+    : DecoderTemplate<VideoDecoderTraits>(script_state, init, exception_state) {
+}
+
+CodecConfigEval VideoDecoder::MakeMediaConfig(const ConfigType& config,
+                                              MediaConfigType* out_media_config,
+                                              String* out_console_message) {
   DCHECK(out_media_config);
   DCHECK(out_console_message);
 
@@ -89,13 +123,32 @@ CodecConfigEval VideoDecoderTraits::CreateMediaConfig(
     }
   }
 
-  // If we allow empty |extra_data| here, FFmpegVideoDecoder will expect an
-  // Annex B formatted stream.
-  if (codec == media::kCodecH264 && extra_data.empty()) {
-    *out_console_message =
-        "H.264 configuration for must include an avcC description.";
-    return CodecConfigEval::kInvalid;
+#if BUILDFLAG(USE_PROPRIETARY_CODECS)
+  if (codec == media::kCodecH264) {
+    if (extra_data.empty()) {
+      *out_console_message =
+          "H.264 configuration must include an avcC description.";
+      return CodecConfigEval::kInvalid;
+    }
+
+    h264_avcc_ = std::make_unique<media::mp4::AVCDecoderConfigurationRecord>();
+    h264_converter_ = std::make_unique<media::H264ToAnnexBBitstreamConverter>();
+    if (!h264_converter_->ParseConfiguration(
+            extra_data.data(), static_cast<uint32_t>(extra_data.size()),
+            h264_avcc_.get())) {
+      *out_console_message = "Failed to parse avcC.";
+      return CodecConfigEval::kInvalid;
+    }
+  } else {
+    h264_avcc_.reset();
+    h264_converter_.reset();
+  }
+#else
+  if (codec == media::kCodecH264) {
+    *out_console_message = "H.264 decoding is not supported.";
+    return CodecConfigEval::kUnsupported;
   }
+#endif  // BUILDFLAG(USE_PROPRIETARY_CODECS)
 
   // TODO(sandersd): Either remove sizes from VideoDecoderConfig (replace with
   // sample aspect) or parse the AvcC here to get the actual size.
@@ -111,29 +164,36 @@ CodecConfigEval VideoDecoderTraits::CreateMediaConfig(
   return CodecConfigEval::kSupported;
 }
 
-// static
-void VideoDecoderTraits::InitializeDecoder(
-    MediaDecoderType& decoder,
-    const MediaConfigType& media_config,
-    MediaDecoderType::InitCB init_cb,
-    MediaDecoderType::OutputCB output_cb) {
-  decoder.Initialize(media_config, false /* low_delay */,
-                     nullptr /* cdm_context */, std::move(init_cb), output_cb,
-                     media::WaitingCB());
-}
+scoped_refptr<media::DecoderBuffer> VideoDecoder::MakeDecoderBuffer(
+    const InputType& chunk) {
+  uint8_t* src = static_cast<uint8_t*>(chunk.data()->Data());
+  size_t src_size = chunk.data()->ByteLengthAsSizeT();
+
+  scoped_refptr<media::DecoderBuffer> decoder_buffer;
+#if BUILDFLAG(USE_PROPRIETARY_CODECS)
+  if (h264_converter_) {
+    // Note: this may not be safe if support for SharedArrayBuffers is added.
+    uint32_t output_size = h264_converter_->CalculateNeededOutputBufferSize(
+        src, static_cast<uint32_t>(src_size), h264_avcc_.get());
+    if (!output_size) {
+      // TODO(sandersd): Provide an error message.
+      return nullptr;
+    }
 
-// static
-int VideoDecoderTraits::GetMaxDecodeRequests(const MediaDecoderType& decoder) {
-  return decoder.GetMaxDecodeRequests();
-}
+    std::vector<uint8_t> buf(output_size);
+    if (!h264_converter_->ConvertNalUnitStreamToByteStream(
+            src, static_cast<uint32_t>(src_size), h264_avcc_.get(), buf.data(),
+            &output_size)) {
+      // TODO(sandersd): Provide an error message.
+      return nullptr;
+    }
+
+    decoder_buffer = media::DecoderBuffer::CopyFrom(buf.data(), output_size);
+  }
+#endif  // BUILDFLAG(USE_PROPRIETARY_CODECS)
+  if (!decoder_buffer)
+    decoder_buffer = media::DecoderBuffer::CopyFrom(src, src_size);
 
-// static
-scoped_refptr<media::DecoderBuffer> VideoDecoderTraits::MakeDecoderBuffer(
-    const InputType& chunk) {
-  // Convert |chunk| to a DecoderBuffer.
-  auto decoder_buffer = media::DecoderBuffer::CopyFrom(
-      static_cast<uint8_t*>(chunk.data()->Data()),
-      chunk.data()->ByteLengthAsSizeT());
   decoder_buffer->set_timestamp(
       base::TimeDelta::FromMicroseconds(chunk.timestamp()));
   // TODO(sandersd): Use kUnknownTimestamp instead of 0?
@@ -144,18 +204,4 @@ scoped_refptr<media::DecoderBuffer> VideoDecoderTraits::MakeDecoderBuffer(
   return decoder_buffer;
 }
 
-// static
-VideoDecoder* VideoDecoder::Create(ScriptState* script_state,
-                                   const VideoDecoderInit* init,
-                                   ExceptionState& exception_state) {
-  return MakeGarbageCollected<VideoDecoder>(script_state, init,
-                                            exception_state);
-}
-
-VideoDecoder::VideoDecoder(ScriptState* script_state,
-                           const VideoDecoderInit* init,
-                           ExceptionState& exception_state)
-    : DecoderTemplate<VideoDecoderTraits>(script_state, init, exception_state) {
-}
-
 }  // namespace blink

third_party/blink/renderer/modules/webcodecs/video_decoder.h

@@ -12,6 +12,7 @@
 #include "media/base/status.h"
 #include "media/base/video_decoder.h"
 #include "media/base/video_decoder_config.h"
+#include "media/media_buildflags.h"
 #include "third_party/blink/renderer/bindings/core/v8/script_promise.h"
 #include "third_party/blink/renderer/bindings/core/v8/script_promise_resolver.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_video_frame_output_callback.h"
@@ -31,6 +32,13 @@ class VideoFrame;
 class DecoderBuffer;
 class MediaLog;
 
+#if BUILDFLAG(USE_PROPRIETARY_CODECS)
+class H264ToAnnexBBitstreamConverter;
+namespace mp4 {
+struct AVCDecoderConfigurationRecord;
+}
+#endif  // BUILDFLAG(USE_PROPRIETARY_CODECS)
+
 }  // namespace media
 
 namespace blink {
@@ -56,16 +64,11 @@ class MODULES_EXPORT VideoDecoderTraits {
   static std::unique_ptr<MediaDecoderType> CreateDecoder(
       ExecutionContext& execution_context,
       media::MediaLog* media_log);
-  static CodecConfigEval CreateMediaConfig(const ConfigType& config,
-                                           MediaConfigType* out_media_config,
-                                           String* out_console_message);
   static void InitializeDecoder(MediaDecoderType& decoder,
                                 const MediaConfigType& media_config,
                                 MediaDecoderType::InitCB init_cb,
                                 MediaDecoderType::OutputCB output_cb);
   static int GetMaxDecodeRequests(const MediaDecoderType& decoder);
-  static scoped_refptr<media::DecoderBuffer> MakeDecoderBuffer(
-      const InputType& input);
 };
 
 class MODULES_EXPORT VideoDecoder : public DecoderTemplate<VideoDecoderTraits> {
@@ -78,6 +81,18 @@ class MODULES_EXPORT VideoDecoder : public DecoderTemplate<VideoDecoderTraits> {
 
   VideoDecoder(ScriptState*, const VideoDecoderInit*, ExceptionState&);
   ~VideoDecoder() override = default;
+
+ protected:
+  CodecConfigEval MakeMediaConfig(const ConfigType& config,
+                                  MediaConfigType* out_media_config,
+                                  String* out_console_message) override;
+  scoped_refptr<media::DecoderBuffer> MakeDecoderBuffer(
+      const InputType& input) override;
+
+#if BUILDFLAG(USE_PROPRIETARY_CODECS)
+  std::unique_ptr<media::H264ToAnnexBBitstreamConverter> h264_converter_;
+  std::unique_ptr<media::mp4::AVCDecoderConfigurationRecord> h264_avcc_;
+#endif  // BUILDFLAG(USE_PROPRIETARY_CODECS)
 };
 
 }  // namespace blink