[MSE][WebCodecs] Stubs new WebCodecsEncodedChunkStreamParser

Adds a new "StreamParser" subtype for buffering WebCodecs encoded audio
or video chunks. Plumbs the initial decoder configuration (to later be
emitted during eventual first ProcessChunks() call on the new parser)
through ChunkDemuxer's AddId to a new instance of one of these parsers.

Includes StreamParserFactory methods to create the new kind of parser.

Updates the web_test to expect a SourceBuffer to be created for valid
WebCodecs config cases (instead of a QuotaExceededError exception as was
stubbed previously.)

Later changes will complete the implementation of
WebCodecsEncodedChunkStreamParser::ProcessChunks() and include more
testing of end-to-end buffering and playback.

BUG=1144908

Change-Id: Ifb5e88457cdbc8ccfe37d5ce2e6bee07fec13726
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2573537
Commit-Queue: Matthew Wolenetz <wolenetz@chromium.org>
Reviewed-by: Dan Sanders <sandersd@chromium.org>
Cr-Commit-Position: refs/heads/master@{#835908}

media/base/stream_parser.cc

@@ -19,6 +19,12 @@ StreamParser::StreamParser() = default;
 
 StreamParser::~StreamParser() = default;
 
+// Default implementation of ProcessChunks() is not fully implemented.
+bool StreamParser::ProcessChunks(std::unique_ptr<BufferQueue> buffer_queue) {
+  NOTIMPLEMENTED();  // Likely the wrong type of parser is being used.
+  return false;
+}
+
 static bool MergeBufferQueuesInternal(
     const std::vector<const StreamParser::BufferQueue*>& buffer_queues,
     StreamParser::BufferQueue* merged_buffers) {

media/base/stream_parser.h

@@ -140,7 +140,12 @@ class MEDIA_EXPORT StreamParser {
   // Called when there is new data to parse.
   //
   // Returns true if the parse succeeds.
+  //
+  // Regular "bytestream-formatted" StreamParsers should fully implement
+  // Parse(), but WebCodecsEncodedChunkStreamParsers should instead fully
+  // implement ProcessChunks().
   virtual bool Parse(const uint8_t* buf, int size) = 0;
+  virtual bool ProcessChunks(std::unique_ptr<BufferQueue> buffer_queue);
 
  private:
   DISALLOW_COPY_AND_ASSIGN(StreamParser);

media/filters/chunk_demuxer.cc

@@ -632,10 +632,22 @@ ChunkDemuxer::Status ChunkDemuxer::AddId(
            << " audio_config=" << audio_config->AsHumanReadableString();
   base::AutoLock auto_lock(lock_);
 
-  // TODO(crbug.com/1144908): Build a new kind of templated StreamParser subtype
-  // which emits configs and streamparserbuffers almost trivially when directed
-  // to by ChunkDemuxer/SourceBufferState. Replace the following:
-  return kReachedIdLimit;
+  // Any valid audio config provided by WC is bufferable here, though decode
+  // error may occur later.
+  if (!audio_config->IsValidConfig())
+    return ChunkDemuxer::kNotSupported;
+
+  if ((state_ != WAITING_FOR_INIT && state_ != INITIALIZING) || IsValidId(id))
+    return kReachedIdLimit;
+
+  DCHECK(init_cb_);
+
+  std::string expected_codec = GetCodecName(audio_config->codec());
+  std::unique_ptr<media::StreamParser> stream_parser(
+      media::StreamParserFactory::Create(std::move(audio_config)));
+  DCHECK(stream_parser);
+
+  return AddIdInternal(id, std::move(stream_parser), expected_codec);
 }
 
 ChunkDemuxer::Status ChunkDemuxer::AddId(
@@ -646,10 +658,22 @@ ChunkDemuxer::Status ChunkDemuxer::AddId(
            << " video_config=" << video_config->AsHumanReadableString();
   base::AutoLock auto_lock(lock_);
 
-  // TODO(crbug.com/1144908): Build a new kind of templated StreamParser subtype
-  // which emits configs and streamparserbuffers almost trivially when directed
-  // to by ChunkDemuxer/SourceBufferState. Replace the following:
-  return kReachedIdLimit;
+  // Any valid video config provided by WC is bufferable here, though decode
+  // error may occur later.
+  if (!video_config->IsValidConfig())
+    return ChunkDemuxer::kNotSupported;
+
+  if ((state_ != WAITING_FOR_INIT && state_ != INITIALIZING) || IsValidId(id))
+    return kReachedIdLimit;
+
+  DCHECK(init_cb_);
+
+  std::string expected_codec = GetCodecName(video_config->codec());
+  std::unique_ptr<media::StreamParser> stream_parser(
+      media::StreamParserFactory::Create(std::move(video_config)));
+  DCHECK(stream_parser);
+
+  return AddIdInternal(id, std::move(stream_parser), expected_codec);
 }
 
 ChunkDemuxer::Status ChunkDemuxer::AddId(const std::string& id,
@@ -674,6 +698,18 @@ ChunkDemuxer::Status ChunkDemuxer::AddId(const std::string& id,
     return ChunkDemuxer::kNotSupported;
   }
 
+  return AddIdInternal(id, std::move(stream_parser),
+                       ExpectedCodecs(content_type, codecs));
+}
+
+ChunkDemuxer::Status ChunkDemuxer::AddIdInternal(
+    const std::string& id,
+    std::unique_ptr<media::StreamParser> stream_parser,
+    std::string expected_codecs) {
+  DVLOG(2) << __func__ << " id=" << id
+           << " expected_codecs=" << expected_codecs;
+  lock_.AssertAcquired();
+
   std::unique_ptr<FrameProcessor> frame_processor =
       std::make_unique<FrameProcessor>(
           base::BindRepeating(&ChunkDemuxer::IncreaseDurationIfNecessary,
@@ -698,8 +734,8 @@ ChunkDemuxer::Status ChunkDemuxer::AddId(const std::string& id,
 
   source_state->Init(base::BindOnce(&ChunkDemuxer::OnSourceInitDone,
                                     base::Unretained(this), id),
-                     ExpectedCodecs(content_type, codecs),
-                     encrypted_media_init_data_cb_, base::NullCallback());
+                     expected_codecs, encrypted_media_init_data_cb_,
+                     base::NullCallback());
 
   // TODO(wolenetz): Change to DCHECKs once less verification in release build
   // is needed. See https://crbug.com/786975.

media/filters/chunk_demuxer.h

@@ -241,8 +241,8 @@ class MEDIA_EXPORT ChunkDemuxer : public Demuxer {
   // type's "codecs" parameter string (if any) for the data that we intend to
   // append for this ID.  kOk is returned if the demuxer has enough resources to
   // support another ID and supports the format indicated by |content_type| and
-  // |codecs|.  kReachedIdLimit is returned if the demuxer cannot handle another
-  // ID right now.  kNotSupported is returned if |content_type| and |codecs| is
+  // |codecs|. kReachedIdLimit is returned if the demuxer cannot handle another
+  // ID right now. kNotSupported is returned if |content_type| and |codecs| is
   // not a supported format.
   // The |audio_config| and |video_config| overloads behave similarly, except
   // the caller must provide valid, supported decoder configs; those overloads'
@@ -251,8 +251,6 @@ class MEDIA_EXPORT ChunkDemuxer : public Demuxer {
   Status AddId(const std::string& id,
                const std::string& content_type,
                const std::string& codecs);
-  // TODO(crbug.com/1144908): Consider templating the following two if they
-  // continue to be so similar except for specific decoder config type.
   Status AddId(const std::string& id,
                std::unique_ptr<AudioDecoderConfig> audio_config);
   Status AddId(const std::string& id,
@@ -406,6 +404,14 @@ class MEDIA_EXPORT ChunkDemuxer : public Demuxer {
     SHUTDOWN,
   };
 
+  // Helper for AddId's creation of FrameProcessor, and
+  // SourceBufferState creation, initialization and tracking in
+  // source_state_map_.
+  ChunkDemuxer::Status AddIdInternal(
+      const std::string& id,
+      std::unique_ptr<media::StreamParser> stream_parser,
+      std::string expected_codecs);
+
   // Helper for vide and audio track changing.
   void FindAndEnableProperTracks(const std::vector<MediaTrack::Id>& track_ids,
                                  base::TimeDelta curr_time,

media/filters/stream_parser_factory.cc

@@ -14,13 +14,16 @@
 #include "base/strings/string_split.h"
 #include "base/strings/string_util.h"
 #include "build/build_config.h"
+#include "media/base/audio_decoder_config.h"
 #include "media/base/media.h"
 #include "media/base/media_switches.h"
 #include "media/base/media_util.h"
 #include "media/base/video_codecs.h"
+#include "media/base/video_decoder_config.h"
 #include "media/formats/mp4/mp4_stream_parser.h"
 #include "media/formats/mpeg/adts_stream_parser.h"
 #include "media/formats/mpeg/mpeg1_audio_stream_parser.h"
+#include "media/formats/webcodecs/webcodecs_encoded_chunk_stream_parser.h"
 #include "media/formats/webm/webm_stream_parser.h"
 #include "media/media_buildflags.h"
 
@@ -517,6 +520,7 @@ static SupportsType CheckTypeAndCodecs(
   return IsNotSupported;
 }
 
+// static
 SupportsType StreamParserFactory::IsTypeSupported(
     const std::string& type,
     const std::vector<std::string>& codecs) {
@@ -526,6 +530,7 @@ SupportsType StreamParserFactory::IsTypeSupported(
                             nullptr);
 }
 
+// static
 std::unique_ptr<StreamParser> StreamParserFactory::Create(
     const std::string& type,
     const std::vector<std::string>& codecs,
@@ -569,4 +574,28 @@ std::unique_ptr<StreamParser> StreamParserFactory::Create(
   return stream_parser;
 }
 
+// static
+std::unique_ptr<StreamParser> StreamParserFactory::Create(
+    std::unique_ptr<AudioDecoderConfig> audio_config) {
+  DCHECK(audio_config);
+
+  // TODO(crbug.com/1144908): Histogram-log the codec used for buffering
+  // WebCodecs in MSE?
+
+  return std::make_unique<media::WebCodecsEncodedChunkStreamParser>(
+      std::move(audio_config));
+}
+
+// static
+std::unique_ptr<StreamParser> StreamParserFactory::Create(
+    std::unique_ptr<VideoDecoderConfig> video_config) {
+  DCHECK(video_config);
+
+  // TODO(crbug.com/1144908): Histogram-log the codec used for buffering
+  // WebCodecs in MSE?
+
+  return std::make_unique<media::WebCodecsEncodedChunkStreamParser>(
+      std::move(video_config));
+}
+
 }  // namespace media

media/filters/stream_parser_factory.h

@@ -15,7 +15,9 @@
 
 namespace media {
 
+class AudioDecoderConfig;
 class StreamParser;
+class VideoDecoderConfig;
 
 class MEDIA_EXPORT StreamParserFactory {
  public:
@@ -36,10 +38,28 @@ class MEDIA_EXPORT StreamParserFactory {
   // Returns a new StreamParser object if |type| and all codecs listed in
   //   |codecs| are supported.
   // Returns NULL otherwise.
+  // The |audio_config| and |video_config| overloads behave similarly, except
+  // the caller must provide a valid, supported decoder config; those overloads'
+  // usage indicates that we intend to buffer WebCodecs encoded audio or video
+  // chunks with this parser's ProcessChunks() method. Note that
+  // these overloads do not check support, unlike the |type| and |codecs|
+  // version. Support checking for WebCodecs-originated decoder configs could be
+  // async, and should be done by the caller if necessary as part of the decoder
+  // config creation rather than relying upon parser creation to do this
+  // potentially expensive step (this step is typically done in a synchronous
+  // API call by the web app, such as addSourceBuffer().) Like |type| and
+  // |codecs| versions, basic IsValidConfig() is done on configs emitted from
+  // the parser. Failing that catching an unsupported config, eventual pipeline
+  // error should occur for unsupported or invalid decoder configs during
+  // attempted decode.
   static std::unique_ptr<StreamParser> Create(
       const std::string& type,
       const std::vector<std::string>& codecs,
       MediaLog* media_log);
+  static std::unique_ptr<StreamParser> Create(
+      std::unique_ptr<AudioDecoderConfig> audio_config);
+  static std::unique_ptr<StreamParser> Create(
+      std::unique_ptr<VideoDecoderConfig> video_config);
 };
 
 }  // namespace media

media/formats/BUILD.gn

@@ -45,6 +45,8 @@ source_set("formats") {
     "mpeg/mpeg1_audio_stream_parser.h",
     "mpeg/mpeg_audio_stream_parser_base.cc",
     "mpeg/mpeg_audio_stream_parser_base.h",
+    "webcodecs/webcodecs_encoded_chunk_stream_parser.cc",
+    "webcodecs/webcodecs_encoded_chunk_stream_parser.h",
     "webm/webm_audio_client.cc",
     "webm/webm_audio_client.h",
     "webm/webm_cluster_parser.cc",

media/formats/webcodecs/webcodecs_encoded_chunk_stream_parser.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/formats/webcodecs/webcodecs_encoded_chunk_stream_parser.h"
+
+#include <string>
+
+#include "base/callback.h"
+#include "base/logging.h"
+#include "base/notreached.h"
+#include "media/base/media_log.h"
+#include "media/base/media_track.h"
+#include "media/base/media_tracks.h"
+#include "media/base/stream_parser_buffer.h"
+#include "media/base/text_track_config.h"
+
+namespace media {
+
+WebCodecsEncodedChunkStreamParser::WebCodecsEncodedChunkStreamParser(
+    std::unique_ptr<AudioDecoderConfig> audio_config)
+    : state_(kWaitingForInit), audio_config_(std::move(audio_config)) {
+  DCHECK(audio_config_ && !video_config_);
+}
+
+WebCodecsEncodedChunkStreamParser::WebCodecsEncodedChunkStreamParser(
+    std::unique_ptr<VideoDecoderConfig> video_config)
+    : state_(kWaitingForInit), video_config_(std::move(video_config)) {
+  DCHECK(video_config_ && !audio_config_);
+}
+
+WebCodecsEncodedChunkStreamParser::~WebCodecsEncodedChunkStreamParser() =
+    default;
+
+void WebCodecsEncodedChunkStreamParser::Init(
+    InitCB init_cb,
+    const NewConfigCB& config_cb,
+    const NewBuffersCB& new_buffers_cb,
+    bool /* ignore_text_tracks */,
+    const EncryptedMediaInitDataCB& /* ignored */,
+    const NewMediaSegmentCB& new_segment_cb,
+    const EndMediaSegmentCB& end_of_segment_cb,
+    MediaLog* media_log) {
+  DCHECK_EQ(state_, kWaitingForInit);
+  DCHECK(!init_cb_);
+  DCHECK(init_cb);
+  DCHECK(config_cb);
+  DCHECK(new_buffers_cb);
+  DCHECK(new_segment_cb);
+  DCHECK(end_of_segment_cb);
+
+  ChangeState(kWaitingForConfigEmission);
+  init_cb_ = std::move(init_cb);
+  config_cb_ = config_cb;
+  new_buffers_cb_ = new_buffers_cb;
+  new_segment_cb_ = new_segment_cb;
+  end_of_segment_cb_ = end_of_segment_cb;
+  media_log_ = media_log;
+}
+
+void WebCodecsEncodedChunkStreamParser::Flush() {
+  DCHECK_NE(state_, kWaitingForInit);
+  if (state_ == kWaitingForEncodedChunks)
+    ChangeState(kWaitingForConfigEmission);
+}
+
+bool WebCodecsEncodedChunkStreamParser::GetGenerateTimestampsFlag() const {
+  return false;
+}
+
+bool WebCodecsEncodedChunkStreamParser::Parse(const uint8_t* /* buf */,
+                                              int /* size */) {
+  NOTREACHED();  // ProcessChunks() is the method to use instead for this
+                 // parser.
+  return false;
+}
+
+bool WebCodecsEncodedChunkStreamParser::ProcessChunks(
+    std::unique_ptr<BufferQueue> buffer_queue) {
+  // TODO(crbug.com/1144908): Implement.
+  NOTIMPLEMENTED();
+  return false;
+}
+
+void WebCodecsEncodedChunkStreamParser::ChangeState(State new_state) {
+  DVLOG(1) << __func__ << ": " << state_ << " -> " << new_state;
+  state_ = new_state;
+}
+
+}  // namespace media

media/formats/webcodecs/webcodecs_encoded_chunk_stream_parser.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef MEDIA_FORMATS_WEBCODECS_WEBCODECS_ENCODED_CHUNK_STREAM_PARSER_H_
+#define MEDIA_FORMATS_WEBCODECS_WEBCODECS_ENCODED_CHUNK_STREAM_PARSER_H_
+
+#include <stdint.h>
+
+#include <memory>
+
+#include "base/callback_forward.h"
+#include "base/macros.h"
+#include "base/memory/ref_counted.h"
+#include "media/base/audio_decoder_config.h"
+#include "media/base/media_export.h"
+#include "media/base/stream_parser.h"
+#include "media/base/video_decoder_config.h"
+
+namespace media {
+
+class MEDIA_EXPORT WebCodecsEncodedChunkStreamParser : public StreamParser {
+ public:
+  explicit WebCodecsEncodedChunkStreamParser(
+      std::unique_ptr<AudioDecoderConfig> audio_config);
+  explicit WebCodecsEncodedChunkStreamParser(
+      std::unique_ptr<VideoDecoderConfig> video_config);
+  ~WebCodecsEncodedChunkStreamParser() override;
+
+  // StreamParser implementation.
+  void Init(InitCB init_cb,
+            const NewConfigCB& config_cb,
+            const NewBuffersCB& new_buffers_cb,
+            bool ignore_text_tracks /* must be true */,
+            const EncryptedMediaInitDataCB& encrypted_media_init_data_cb,
+            const NewMediaSegmentCB& new_segment_cb,
+            const EndMediaSegmentCB& end_of_segment_cb,
+            MediaLog* media_log) override;
+  void Flush() override;
+  bool GetGenerateTimestampsFlag() const override;
+  bool Parse(const uint8_t* buf, int size) override;
+
+  // Processes and emits buffers from |buffer_queue|. If state is
+  // kWaitingForConfigEmission, first emit the config.
+  bool ProcessChunks(std::unique_ptr<BufferQueue> buffer_queue) override;
+
+ private:
+  enum State {
+    kWaitingForInit,
+    kWaitingForConfigEmission,
+    kWaitingForEncodedChunks,
+    kError
+  };
+
+  void ChangeState(State new_state);
+
+  State state_;
+
+  // These configs are populated during ctor. A copy of the appropriate config
+  // is emitted on demand when "parsing" newly appended encoded chunks if that
+  // append occurs when state is kWaitingForConfigEmission. Note, only one type
+  // of config can be emitted (not both), for an instance of this parser.
+  std::unique_ptr<AudioDecoderConfig> audio_config_;
+  std::unique_ptr<VideoDecoderConfig> video_config_;
+
+  InitCB init_cb_;
+  NewConfigCB config_cb_;
+  NewBuffersCB new_buffers_cb_;
+
+  NewMediaSegmentCB new_segment_cb_;
+  EndMediaSegmentCB end_of_segment_cb_;
+  MediaLog* media_log_;
+
+  DISALLOW_COPY_AND_ASSIGN(WebCodecsEncodedChunkStreamParser);
+};
+
+}  // namespace media
+
+#endif  // MEDIA_FORMATS_WEBCODECS_WEBCODECS_ENCODED_CHUNK_STREAM_PARSER_H_

third_party/blink/web_tests/external/wpt/media-source/mse-for-webcodecs/tentative/mediasource-webcodecs-addsourcebuffer.html

@@ -162,19 +162,15 @@ function runAddSourceBufferTest(argument, isValidArgument, argumentDescription,
         assert_equals(mediaSource.readyState, 'open', 'MediaSource is open');
         let sourceBuffer;
         if (isValidArgument) {
-          // TODO(crbug.com/1144908): Update to expect success once the impl is
-          // more complete.
-          assert_throws_dom('QuotaExceededError',
-            () => { sourceBuffer = mediaSource.addSourceBuffer(argument); },
-            'addSourceBuffer(valid config) throws QuotaExceededError');
-          assert_equals(sourceBuffer, undefined,
-            'addSourceBuffer result for valid config while "open" should be exception');
+          sourceBuffer = mediaSource.addSourceBuffer(argument);
+          assert_true(sourceBuffer instanceof SourceBuffer,
+              'addSourceBuffer result for valid config while "open" should be a SourceBuffer instance');
         } else {
           assert_throws_js(TypeError,
-            () => { sourceBuffer = mediaSource.addSourceBuffer(argument); },
-            'addSourceBuffer(invalid config) throws TypeError if MediaSource is "open"');
+              () => { sourceBuffer = mediaSource.addSourceBuffer(argument); },
+              'addSourceBuffer(invalid config) throws TypeError if MediaSource is "open"');
           assert_equals(sourceBuffer, undefined,
-            'addSourceBufferResult for invalid config while "open" should be exception');
+              'addSourceBufferResult for invalid config while "open" should be exception');
         }
       }, testDescription);
       break;