VideoDecoderBroker: Connect WebCodecs to media::VideoDecoder(s)

"Broker" acts as a simple decoder, but under the hood manages creation
and selection of of underlying media::VideoDecoders, and thread-hopping
required to use them.

This enables WebCodecs to use hw accelerated decoders (depending on
platform/gpu) when not in a Worker context. For Workers, we still need
to expose the relevant Mojo interfaces (currently tied to the
RenderFrame). The code in webcodecs/ will then support hw-decode in
workers automatically.

Bug: 1094103

Change-Id: I9d1abcae842b16cbb095e2e24dc095c284075473
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2152106
Commit-Queue: Chrome Cunningham <chcunningham@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Dan Sanders <sandersd@chromium.org>
Cr-Commit-Position: refs/heads/master@{#782793}

media/BUILD.gn

@@ -157,7 +157,6 @@ test("media_unittests") {
     "//media/test:pipeline_integration_tests",
     "//media/test:run_all_unittests",
     "//media/video:unit_tests",
-    "//media/webcodecs:unit_tests",
     "//media/webrtc:unit_tests",
   ]
 

media/filters/BUILD.gn

@@ -14,7 +14,6 @@ jumbo_source_set("filters") {
   visibility = [
     "//media",
     "//media/renderers",
-    "//media/webcodecs",
   ]
 
   sources = [
@@ -262,6 +261,8 @@ static_library("test_support") {
   visibility = [ "//media:test_support" ]
 
   sources = [
+    "fake_video_decoder.cc",
+    "fake_video_decoder.h",
     "ivf_parser.cc",
     "ivf_parser.h",
   ]
@@ -290,8 +291,6 @@ source_set("unit_tests") {
     "decrypting_demuxer_stream_unittest.cc",
     "decrypting_media_resource_unittest.cc",
     "decrypting_video_decoder_unittest.cc",
-    "fake_video_decoder.cc",
-    "fake_video_decoder.h",
     "fake_video_decoder_unittest.cc",
     "file_data_source_unittest.cc",
     "frame_buffer_pool_unittest.cc",

media/filters/fake_video_decoder.cc

@@ -122,14 +122,18 @@ void FakeVideoDecoder::Decode(scoped_refptr<DecoderBuffer> buffer,
     state_ = STATE_END_OF_STREAM;
   } else {
     DCHECK(VerifyFakeVideoBufferForTest(*buffer, current_config_));
-    scoped_refptr<VideoFrame> video_frame = VideoFrame::CreateColorFrame(
-        current_config_.coded_size(), 0, 0, 0, buffer->timestamp());
-    decoded_frames_.push_back(video_frame);
+    decoded_frames_.push_back(MakeVideoFrame(*buffer));
   }
 
   RunOrHoldDecode(std::move(wrapped_decode_cb));
 }
 
+scoped_refptr<VideoFrame> FakeVideoDecoder::MakeVideoFrame(
+    const DecoderBuffer& buffer) {
+  return VideoFrame::CreateColorFrame(current_config_.coded_size(), 0, 0, 0,
+                                      buffer.timestamp());
+}
+
 void FakeVideoDecoder::Reset(base::OnceClosure closure) {
   DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK(reset_cb_.IsNull());
@@ -222,7 +226,8 @@ void FakeVideoDecoder::OnFrameDecoded(int buffer_size,
 
   if (status == DecodeStatus::OK) {
     total_bytes_decoded_ += buffer_size;
-    bytes_decoded_cb_.Run(buffer_size);
+    if (bytes_decoded_cb_)
+      bytes_decoded_cb_.Run(buffer_size);
   }
   std::move(decode_cb).Run(status);
 }

media/filters/fake_video_decoder.h

@@ -75,7 +75,7 @@ class FakeVideoDecoder : public VideoDecoder {
 
   int total_bytes_decoded() const { return total_bytes_decoded_; }
 
- private:
+ protected:
   enum State {
     STATE_UNINITIALIZED,
     STATE_NORMAL,
@@ -83,6 +83,9 @@ class FakeVideoDecoder : public VideoDecoder {
     STATE_ERROR,
   };
 
+  // Derived classes may override to customize the VideoFrame.
+  virtual scoped_refptr<VideoFrame> MakeVideoFrame(const DecoderBuffer& buffer);
+
   // Callback for updating |total_bytes_decoded_|.
   void OnFrameDecoded(int buffer_size, DecodeCB decode_cb, DecodeStatus status);
 

media/media_options.gni

@@ -29,7 +29,6 @@ media_subcomponent_deps = [
   "//media/muxers",
   "//media/renderers",
   "//media/video",
-  "//media/webcodecs",
 ]
 
 if (is_fuchsia) {

media/mojo/clients/BUILD.gn

@@ -27,6 +27,8 @@ jumbo_source_set("clients") {
 
     # TODO(liberato): can we avoid this?
     "//content/test/*",
+
+    "//third_party/blink/renderer/modules/webcodecs",
   ]
 
   sources = [

media/webcodecs/BUILD.gn

-# Copyright 2020 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import("//testing/test.gni")
-
-source_set("webcodecs") {
-  # Do not expand the visibility here without double-checking with OWNERS, this
-  # is a roll-up target which is part of the //media component. Most other DEPs
-  # should be using //media and not directly DEP this roll-up target.
-  visibility = [ "//media" ]
-
-  sources = [
-    "wc_decoder_selector.cc",
-    "wc_decoder_selector.h",
-  ]
-
-  public_deps = [
-    "//base",
-    "//media/base",
-    "//media/filters",
-  ]
-
-  deps = []
-
-  configs += [ "//media:subcomponent_config" ]
-}
-
-source_set("unit_tests") {
-  testonly = true
-  sources = [ "wc_decoder_selector_unittest.cc" ]
-
-  deps = [
-    "//base/test:test_support",
-    "//media:test_support",
-    "//testing/gmock",
-    "//testing/gtest",
-  ]
-}

third_party/blink/PRESUBMIT.py

@@ -57,12 +57,13 @@ def _CheckForWrongMojomIncludes(input_api, output_api):
     # - It uses POD types that will not import STL (or base string) types into blink
     #   (such as no strings or vectors).
     #
-    # So far, non-blink interfaces are allowed only for loading / loader
+    # So far, non-blink interfaces are allowed only for loading / loader and media
     # interfaces so that we don't need type conversions to get through the
     # boundary between Blink and non-Blink.
-    allowed_interfaces = (r'services/network/public/mojom/cross_origin_embedder_policy', r'services/network/public/mojom/fetch_api',
-                          r'services/network/public/mojom/load_timing_info',
-                          r'third_party/blink/public/mojom/worker/subresource_loader_updater')
+    allowed_interfaces = ('services/network/public/mojom/cross_origin_embedder_policy', 'services/network/public/mojom/fetch_api',
+                          'services/network/public/mojom/load_timing_info',
+                          'third_party/blink/public/mojom/worker/subresource_loader_updater', 'media/mojo/mojom/interface_factory',
+                          'media/mojo/mojom/video_decoder')
 
     for f in input_api.AffectedFiles(file_filter=source_file_filter):
         for line_num, line in f.ChangedContents():

third_party/blink/renderer/modules/webcodecs/BUILD.gn

@@ -6,6 +6,8 @@ import("//third_party/blink/renderer/modules/modules.gni")
 
 blink_modules_sources("webcodecs") {
   sources = [
+    "decoder_selector.cc",
+    "decoder_selector.h",
     "encoded_video_chunk.cc",
     "encoded_video_chunk.h",
     "encoded_video_metadata.h",
@@ -13,6 +15,8 @@ blink_modules_sources("webcodecs") {
     "image_decoder_external.h",
     "video_decoder.cc",
     "video_decoder.h",
+    "video_decoder_broker.cc",
+    "video_decoder_broker.h",
     "video_encoder.cc",
     "video_encoder.h",
     "video_frame.cc",
@@ -22,12 +26,20 @@ blink_modules_sources("webcodecs") {
     "video_track_writer.cc",
     "video_track_writer.h",
   ]
+  deps = [
+    "//media",
+    "//media/mojo:buildflags",
+    "//media/mojo/clients",
+    "//media/mojo/mojom",
+  ]
 }
 
 source_set("unit_tests") {
   testonly = true
   sources = [
+    "decoder_selector_test.cc",
     "encoded_video_chunk_test.cc",
+    "video_decoder_broker_test.cc",
     "video_frame_test.cc",
     "video_track_reader_writer_test.cc",
   ]
@@ -39,6 +51,9 @@ source_set("unit_tests") {
   ]
 
   deps = [
+    "//base/test:test_support",
+    "//gpu/command_buffer/common",
+    "//media:test_support",
     "//testing/gmock",
     "//testing/gtest",
     "//third_party/blink/renderer/modules",

third_party/blink/renderer/modules/webcodecs/DEPS

@@ -9,11 +9,13 @@ include_rules = [
     "+media/base",
     "+media/filters",
     "+media/media_buildflags.h",
+    "+media/mojo",
     "+media/renderers",
     "+media/video",
 
     "+third_party/libyuv",
 
+    "+ui/gfx/color_space.h",
     "+ui/gfx/geometry/rect.h",
     "+ui/gfx/geometry/size.h",
 ]
@@ -22,4 +24,9 @@ specific_include_rules = {
   "video_track_reader_writer_test\.cc": [
     "+base/run_loop.h",
   ],
-}
\ No newline at end of file
+  "video_decoder_broker_test\.cc": [
+    "+base/run_loop.h",
+    "+base/threading/thread.h",
+    "+gpu/command_buffer/common/mailbox_holder.h",
+  ],
+}

media/webcodecs/wc_decoder_selector.cc → third_party/blink/renderer/modules/webcodecs/decoder_selector.cc

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-#include "media/webcodecs/wc_decoder_selector.h"
+#include "third_party/blink/renderer/modules/webcodecs/decoder_selector.h"
 
 #include "base/bind.h"
 #include "base/check_op.h"
@@ -11,21 +11,23 @@
 #include "media/base/channel_layout.h"
 #include "media/base/demuxer_stream.h"
 #include "media/filters/decrypting_demuxer_stream.h"
+#include "third_party/blink/renderer/modules/modules_export.h"
+#include "third_party/blink/renderer/platform/wtf/functional.h"
 
-namespace media {
+namespace blink {
 
 // Demuxing isn't part of WebCodecs. This shim allows us to reuse decoder
 // selection logic from <video>.
 // TODO(chcunningham): Maybe refactor DecoderSelector to separate dependency on
-// DemuxerStream. DecoderSelection doesn't conceptually require a Demuxer. The
-// tough part is re-working DecryptingDemuxerStream.
-template <DemuxerStream::Type StreamType>
-class ShimDemuxerStream : public DemuxerStream {
+// media::DemuxerStream. DecoderSelection doesn't conceptually require a
+// Demuxer. The tough part is re-working Decryptingmedia::DemuxerStream.
+template <media::DemuxerStream::Type StreamType>
+class NullDemuxerStream : public media::DemuxerStream {
  public:
   using DecoderConfigType =
-      typename DecoderStreamTraits<StreamType>::DecoderConfigType;
+      typename media::DecoderStreamTraits<StreamType>::DecoderConfigType;
 
-  ~ShimDemuxerStream() override = default;
+  ~NullDemuxerStream() override = default;
 
   void Read(ReadCB read_cb) override { NOTREACHED(); }
   bool IsReadPending() const override {
@@ -35,13 +37,13 @@ class ShimDemuxerStream : public DemuxerStream {
 
   void Configure(DecoderConfigType config);
 
-  AudioDecoderConfig audio_decoder_config() override {
-    DCHECK_EQ(type(), DemuxerStream::AUDIO);
+  media::AudioDecoderConfig audio_decoder_config() override {
+    DCHECK_EQ(type(), media::DemuxerStream::AUDIO);
     return audio_decoder_config_;
   }
 
-  VideoDecoderConfig video_decoder_config() override {
-    DCHECK_EQ(type(), DemuxerStream::VIDEO);
+  media::VideoDecoderConfig video_decoder_config() override {
+    DCHECK_EQ(type(), media::DemuxerStream::VIDEO);
     return video_decoder_config_;
   }
 
@@ -53,90 +55,88 @@ class ShimDemuxerStream : public DemuxerStream {
   }
 
  private:
-  static const DemuxerStream::Type stream_type = StreamType;
+  static const media::DemuxerStream::Type stream_type = StreamType;
 
-  AudioDecoderConfig audio_decoder_config_;
-  VideoDecoderConfig video_decoder_config_;
+  media::AudioDecoderConfig audio_decoder_config_;
+  media::VideoDecoderConfig video_decoder_config_;
 };
 
 template <>
-void ShimDemuxerStream<DemuxerStream::AUDIO>::Configure(
+void NullDemuxerStream<media::DemuxerStream::AUDIO>::Configure(
     DecoderConfigType config) {
   audio_decoder_config_ = config;
 }
 
 template <>
-void ShimDemuxerStream<DemuxerStream::VIDEO>::Configure(
+void NullDemuxerStream<media::DemuxerStream::VIDEO>::Configure(
     DecoderConfigType config) {
   video_decoder_config_ = config;
 }
 
-template <DemuxerStream::Type StreamType>
-WebCodecsDecoderSelector<StreamType>::WebCodecsDecoderSelector(
+template <media::DemuxerStream::Type StreamType>
+DecoderSelector<StreamType>::DecoderSelector(
     scoped_refptr<base::SingleThreadTaskRunner> task_runner,
     CreateDecodersCB create_decoders_cb,
     typename Decoder::OutputCB output_cb)
     : impl_(std::move(task_runner),
             std::move(create_decoders_cb),
             &null_media_log_),
-      demuxer_stream_(new ShimDemuxerStream<StreamType>()),
+      demuxer_stream_(new NullDemuxerStream<StreamType>()),
       stream_traits_(CreateStreamTraits()),
       output_cb_(output_cb) {
   impl_.Initialize(stream_traits_.get(), demuxer_stream_.get(),
-                   nullptr /*CdmContext*/, WaitingCB());
+                   nullptr /*CdmContext*/, media::WaitingCB());
 }
 
-template <DemuxerStream::Type StreamType>
-WebCodecsDecoderSelector<StreamType>::~WebCodecsDecoderSelector() {}
+template <media::DemuxerStream::Type StreamType>
+DecoderSelector<StreamType>::~DecoderSelector() = default;
 
-template <DemuxerStream::Type StreamType>
-void WebCodecsDecoderSelector<StreamType>::SelectDecoder(
+template <media::DemuxerStream::Type StreamType>
+void DecoderSelector<StreamType>::SelectDecoder(
     const DecoderConfig& config,
     SelectDecoderCB select_decoder_cb) {
-  // |impl_| will internally use this the |config| from our ShimDemuxerStream.
+  // |impl_| will internally use this the |config| from our NullDemuxerStream.
   demuxer_stream_->Configure(config);
 
-  // |impl_| uses a WeakFactory for its SelectDecoderCB, so we're safe to use
-  // Unretained here.
+  // Destroying |impl_| will cancel pending operations, so it's safe to use
+  // Unretained() with |select_decoder_cb|.
   impl_.SelectDecoder(
-      base::BindOnce(&WebCodecsDecoderSelector<StreamType>::OnDecoderSelected,
-                     base::Unretained(this), std::move(select_decoder_cb)),
+      WTF::Bind(&DecoderSelector<StreamType>::OnDecoderSelected,
+                WTF::Unretained(this), std::move(select_decoder_cb)),
       output_cb_);
 }
 
 template <>
 std::unique_ptr<WebCodecsAudioDecoderSelector::StreamTraits>
-WebCodecsDecoderSelector<DemuxerStream::AUDIO>::CreateStreamTraits() {
+DecoderSelector<media::DemuxerStream::AUDIO>::CreateStreamTraits() {
   // TODO(chcunningham): Consider plumbing real hw channel layout.
-  return std::make_unique<WebCodecsDecoderSelector::StreamTraits>(
-      &null_media_log_, CHANNEL_LAYOUT_NONE);
+  return std::make_unique<DecoderSelector::StreamTraits>(
+      &null_media_log_, media::CHANNEL_LAYOUT_NONE);
 }
 
 template <>
 std::unique_ptr<WebCodecsVideoDecoderSelector::StreamTraits>
-WebCodecsDecoderSelector<DemuxerStream::VIDEO>::CreateStreamTraits() {
-  return std::make_unique<WebCodecsDecoderSelector::StreamTraits>(
-      &null_media_log_);
+DecoderSelector<media::DemuxerStream::VIDEO>::CreateStreamTraits() {
+  return std::make_unique<DecoderSelector::StreamTraits>(&null_media_log_);
 }
 
-template <DemuxerStream::Type StreamType>
-void WebCodecsDecoderSelector<StreamType>::OnDecoderSelected(
+template <media::DemuxerStream::Type StreamType>
+void DecoderSelector<StreamType>::OnDecoderSelected(
     SelectDecoderCB select_decoder_cb,
     std::unique_ptr<Decoder> decoder,
-    std::unique_ptr<DecryptingDemuxerStream> decrypting_demuxer_stream) {
+    std::unique_ptr<media::DecryptingDemuxerStream> decrypting_demuxer_stream) {
   DCHECK(!decrypting_demuxer_stream);
 
-  // We immediately finalize decoder selection. From a spec POV we strongly
-  // prefer to avoid replicating our internal design of having to wait for the
-  // first frame to arrive before we consider configuration successful.
-  // TODO(chcunningham): Measure first frame decode failures and find other ways
-  // to solve (or minimize) the problem.
+  // We immediately finalize decoder selection.
+  // TODO(chcunningham): Rework this to do finalize after first frame
+  // successfully decoded. This updates to match latest plans for spec
+  // (configure() no longer takes a promise).
   impl_.FinalizeDecoderSelection();
 
   std::move(select_decoder_cb).Run(std::move(decoder));
 }
 
-template class WebCodecsDecoderSelector<DemuxerStream::VIDEO>;
-template class WebCodecsDecoderSelector<DemuxerStream::AUDIO>;
+template class MODULES_EXPORT DecoderSelector<media::DemuxerStream::VIDEO>;
+template class MODULES_EXPORT DecoderSelector<media::DemuxerStream::AUDIO>;
 
-}  // namespace media
+}  // namespace blink

media/webcodecs/wc_decoder_selector.h → third_party/blink/renderer/modules/webcodecs/decoder_selector.h

@@ -2,26 +2,26 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-#ifndef MEDIA_WEBCODECS_WC_DECODER_SELECTOR_H_
-#define MEDIA_WEBCODECS_WC_DECODER_SELECTOR_H_
+#ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_DECODER_SELECTOR_H_
+#define THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_DECODER_SELECTOR_H_
 
 #include <memory>
 
 #include "media/base/demuxer_stream.h"
-#include "media/base/media_export.h"
 #include "media/base/media_util.h"
 #include "media/filters/decoder_selector.h"
 #include "media/filters/decoder_stream_traits.h"
+#include "third_party/blink/renderer/modules/modules_export.h"
 
-namespace media {
+namespace blink {
 
-template <DemuxerStream::Type StreamType>
-class ShimDemuxerStream;
+template <media::DemuxerStream::Type StreamType>
+class NullDemuxerStream;
 
-template <DemuxerStream::Type StreamType>
-class MEDIA_EXPORT WebCodecsDecoderSelector {
+template <media::DemuxerStream::Type StreamType>
+class DecoderSelector {
  public:
-  typedef DecoderStreamTraits<StreamType> StreamTraits;
+  typedef media::DecoderStreamTraits<StreamType> StreamTraits;
   typedef typename StreamTraits::DecoderType Decoder;
   typedef typename StreamTraits::DecoderConfigType DecoderConfig;
 
@@ -34,13 +34,20 @@ class MEDIA_EXPORT WebCodecsDecoderSelector {
   // Decoder.
   using SelectDecoderCB = base::OnceCallback<void(std::unique_ptr<Decoder>)>;
 
-  WebCodecsDecoderSelector(
-      scoped_refptr<base::SingleThreadTaskRunner> task_runner,
-      CreateDecodersCB create_decoders_cb,
-      typename Decoder::OutputCB output_cb);
+  // Construction can happen on any thread, but all subsequent API calls
+  // including destruction must use |task_runner| thread.
+  // Provided callbacks will be called on |task_runner|. |output_cb| will always
+  // be Post()'ed.
+  DecoderSelector(scoped_refptr<base::SingleThreadTaskRunner> task_runner,
+                  CreateDecodersCB create_decoders_cb,
+                  typename Decoder::OutputCB output_cb);
 
   // Aborts any pending decoder selection.
-  ~WebCodecsDecoderSelector();
+  ~DecoderSelector();
+
+  // Disallow copy and assign.
+  DecoderSelector(const DecoderSelector&) = delete;
+  DecoderSelector& operator=(const DecoderSelector&) = delete;
 
   // Selects and initializes a decoder using |config|. Decoder will
   // be returned via |select_decoder_cb| posted to |task_runner_|. Subsequent
@@ -55,13 +62,13 @@ class MEDIA_EXPORT WebCodecsDecoderSelector {
   // Proxy SelectDecoderCB from impl_ to our |select_decoder_cb|.
   void OnDecoderSelected(SelectDecoderCB select_decoder_cb,
                          std::unique_ptr<Decoder> decoder,
-                         std::unique_ptr<DecryptingDemuxerStream>);
+                         std::unique_ptr<media::DecryptingDemuxerStream>);
 
   // Implements heavy lifting for decoder selection.
-  DecoderSelector<StreamType> impl_;
+  media::DecoderSelector<StreamType> impl_;
 
   // Shim to satisfy dependencies of |impl_|. Provides DecoderConfig to |impl_|.
-  std::unique_ptr<ShimDemuxerStream<StreamType>> demuxer_stream_;
+  std::unique_ptr<NullDemuxerStream<StreamType>> demuxer_stream_;
 
   // Helper to unify API for configuring audio/video decoders.
   std::unique_ptr<StreamTraits> stream_traits_;
@@ -70,14 +77,14 @@ class MEDIA_EXPORT WebCodecsDecoderSelector {
   typename Decoder::OutputCB output_cb_;
 
   // TODO(chcunningham): Route MEDIA_LOG for WebCodecs.
-  NullMediaLog null_media_log_;
+  media::NullMediaLog null_media_log_;
 };
 
-typedef WebCodecsDecoderSelector<DemuxerStream::VIDEO>
+typedef DecoderSelector<media::DemuxerStream::VIDEO>
     WebCodecsVideoDecoderSelector;
-typedef WebCodecsDecoderSelector<DemuxerStream::AUDIO>
+typedef DecoderSelector<media::DemuxerStream::AUDIO>
     WebCodecsAudioDecoderSelector;
 
-}  // namespace media
+}  // namespace blink
 
-#endif  // MEDIA_WEBCODECS_WC_DECODER_SELECTOR_H_
+#endif  // THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_DECODER_SELECTOR_H_

media/webcodecs/wc_decoder_selector_unittest.cc → third_party/blink/renderer/modules/webcodecs/decoder_selector_test.cc

@@ -4,7 +4,6 @@
 
 #include <vector>
 
-#include "base/test/task_environment.h"
 #include "media/base/demuxer_stream.h"
 #include "media/base/media_util.h"
 #include "media/base/mock_filters.h"
@@ -14,14 +13,16 @@
 #include "media/filters/decoder_stream.h"
 #include "testing/gmock/include/gmock/gmock.h"
 #include "testing/gtest/include/gtest/gtest.h"
+#include "third_party/blink/public/platform/scheduler/test/renderer_scheduler_test_support.h"
+#include "third_party/blink/renderer/platform/testing/testing_platform_support.h"
 
-#include "media/webcodecs/wc_decoder_selector.h"
+#include "third_party/blink/renderer/modules/webcodecs/decoder_selector.h"
 
 using ::testing::_;
 using ::testing::IsNull;
 using ::testing::StrictMock;
 
-namespace media {
+namespace blink {
 
 namespace {
 
@@ -37,32 +38,37 @@ const char kDecoder2[] = "Decoder2";
 // Specializations for the AUDIO version of the test.
 class AudioDecoderSelectorTestParam {
  public:
-  static constexpr DemuxerStream::Type kStreamType = DemuxerStream::AUDIO;
+  static constexpr media::DemuxerStream::Type kStreamType =
+      media::DemuxerStream::AUDIO;
 
-  using DecoderSelector = WebCodecsDecoderSelector<DemuxerStream::AUDIO>;
-  using MockDecoder = MockAudioDecoder;
-  using Output = AudioBuffer;
+  using DecoderSelector = DecoderSelector<media::DemuxerStream::AUDIO>;
+  using MockDecoder = media::MockAudioDecoder;
+  using Output = media::AudioBuffer;
 
-  static AudioDecoderConfig CreateConfig() { return TestAudioConfig::Normal(); }
+  static media::AudioDecoderConfig CreateConfig() {
+    return media::TestAudioConfig::Normal();
+  }
 
   // Create a config that won't match the return of CreateConfig().
-  static AudioDecoderConfig CreateAlternateConfig() {
-    return TestAudioConfig::NormalEncrypted();
+  static media::AudioDecoderConfig CreateAlternateConfig() {
+    return media::TestAudioConfig::NormalEncrypted();
   }
 
   // Decoder::Initialize() takes different parameters depending on the type.
   static void ExpectInitialize(MockDecoder* decoder,
                                DecoderCapability capability,
-                               AudioDecoderConfig expected_config) {
+                               media::AudioDecoderConfig expected_config) {
     EXPECT_CALL(*decoder, Initialize_(_, _, _, _, _))
         .WillRepeatedly([capability, expected_config](
-                            const AudioDecoderConfig& config, CdmContext*,
-                            AudioDecoder::InitCB& init_cb,
-                            const AudioDecoder::OutputCB&, const WaitingCB&) {
+                            const media::AudioDecoderConfig& config,
+                            media::CdmContext*,
+                            media::AudioDecoder::InitCB& init_cb,
+                            const media::AudioDecoder::OutputCB&,
+                            const media::WaitingCB&) {
           EXPECT_TRUE(config.Matches(expected_config));
           std::move(init_cb).Run(capability == kSucceed
-                                     ? OkStatus()
-                                     : StatusCode::kCodeOnlyForTesting);
+                                     ? media::OkStatus()
+                                     : media::StatusCode::kCodeOnlyForTesting);
         });
   }
 };
@@ -70,38 +76,43 @@ class AudioDecoderSelectorTestParam {
 // Specializations for the VIDEO version of the test.
 class VideoDecoderSelectorTestParam {
  public:
-  static constexpr DemuxerStream::Type kStreamType = DemuxerStream::VIDEO;
+  static constexpr media::DemuxerStream::Type kStreamType =
+      media::DemuxerStream::VIDEO;
 
-  using DecoderSelector = WebCodecsDecoderSelector<DemuxerStream::VIDEO>;
-  using MockDecoder = MockVideoDecoder;
-  using Output = VideoFrame;
+  using DecoderSelector = DecoderSelector<media::DemuxerStream::VIDEO>;
+  using MockDecoder = media::MockVideoDecoder;
+  using Output = media::VideoFrame;
 
-  static VideoDecoderConfig CreateConfig() { return TestVideoConfig::Normal(); }
+  static media::VideoDecoderConfig CreateConfig() {
+    return media::TestVideoConfig::Normal();
+  }
 
   // Create a config that won't match the return of CreateConfig().
-  static VideoDecoderConfig CreateAlternateConfig() {
-    return TestVideoConfig::LargeEncrypted();
+  static media::VideoDecoderConfig CreateAlternateConfig() {
+    return media::TestVideoConfig::LargeEncrypted();
   }
 
   static void ExpectInitialize(MockDecoder* decoder,
                                DecoderCapability capability,
-                               VideoDecoderConfig expected_config) {
+                               media::VideoDecoderConfig expected_config) {
     EXPECT_CALL(*decoder, Initialize_(_, _, _, _, _, _))
         .WillRepeatedly([capability, expected_config](
-                            const VideoDecoderConfig& config, bool low_delay,
-                            CdmContext*, VideoDecoder::InitCB& init_cb,
-                            const VideoDecoder::OutputCB&, const WaitingCB&) {
+                            const media::VideoDecoderConfig& config,
+                            bool low_delay, media::CdmContext*,
+                            media::VideoDecoder::InitCB& init_cb,
+                            const media::VideoDecoder::OutputCB&,
+                            const media::WaitingCB&) {
           EXPECT_TRUE(config.Matches(expected_config));
           std::move(init_cb).Run(capability == kSucceed
-                                     ? OkStatus()
-                                     : StatusCode::kCodeOnlyForTesting);
+                                     ? media::OkStatus()
+                                     : media::StatusCode::kCodeOnlyForTesting);
         });
   }
 };
 
 // Allocate storage for the member variables.
-constexpr DemuxerStream::Type AudioDecoderSelectorTestParam::kStreamType;
-constexpr DemuxerStream::Type VideoDecoderSelectorTestParam::kStreamType;
+constexpr media::DemuxerStream::Type AudioDecoderSelectorTestParam::kStreamType;
+constexpr media::DemuxerStream::Type VideoDecoderSelectorTestParam::kStreamType;
 
 }  // namespace
 
@@ -154,8 +165,8 @@ class WebCodecsDecoderSelectorTest : public ::testing::Test {
 
   void CreateDecoderSelector() {
     decoder_selector_ =
-        std::make_unique<WebCodecsDecoderSelector<TypeParam::kStreamType>>(
-            task_environment_.GetMainThreadTaskRunner(),
+        std::make_unique<DecoderSelector<TypeParam::kStreamType>>(
+            scheduler::GetSingleThreadTaskRunnerForTesting(),
             base::BindRepeating(&Self::CreateDecoders, base::Unretained(this)),
             base::BindRepeating(&Self::OnOutput, base::Unretained(this)));
   }
@@ -168,15 +179,14 @@ class WebCodecsDecoderSelectorTest : public ::testing::Test {
     RunUntilIdle();
   }
 
-  void RunUntilIdle() { task_environment_.RunUntilIdle(); }
+  void RunUntilIdle() { platform_->RunUntilIdle(); }
 
-  base::test::TaskEnvironment task_environment_;
-  NullMediaLog media_log_;
+  ScopedTestingPlatformSupport<TestingPlatformSupport> platform_;
+  media::NullMediaLog media_log_;
 
   DecoderConfig last_set_decoder_config_;
 
-  std::unique_ptr<WebCodecsDecoderSelector<TypeParam::kStreamType>>
-      decoder_selector_;
+  std::unique_ptr<DecoderSelector<TypeParam::kStreamType>> decoder_selector_;
 
   std::vector<std::pair<std::string, DecoderCapability>>
       mock_decoders_to_create_;
@@ -237,4 +247,4 @@ TYPED_TEST(WebCodecsDecoderSelectorTest, TwoDecoders_NewConfigSelectAgain) {
   this->SelectDecoder(TypeParam::CreateAlternateConfig());
 }
 
-}  // namespace media
+}  // namespace blink

third_party/blink/renderer/modules/webcodecs/video_decoder.cc

@@ -15,11 +15,13 @@
 #include "media/base/video_decoder.h"
 #include "media/filters/ffmpeg_video_decoder.h"
 #include "media/media_buildflags.h"
+#include "third_party/blink/public/platform/platform.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_encoded_video_chunk.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_encoded_video_config.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_video_decoder_init.h"
 #include "third_party/blink/renderer/core/dom/dom_exception.h"
 #include "third_party/blink/renderer/modules/webcodecs/encoded_video_chunk.h"
+#include "third_party/blink/renderer/modules/webcodecs/video_decoder_broker.h"
 #include "third_party/blink/renderer/modules/webcodecs/video_frame.h"
 #include "third_party/blink/renderer/platform/bindings/exception_code.h"
 #include "third_party/blink/renderer/platform/bindings/exception_state.h"
@@ -64,15 +66,6 @@ media::VideoDecoderConfig ToVideoDecoderConfig(
       media::EncryptionScheme::kUnencrypted);
 }
 
-std::unique_ptr<media::VideoDecoder> CreateVideoDecoder(
-    media::MediaLog* media_log) {
-#if BUILDFLAG(ENABLE_FFMPEG_VIDEO_DECODERS)
-  return std::make_unique<media::FFmpegVideoDecoder>(media_log);
-#else
-  return nullptr;
-#endif  // BUILDFLAG(ENABLE_FFMPEG_VIDEO_DECODERS)
-}
-
 }  // namespace
 
 // static
@@ -183,13 +176,9 @@ bool VideoDecoder::ProcessConfigureRequest(Request* request) {
 
   if (!decoder_) {
     media_log_ = std::make_unique<media::NullMediaLog>();
-    decoder_ = CreateVideoDecoder(media_log_.get());
-    if (!decoder_) {
-      // TODO(sandersd): This is a bit awkward because |request| is still in the
-      // queue.
-      HandleError();
-      return false;
-    }
+    decoder_ = std::make_unique<VideoDecoderBroker>(
+        *ExecutionContext::From(script_state_),
+        Platform::Current()->GetGpuFactories());
 
     // Processing continues in OnInitializeDone().
     // TODO(sandersd): OnInitializeDone() may be called reentrantly, in which

third_party/blink/renderer/modules/webcodecs/video_decoder_broker.h

+// Copyright (c) 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_VIDEO_DECODER_BROKER_H_
+#define THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_VIDEO_DECODER_BROKER_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "base/callback.h"
+#include "base/memory/scoped_refptr.h"
+#include "base/memory/weak_ptr.h"
+#include "base/optional.h"
+#include "base/sequence_checker.h"
+#include "base/single_thread_task_runner.h"
+#include "media/base/decode_status.h"
+#include "media/base/video_decoder.h"
+#include "media/base/video_frame.h"
+#include "media/video/gpu_video_accelerator_factories.h"
+#include "third_party/blink/renderer/core/execution_context/execution_context.h"
+#include "third_party/blink/renderer/modules/modules_export.h"
+
+namespace blink {
+
+// Implementation detail of VideoDecoderBroker. Helps safely perform decoder
+// tasks on the media thread.
+class MediaVideoTaskWrapper;
+
+// Client interface for MediaVideoTaskWrapper. Implementation detail of
+// VideoDecoderBroker, but we need to define it here to implement it below.
+class CrossThreadVideoDecoderClient {
+ public:
+  virtual void OnDecodeOutput(scoped_refptr<media::VideoFrame> frame,
+                              bool can_read_without_stalling) = 0;
+};
+
+// This class brokers the connection between WebCodecs and an underlying
+// media::VideoDecoder. It abstracts away details of construction and selection
+// of the media/ decoder. It also handles thread-hopping as required by
+// underlying APIS.
+//
+// A new underlying decoder is selected anytime Initialize() is called.
+// TODO(chcunningham): Elide re-selection if the config has not significantly
+// changed.
+//
+// All API calls and callbacks must occur on the main thread.
+class MODULES_EXPORT VideoDecoderBroker : public media::VideoDecoder,
+                                          public CrossThreadVideoDecoderClient {
+ public:
+  static constexpr char kDefaultDisplayName[] = "EmptyWebCodecsVideoDecoder";
+
+  struct DecoderDetails {
+    std::string display_name;
+    bool is_platform_decoder;
+    bool needs_bitstream_conversion;
+    int max_decode_requests;
+  };
+
+  // |gpu_factories| may be null when GPU accelerated decoding is not available.
+  explicit VideoDecoderBroker(
+      ExecutionContext& execution_context,
+      media::GpuVideoAcceleratorFactories* gpu_factories);
+  ~VideoDecoderBroker() override;
+
+  // Disallow copy and assign.
+  VideoDecoderBroker(const VideoDecoderBroker&) = delete;
+  VideoDecoderBroker& operator=(const VideoDecoderBroker&) = delete;
+
+  // VideoDecoder implementation.
+  std::string GetDisplayName() const override;
+  bool IsPlatformDecoder() const override;
+  void Initialize(const media::VideoDecoderConfig& config,
+                  bool low_delay,
+                  media::CdmContext* cdm_context,
+                  InitCB init_cb,
+                  const OutputCB& output_cb,
+                  const media::WaitingCB& waiting_cb) override;
+  void Decode(scoped_refptr<media::DecoderBuffer> buffer,
+              DecodeCB decode_cb) override;
+  void Reset(base::OnceClosure reset_cb) override;
+  bool NeedsBitstreamConversion() const override;
+  bool CanReadWithoutStalling() const override;
+  int GetMaxDecodeRequests() const override;
+
+ private:
+  void OnInitialize(InitCB init_cb,
+                    media::Status status,
+                    base::Optional<DecoderDetails> details);
+  void OnDecodeDone(DecodeCB decode_cb, media::DecodeStatus status);
+  void OnReset(base::OnceClosure reset_cb);
+
+  // MediaVideoTaskWrapper::CrossThreadVideoDecoderClient
+  void OnDecodeOutput(scoped_refptr<media::VideoFrame> frame,
+                      bool can_read_without_stalling) override;
+
+  // When media::GpuVideoAcceleratorFactories is provided, its API requires
+  // that we use its TaskRunner (the media thread). When not provided, this task
+  // runner will still be used to reduce contention on the main thread.
+  // TODO(chcunningham): Try to eliminate the Post(). Most of the
+  // underlying::VideoDecoders already offload their work, so this just adds
+  // overhead.
+  scoped_refptr<base::SingleThreadTaskRunner> media_task_runner_;
+
+  // Owner of state and methods to be used on media_task_runner_;
+  std::unique_ptr<MediaVideoTaskWrapper> media_tasks_;
+
+  // Display name for current underlying decoder. Will be kDefaultDisplayName
+  // if no decoder is currently initialized.
+  std::string display_name_ = kDefaultDisplayName;
+
+  // Wrapper state for GetDisplayName(), IsPlatformDecoder() and others.
+  base::Optional<DecoderDetails> decoder_details_;
+
+  // Set to match the underlying decoder's answer at every OnDecodeOutput().
+  bool can_read_without_stalling_ = true;
+
+  // OutputCB saved from last call to Initialize().
+  OutputCB output_cb_;
+
+  SEQUENCE_CHECKER(sequence_checker_);
+
+  base::WeakPtrFactory<VideoDecoderBroker> weak_factory_{this};
+};
+
+}  // namespace blink
+
+#endif  // THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_VIDEO_DECODER_BROKER_H_