AudioDecoderBroker: Connect WebCodecs to media::AudioDecoder(s)

"Broker" acts as a simple decoder, but under the hood manages creation
and selection of of underlying media::AudioDecoders, and thread-hopping.

This also enables WebCodecs to use MojoAudioDecoder (android) when not
in a Worker context. For Workers, we still need to expose the relevant
Mojo interfaces (currently tied to the RenderFrame).

Bug: 1094091
Change-Id: I3743de18aa6701428fc3655a2e917c3b447c4a3f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2333379
Commit-Queue: Chrome Cunningham <chcunningham@chromium.org>
Reviewed-by: Dan Sanders <sandersd@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Cr-Commit-Position: refs/heads/master@{#795611}

third_party/blink/PRESUBMIT.py

@@ -62,7 +62,7 @@ def _CheckForWrongMojomIncludes(input_api, output_api):
     allowed_interfaces = ('services/network/public/mojom/cross_origin_embedder_policy', 'services/network/public/mojom/fetch_api',
                           'services/network/public/mojom/load_timing_info',
                           'third_party/blink/public/mojom/worker/subresource_loader_updater', 'media/mojo/mojom/interface_factory',
-                          'media/mojo/mojom/video_decoder')
+                          'media/mojo/mojom/audio_decoder', 'media/mojo/mojom/video_decoder')
 
     for f in input_api.AffectedFiles(file_filter=source_file_filter):
         for line_num, line in f.ChangedContents():

third_party/blink/renderer/modules/webcodecs/BUILD.gn

@@ -8,6 +8,8 @@ blink_modules_sources("webcodecs") {
   sources = [
     "audio_decoder.cc",
     "audio_decoder.h",
+    "audio_decoder_broker.cc",
+    "audio_decoder_broker.h",
     "audio_frame.cc",
     "audio_frame.h",
     "decoder_selector.cc",
@@ -46,6 +48,7 @@ blink_modules_sources("webcodecs") {
 source_set("unit_tests") {
   testonly = true
   sources = [
+    "audio_decoder_broker_test.cc",
     "audio_decoder_test.cc",
     "decoder_selector_test.cc",
     "encoded_video_chunk_test.cc",

third_party/blink/renderer/modules/webcodecs/DEPS

@@ -29,4 +29,8 @@ specific_include_rules = {
     "+base/threading/thread.h",
     "+gpu/command_buffer/common/mailbox_holder.h",
   ],
+  "audio_decoder_broker_test\.cc": [
+    "+base/run_loop.h",
+    "+base/files/file_util.h",
+  ],
 }

third_party/blink/renderer/modules/webcodecs/audio_decoder.cc

@@ -11,12 +11,10 @@
 #include "media/base/encryption_scheme.h"
 #include "media/base/media_util.h"
 #include "media/base/waiting.h"
-#if BUILDFLAG(ENABLE_FFMPEG)
-#include "media/filters/ffmpeg_audio_decoder.h"
-#endif
 #include "third_party/blink/renderer/bindings/modules/v8/v8_audio_decoder_init.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_encoded_audio_chunk.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_encoded_audio_config.h"
+#include "third_party/blink/renderer/modules/webcodecs/audio_decoder_broker.h"
 
 #include <memory>
 #include <vector>
@@ -27,12 +25,7 @@ namespace blink {
 std::unique_ptr<AudioDecoderTraits::MediaDecoderType>
 AudioDecoderTraits::CreateDecoder(ExecutionContext& execution_context,
                                   media::MediaLog* media_log) {
-#if BUILDFLAG(ENABLE_FFMPEG)
-  return std::make_unique<media::FFmpegAudioDecoder>(
-      execution_context.GetTaskRunner(TaskType::kInternalMedia), media_log);
-#else
-  return nullptr;
-#endif
+  return std::make_unique<AudioDecoderBroker>(execution_context);
 }
 
 // static

third_party/blink/renderer/modules/webcodecs/audio_decoder_broker.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_AUDIO_DECODER_BROKER_H_
+#define THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_AUDIO_DECODER_BROKER_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "base/callback.h"
+#include "base/memory/scoped_refptr.h"
+#include "base/memory/weak_ptr.h"
+#include "base/optional.h"
+#include "base/sequence_checker.h"
+#include "base/single_thread_task_runner.h"
+#include "media/base/audio_buffer.h"
+#include "media/base/audio_decoder.h"
+#include "media/base/decode_status.h"
+#include "third_party/blink/renderer/core/execution_context/execution_context.h"
+#include "third_party/blink/renderer/modules/modules_export.h"
+
+namespace blink {
+
+// Implementation detail of AudioDecoderBroker. Helps safely perform decoder
+// tasks on the media thread.
+class MediaAudioTaskWrapper;
+
+// Client interface for MediaAudioTaskWrapper. Implementation detail of
+// AudioDecoderBroker, but we need to define it here to implement it below.
+class CrossThreadAudioDecoderClient {
+ public:
+  virtual void OnDecodeOutput(scoped_refptr<media::AudioBuffer> buffer) = 0;
+};
+
+// This class brokers the connection between WebCodecs and an underlying
+// media::AudioDecoder. It abstracts away details of construction and selection
+// of the media/ decoder. It also handles thread-hopping as required by
+// underlying APIS.
+//
+// A new underlying decoder is selected anytime Initialize() is called.
+// TODO(chcunningham): Elide re-selection if the config has not significantly
+// changed.
+//
+// All API calls and callbacks must occur on the main thread.
+class MODULES_EXPORT AudioDecoderBroker : public media::AudioDecoder,
+                                          public CrossThreadAudioDecoderClient {
+ public:
+  static constexpr char kDefaultDisplayName[] = "EmptyWebCodecsAudioDecoder";
+
+  struct DecoderDetails {
+    std::string display_name;
+    bool is_platform_decoder;
+    bool needs_bitstream_conversion;
+  };
+
+  explicit AudioDecoderBroker(ExecutionContext& execution_context);
+  ~AudioDecoderBroker() override;
+
+  // Disallow copy and assign.
+  AudioDecoderBroker(const AudioDecoderBroker&) = delete;
+  AudioDecoderBroker& operator=(const AudioDecoderBroker&) = delete;
+
+  // AudioDecoder implementation.
+  std::string GetDisplayName() const override;
+  bool IsPlatformDecoder() const override;
+  void Initialize(const media::AudioDecoderConfig& config,
+                  media::CdmContext* cdm_context,
+                  InitCB init_cb,
+                  const OutputCB& output_cb,
+                  const media::WaitingCB& waiting_cb) override;
+  void Decode(scoped_refptr<media::DecoderBuffer> buffer,
+              DecodeCB decode_cb) override;
+  void Reset(base::OnceClosure reset_cb) override;
+  bool NeedsBitstreamConversion() const override;
+
+ private:
+  void OnInitialize(InitCB init_cb,
+                    media::Status status,
+                    base::Optional<DecoderDetails> details);
+  void OnDecodeDone(DecodeCB decode_cb, media::DecodeStatus status);
+  void OnReset(base::OnceClosure reset_cb);
+
+  // MediaAudioTaskWrapper::CrossThreadAudioDecoderClient
+  void OnDecodeOutput(scoped_refptr<media::AudioBuffer> buffer) override;
+
+  // Task runner for running codec work (traditionally the media thread).
+  scoped_refptr<base::SingleThreadTaskRunner> media_task_runner_;
+
+  // Owner of state and methods to be used on media_task_runner_;
+  std::unique_ptr<MediaAudioTaskWrapper> media_tasks_;
+
+  // Wrapper state for GetDisplayName(), IsPlatformDecoder() and others.
+  base::Optional<DecoderDetails> decoder_details_;
+
+  // OutputCB saved from last call to Initialize().
+  OutputCB output_cb_;
+
+  SEQUENCE_CHECKER(sequence_checker_);
+
+  base::WeakPtrFactory<AudioDecoderBroker> weak_factory_{this};
+};
+
+}  // namespace blink
+
+#endif  // THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_AUDIO_DECODER_BROKER_H_

third_party/blink/renderer/modules/webcodecs/video_decoder_broker.cc

-// Copyright (c) 2020 The Chromium Authors. All rights reserved.
+// Copyright 2020 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
@@ -78,7 +78,7 @@ class MediaVideoTaskWrapper {
       media::GpuVideoAcceleratorFactories* gpu_factories,
       scoped_refptr<base::SingleThreadTaskRunner> media_task_runner,
       scoped_refptr<base::SingleThreadTaskRunner> main_task_runner)
-      : weak_client_(weak_client),
+      : weak_client_(std::move(weak_client)),
         media_task_runner_(std::move(media_task_runner)),
         main_task_runner_(std::move(main_task_runner)),
         gpu_factories_(gpu_factories) {
@@ -148,7 +148,7 @@ class MediaVideoTaskWrapper {
       return;
     }
 
-    decoder_->Decode(buffer,
+    decoder_->Decode(std::move(buffer),
                      WTF::Bind(&MediaVideoTaskWrapper::OnDecodeDone,
                                WTF::Unretained(this), std::move(decode_cb)));
   }
@@ -173,7 +173,8 @@ class MediaVideoTaskWrapper {
     DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
     media_interface_factory_.Bind(std::move(interface_factory));
 
-    // This setup is blocked on the Bind() above.
+    // Bind the |interface_factory_| above before passing to
+    // |external_decoder_factory|.
     std::unique_ptr<media::DecoderFactory> external_decoder_factory;
 #if BUILDFLAG(ENABLE_MOJO_VIDEO_DECODER)
     external_decoder_factory = std::make_unique<media::MojoDecoderFactory>(

third_party/blink/renderer/modules/webcodecs/video_decoder_broker.h

-// Copyright (c) 2020 The Chromium Authors. All rights reserved.
+// Copyright 2020 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.