Add assistant audio decoder

This cl creates a mojo service for assistant audio decoder. The decoder
is running in utility process. This is a temporary solution until
MediaService for Chrome OS is implemented.

Bug: b/80315134
Test: manual tested can create the audio decoder for assistant.
Change-Id: Ib4d1a8d3e6d38b0cbb436e20e6d12033d74ed7cf
Reviewed-on: https://chromium-review.googlesource.com/1137020Reviewed-by: Kinuko Yasuda <kinuko@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Dan Sanders <sandersd@chromium.org>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Commit-Queue: Tao Wu <wutao@chromium.org>
Cr-Commit-Position: refs/heads/master@{#577713}

content/utility/utility_service_factory.cc

@@ -34,6 +34,10 @@
 #include "services/viz/public/interfaces/constants.mojom.h"
 #include "services/viz/service.h"
 
+#if BUILDFLAG(ENABLE_ASSISTANT_AUDIO_DECODER)
+#include "media/mojo/services/media_service_factory.h"  // nogncheck
+#endif
+
 #if BUILDFLAG(ENABLE_LIBRARY_CDMS)
 #include "media/cdm/cdm_adapter_factory.h"           // nogncheck
 #include "media/mojo/interfaces/constants.mojom.h"   // nogncheck
@@ -96,6 +100,12 @@ std::unique_ptr<service_manager::Service> CreateCdmService() {
 }
 #endif  // BUILDFLAG(ENABLE_LIBRARY_CDMS)
 
+#if BUILDFLAG(ENABLE_ASSISTANT_AUDIO_DECODER)
+std::unique_ptr<service_manager::Service> CreateMediaService() {
+  return ::media::CreateMediaService();
+}
+#endif  // BUILDFLAG(ENABLE_ASSISTANT_AUDIO_DECODER)
+
 std::unique_ptr<service_manager::Service> CreateDataDecoderService() {
   content::UtilityThread::Get()->EnsureBlinkInitialized();
   return data_decoder::DataDecoderService::Create();
@@ -145,6 +155,14 @@ void UtilityServiceFactory::RegisterServices(ServiceMap* services) {
   services->emplace(media::mojom::kCdmServiceName, info);
 #endif
 
+#if BUILDFLAG(ENABLE_ASSISTANT_AUDIO_DECODER)
+  service_manager::EmbeddedServiceInfo assistant_media_service_info;
+  assistant_media_service_info.factory =
+      base::BindRepeating(&CreateMediaService);
+  services->emplace(media::mojom::kMediaServiceName,
+                    assistant_media_service_info);
+#endif
+
   service_manager::EmbeddedServiceInfo shape_detection_info;
   shape_detection_info.factory =
       base::Bind(&shape_detection::ShapeDetectionService::Create);

media/BUILD.gn

@@ -34,6 +34,7 @@ buildflag_header("media_buildflags") {
     "ENABLE_CDM_STORAGE_ID=$enable_cdm_storage_id",
     "ENABLE_MEDIA_REMOTING=$enable_media_remoting",
     "ENABLE_MEDIA_REMOTING_RPC=$enable_media_remoting_rpc",
+    "ENABLE_ASSISTANT_AUDIO_DECODER=$enable_assistant_audio_decoder",
     "USE_PROPRIETARY_CODECS=$proprietary_codecs",
   ]
 }

media/media_options.gni

@@ -5,6 +5,7 @@
 import("//build/config/chrome_build.gni")
 import("//build/config/chromecast_build.gni")
 import("//build/config/features.gni")
+import("//chromeos/assistant/assistant.gni")
 import("//testing/libfuzzer/fuzzer_test.gni")
 
 # Do not expand this list without double-checking with OWNERS, this is a list of
@@ -173,6 +174,13 @@ assert(!enable_cdm_host_verification || is_mac || is_win,
 _default_mojo_media_services = []
 _default_mojo_media_host = "none"
 
+# Mojo media host for Chrome OS Assistant.
+_default_assistant_mojo_media_host = "none"
+
+# Enables the use of Assistant Audio Decoder Service. If true, it will be hosted
+# in the mojo MediaService running in the utility process.
+enable_assistant_audio_decoder = is_chromeos && enable_cros_libassistant
+
 # Default mojo_media_services and mojo_media_host on various platforms.
 # Can be overridden by gn build arguments from the --args command line flag
 # for local testing.
@@ -196,6 +204,12 @@ if (enable_mojo_media) {
     _default_mojo_media_host = "gpu"
   }
 
+  if (enable_assistant_audio_decoder) {
+    # Chrome OS Assistant uses audio decoder running in utility process.
+    _default_mojo_media_services += [ "assistant_audio_decoder" ]
+    _default_assistant_mojo_media_host = "utility"
+  }
+
   if (enable_library_cdms) {
     _default_mojo_media_services += [ "cdm" ]
 
@@ -235,6 +249,7 @@ declare_args() {
   # - "gpu": Use mojo media service hosted in the gpu process.
   # - "utility": Use mojo media service hosted in the utility process.
   mojo_media_host = _default_mojo_media_host
+  assistant_mojo_media_host = _default_assistant_mojo_media_host
 }
 
 declare_args() {

media/mojo/BUILD.gn

@@ -12,6 +12,7 @@ buildflag_header("buildflags") {
   enable_mojo_renderer = false
   enable_mojo_cdm = false
   enable_mojo_audio_decoder = false
+  enable_assistant_mojo_audio_decoder = false
   enable_mojo_video_decoder = false
   enable_mojo_media_in_browser_process = false
   enable_mojo_media_in_gpu_process = false
@@ -30,6 +31,8 @@ buildflag_header("buildflags") {
         enable_mojo_cdm = true
       } else if (service == "audio_decoder") {
         enable_mojo_audio_decoder = true
+      } else if (service == "assistant_audio_decoder") {
+        enable_assistant_mojo_audio_decoder = true
       } else if (service == "video_decoder") {
         enable_mojo_video_decoder = true
       } else {
@@ -49,6 +52,10 @@ buildflag_header("buildflags") {
     } else {
       assert(false, "Invalid mojo media host: $mojo_media_host")
     }
+
+    if (assistant_mojo_media_host == "utility") {
+      enable_mojo_media_in_utility_process = true
+    }
   }
 
   flags = [
@@ -57,6 +64,7 @@ buildflag_header("buildflags") {
     "ENABLE_MOJO_RENDERER=$enable_mojo_renderer",
     "ENABLE_MOJO_CDM=$enable_mojo_cdm",
     "ENABLE_MOJO_AUDIO_DECODER=$enable_mojo_audio_decoder",
+    "ENABLE_ASSISTANT_MOJO_AUDIO_DECODER=$enable_assistant_mojo_audio_decoder",
     "ENABLE_MOJO_VIDEO_DECODER=$enable_mojo_video_decoder",
     "ENABLE_MOJO_MEDIA_IN_BROWSER_PROCESS=$enable_mojo_media_in_browser_process",
     "ENABLE_MOJO_MEDIA_IN_GPU_PROCESS=$enable_mojo_media_in_gpu_process",

media/mojo/services/BUILD.gn

@@ -142,6 +142,13 @@ component("services") {
       deps += [ "//sandbox" ]
     }
   }
+
+  if (enable_assistant_audio_decoder) {
+    sources += [
+      "assistant_mojo_media_client.cc",
+      "assistant_mojo_media_client.h",
+    ]
+  }
 }
 
 service_manifest("cdm_manifest") {

media/mojo/services/android_mojo_media_client.cc

@@ -49,7 +49,8 @@ AndroidMojoMediaClient::~AndroidMojoMediaClient() {}
 // MojoMediaClient overrides.
 
 std::unique_ptr<AudioDecoder> AndroidMojoMediaClient::CreateAudioDecoder(
-    scoped_refptr<base::SingleThreadTaskRunner> task_runner) {
+    scoped_refptr<base::SingleThreadTaskRunner> task_runner,
+    MediaLog* media_log) {
   return std::make_unique<MediaCodecAudioDecoder>(task_runner);
 }
 

media/mojo/services/android_mojo_media_client.h

@@ -20,7 +20,8 @@ class AndroidMojoMediaClient : public MojoMediaClient {
 
   // MojoMediaClient implementation.
   std::unique_ptr<AudioDecoder> CreateAudioDecoder(
-      scoped_refptr<base::SingleThreadTaskRunner> task_runner) final;
+      scoped_refptr<base::SingleThreadTaskRunner> task_runner,
+      MediaLog* media_log) final;
 
   std::unique_ptr<CdmFactory> CreateCdmFactory(
       service_manager::mojom::InterfaceProvider* host_interfaces) final;

media/mojo/services/assistant_mojo_media_client.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/mojo/services/assistant_mojo_media_client.h"
+
+#include "base/single_thread_task_runner.h"
+#include "media/base/audio_decoder.h"
+#include "media/filters/ffmpeg_audio_decoder.h"
+
+namespace media {
+
+AssistantMojoMediaClient::AssistantMojoMediaClient() = default;
+
+AssistantMojoMediaClient::~AssistantMojoMediaClient() = default;
+
+std::unique_ptr<AudioDecoder> AssistantMojoMediaClient::CreateAudioDecoder(
+    scoped_refptr<base::SingleThreadTaskRunner> task_runner,
+    MediaLog* media_log) {
+  return std::make_unique<FFmpegAudioDecoder>(std::move(task_runner),
+                                              media_log);
+}
+
+}  // namespace media

media/mojo/services/assistant_mojo_media_client.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef MEDIA_MOJO_SERVICES_ASSISTANT_MOJO_MEDIA_CLIENT_H_
+#define MEDIA_MOJO_SERVICES_ASSISTANT_MOJO_MEDIA_CLIENT_H_
+
+#include <memory>
+
+#include "base/macros.h"
+#include "base/memory/scoped_refptr.h"
+#include "media/base/media_log.h"
+#include "media/mojo/services/mojo_media_client.h"
+
+namespace base {
+class SingleThreadTaskRunner;
+}  // namespace base
+
+namespace media {
+
+class MediaLog;
+
+class AssistantMojoMediaClient : public MojoMediaClient {
+ public:
+  AssistantMojoMediaClient();
+  ~AssistantMojoMediaClient() final;
+
+  // MojoMediaClient implementation.
+  std::unique_ptr<AudioDecoder> CreateAudioDecoder(
+      scoped_refptr<base::SingleThreadTaskRunner> task_runner,
+      MediaLog* media_log) final;
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(AssistantMojoMediaClient);
+};
+
+}  // namespace media
+
+#endif  // MEDIA_MOJO_SERVICES_ASSISTANT_MOJO_MEDIA_CLIENT_H_

media/mojo/services/gpu_mojo_media_client.cc

@@ -102,7 +102,8 @@ GpuMojoMediaClient::~GpuMojoMediaClient() = default;
 void GpuMojoMediaClient::Initialize(service_manager::Connector* connector) {}
 
 std::unique_ptr<AudioDecoder> GpuMojoMediaClient::CreateAudioDecoder(
-    scoped_refptr<base::SingleThreadTaskRunner> task_runner) {
+    scoped_refptr<base::SingleThreadTaskRunner> task_runner,
+    MediaLog* media_log) {
 #if defined(OS_ANDROID)
   return std::make_unique<MediaCodecAudioDecoder>(task_runner);
 #else

media/mojo/services/gpu_mojo_media_client.h

@@ -39,7 +39,8 @@ class GpuMojoMediaClient : public MojoMediaClient {
   // MojoMediaClient implementation.
   void Initialize(service_manager::Connector* connector) final;
   std::unique_ptr<AudioDecoder> CreateAudioDecoder(
-      scoped_refptr<base::SingleThreadTaskRunner> task_runner) final;
+      scoped_refptr<base::SingleThreadTaskRunner> task_runner,
+      MediaLog* media_log) final;
   std::unique_ptr<VideoDecoder> CreateVideoDecoder(
       scoped_refptr<base::SingleThreadTaskRunner> task_runner,
       MediaLog* media_log,

media/mojo/services/interface_factory_impl.cc

@@ -16,9 +16,11 @@
 #include "mojo/public/cpp/bindings/strong_binding.h"
 #include "services/service_manager/public/mojom/interface_provider.mojom.h"
 
-#if BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER)
+#if BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER) || \
+    BUILDFLAG(ENABLE_ASSISTANT_MOJO_AUDIO_DECODER)
 #include "media/mojo/services/mojo_audio_decoder_service.h"
-#endif  // BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER)
+#endif  // BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER) ||
+        // BUILDFLAG(ENABLE_ASSISTANT_MOJO_AUDIO_DECODER)
 
 #if BUILDFLAG(ENABLE_MOJO_VIDEO_DECODER)
 #include "media/mojo/services/mojo_video_decoder_service.h"
@@ -47,7 +49,8 @@ InterfaceFactoryImpl::InterfaceFactoryImpl(
     std::unique_ptr<service_manager::ServiceContextRef> connection_ref,
     MojoMediaClient* mojo_media_client)
     :
-#if BUILDFLAG(ENABLE_MOJO_RENDERER)
+#if BUILDFLAG(ENABLE_MOJO_RENDERER) || BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER) || \
+    BUILDFLAG(ENABLE_ASSISTANT_MOJO_AUDIO_DECODER)
       media_log_(media_log),
 #endif
 #if BUILDFLAG(ENABLE_MOJO_CDM)
@@ -70,12 +73,13 @@ InterfaceFactoryImpl::~InterfaceFactoryImpl() {
 void InterfaceFactoryImpl::CreateAudioDecoder(
     mojo::InterfaceRequest<mojom::AudioDecoder> request) {
   DVLOG(2) << __func__;
-#if BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER)
+#if BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER) || \
+    BUILDFLAG(ENABLE_ASSISTANT_MOJO_AUDIO_DECODER)
   scoped_refptr<base::SingleThreadTaskRunner> task_runner(
       base::ThreadTaskRunnerHandle::Get());
 
   std::unique_ptr<AudioDecoder> audio_decoder =
-      mojo_media_client_->CreateAudioDecoder(task_runner);
+      mojo_media_client_->CreateAudioDecoder(task_runner, media_log_);
   if (!audio_decoder) {
     DLOG(ERROR) << "AudioDecoder creation failed.";
     return;
@@ -85,7 +89,8 @@ void InterfaceFactoryImpl::CreateAudioDecoder(
       std::make_unique<MojoAudioDecoderService>(&cdm_service_context_,
                                                 std::move(audio_decoder)),
       std::move(request));
-#endif  // BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER)
+#endif  // BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER) ||
+        // BUILDFLAG(ENABLE_ASSISTANT_MOJO_AUDIO_DECODER)
 }
 
 void InterfaceFactoryImpl::CreateVideoDecoder(
@@ -202,10 +207,12 @@ void InterfaceFactoryImpl::OnDestroyPending(base::OnceClosure destroy_cb) {
 }
 
 bool InterfaceFactoryImpl::IsEmpty() {
-#if BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER)
+#if BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER) || \
+    BUILDFLAG(ENABLE_ASSISTANT_MOJO_AUDIO_DECODER)
   if (!audio_decoder_bindings_.empty())
     return false;
-#endif  // BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER)
+#endif  // BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER) ||
+        // BUILDFLAG(ENABLE_ASSISTANT_MOJO_AUDIO_DECODER)
 
 #if BUILDFLAG(ENABLE_MOJO_VIDEO_DECODER)
   if (!video_decoder_bindings_.empty())
@@ -240,9 +247,11 @@ void InterfaceFactoryImpl::SetBindingConnectionErrorHandler() {
   auto connection_error_cb = base::BindRepeating(
       &InterfaceFactoryImpl::OnBindingConnectionError, base::Unretained(this));
 
-#if BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER)
+#if BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER) || \
+    BUILDFLAG(ENABLE_ASSISTANT_MOJO_AUDIO_DECODER)
   audio_decoder_bindings_.set_connection_error_handler(connection_error_cb);
-#endif  // BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER)
+#endif  // BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER) ||
+        // BUILDFLAG(ENABLE_ASSISTANT_MOJO_AUDIO_DECODER)
 
 #if BUILDFLAG(ENABLE_MOJO_VIDEO_DECODER)
   video_decoder_bindings_.set_connection_error_handler(connection_error_cb);

media/mojo/services/interface_factory_impl.h

@@ -63,16 +63,24 @@ class InterfaceFactoryImpl : public DeferredDestroy<mojom::InterfaceFactory> {
   // available.
   MojoCdmServiceContext cdm_service_context_;
 
-#if BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER)
+#if BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER) || \
+    BUILDFLAG(ENABLE_ASSISTANT_MOJO_AUDIO_DECODER)
   mojo::StrongBindingSet<mojom::AudioDecoder> audio_decoder_bindings_;
-#endif  // BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER)
+#endif  // BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER) ||
+        // BUILDFLAG(ENABLE_ASSISTANT_MOJO_AUDIO_DECODER)
 
 #if BUILDFLAG(ENABLE_MOJO_VIDEO_DECODER)
   mojo::StrongBindingSet<mojom::VideoDecoder> video_decoder_bindings_;
 #endif  // BUILDFLAG(ENABLE_MOJO_VIDEO_DECODER)
 
-#if BUILDFLAG(ENABLE_MOJO_RENDERER)
+#if BUILDFLAG(ENABLE_MOJO_RENDERER) || BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER) || \
+    BUILDFLAG(ENABLE_ASSISTANT_MOJO_AUDIO_DECODER)
   MediaLog* media_log_;
+#endif  // BUILDFLAG(ENABLE_MOJO_RENDERER) ||
+        // BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER) ||
+        // BUILDFLAG(ENABLE_ASSISTANT_MOJO_AUDIO_DECODER)
+
+#if BUILDFLAG(ENABLE_MOJO_RENDERER)
   mojo::StrongBindingSet<mojom::Renderer> renderer_bindings_;
 #endif  // BUILDFLAG(ENABLE_MOJO_RENDERER)
 

media/mojo/services/media_service_factory.cc

@@ -7,10 +7,15 @@
 #include <memory>
 
 #include "base/logging.h"
+#include "media/mojo/buildflags.h"
 #include "media/mojo/services/gpu_mojo_media_client.h"
 #include "media/mojo/services/media_service.h"
 #include "media/mojo/services/test_mojo_media_client.h"
 
+#if BUILDFLAG(ENABLE_MOJO_MEDIA_IN_UTILITY_PROCESS)
+#include "media/mojo/services/assistant_mojo_media_client.h"  // nogncheck
+#endif
+
 #if defined(OS_ANDROID)
 #include "media/mojo/services/android_mojo_media_client.h"  // nogncheck
 #endif
@@ -23,6 +28,9 @@ std::unique_ptr<service_manager::Service> CreateMediaService() {
 #elif defined(OS_ANDROID)
   return std::unique_ptr<service_manager::Service>(
       new MediaService(std::make_unique<AndroidMojoMediaClient>()));
+#elif BUILDFLAG(ENABLE_ASSISTANT_MOJO_AUDIO_DECODER)
+  return std::make_unique<MediaService>(
+      std::make_unique<AssistantMojoMediaClient>());
 #else
   NOTREACHED() << "No MediaService implementation available.";
   return nullptr;

media/mojo/services/mojo_media_client.cc

@@ -24,7 +24,8 @@ MojoMediaClient::~MojoMediaClient() = default;
 void MojoMediaClient::Initialize(service_manager::Connector* connector) {}
 
 std::unique_ptr<AudioDecoder> MojoMediaClient::CreateAudioDecoder(
-    scoped_refptr<base::SingleThreadTaskRunner> task_runner) {
+    scoped_refptr<base::SingleThreadTaskRunner> task_runner,
+    MediaLog* media_log) {
   return nullptr;
 }
 

media/mojo/services/mojo_media_client.h

@@ -52,7 +52,8 @@ class MEDIA_MOJO_EXPORT MojoMediaClient {
   virtual void Initialize(service_manager::Connector* connector);
 
   virtual std::unique_ptr<AudioDecoder> CreateAudioDecoder(
-      scoped_refptr<base::SingleThreadTaskRunner> task_runner);
+      scoped_refptr<base::SingleThreadTaskRunner> task_runner,
+      MediaLog* media_log);
 
   virtual std::unique_ptr<VideoDecoder> CreateVideoDecoder(
       scoped_refptr<base::SingleThreadTaskRunner> task_runner,