Introduce AudioInputController Mojom API

This CL is part of a migration of all interactions with the Libassistant
3rd party library to a mojom service. Currently Libassistant is invoked
directly from the browser thread.

This specific CL focuses on the audio input, aka the microphone.
Libassistant needs to open the microphone both for hotword detection
and for listening to actual user queries (for example
'ok google; set a timer for 5 minutes').

The idea behind the mojom interface is that we tell Libassistant
    - What audio devices to use.
    - When it is allowed to open the microphone (for example we don't
      want Libassistant listening if the lid of the device is closed).

We also want to keep creating the |AudioStreamFactory| instances on the
browser thread, so this mojom API contains a delegate to allow our
service to request a |AudioStreamFactory| from the Browser thread.

Bug: b/171748795
Test: N/A
Change-Id: I3b75d9424e78fe5af8793395e25485168bc3c999
Cq-Include-Trybots: luci.chrome.try:linux-chromeos-chrome
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2600332
Commit-Queue: Jeroen Dhollander <jeroendh@chromium.org>
Reviewed-by: Meilin Wang <meilinw@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Reviewed-by: Xiaohui Chen <xiaohuic@chromium.org>
Cr-Commit-Position: refs/heads/master@{#843078}

chromeos/services/assistant/proxy/assistant_proxy.cc

@@ -29,11 +29,7 @@ void AssistantProxy::Initialize(LibassistantServiceHost* host) {
   libassistant_service_host_ = host;
   LaunchLibassistantService();
 
-  service_controller_proxy_ =
-      std::make_unique<ServiceControllerProxy>(host, BindServiceController());
-  conversation_controller_proxy_ =
-      std::make_unique<ConversationControllerProxy>(
-          BindConversationController());
+  BindControllers(host);
 }
 
 void AssistantProxy::LaunchLibassistantService() {
@@ -74,20 +70,31 @@ void AssistantProxy::StopLibassistantServiceOnBackgroundThread() {
   libassistant_service_host_->Stop();
 }
 
-mojo::PendingRemote<AssistantProxy::ServiceControllerMojom>
-AssistantProxy::BindServiceController() {
-  mojo::PendingRemote<ServiceControllerMojom> pending_remote;
-  libassistant_service_remote_->BindServiceController(
-      pending_remote.InitWithNewPipeAndPassReceiver());
-  return pending_remote;
-}
-
-mojo::PendingRemote<AssistantProxy::ConversationControllerMojom>
-AssistantProxy::BindConversationController() {
-  mojo::PendingRemote<ConversationControllerMojom> pending_remote;
-  libassistant_service_remote_->BindConversationController(
-      pending_remote.InitWithNewPipeAndPassReceiver());
-  return pending_remote;
+void AssistantProxy::BindControllers(LibassistantServiceHost* host) {
+  mojo::PendingRemote<AudioInputControllerMojom>
+      pending_audio_input_controller_remote;
+  mojo::PendingRemote<AudioStreamFactoryDelegateMojom>
+      pending_audio_stream_factory_delegate_remote;
+  mojo::PendingRemote<ServiceControllerMojom> pending_service_controller_remote;
+  mojo::PendingRemote<ConversationControllerMojom>
+      pending_conversation_controller_remote;
+
+  mojo::PendingReceiver<AudioStreamFactoryDelegateMojom>
+      pending_audio_stream_factory_delegate_receiver =
+          pending_audio_stream_factory_delegate_remote
+              .InitWithNewPipeAndPassReceiver();
+
+  libassistant_service_remote_->Bind(
+      pending_audio_input_controller_remote.InitWithNewPipeAndPassReceiver(),
+      std::move(pending_audio_stream_factory_delegate_remote),
+      pending_conversation_controller_remote.InitWithNewPipeAndPassReceiver(),
+      pending_service_controller_remote.InitWithNewPipeAndPassReceiver());
+
+  service_controller_proxy_ = std::make_unique<ServiceControllerProxy>(
+      host, std::move(pending_service_controller_remote));
+  conversation_controller_proxy_ =
+      std::make_unique<ConversationControllerProxy>(
+          std::move(pending_conversation_controller_remote));
 }
 
 scoped_refptr<base::SingleThreadTaskRunner>

chromeos/services/assistant/proxy/assistant_proxy.h

@@ -48,12 +48,16 @@ class AssistantProxy {
   base::Thread& background_thread() { return background_thread_; }
 
  private:
+  using AudioInputControllerMojom =
+      chromeos::libassistant::mojom::AudioInputController;
+  using AudioStreamFactoryDelegateMojom =
+      chromeos::libassistant::mojom::AudioStreamFactoryDelegate;
+  using ConversationControllerMojom =
+      chromeos::libassistant::mojom::ConversationController;
   using LibassistantServiceMojom =
       chromeos::libassistant::mojom::LibassistantService;
   using ServiceControllerMojom =
       chromeos::libassistant::mojom::ServiceController;
-  using ConversationControllerMojom =
-      chromeos::libassistant::mojom::ConversationController;
 
   scoped_refptr<base::SingleThreadTaskRunner> background_task_runner();
 
@@ -63,8 +67,7 @@ class AssistantProxy {
   void StopLibassistantService();
   void StopLibassistantServiceOnBackgroundThread();
 
-  mojo::PendingRemote<ServiceControllerMojom> BindServiceController();
-  mojo::PendingRemote<ConversationControllerMojom> BindConversationController();
+  void BindControllers(LibassistantServiceHost* host);
 
   // Owned by |AssistantManagerServiceImpl|.
   LibassistantServiceHost* libassistant_service_host_ = nullptr;

chromeos/services/assistant/test_support/fake_libassistant_service.cc

@@ -26,9 +26,16 @@ void FakeLibassistantService::Unbind() {
   service_controller().Unbind();
 }
 
-void FakeLibassistantService::BindServiceController(
-    mojo::PendingReceiver<libassistant::mojom::ServiceController> receiver) {
-  service_controller_.Bind(std::move(receiver));
+void FakeLibassistantService::Bind(
+    mojo::PendingReceiver<libassistant::mojom::AudioInputController>
+        audio_input_controller,
+    mojo::PendingRemote<libassistant::mojom::AudioStreamFactoryDelegate>
+        audio_stream_factory_delegate,
+    mojo::PendingReceiver<libassistant::mojom::ConversationController>
+        conversation_controller,
+    mojo::PendingReceiver<libassistant::mojom::ServiceController>
+        service_controller) {
+  service_controller_.Bind(std::move(service_controller));
 }
 
 }  // namespace assistant

chromeos/services/assistant/test_support/fake_libassistant_service.h

@@ -30,14 +30,14 @@ class FakeLibassistantService
   FakeServiceController& service_controller() { return service_controller_; }
 
   // mojom::LibassistantService implementation:
-  void BindServiceController(
-      mojo::PendingReceiver<libassistant::mojom::ServiceController> receiver)
-      override;
-  void BindConversationController(
+  void Bind(mojo::PendingReceiver<libassistant::mojom::AudioInputController>
+                audio_input_controller,
+            mojo::PendingRemote<libassistant::mojom::AudioStreamFactoryDelegate>
+                audio_stream_factory_delegate,
             mojo::PendingReceiver<libassistant::mojom::ConversationController>
-          receiver) override {}
-  void BindAudioInputController() override {}
-  void BindAudioOutputController() override {}
+                conversation_controller,
+            mojo::PendingReceiver<libassistant::mojom::ServiceController>
+                service_controller) override;
 
  private:
   mojo::Receiver<libassistant::mojom::LibassistantService> receiver_;

chromeos/services/libassistant/libassistant_service.cc

@@ -37,19 +37,20 @@ LibassistantService::LibassistantService(
 
 LibassistantService::~LibassistantService() = default;
 
-void LibassistantService::BindServiceController(
-    mojo::PendingReceiver<mojom::ServiceController> receiver) {
-  service_controller_->Bind(std::move(receiver));
+void LibassistantService::Bind(
+    mojo::PendingReceiver<mojom::AudioInputController> audio_input_controller,
+    mojo::PendingRemote<mojom::AudioStreamFactoryDelegate>
+        audio_stream_factory_delegate,
+    mojo::PendingReceiver<mojom::ConversationController>
+        conversation_controller,
+    mojo::PendingReceiver<mojom::ServiceController> service_controller) {
+  service_controller_->Bind(std::move(service_controller));
+  conversation_controller_->Bind(std::move(conversation_controller));
 }
 
 void LibassistantService::SetInitializeCallback(InitializeCallback callback) {
   service_controller().SetInitializeCallback(std::move(callback));
 }
 
-void LibassistantService::BindConversationController(
-    mojo::PendingReceiver<mojom::ConversationController> receiver) {
-  conversation_controller_->Bind(std::move(receiver));
-}
-
 }  // namespace libassistant
 }  // namespace chromeos

chromeos/services/libassistant/libassistant_service.h

@@ -8,7 +8,8 @@
 #include <memory>
 
 #include "base/component_export.h"
-#include "chromeos/services/libassistant/public/mojom/conversation_controller.mojom.h"
+#include "chromeos/services/libassistant/public/mojom/audio_input_controller.mojom-forward.h"
+#include "chromeos/services/libassistant/public/mojom/conversation_controller.mojom-forward.h"
 #include "chromeos/services/libassistant/public/mojom/service.mojom.h"
 #include "mojo/public/cpp/bindings/pending_receiver.h"
 #include "mojo/public/cpp/bindings/receiver.h"
@@ -53,12 +54,14 @@ class COMPONENT_EXPORT(LIBASSISTANT_SERVICE) LibassistantService
   ServiceController& service_controller() { return *service_controller_; }
 
   // mojom::LibassistantService implementation:
-  void BindServiceController(
-      mojo::PendingReceiver<mojom::ServiceController> receiver) override;
-  void BindConversationController(
-      mojo::PendingReceiver<mojom::ConversationController> receiver) override;
-  void BindAudioInputController() override {}
-  void BindAudioOutputController() override {}
+  void Bind(
+      mojo::PendingReceiver<mojom::AudioInputController> audio_input_controller,
+      mojo::PendingRemote<mojom::AudioStreamFactoryDelegate>
+          audio_stream_factory_delegate,
+      mojo::PendingReceiver<mojom::ConversationController>
+          conversation_controller,
+      mojo::PendingReceiver<mojom::ServiceController> service_controller)
+      override;
 
   mojo::Receiver<mojom::LibassistantService> receiver_;
   std::unique_ptr<PlatformApi> platform_api_;

chromeos/services/libassistant/public/mojom/BUILD.gn

@@ -7,7 +7,10 @@ import("//mojo/public/tools/bindings/mojom.gni")
 mojom("mojom") {
   sources = [
     "conversation_controller.mojom",
+    "audio_input_controller.mojom",
     "service.mojom",
     "service_controller.mojom",
   ]
+
+  deps = [ "//services/audio/public/mojom" ]
 }

chromeos/services/libassistant/public/mojom/audio_input_controller.mojom

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+module chromeos.libassistant.mojom;
+
+import "services/audio/public/mojom/stream_factory.mojom";
+
+// Interface managing Libassistant audio input, allowing Libassistant to open
+// the microphone both for hotword detection ('ok google') as well for the
+// actual voice queries.
+// The Libassistant service will be responsible for opening/closing its own
+// audio streams.
+// This controller is hosted in the Libassistant mojom service, and will be
+// invoked from the browser process.
+interface AudioInputController {
+
+  // Called when the user manually opens/closes the microphone, for example by
+  // pressing the microphone icon in the Assistant UI.
+  SetMicOpen(bool mic_open);
+
+  // Called when the user enables/disables hotword detection.
+  SetHotwordEnabled(bool enable);
+
+  // Set/Unset the device id that should be used for voice queries.
+  // The device-id provided must be a valid audio device id (or null to remove
+  // the configured device-id).
+  SetDeviceId(string? device_id);
+
+  // Set/Unset the device id that should be used for the hotword detection.
+  // The device-id provided must be a valid audio device id (or null to remove
+  // the configured device-id).
+  SetHotwordDeviceId(string? device_id);
+
+  // Called when the user opens/closes the lid.
+  // When the lid is closed no audio should be recorded.
+  // This is initially set to |kClosed|.
+  SetLidState(LidState new_state);
+
+  // TODO(jeroendh): It doesn't really make sense that our browser thread must
+  // inform the Libassistant mojom service that Libassistant has
+  // started/stopped a conversation.
+  // However, this is required until we migrate the conversation listening APIs
+  // to the Libassistant mojom service.
+  OnConversationTurnStarted();
+  // Called when the conversation is finished.
+  OnConversationTurnFinished();
+};
+
+// Whether the lid of the current device is open or closed.
+// If the device has no lid it is considered to be open.
+enum LidState {
+  kOpen,
+  kClosed,
+};
+
+// Interface that exposes a method to create a |StreamFactory|.
+// This interface is passed to the Libassistant mojom service so it can open
+// audio streams.
+interface AudioStreamFactoryDelegate  {
+
+  // Must return a stream factory that Libassistant can use,
+  // or null if no stream factory can be bound for any reason.
+  GetAudioStreamFactory() => (pending_remote<audio.mojom.StreamFactory>? result);
+
+};

chromeos/services/libassistant/public/mojom/service.mojom

@@ -4,8 +4,9 @@
 
 module chromeos.libassistant.mojom;
 
-import "chromeos/services/libassistant/public/mojom/service_controller.mojom";
+import "chromeos/services/libassistant/public/mojom/audio_input_controller.mojom";
 import "chromeos/services/libassistant/public/mojom/conversation_controller.mojom";
+import "chromeos/services/libassistant/public/mojom/service_controller.mojom";
 
 // The main interface to the Libassistant service on ChromeOS.
 // Libassistant provides access to the Google Assistant.
@@ -14,16 +15,12 @@ import "chromeos/services/libassistant/public/mojom/conversation_controller.mojo
 // It is used by //chromeos/services/assistant/proxy, which is a trusted
 // service running in the browser process.
 interface LibassistantService {
-  // Bind the service controller, which provides control over the lifetime
-  // of the Libassistant process (start/stop).
-  BindServiceController(pending_receiver<ServiceController> receiver);
-
-  // Bind the conversation controller, which in charge of handling
-  // conversations with Libassistant.
-  BindConversationController(pending_receiver<ConversationController> receiver);
 
-  // This service will further expose methods to bind all other helper
-  // controllers.
-  BindAudioInputController(/* pending_receiver */);
-  BindAudioOutputController(/* pending_receiver */);
+  // Bind everything needed to start the service.
+  Bind(
+    pending_receiver<AudioInputController> audio_input_controller,
+    pending_remote<AudioStreamFactoryDelegate> audio_stream_factory_delegate,
+    pending_receiver<ConversationController> conversation_controller,
+    pending_receiver<ServiceController> service_controller
+  );
 };