[LibassistantV2] Introduce mojom and proxy for |ConversationController|.

Introduces |ConversationController| (and its proxy) as a sub controller
responsible for handling conversations between our Assistant service and
Libassistant. Adds its first mojom API |SendTextQuery()| which sends a
text query to Libassistant upon called.

Bug: b/171748795

Change-Id: I9b86f9323c4e34ebce6fe78bfda487ad81d1f87a
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2545648
Commit-Queue: Meilin Wang <meilinw@chromium.org>
Reviewed-by: Sam McNally <sammc@chromium.org>
Reviewed-by: Xiaohui Chen <xiaohuic@chromium.org>
Reviewed-by: Jeroen Dhollander <jeroendh@chromium.org>
Reviewed-by: Tao Wu <wutao@chromium.org>
Cr-Commit-Position: refs/heads/master@{#842302}

chromeos/services/assistant/assistant_manager_service_impl.cc

@@ -35,6 +35,7 @@
 #include "chromeos/services/assistant/libassistant_service_host_impl.h"
 #include "chromeos/services/assistant/media_session/assistant_media_session.h"
 #include "chromeos/services/assistant/platform_api_impl.h"
+#include "chromeos/services/assistant/proxy/conversation_controller_proxy.h"
 #include "chromeos/services/assistant/proxy/service_controller_proxy.h"
 #include "chromeos/services/assistant/public/cpp/assistant_client.h"
 #include "chromeos/services/assistant/public/cpp/device_actions.h"
@@ -506,22 +507,10 @@ void AssistantManagerServiceImpl::StartTextInteraction(
     AssistantQuerySource source,
     bool allow_tts) {
   DVLOG(1) << __func__;
-  assistant_client::VoicelessOptions options;
-  options.is_user_initiated = true;
-
-  if (!allow_tts) {
-    options.modality =
-        assistant_client::VoicelessOptions::Modality::TYPING_MODALITY;
-  }
 
-  // Cache metadata about this interaction that can be resolved when the
-  // associated conversation turn starts in LibAssistant.
-  options.conversation_turn_id =
-      NewPendingInteraction(AssistantInteractionType::kText, source, query);
-
-  std::string interaction = CreateTextQueryInteraction(query);
-  assistant_manager_internal()->SendVoicelessInteraction(
-      interaction, /*description=*/"text_query", options, [](auto) {});
+  conversation_controller_proxy().SendTextQuery(
+      query, allow_tts,
+      NewPendingInteraction(AssistantInteractionType::kText, source, query));
 }
 
 void AssistantManagerServiceImpl::AddAssistantInteractionSubscriber(
@@ -1403,6 +1392,11 @@ void AssistantManagerServiceImpl::SetMicState(bool mic_open) {
   audio_input_host_->SetMicState(mic_open);
 }
 
+ConversationControllerProxy&
+AssistantManagerServiceImpl::conversation_controller_proxy() {
+  return assistant_proxy_->conversation_controller_proxy();
+}
+
 ServiceControllerProxy& AssistantManagerServiceImpl::service_controller() {
   return assistant_proxy_->service_controller();
 }

chromeos/services/assistant/assistant_manager_service_impl.h

@@ -24,6 +24,7 @@
 #include "chromeos/services/assistant/assistant_settings_impl.h"
 #include "chromeos/services/assistant/chromium_api_delegate.h"
 #include "chromeos/services/assistant/proxy/assistant_proxy.h"
+#include "chromeos/services/assistant/proxy/conversation_controller_proxy.h"
 #include "chromeos/services/assistant/proxy/libassistant_service_host.h"
 #include "chromeos/services/assistant/public/cpp/assistant_notification.h"
 #include "chromeos/services/assistant/public/cpp/assistant_service.h"
@@ -299,6 +300,7 @@ class COMPONENT_EXPORT(ASSISTANT_SERVICE) AssistantManagerServiceImpl
   DeviceActions* device_actions();
   scoped_refptr<base::SequencedTaskRunner> main_task_runner();
 
+  ConversationControllerProxy& conversation_controller_proxy();
   CrosDisplayConnection* display_connection();
   ServiceControllerProxy& service_controller();
   const ServiceControllerProxy& service_controller() const;

chromeos/services/assistant/proxy/BUILD.gn

@@ -11,6 +11,8 @@ source_set("proxy") {
   sources = [
     "assistant_proxy.cc",
     "assistant_proxy.h",
+    "conversation_controller_proxy.cc",
+    "conversation_controller_proxy.h",
     "libassistant_service_host.h",
     "service_controller_proxy.cc",
     "service_controller_proxy.h",

chromeos/services/assistant/proxy/assistant_proxy.cc

@@ -8,6 +8,7 @@
 
 #include "base/bind.h"
 #include "base/check.h"
+#include "chromeos/services/assistant/proxy/conversation_controller_proxy.h"
 #include "chromeos/services/assistant/proxy/libassistant_service_host.h"
 #include "chromeos/services/assistant/proxy/service_controller_proxy.h"
 #include "chromeos/services/libassistant/libassistant_service.h"
@@ -30,6 +31,9 @@ void AssistantProxy::Initialize(LibassistantServiceHost* host) {
 
   service_controller_proxy_ =
       std::make_unique<ServiceControllerProxy>(host, BindServiceController());
+  conversation_controller_proxy_ =
+      std::make_unique<ConversationControllerProxy>(
+          BindConversationController());
 }
 
 void AssistantProxy::LaunchLibassistantService() {
@@ -78,6 +82,14 @@ AssistantProxy::BindServiceController() {
   return pending_remote;
 }
 
+mojo::PendingRemote<AssistantProxy::ConversationControllerMojom>
+AssistantProxy::BindConversationController() {
+  mojo::PendingRemote<ConversationControllerMojom> pending_remote;
+  libassistant_service_remote_->BindConversationController(
+      pending_remote.InitWithNewPipeAndPassReceiver());
+  return pending_remote;
+}
+
 scoped_refptr<base::SingleThreadTaskRunner>
 AssistantProxy::background_task_runner() {
   return background_thread_.task_runner();
@@ -88,5 +100,10 @@ ServiceControllerProxy& AssistantProxy::service_controller() {
   return *service_controller_proxy_;
 }
 
+ConversationControllerProxy& AssistantProxy::conversation_controller_proxy() {
+  DCHECK(conversation_controller_proxy_);
+  return *conversation_controller_proxy_;
+}
+
 }  // namespace assistant
 }  // namespace chromeos

chromeos/services/assistant/proxy/assistant_proxy.h

@@ -8,6 +8,7 @@
 #include <memory>
 
 #include "base/threading/thread.h"
+#include "chromeos/services/libassistant/public/mojom/conversation_controller.mojom.h"
 #include "chromeos/services/libassistant/public/mojom/service.mojom.h"
 #include "mojo/public/cpp/bindings/remote.h"
 
@@ -20,6 +21,7 @@ class LibassistantService;
 namespace chromeos {
 namespace assistant {
 
+class ConversationControllerProxy;
 class LibassistantServiceHost;
 class ServiceControllerProxy;
 
@@ -38,6 +40,9 @@ class AssistantProxy {
   // service.
   ServiceControllerProxy& service_controller();
 
+  // Returns the controller that manages conversations with Libassistant.
+  ConversationControllerProxy& conversation_controller_proxy();
+
   // The background thread is temporary exposed until the entire Libassistant
   // API is hidden behind this proxy API.
   base::Thread& background_thread() { return background_thread_; }
@@ -47,6 +52,8 @@ class AssistantProxy {
       chromeos::libassistant::mojom::LibassistantService;
   using ServiceControllerMojom =
       chromeos::libassistant::mojom::ServiceController;
+  using ConversationControllerMojom =
+      chromeos::libassistant::mojom::ConversationController;
 
   scoped_refptr<base::SingleThreadTaskRunner> background_task_runner();
 
@@ -57,12 +64,14 @@ class AssistantProxy {
   void StopLibassistantServiceOnBackgroundThread();
 
   mojo::PendingRemote<ServiceControllerMojom> BindServiceController();
+  mojo::PendingRemote<ConversationControllerMojom> BindConversationController();
 
   // Owned by |AssistantManagerServiceImpl|.
   LibassistantServiceHost* libassistant_service_host_ = nullptr;
   mojo::Remote<LibassistantServiceMojom> libassistant_service_remote_;
 
   std::unique_ptr<ServiceControllerProxy> service_controller_proxy_;
+  std::unique_ptr<ConversationControllerProxy> conversation_controller_proxy_;
 
   // The thread on which the Libassistant service runs.
   // Warning: must be the last object, so it is destroyed (and flushed) first.

chromeos/services/assistant/proxy/conversation_controller_proxy.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chromeos/services/assistant/proxy/conversation_controller_proxy.h"
+
+#include "chromeos/assistant/internal/internal_util.h"
+
+namespace chromeos {
+namespace assistant {
+
+ConversationControllerProxy::ConversationControllerProxy(
+    mojo::PendingRemote<ConversationController> conversation_controller_remote)
+    : conversation_controller_remote_(
+          std::move(conversation_controller_remote)) {}
+
+ConversationControllerProxy::~ConversationControllerProxy() = default;
+
+void ConversationControllerProxy::SendTextQuery(
+    const std::string& query,
+    bool allow_tts,
+    const std::string& conversation_id) {
+  conversation_controller_remote_->SendTextQuery(query, allow_tts,
+                                                 conversation_id);
+}
+
+}  // namespace assistant
+}  // namespace chromeos

chromeos/services/assistant/proxy/conversation_controller_proxy.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROMEOS_SERVICES_ASSISTANT_PROXY_CONVERSATION_CONTROLLER_PROXY_H_
+#define CHROMEOS_SERVICES_ASSISTANT_PROXY_CONVERSATION_CONTROLLER_PROXY_H_
+
+#include <string>
+
+#include "chromeos/services/libassistant/public/mojom/conversation_controller.mojom.h"
+#include "mojo/public/cpp/bindings/remote.h"
+
+namespace chromeos {
+namespace assistant {
+
+using chromeos::libassistant::mojom::ConversationController;
+
+class ConversationControllerProxy {
+ public:
+  explicit ConversationControllerProxy(
+      mojo::PendingRemote<ConversationController>
+          conversation_controller_remote);
+  ConversationControllerProxy(const ConversationControllerProxy&) = delete;
+  ConversationControllerProxy& operator=(const ConversationControllerProxy&) =
+      delete;
+  ~ConversationControllerProxy();
+
+  void SendTextQuery(const std::string& query,
+                     bool allow_tts,
+                     const std::string& conversation_id);
+
+ private:
+  mojo::Remote<ConversationController> conversation_controller_remote_;
+};
+
+}  // namespace assistant
+}  // namespace chromeos
+
+#endif  // CHROMEOS_SERVICES_ASSISTANT_PROXY_CONVERSATION_CONTROLLER_PROXY_H_

chromeos/services/assistant/test_support/fake_libassistant_service.h

@@ -33,9 +33,11 @@ class FakeLibassistantService
   void BindServiceController(
       mojo::PendingReceiver<libassistant::mojom::ServiceController> receiver)
       override;
+  void BindConversationController(
+      mojo::PendingReceiver<libassistant::mojom::ConversationController>
+          receiver) override {}
   void BindAudioInputController() override {}
   void BindAudioOutputController() override {}
-  void BindInteractionController() override {}
 
  private:
   mojo::Receiver<libassistant::mojom::LibassistantService> receiver_;

chromeos/services/libassistant/BUILD.gn

@@ -33,6 +33,8 @@ source_set("internal") {
 
   sources = [
     "assistant_manager_observer.h",
+    "conversation_controller.cc",
+    "conversation_controller.h",
     "platform_api.cc",
     "platform_api.h",
     "service_controller.cc",
@@ -40,9 +42,12 @@ source_set("internal") {
   ]
 
   deps = [
+    "//chromeos/assistant/internal",
+    "//chromeos/assistant/internal/proto/google3",
     "//chromeos/services/assistant/public/cpp",
     "//chromeos/services/assistant/public/cpp/migration",
     "//chromeos/services/libassistant/public/mojom",
+    "//libassistant/shared/internal_api:assistant_manager_internal",
     "//libassistant/shared/public",
   ]
 

chromeos/services/libassistant/conversation_controller.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chromeos/services/libassistant/conversation_controller.h"
+
+#include "chromeos/assistant/internal/internal_util.h"
+#include "chromeos/services/libassistant/public/mojom/conversation_controller.mojom.h"
+#include "chromeos/services/libassistant/service_controller.h"
+#include "libassistant/shared/internal_api/assistant_manager_internal.h"
+#include "mojo/public/cpp/bindings/pending_receiver.h"
+
+namespace chromeos {
+namespace libassistant {
+
+ConversationController::ConversationController(
+    ServiceController* service_controller)
+    : receiver_(this), service_controller_(service_controller) {
+  DCHECK(service_controller_);
+}
+
+ConversationController::~ConversationController() = default;
+
+void ConversationController::Bind(
+    mojo::PendingReceiver<mojom::ConversationController> receiver) {
+  // Cannot bind the receiver twice.
+  DCHECK(!receiver_.is_bound());
+  receiver_.Bind(std::move(receiver));
+}
+
+void ConversationController::SendTextQuery(
+    const std::string& query,
+    bool allow_tts,
+    const base::Optional<std::string>& conversation_id) {
+  // DCHECKs if this function gets invoked after the service has been fully
+  // started.
+  // TODO(meilinw): only check for the |ServiceState::kRunning| state instead
+  // after it has been wired up.
+  DCHECK(service_controller_->IsStarted())
+      << "Libassistant service is not ready to handle queries.";
+  DCHECK(assistant_manager_internal());
+
+  // Configs |VoicelessOptions|.
+  assistant_client::VoicelessOptions options;
+  options.is_user_initiated = true;
+  if (!allow_tts) {
+    options.modality =
+        assistant_client::VoicelessOptions::Modality::TYPING_MODALITY;
+  }
+  // Ensure LibAssistant uses the requested conversation id.
+  if (conversation_id.has_value())
+    options.conversation_turn_id = conversation_id.value();
+
+  // Builds text interaction.
+  std::string interaction =
+      chromeos::assistant::CreateTextQueryInteraction(query);
+
+  assistant_manager_internal()->SendVoicelessInteraction(
+      interaction, /*description=*/"text_query", options, [](auto) {});
+}
+
+assistant_client::AssistantManagerInternal*
+ConversationController::assistant_manager_internal() {
+  return service_controller_->assistant_manager_internal();
+}
+
+}  // namespace libassistant
+}  // namespace chromeos

chromeos/services/libassistant/conversation_controller.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROMEOS_SERVICES_LIBASSISTANT_CONVERSATION_CONTROLLER_H_
+#define CHROMEOS_SERVICES_LIBASSISTANT_CONVERSATION_CONTROLLER_H_
+
+#include "base/component_export.h"
+#include "base/optional.h"
+#include "chromeos/services/libassistant/public/mojom/conversation_controller.mojom.h"
+#include "mojo/public/cpp/bindings/receiver.h"
+
+namespace assistant_client {
+class AssistantManagerInternal;
+}  // namespace assistant_client
+
+namespace chromeos {
+namespace libassistant {
+
+class ServiceController;
+
+class COMPONENT_EXPORT(LIBASSISTANT_SERVICE) ConversationController
+    : public mojom::ConversationController {
+ public:
+  explicit ConversationController(ServiceController* service_controller);
+  ConversationController(const ConversationController&) = delete;
+  ConversationController& operator=(const ConversationController&) = delete;
+  ~ConversationController() override;
+
+  void Bind(mojo::PendingReceiver<mojom::ConversationController> receiver);
+
+  // mojom::ConversationController implementation:
+  void SendTextQuery(
+      const std::string& query,
+      bool allow_tts,
+      const base::Optional<std::string>& conversation_id) override;
+
+ private:
+  assistant_client::AssistantManagerInternal* assistant_manager_internal();
+
+  mojo::Receiver<mojom::ConversationController> receiver_;
+
+  // Owned by |LibassistantService|.
+  ServiceController* const service_controller_;
+};
+
+}  // namespace libassistant
+}  // namespace chromeos
+
+#endif  //  CHROMEOS_SERVICES_LIBASSISTANT_CONVERSATION_CONTROLLER_H_

chromeos/services/libassistant/libassistant_service.cc

@@ -10,6 +10,7 @@
 #include "base/check.h"
 #include "base/logging.h"
 #include "chromeos/services/assistant/public/cpp/migration/cros_platform_api.h"
+#include "chromeos/services/libassistant/conversation_controller.h"
 #include "chromeos/services/libassistant/platform_api.h"
 #include "chromeos/services/libassistant/service_controller.h"
 
@@ -23,7 +24,9 @@ LibassistantService::LibassistantService(
     : receiver_(this, std::move(receiver)),
       platform_api_(std::make_unique<PlatformApi>()),
       service_controller_(
-          std::make_unique<ServiceController>(delegate, platform_api_.get())) {
+          std::make_unique<ServiceController>(delegate, platform_api_.get())),
+      conversation_controller_(
+          std::make_unique<ConversationController>(service_controller_.get())) {
   platform_api_->SetAudioInputProvider(&platform_api->GetAudioInputProvider())
       .SetAudioOutputProvider(&platform_api->GetAudioOutputProvider())
       .SetAuthProvider(&platform_api->GetAuthProvider())
@@ -43,5 +46,10 @@ void LibassistantService::SetInitializeCallback(InitializeCallback callback) {
   service_controller().SetInitializeCallback(std::move(callback));
 }
 
+void LibassistantService::BindConversationController(
+    mojo::PendingReceiver<mojom::ConversationController> receiver) {
+  conversation_controller_->Bind(std::move(receiver));
+}
+
 }  // namespace libassistant
 }  // namespace chromeos

chromeos/services/libassistant/libassistant_service.h

@@ -8,7 +8,9 @@
 #include <memory>
 
 #include "base/component_export.h"
+#include "chromeos/services/libassistant/public/mojom/conversation_controller.mojom.h"
 #include "chromeos/services/libassistant/public/mojom/service.mojom.h"
+#include "mojo/public/cpp/bindings/pending_receiver.h"
 #include "mojo/public/cpp/bindings/receiver.h"
 
 namespace assistant_client {
@@ -26,6 +28,7 @@ class CrosPlatformApi;
 namespace chromeos {
 namespace libassistant {
 
+class ConversationController;
 class PlatformApi;
 class ServiceController;
 
@@ -52,13 +55,15 @@ class COMPONENT_EXPORT(LIBASSISTANT_SERVICE) LibassistantService
   // mojom::LibassistantService implementation:
   void BindServiceController(
       mojo::PendingReceiver<mojom::ServiceController> receiver) override;
+  void BindConversationController(
+      mojo::PendingReceiver<mojom::ConversationController> receiver) override;
   void BindAudioInputController() override {}
   void BindAudioOutputController() override {}
-  void BindInteractionController() override {}
 
   mojo::Receiver<mojom::LibassistantService> receiver_;
   std::unique_ptr<PlatformApi> platform_api_;
   std::unique_ptr<ServiceController> service_controller_;
+  std::unique_ptr<ConversationController> conversation_controller_;
 };
 
 }  // namespace libassistant

chromeos/services/libassistant/public/mojom/BUILD.gn

@@ -6,6 +6,7 @@ import("//mojo/public/tools/bindings/mojom.gni")
 
 mojom("mojom") {
   sources = [
+    "conversation_controller.mojom",
     "service.mojom",
     "service_controller.mojom",
   ]

chromeos/services/libassistant/public/mojom/conversation_controller.mojom

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+module chromeos.libassistant.mojom;
+
+// Interface for controller in charge of Assistant conversations.
+interface ConversationController {
+  // Sends the specific text query to Libassistant. |conversation_id| is an
+  // identifier that will be propagated through the speech processor event
+  // pipeline to uniquely identify a conversation turn. If omitted, a unique
+  // identifier will be automatically generated.
+  SendTextQuery(string query, bool allow_tts, string? conversation_id);
+};

chromeos/services/libassistant/public/mojom/service.mojom

@@ -5,6 +5,7 @@
 module chromeos.libassistant.mojom;
 
 import "chromeos/services/libassistant/public/mojom/service_controller.mojom";
+import "chromeos/services/libassistant/public/mojom/conversation_controller.mojom";
 
 // The main interface to the Libassistant service on ChromeOS.
 // Libassistant provides access to the Google Assistant.
@@ -17,9 +18,12 @@ interface LibassistantService {
   // of the Libassistant process (start/stop).
   BindServiceController(pending_receiver<ServiceController> receiver);
 
+  // Bind the conversation controller, which in charge of handling
+  // conversations with Libassistant.
+  BindConversationController(pending_receiver<ConversationController> receiver);
+
   // This service will further expose methods to bind all other helper
   // controllers.
   BindAudioInputController(/* pending_receiver */);
   BindAudioOutputController(/* pending_receiver */);
-  BindInteractionController(/* pending_receiver */);
 };

chromeos/services/libassistant/service_controller.cc

@@ -124,7 +124,7 @@ void ServiceController::RemoveAssistantManagerObserver(
 }
 
 bool ServiceController::IsStarted() const {
-  return state_ != mojom::ServiceState::kStopped;
+  return state_ != ServiceState::kStopped;
 }
 
 bool ServiceController::IsInitialized() const {