Introduce DisplayController mojom API

This CL is part of a migration of all interactions with the Libassistant
3rd party library to a mojom service. Currently Libassistant is invoked
directly from the browser thread.

This specific CL focuses on the display controller, which in the current
Libassistant V1 API has 2 jobs:
   - Configure Libassistant parameters that only apply to devices which
     have a display (as opposed to for example Google home devices).
   - Receive updates about the user speech input volume.

In the new Libassistant V2 API, the first job is handled by the display
service (http://shortn/_XZThaXrQKb), while the second job is handled by
a newly introduce recognition event API (http://shortn/_GyiHLrIRtn).

As such, this CL also introduces 2 distinct mojom files to handle these
different jobs.

Bug: b/177471991
Bug: 1168230
Test: compiled & deployed on real hardware
Change-Id: Ic404db5b29d9c12332b584501a43012b9ee8312d
Cq-Include-Trybots: luci.chrome.try:linux-chromeos-chrome
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2625963
Commit-Queue: Jeroen Dhollander <jeroendh@chromium.org>
Reviewed-by: Sam McNally <sammc@chromium.org>
Reviewed-by: Meilin Wang <meilinw@chromium.org>
Reviewed-by: Xiaohui Chen <xiaohuic@chromium.org>
Cr-Commit-Position: refs/heads/master@{#845291}

chromeos/services/assistant/assistant_manager_service_impl.cc

@@ -21,6 +21,7 @@
 #include "base/i18n/rtl.h"
 #include "base/logging.h"
 #include "base/metrics/histogram_macros.h"
+#include "base/notreached.h"
 #include "base/run_loop.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/utf_string_conversions.h"
@@ -47,6 +48,7 @@
 #include "chromeos/services/assistant/public/shared/utils.h"
 #include "chromeos/services/assistant/service_context.h"
 #include "chromeos/services/assistant/utils.h"
+#include "chromeos/services/libassistant/public/mojom/speech_recognition_observer.mojom.h"
 #include "chromeos/strings/grit/chromeos_strings.h"
 #include "libassistant/shared/internal_api/alarm_timer_manager.h"
 #include "libassistant/shared/internal_api/alarm_timer_types.h"
@@ -163,8 +165,49 @@ bool ShouldPutLogsInHomeDirectory() {
   return !redirect_logging;
 }
 
+libassistant::mojom::AndroidAppInfoPtr ToAndroidAppInfoPtr(
+    const AndroidAppInfo& app_info) {
+  return libassistant::mojom::AndroidAppInfo::New(
+      app_info.package_name, app_info.version, app_info.localized_app_name);
+}
+
 }  // namespace
 
+// Observer that will receive all speech recognition related events,
+// and forwards them to all |AssistantInteractionSubscriber|.
+class SpeechRecognitionObserverWrapper
+    : public libassistant::mojom::SpeechRecognitionObserver {
+ public:
+  explicit SpeechRecognitionObserverWrapper(
+      const base::ObserverList<AssistantInteractionSubscriber>* observers)
+      : interaction_subscribers_(*observers) {
+    DCHECK(observers);
+  }
+  SpeechRecognitionObserverWrapper(const SpeechRecognitionObserverWrapper&) =
+      delete;
+  SpeechRecognitionObserverWrapper& operator=(
+      const SpeechRecognitionObserverWrapper&) = delete;
+  ~SpeechRecognitionObserverWrapper() override = default;
+
+  mojo::PendingRemote<chromeos::libassistant::mojom::SpeechRecognitionObserver>
+  BindNewPipeAndPassRemote() {
+    return receiver_.BindNewPipeAndPassRemote();
+  }
+
+  // libassistant::mojom::SpeechRecognitionObserver implementation:
+  void OnSpeechLevelUpdated(float speech_level_in_decibels) override {
+    for (auto& it : interaction_subscribers_)
+      it.OnSpeechLevelUpdated(speech_level_in_decibels);
+  }
+
+ private:
+  // Owned by our parent, |AssistantManagerServiceImpl|.
+  const base::ObserverList<AssistantInteractionSubscriber>&
+      interaction_subscribers_;
+
+  mojo::Receiver<chromeos::libassistant::mojom::SpeechRecognitionObserver>
+      receiver_{this};
+};
 AssistantManagerServiceImpl::AssistantManagerServiceImpl(
     ServiceContext* context,
     std::unique_ptr<AssistantManagerServiceDelegate> delegate,
@@ -183,6 +226,9 @@ AssistantManagerServiceImpl::AssistantManagerServiceImpl(
       assistant_proxy_(std::make_unique<AssistantProxy>()),
       context_(context),
       delegate_(std::move(delegate)),
+      speech_recognition_observer_(
+          std::make_unique<SpeechRecognitionObserverWrapper>(
+              &interaction_subscribers_)),
       bootup_config_(ServiceControllerProxy::BootupConfig::New(
           s3_server_uri_override,
           device_id_override,
@@ -209,6 +255,9 @@ AssistantManagerServiceImpl::AssistantManagerServiceImpl(
   assistant_proxy_->Initialize(libassistant_service_host_.get(),
                                std::move(pending_url_loader_factory));
 
+  assistant_proxy_->AddSpeechRecognitionObserver(
+      speech_recognition_observer_->BindNewPipeAndPassRemote());
+
   audio_input_host_ = delegate_->CreateAudioInputHost();
 
   platform_api_->InitializeAudioInputHost(*audio_input_host_);
@@ -382,7 +431,7 @@ void AssistantManagerServiceImpl::EnableHotword(bool enable) {
 void AssistantManagerServiceImpl::SetArcPlayStoreEnabled(bool enable) {
   DCHECK(GetState() == State::RUNNING);
   if (assistant::features::IsAppSupportEnabled())
-    display_connection()->SetArcPlayStoreEnabled(enable);
+    display_controller().SetArcPlayStoreEnabled(enable);
 }
 
 void AssistantManagerServiceImpl::SetAssistantContextEnabled(bool enable) {
@@ -395,7 +444,7 @@ void AssistantManagerServiceImpl::SetAssistantContextEnabled(bool enable) {
     ResetMediaState();
   }
 
-  display_connection()->SetAssistantContextEnabled(enable);
+  display_controller().SetRelatedInfoEnabled(enable);
 }
 
 AssistantSettings* AssistantManagerServiceImpl::GetAssistantSettings() {
@@ -933,15 +982,6 @@ void AssistantManagerServiceImpl::OnRespondingStarted(bool is_error_response) {
     it.OnTtsStarted(is_error_response);
 }
 
-void AssistantManagerServiceImpl::OnSpeechLevelUpdated(
-    const float speech_level) {
-  ENSURE_MAIN_THREAD(&AssistantManagerServiceImpl::OnSpeechLevelUpdated,
-                     speech_level);
-
-  for (auto& it : interaction_subscribers_)
-    it.OnSpeechLevelUpdated(speech_level);
-}
-
 void AssistantManagerServiceImpl::OnModifyDeviceSetting(
     const api::client_op::ModifySettingArgs& modify_setting_args) {
   ENSURE_MAIN_THREAD(&AssistantManagerServiceImpl::OnModifyDeviceSetting,
@@ -998,8 +1038,7 @@ void AssistantManagerServiceImpl::InitAssistant(
       action_module_.get(),
       /*assistant_manager_delegate=*/this,
       /*conversation_state_listener=*/this,
-      /*device_state_listener=*/this,
-      /*event_observer=*/this, std::move(bootup_config_), locale,
+      /*device_state_listener=*/this, std::move(bootup_config_), locale,
       GetLocaleOrDefault(assistant_state()->locale().value()),
       spoken_feedback_enabled_, ToAuthTokensOrEmpty(user),
       base::BindOnce(&AssistantManagerServiceImpl::PostInitAssistant,
@@ -1080,15 +1119,15 @@ void AssistantManagerServiceImpl::OnStartFinished() {
 
 void AssistantManagerServiceImpl::OnAndroidAppListRefreshed(
     const std::vector<AndroidAppInfo>& apps_info) {
-  std::vector<AndroidAppInfo> filtered_apps_info;
+  std::vector<libassistant::mojom::AndroidAppInfoPtr> filtered_apps_info;
   for (const auto& app_info : apps_info) {
     // TODO(b/146355799): Remove the special handling for Android settings app.
     if (app_info.package_name == kAndroidSettingsAppPackage)
       continue;
 
-    filtered_apps_info.emplace_back(app_info);
+    filtered_apps_info.emplace_back(ToAndroidAppInfoPtr(app_info));
   }
-  display_connection()->OnAndroidAppListRefreshed(filtered_apps_info);
+  display_controller().SetAndroidAppList(std::move(filtered_apps_info));
 }
 
 void AssistantManagerServiceImpl::OnPlaybackStateChange(
@@ -1187,7 +1226,7 @@ void AssistantManagerServiceImpl::OnDeviceAppsEnabled(bool enabled) {
   if (GetState() != State::RUNNING)
     return;
 
-  display_connection()->SetDeviceAppsEnabled(enabled);
+  display_controller().SetDeviceAppsEnabled(enabled);
   action_module_->SetAppSupportEnabled(
       assistant::features::IsAppSupportEnabled() && enabled);
 }
@@ -1385,8 +1424,9 @@ AssistantManagerServiceImpl::main_task_runner() {
   return context_->main_task_runner();
 }
 
-CrosDisplayConnection* AssistantManagerServiceImpl::display_connection() {
-  return service_controller().display_connection();
+AssistantProxy::DisplayController&
+AssistantManagerServiceImpl::display_controller() {
+  return assistant_proxy_->display_controller();
 }
 
 assistant_client::AssistantManager*

chromeos/services/assistant/assistant_manager_service_impl.h

@@ -19,7 +19,6 @@
 #include "base/synchronization/lock.h"
 #include "base/threading/thread.h"
 #include "chromeos/assistant/internal/action/cros_action_module.h"
-#include "chromeos/assistant/internal/cros_display_connection.h"
 #include "chromeos/assistant/internal/internal_util.h"
 #include "chromeos/services/assistant/assistant_manager_service.h"
 #include "chromeos/services/assistant/assistant_settings_impl.h"
@@ -69,6 +68,7 @@ class AudioInputHost;
 class CrosPlatformApi;
 class ServiceContext;
 class ServiceControllerProxy;
+class SpeechRecognitionObserverWrapper;
 
 // Enumeration of Assistant query response type, also recorded in histograms.
 // These values are persisted to logs. Entries should not be renumbered and
@@ -104,7 +104,6 @@ enum class AssistantQueryResponseType {
 class COMPONENT_EXPORT(ASSISTANT_SERVICE) AssistantManagerServiceImpl
     : public AssistantManagerService,
       public ::chromeos::assistant::action::AssistantActionObserver,
-      public AssistantEventObserver,
       public assistant_client::ConversationStateListener,
       public assistant_client::AssistantManagerDelegate,
       public assistant_client::DeviceStateListener,
@@ -193,9 +192,6 @@ class COMPONENT_EXPORT(ASSISTANT_SERVICE) AssistantManagerServiceImpl
       int interaction_id,
       const ::assistant::api::client_op::GetDeviceSettingsArgs& args) override;
 
-  // AssistantEventObserver overrides:
-  void OnSpeechLevelUpdated(float speech_level) override;
-
   // assistant_client::ConversationStateListener overrides:
   void OnConversationTurnFinished(
       assistant_client::ConversationStateListener::Resolution resolution)
@@ -306,7 +302,7 @@ class COMPONENT_EXPORT(ASSISTANT_SERVICE) AssistantManagerServiceImpl
   scoped_refptr<base::SequencedTaskRunner> main_task_runner();
 
   ConversationControllerProxy& conversation_controller_proxy();
-  CrosDisplayConnection* display_connection();
+  AssistantProxy::DisplayController& display_controller();
   ServiceControllerProxy& service_controller();
   const ServiceControllerProxy& service_controller() const;
   base::Thread& background_thread();
@@ -334,6 +330,8 @@ class COMPONENT_EXPORT(ASSISTANT_SERVICE) AssistantManagerServiceImpl
   std::unique_ptr<AssistantManagerServiceDelegate> delegate_;
   std::unique_ptr<LibassistantServiceHost> libassistant_service_host_;
   std::unique_ptr<AssistantDeviceSettingsDelegate> settings_delegate_;
+  std::unique_ptr<SpeechRecognitionObserverWrapper>
+      speech_recognition_observer_;
 
   bool spoken_feedback_enabled_ = false;
 

chromeos/services/assistant/proxy/assistant_proxy.cc

@@ -95,6 +95,7 @@ void AssistantProxy::BindControllers(
       pending_audio_input_controller_remote.InitWithNewPipeAndPassReceiver(),
       std::move(pending_audio_stream_factory_delegate_remote),
       pending_conversation_controller_remote.InitWithNewPipeAndPassReceiver(),
+      display_controller_remote_.BindNewPipeAndPassReceiver(),
       pending_service_controller_remote.InitWithNewPipeAndPassReceiver());
 
   service_controller_proxy_ = std::make_unique<ServiceControllerProxy>(
@@ -120,5 +121,17 @@ ConversationControllerProxy& AssistantProxy::conversation_controller_proxy() {
   return *conversation_controller_proxy_;
 }
 
+AssistantProxy::DisplayController& AssistantProxy::display_controller() {
+  DCHECK(display_controller_remote_.is_bound());
+  return *display_controller_remote_.get();
+}
+
+void AssistantProxy::AddSpeechRecognitionObserver(
+    mojo::PendingRemote<
+        chromeos::libassistant::mojom::SpeechRecognitionObserver> observer) {
+  libassistant_service_remote_->AddSpeechRecognitionObserver(
+      std::move(observer));
+}
+
 }  // namespace assistant
 }  // namespace chromeos

chromeos/services/assistant/proxy/assistant_proxy.h

@@ -33,6 +33,8 @@ class ServiceControllerProxy;
 // access point to the entire Assistant API.
 class AssistantProxy {
  public:
+  using DisplayController = chromeos::libassistant::mojom::DisplayController;
+
   AssistantProxy();
   AssistantProxy(AssistantProxy&) = delete;
   AssistantProxy& operator=(AssistantProxy&) = delete;
@@ -49,10 +51,19 @@ class AssistantProxy {
   // Returns the controller that manages conversations with Libassistant.
   ConversationControllerProxy& conversation_controller_proxy();
 
+  // Returns the controller that manages display related settings.
+  DisplayController& display_controller();
+
   // The background thread is temporary exposed until the entire Libassistant
   // API is hidden behind this proxy API.
   base::Thread& background_thread() { return background_thread_; }
 
+  // Add an observer that will be informed of all speech recognition related
+  // updates.
+  void AddSpeechRecognitionObserver(
+      mojo::PendingRemote<
+          chromeos::libassistant::mojom::SpeechRecognitionObserver> observer);
+
  private:
   using AudioInputControllerMojom =
       chromeos::libassistant::mojom::AudioInputController;
@@ -60,6 +71,8 @@ class AssistantProxy {
       chromeos::libassistant::mojom::AudioStreamFactoryDelegate;
   using ConversationControllerMojom =
       chromeos::libassistant::mojom::ConversationController;
+  using DisplayControllerMojom =
+      chromeos::libassistant::mojom::DisplayController;
   using LibassistantServiceMojom =
       chromeos::libassistant::mojom::LibassistantService;
   using ServiceControllerMojom =
@@ -80,6 +93,7 @@ class AssistantProxy {
   // Owned by |AssistantManagerServiceImpl|.
   LibassistantServiceHost* libassistant_service_host_ = nullptr;
   mojo::Remote<LibassistantServiceMojom> libassistant_service_remote_;
+  mojo::Remote<DisplayControllerMojom> display_controller_remote_;
 
   std::unique_ptr<ServiceControllerProxy> service_controller_proxy_;
   std::unique_ptr<ConversationControllerProxy> conversation_controller_proxy_;

chromeos/services/assistant/proxy/service_controller_proxy.cc

@@ -8,7 +8,6 @@
 #include "base/logging.h"
 #include "base/notreached.h"
 #include "base/optional.h"
-#include "chromeos/assistant/internal/cros_display_connection.h"
 #include "chromeos/assistant/internal/internal_util.h"
 #include "chromeos/constants/chromeos_features.h"
 #include "chromeos/services/assistant/proxy/libassistant_service_host.h"
@@ -48,7 +47,6 @@ struct StartArguments {
   assistant_client::AssistantManagerDelegate* assistant_manager_delegate;
   assistant_client::ConversationStateListener* conversation_state_listener;
   assistant_client::DeviceStateListener* device_state_listener;
-  CrosDisplayConnection* display_connection;
 };
 
 void FillServerExperimentIds(std::vector<std::string>* server_experiment_ids) {
@@ -81,8 +79,6 @@ void InitializeAssistantManager(
     StartArguments arguments,
     assistant_client::AssistantManager* assistant_manager,
     assistant_client::AssistantManagerInternal* assistant_manager_internal) {
-  assistant_manager_internal->SetDisplayConnection(
-      arguments.display_connection);
   assistant_manager_internal->RegisterActionModule(arguments.action_module);
   assistant_manager_internal->SetAssistantManagerDelegate(
       arguments.assistant_manager_delegate);
@@ -129,7 +125,6 @@ void ServiceControllerProxy::Start(
     assistant_client::AssistantManagerDelegate* assistant_manager_delegate,
     assistant_client::ConversationStateListener* conversation_state_listener,
     assistant_client::DeviceStateListener* device_state_listener,
-    AssistantEventObserver* event_observer,
     BootupConfigPtr bootup_config,
     const std::string& locale,
     const std::string& locale_override,
@@ -140,10 +135,6 @@ void ServiceControllerProxy::Start(
   DCHECK_EQ(state_, State::kStopped);
   state_ = State::kStarting;
 
-  pending_display_connection_ = std::make_unique<CrosDisplayConnection>(
-      event_observer, /*feedback_ui_enabled=*/true,
-      assistant::features::IsMediaSessionIntegrationEnabled());
-
   // The mojom service will create the |AssistantManager|.
   service_controller_remote_->Initialize(std::move(bootup_config),
                                          BindURLLoaderFactory());
@@ -159,7 +150,6 @@ void ServiceControllerProxy::Start(
   arguments.assistant_manager_delegate = assistant_manager_delegate;
   arguments.conversation_state_listener = conversation_state_listener;
   arguments.device_state_listener = device_state_listener;
-  arguments.display_connection = pending_display_connection_.get();
 
   host_->SetInitializeCallback(
       base::BindOnce(InitializeAssistantManager, std::move(arguments)));
@@ -205,7 +195,6 @@ void ServiceControllerProxy::OnStateChanged(ServiceState new_state) {
       NOTIMPLEMENTED();
       break;
     case ServiceState::kStopped:
-      display_connection_ = nullptr;
       break;
   }
 }
@@ -229,15 +218,12 @@ void ServiceControllerProxy::FinishCreatingAssistant() {
     // This means the |AssistantManager| could be destroyed at any second,
     // so we simply clean up and bail out.
     on_start_done_callback_.reset();
-    pending_display_connection_ = nullptr;
     return;
   }
 
   DCHECK(on_start_done_callback_.has_value());
-  DCHECK_NE(pending_display_connection_, nullptr);
 
   state_ = State::kStarted;
-  display_connection_ = std::move(pending_display_connection_);
   std::move(on_start_done_callback_.value()).Run();
 }
 

chromeos/services/assistant/proxy/service_controller_proxy.h

@@ -5,7 +5,6 @@
 #ifndef CHROMEOS_SERVICES_ASSISTANT_PROXY_SERVICE_CONTROLLER_PROXY_H_
 #define CHROMEOS_SERVICES_ASSISTANT_PROXY_SERVICE_CONTROLLER_PROXY_H_
 
-#include <memory>
 #include <string>
 #include <utility>
 #include <vector>
@@ -34,8 +33,6 @@ class SharedURLLoaderFactory;
 namespace chromeos {
 namespace assistant {
 
-class AssistantEventObserver;
-class CrosDisplayConnection;
 class LibassistantServiceHost;
 
 // Component managing the lifecycle of Libassistant,
@@ -58,12 +55,6 @@ class ServiceControllerProxy : private libassistant::mojom::StateObserver {
   ServiceControllerProxy& operator=(ServiceControllerProxy&) = delete;
   ~ServiceControllerProxy() override;
 
-  // Can not be invoked before Start() has finished.
-  CrosDisplayConnection* display_connection() {
-    DCHECK(display_connection_);
-    return display_connection_.get();
-  }
-
   // Initialize the |AssistantManager| and all related objects.
   // Will signal the objects exist and can be accessed by calling the
   // |done_callback|.
@@ -74,7 +65,6 @@ class ServiceControllerProxy : private libassistant::mojom::StateObserver {
       assistant_client::AssistantManagerDelegate* assistant_manager_delegate,
       assistant_client::ConversationStateListener* conversation_state_listener,
       assistant_client::DeviceStateListener* device_state_listener,
-      AssistantEventObserver* event_observer,
       BootupConfigPtr bootup_config,
       const std::string& locale,
       const std::string& locale_override,
@@ -142,11 +132,6 @@ class ServiceControllerProxy : private libassistant::mojom::StateObserver {
   // has started.
   base::Optional<base::OnceClosure> on_start_done_callback_;
 
-  std::unique_ptr<CrosDisplayConnection> display_connection_;
-  // Populated when we're starting but not started yet, so after Start() has
-  // been called but before the mojom service signalled it has started.
-  std::unique_ptr<CrosDisplayConnection> pending_display_connection_;
-
   base::WeakPtrFactory<ServiceControllerProxy> weak_factory_{this};
 };
 }  // namespace assistant

chromeos/services/assistant/test_support/fake_libassistant_service.cc

@@ -33,6 +33,8 @@ void FakeLibassistantService::Bind(
         audio_stream_factory_delegate,
     mojo::PendingReceiver<libassistant::mojom::ConversationController>
         conversation_controller,
+    mojo::PendingReceiver<libassistant::mojom::DisplayController>
+        display_controller,
     mojo::PendingReceiver<libassistant::mojom::ServiceController>
         service_controller) {
   service_controller_.Bind(std::move(service_controller));

chromeos/services/assistant/test_support/fake_libassistant_service.h

@@ -36,8 +36,13 @@ class FakeLibassistantService
                 audio_stream_factory_delegate,
             mojo::PendingReceiver<libassistant::mojom::ConversationController>
                 conversation_controller,
+            mojo::PendingReceiver<libassistant::mojom::DisplayController>
+                display_controller,
             mojo::PendingReceiver<libassistant::mojom::ServiceController>
                 service_controller) override;
+  void AddSpeechRecognitionObserver(
+      mojo::PendingRemote<libassistant::mojom::SpeechRecognitionObserver>
+          observer) override {}
 
  private:
   mojo::Receiver<libassistant::mojom::LibassistantService> receiver_;

chromeos/services/libassistant/BUILD.gn

@@ -48,6 +48,8 @@ source_set("internal") {
     "chromium_http_connection.h",
     "conversation_controller.cc",
     "conversation_controller.h",
+    "display_controller.cc",
+    "display_controller.h",
     "platform_api.cc",
     "platform_api.h",
     "service_controller.cc",

chromeos/services/libassistant/display_controller.cc

+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chromeos/services/libassistant/display_controller.h"
+
+#include <memory>
+
+#include "chromeos/assistant/internal/cros_display_connection.h"
+#include "chromeos/services/assistant/public/cpp/features.h"
+#include "chromeos/services/libassistant/public/mojom/speech_recognition_observer.mojom.h"
+#include "libassistant/shared/internal_api/assistant_manager_internal.h"
+
+namespace chromeos {
+namespace libassistant {
+
+namespace {
+
+assistant::AndroidAppInfo ToAndroidAppInfo(
+    const mojom::AndroidAppInfoPtr& app) {
+  assistant::AndroidAppInfo result;
+  result.package_name = app->package_name;
+  result.version = app->version;
+  result.localized_app_name = app->localized_app_name;
+  return result;
+}
+
+std::vector<assistant::AndroidAppInfo> ToAndroidAppInfoList(
+    const std::vector<mojom::AndroidAppInfoPtr>& apps) {
+  std::vector<assistant::AndroidAppInfo> result;
+
+  for (const auto& app : apps)
+    result.push_back(ToAndroidAppInfo(app));
+
+  return result;
+}
+
+}  // namespace
+
+class DisplayController::EventObserver
+    : public assistant::AssistantEventObserver {
+ public:
+  explicit EventObserver(DisplayController* parent) : parent_(parent) {}
+  EventObserver(const EventObserver&) = delete;
+  EventObserver& operator=(const EventObserver&) = delete;
+  ~EventObserver() override = default;
+
+  void OnSpeechLevelUpdated(const float speech_level) override {
+    for (auto& observer : parent_->speech_recognition_observers_)
+      observer->OnSpeechLevelUpdated(speech_level);
+  }
+
+ private:
+  DisplayController* const parent_;
+};
+
+DisplayController::DisplayController(
+    mojo::RemoteSet<mojom::SpeechRecognitionObserver>*
+        speech_recognition_observers)
+    : event_observer_(std::make_unique<EventObserver>(this)),
+      display_connection_(std::make_unique<assistant::CrosDisplayConnection>(
+          event_observer_.get(),
+          /*feedback_ui_enabled=*/true,
+          assistant::features::IsMediaSessionIntegrationEnabled())),
+      speech_recognition_observers_(*speech_recognition_observers) {
+  DCHECK(speech_recognition_observers);
+}
+
+DisplayController::~DisplayController() = default;
+
+void DisplayController::Bind(
+    mojo::PendingReceiver<mojom::DisplayController> receiver) {
+  receiver_.Bind(std::move(receiver));
+}
+
+void DisplayController::SetArcPlayStoreEnabled(bool enabled) {
+  display_connection_->SetArcPlayStoreEnabled(enabled);
+}
+
+void DisplayController::SetDeviceAppsEnabled(bool enabled) {
+  display_connection_->SetDeviceAppsEnabled(enabled);
+}
+
+void DisplayController::SetRelatedInfoEnabled(bool enabled) {
+  display_connection_->SetAssistantContextEnabled(enabled);
+}
+
+void DisplayController::SetAndroidAppList(
+    std::vector<mojom::AndroidAppInfoPtr> apps) {
+  display_connection_->OnAndroidAppListRefreshed(ToAndroidAppInfoList(apps));
+}
+
+void DisplayController::OnAssistantManagerCreated(
+    assistant_client::AssistantManager* assistant_manager,
+    assistant_client::AssistantManagerInternal* assistant_manager_internal) {
+  assistant_manager_internal->SetDisplayConnection(display_connection_.get());
+}
+
+}  // namespace libassistant
+}  // namespace chromeos

chromeos/services/libassistant/display_controller.h

+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROMEOS_SERVICES_LIBASSISTANT_DISPLAY_CONTROLLER_H_
+#define CHROMEOS_SERVICES_LIBASSISTANT_DISPLAY_CONTROLLER_H_
+
+#include "chromeos/services/libassistant/assistant_manager_observer.h"
+#include "chromeos/services/libassistant/public/mojom/display_controller.mojom.h"
+#include "mojo/public/cpp/bindings/receiver.h"
+#include "mojo/public/cpp/bindings/remote_set.h"
+
+namespace chromeos {
+namespace libassistant {
+namespace mojom {
+class SpeechRecognitionObserver;
+}  // namespace mojom
+}  // namespace libassistant
+}  // namespace chromeos
+
+namespace chromeos {
+namespace assistant {
+class CrosDisplayConnection;
+}  // namespace assistant
+}  // namespace chromeos
+
+namespace chromeos {
+namespace libassistant {
+
+class DisplayController : public mojom::DisplayController,
+                          public AssistantManagerObserver {
+ public:
+  explicit DisplayController(mojo::RemoteSet<mojom::SpeechRecognitionObserver>*
+                                 speech_recognition_observers);
+  DisplayController(const DisplayController&) = delete;
+  DisplayController& operator=(const DisplayController&) = delete;
+  ~DisplayController() override;
+
+  void Bind(mojo::PendingReceiver<mojom::DisplayController> receiver);
+
+  // mojom::DisplayController implementation:
+  void SetArcPlayStoreEnabled(bool enabled) override;
+  void SetDeviceAppsEnabled(bool enabled) override;
+  void SetRelatedInfoEnabled(bool enabled) override;
+  void SetAndroidAppList(std::vector<mojom::AndroidAppInfoPtr> apps) override;
+
+  // AssistantManagerObserver implementation:
+  void OnAssistantManagerCreated(
+      assistant_client::AssistantManager* assistant_manager,
+      assistant_client::AssistantManagerInternal* assistant_manager_internal)
+      override;
+
+ private:
+  class EventObserver;
+
+  mojo::Receiver<mojom::DisplayController> receiver_{this};
+  std::unique_ptr<EventObserver> event_observer_;
+  std::unique_ptr<assistant::CrosDisplayConnection> display_connection_;
+
+  // Owned by |LibassistantService|.
+  mojo::RemoteSet<mojom::SpeechRecognitionObserver>&
+      speech_recognition_observers_;
+};
+
+}  // namespace libassistant
+}  // namespace chromeos
+
+#endif  // CHROMEOS_SERVICES_LIBASSISTANT_DISPLAY_CONTROLLER_H_

chromeos/services/libassistant/libassistant_service.cc

@@ -12,6 +12,7 @@
 #include "chromeos/services/assistant/public/cpp/migration/cros_platform_api.h"
 #include "chromeos/services/libassistant/audio_input_controller.h"
 #include "chromeos/services/libassistant/conversation_controller.h"
+#include "chromeos/services/libassistant/display_controller.h"
 #include "chromeos/services/libassistant/platform_api.h"
 #include "chromeos/services/libassistant/service_controller.h"
 
@@ -24,11 +25,16 @@ LibassistantService::LibassistantService(
     assistant::AssistantManagerServiceDelegate* delegate)
     : receiver_(this, std::move(receiver)),
       platform_api_(std::make_unique<PlatformApi>()),
+      display_controller_(
+          std::make_unique<DisplayController>(&speech_recognition_observers_)),
       service_controller_(
           std::make_unique<ServiceController>(delegate, platform_api_.get())),
       conversation_controller_(
           std::make_unique<ConversationController>(service_controller_.get())),
       audio_input_controller_(std::make_unique<AudioInputController>()) {
+  service_controller_->AddAndFireAssistantManagerObserver(
+      display_controller_.get());
+
   platform_api_->SetAudioInputProvider(&platform_api->GetAudioInputProvider())
       .SetAudioOutputProvider(&platform_api->GetAudioOutputProvider())
       .SetAuthProvider(&platform_api->GetAuthProvider())
@@ -37,7 +43,10 @@ LibassistantService::LibassistantService(
       .SetSystemProvider(&platform_api->GetSystemProvider());
 }
 
-LibassistantService::~LibassistantService() = default;
+LibassistantService::~LibassistantService() {
+  service_controller_->RemoveAssistantManagerObserver(
+      display_controller_.get());
+}
 
 void LibassistantService::Bind(
     mojo::PendingReceiver<mojom::AudioInputController> audio_input_controller,
@@ -45,16 +54,23 @@ void LibassistantService::Bind(
         audio_stream_factory_delegate,
     mojo::PendingReceiver<mojom::ConversationController>
         conversation_controller,
+    mojo::PendingReceiver<mojom::DisplayController> display_controller,
     mojo::PendingReceiver<mojom::ServiceController> service_controller) {
   audio_input_controller_->Bind(std::move(audio_input_controller),
                                 std::move(audio_stream_factory_delegate));
-  service_controller_->Bind(std::move(service_controller));
   conversation_controller_->Bind(std::move(conversation_controller));
+  display_controller_->Bind(std::move(display_controller));
+  service_controller_->Bind(std::move(service_controller));
 }
 
 void LibassistantService::SetInitializeCallback(InitializeCallback callback) {
   service_controller().SetInitializeCallback(std::move(callback));
 }
 
+void LibassistantService::AddSpeechRecognitionObserver(
+    mojo::PendingRemote<mojom::SpeechRecognitionObserver> observer) {
+  speech_recognition_observers_.Add(std::move(observer));
+}
+
 }  // namespace libassistant
 }  // namespace chromeos

chromeos/services/libassistant/libassistant_service.h

@@ -13,6 +13,7 @@
 #include "chromeos/services/libassistant/public/mojom/service.mojom.h"
 #include "mojo/public/cpp/bindings/pending_receiver.h"
 #include "mojo/public/cpp/bindings/receiver.h"
+#include "mojo/public/cpp/bindings/remote_set.h"
 
 namespace assistant_client {
 class AssistantManager;
@@ -31,6 +32,7 @@ namespace libassistant {
 
 class AudioInputController;
 class ConversationController;
+class DisplayController;
 class PlatformApi;
 class ServiceController;
 
@@ -61,11 +63,19 @@ class COMPONENT_EXPORT(LIBASSISTANT_SERVICE) LibassistantService
           audio_stream_factory_delegate,
       mojo::PendingReceiver<mojom::ConversationController>
           conversation_controller,
+      mojo::PendingReceiver<mojom::DisplayController> display_controller,
       mojo::PendingReceiver<mojom::ServiceController> service_controller)
       override;
+  void AddSpeechRecognitionObserver(
+      mojo::PendingRemote<mojom::SpeechRecognitionObserver> observer) override;
 
   mojo::Receiver<mojom::LibassistantService> receiver_;
+
+  mojo::RemoteSet<mojom::SpeechRecognitionObserver>
+      speech_recognition_observers_;
+
   std::unique_ptr<PlatformApi> platform_api_;
+  std::unique_ptr<DisplayController> display_controller_;
   std::unique_ptr<ServiceController> service_controller_;
   std::unique_ptr<ConversationController> conversation_controller_;
   std::unique_ptr<AudioInputController> audio_input_controller_;

chromeos/services/libassistant/public/mojom/BUILD.gn

@@ -8,8 +8,10 @@ mojom("mojom") {
   sources = [
     "audio_input_controller.mojom",
     "conversation_controller.mojom",
+    "display_controller.mojom",
     "service.mojom",
     "service_controller.mojom",
+    "speech_recognition_observer.mojom",
   ]
 
   deps = [

chromeos/services/libassistant/public/mojom/display_controller.mojom

+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+module chromeos.libassistant.mojom;
+
+// Interface grouping display related settings for Libassistant.
+interface DisplayController {
+  // Enable/disable ARC play store.
+  // This is disabled by default.
+  SetArcPlayStoreEnabled(bool enabled);
+
+  // Enable/disable device apps.
+  // Device Apps stores user's apps which could help improve user experience
+  // across Google services.
+  // This is disabled by default, and can not be disabled once it has been
+  // enabled.
+  SetDeviceAppsEnabled(bool enabled);
+
+  // Enable/disable Assistant related information.
+  // This is disabled by default.
+  SetRelatedInfoEnabled(bool enabled);
+
+  // Send the list with installed Android apps to Libassistant.
+  SetAndroidAppList(array<AndroidAppInfo> apps);
+};
+
+
+// Information about an installed Android app.
+struct AndroidAppInfo {
+  // Unique name to identify a specific app.
+  string package_name;
+
+  // Version number of the app.
+  int32 version;
+
+  // Localized app name.
+  string localized_app_name;
+};

chromeos/services/libassistant/public/mojom/service.mojom

@@ -6,7 +6,9 @@ module chromeos.libassistant.mojom;
 
 import "chromeos/services/libassistant/public/mojom/audio_input_controller.mojom";
 import "chromeos/services/libassistant/public/mojom/conversation_controller.mojom";
+import "chromeos/services/libassistant/public/mojom/display_controller.mojom";
 import "chromeos/services/libassistant/public/mojom/service_controller.mojom";
+import "chromeos/services/libassistant/public/mojom/speech_recognition_observer.mojom";
 
 // The main interface to the Libassistant service on ChromeOS.
 // Libassistant provides access to the Google Assistant.
@@ -21,6 +23,12 @@ interface LibassistantService {
     pending_receiver<AudioInputController> audio_input_controller,
     pending_remote<AudioStreamFactoryDelegate> audio_stream_factory_delegate,
     pending_receiver<ConversationController> conversation_controller,
+    pending_receiver<DisplayController> display_controller,
     pending_receiver<ServiceController> service_controller
   );
+
+  // Add a speech recognition observer, which will be informed of all speech
+  // recognition related updates.
+  AddSpeechRecognitionObserver(pending_remote<SpeechRecognitionObserver>
+    observer);
 };

chromeos/services/libassistant/public/mojom/speech_recognition_observer.mojom

+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+module chromeos.libassistant.mojom;
+
+// Observer that will be informed about all speech related updates from
+// the Libassistant mojom service.
+interface SpeechRecognitionObserver {
+  // Provides periodic updates on the user's speech signal power, when the
+  // microphone is open and streaming to the Assistant backend.
+  OnSpeechLevelUpdated(float speech_level_in_decibels);
+};

chromeos/services/libassistant/service_controller.h

@@ -10,8 +10,7 @@
 #include "base/scoped_observation.h"
 #include "chromeos/services/libassistant/assistant_manager_observer.h"
 #include "chromeos/services/libassistant/public/mojom/service.mojom.h"
-#include "chromeos/services/libassistant/public/mojom/service_controller.mojom-forward.h"
-#include "chromeos/services/libassistant/public/mojom/service_controller.mojom-shared.h"
+#include "chromeos/services/libassistant/public/mojom/service_controller.mojom.h"
 #include "libassistant/shared/public/assistant_manager.h"
 #include "mojo/public/cpp/bindings/receiver.h"
 #include "mojo/public/cpp/bindings/remote_set.h"