[LibassistantV2] Move |SendVoicelessInteraction| to Mojom service.

This CL is the part 1 of moving |SendVoicelessInteraction| calls to
Libassistant Mojom service. It defines and implements four more APIs
(listed below) under |ConversationController| Mojom service, and hooks
them up in the browser process by replacing |SendVoicelessInteraction|
calls in |AssistantManagerServiceImpl| with the corresponding Mojom
method.

* StartEditReminderInteraction
* RetrieveNotification
* DismissNotification
* SendAssistantFeedback

Note that in the CL, |ConversationControllerProxy| will forward all
calls through Mojom to the service implementation, where the actual
proto message gets encoded. In this way, we can remove Libassistant
dependencies from the Assistant service side (classes under
/chromeos/services/assistant).

Bug: b/171748795
Change-Id: I350cf49b898fffa798b73bb7720ae1583ce84ec1
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2625327Reviewed-by: Sam McNally <sammc@chromium.org>
Reviewed-by: Xiaohui Chen <xiaohuic@chromium.org>
Reviewed-by: Jeroen Dhollander <jeroendh@chromium.org>
Commit-Queue: Meilin Wang <meilinw@chromium.org>
Cr-Commit-Position: refs/heads/master@{#844984}

chromeos/services/assistant/assistant_manager_service_impl.cc

@@ -503,9 +503,7 @@ void AssistantManagerServiceImpl::StopActiveInteraction(
 
 void AssistantManagerServiceImpl::StartEditReminderInteraction(
     const std::string& client_id) {
-  SendVoicelessInteraction(CreateEditReminderInteraction(client_id),
-                           /*description=*/std::string(),
-                           /*is_user_initiated=*/true);
+  conversation_controller_proxy().StartEditReminderInteraction(client_id);
 }
 
 void AssistantManagerServiceImpl::StartScreenContextInteraction(
@@ -560,41 +558,13 @@ void AssistantManagerServiceImpl::RemoveAssistantInteractionSubscriber(
 void AssistantManagerServiceImpl::RetrieveNotification(
     const AssistantNotification& notification,
     int action_index) {
-  const std::string& notification_id = notification.server_id;
-  const std::string& consistency_token = notification.consistency_token;
-  const std::string& opaque_token = notification.opaque_token;
-
-  const std::string request_interaction =
-      SerializeNotificationRequestInteraction(
-          notification_id, consistency_token, opaque_token, action_index);
-
-  SendVoicelessInteraction(request_interaction,
-                           /*description=*/"RequestNotification",
-                           /*is_user_initiated=*/true);
+  conversation_controller_proxy().RetrieveNotification(notification,
+                                                       action_index);
 }
 
 void AssistantManagerServiceImpl::DismissNotification(
     const AssistantNotification& notification) {
-  // |assistant_manager_internal()| may not exist if we are dismissing
-  // notifications as part of a shutdown sequence.
-  if (!assistant_manager_internal())
-    return;
-
-  const std::string& notification_id = notification.server_id;
-  const std::string& consistency_token = notification.consistency_token;
-  const std::string& opaque_token = notification.opaque_token;
-  const std::string& grouping_key = notification.grouping_key;
-
-  const std::string dismissed_interaction =
-      SerializeNotificationDismissedInteraction(
-          notification_id, consistency_token, opaque_token, {grouping_key});
-
-  assistant_client::VoicelessOptions options;
-  options.obfuscated_gaia_id = notification.obfuscated_gaia_id;
-
-  assistant_manager_internal()->SendVoicelessInteraction(
-      dismissed_interaction, /*description=*/"DismissNotification", options,
-      [](auto) {});
+  conversation_controller_proxy().DismissNotification(notification);
 }
 
 void AssistantManagerServiceImpl::OnConversationTurnStartedInternal(
@@ -1309,13 +1279,7 @@ void AssistantManagerServiceImpl::RecordQueryResponseTypeUMA() {
 
 void AssistantManagerServiceImpl::SendAssistantFeedback(
     const AssistantFeedback& assistant_feedback) {
-  const std::string interaction = CreateSendFeedbackInteraction(
-      assistant_feedback.assistant_debug_info_allowed,
-      assistant_feedback.description, assistant_feedback.screenshot_png);
-
-  SendVoicelessInteraction(interaction,
-                           /*description=*/"send feedback with details",
-                           /*is_user_initiated=*/false);
+  conversation_controller_proxy().SendAssistantFeedback(assistant_feedback);
 }
 
 void AssistantManagerServiceImpl::UpdateMediaState() {

chromeos/services/assistant/proxy/conversation_controller_proxy.cc

@@ -9,6 +9,35 @@
 namespace chromeos {
 namespace assistant {
 
+namespace {
+
+// Converts |AssistantNotification| struct to the corresponding Mojom struct
+// (the Mojom-generated |AssistantNotificationPtr|).
+libassistant::mojom::AssistantNotificationPtr ConvertNotificationToMojomStruct(
+    const AssistantNotification& notification) {
+  auto ptr = libassistant::mojom::AssistantNotification::New();
+  ptr->server_id = notification.server_id;
+  ptr->consistency_token = notification.consistency_token;
+  ptr->opaque_token = notification.opaque_token;
+  ptr->grouping_key = notification.grouping_key;
+  ptr->obfuscated_gaia_id = notification.obfuscated_gaia_id;
+  return ptr;
+}
+
+// Coverts |AssistantFeedback| struct to the corresponding Mojom struct (the
+// Mojom-generated |AssistantFeedbackPtr|).
+libassistant::mojom::AssistantFeedbackPtr ConvertFeedbackToMojomStruct(
+    const AssistantFeedback& feedback) {
+  auto ptr = libassistant::mojom::AssistantFeedback::New();
+  ptr->description = feedback.description;
+  ptr->assistant_debug_info_allowed = feedback.assistant_debug_info_allowed;
+  ptr->screenshot_png = std::vector<uint8_t>(feedback.screenshot_png.begin(),
+                                             feedback.screenshot_png.end());
+  return ptr;
+}
+
+}  // namespace
+
 ConversationControllerProxy::ConversationControllerProxy(
     mojo::PendingRemote<ConversationController> conversation_controller_remote)
     : conversation_controller_remote_(
@@ -24,5 +53,29 @@ void ConversationControllerProxy::SendTextQuery(
                                                  conversation_id);
 }
 
+void ConversationControllerProxy::StartEditReminderInteraction(
+    const std::string& client_id) {
+  conversation_controller_remote_->StartEditReminderInteraction(client_id);
+}
+
+void ConversationControllerProxy::RetrieveNotification(
+    const AssistantNotification& notification,
+    int32_t action_index) {
+  conversation_controller_remote_->RetrieveNotification(
+      ConvertNotificationToMojomStruct(notification), action_index);
+}
+
+void ConversationControllerProxy::DismissNotification(
+    const AssistantNotification& notification) {
+  conversation_controller_remote_->DismissNotification(
+      ConvertNotificationToMojomStruct(notification));
+}
+
+void ConversationControllerProxy::SendAssistantFeedback(
+    const AssistantFeedback& feedback) {
+  conversation_controller_remote_->SendAssistantFeedback(
+      ConvertFeedbackToMojomStruct(feedback));
+}
+
 }  // namespace assistant
 }  // namespace chromeos

chromeos/services/assistant/proxy/conversation_controller_proxy.h

@@ -7,6 +7,8 @@
 
 #include <string>
 
+#include "chromeos/services/assistant/public/cpp/assistant_notification.h"
+#include "chromeos/services/assistant/public/cpp/assistant_service.h"
 #include "chromeos/services/libassistant/public/mojom/conversation_controller.mojom.h"
 #include "mojo/public/cpp/bindings/remote.h"
 
@@ -15,6 +17,8 @@ namespace assistant {
 
 using chromeos::libassistant::mojom::ConversationController;
 
+// Component supporting conversation related functionalities, includes the
+// ability to start an interaction, register action module, etc.
 class ConversationControllerProxy {
  public:
   explicit ConversationControllerProxy(
@@ -25,10 +29,27 @@ class ConversationControllerProxy {
       delete;
   ~ConversationControllerProxy();
 
+  // Starts a new Assistant text interaction. If |allow_tts| is true, the
+  // result will contain TTS. |conversation_id| is a unique identifier of
+  // a conversation turn.
   void SendTextQuery(const std::string& query,
                      bool allow_tts,
                      const std::string& conversation_id);
 
+  // Starts an interaction to edit the reminder uniquely identified by
+  // |client_id|.
+  void StartEditReminderInteraction(const std::string& client_id);
+
+  // Retrieves a notification identified by |action_index|.
+  void RetrieveNotification(const AssistantNotification& notification,
+                            int32_t action_index);
+
+  // Dismisses a notification.
+  void DismissNotification(const AssistantNotification& notification);
+
+  // Sends an Assistant feedback to Assistant server.
+  void SendAssistantFeedback(const AssistantFeedback& feedback);
+
  private:
   mojo::Remote<ConversationController> conversation_controller_remote_;
 };

chromeos/services/libassistant/conversation_controller.cc

@@ -52,13 +52,76 @@ void ConversationController::SendTextQuery(
     options.conversation_turn_id = conversation_id.value();
 
   // Builds text interaction.
-  std::string interaction =
-      chromeos::assistant::CreateTextQueryInteraction(query);
+  std::string interaction = assistant::CreateTextQueryInteraction(query);
 
   assistant_manager_internal()->SendVoicelessInteraction(
       interaction, /*description=*/"text_query", options, [](auto) {});
 }
 
+void ConversationController::StartEditReminderInteraction(
+    const std::string& client_id) {
+  SendVoicelessInteraction(assistant::CreateEditReminderInteraction(client_id),
+                           /*description=*/std::string(),
+                           /*is_user_initiated=*/true);
+}
+
+void ConversationController::RetrieveNotification(
+    mojom::AssistantNotificationPtr notification,
+    int32_t action_index) {
+  const std::string request_interaction =
+      assistant::SerializeNotificationRequestInteraction(
+          notification->server_id, notification->consistency_token,
+          notification->opaque_token, action_index);
+
+  SendVoicelessInteraction(request_interaction,
+                           /*description=*/"RequestNotification",
+                           /*is_user_initiated=*/true);
+}
+
+void ConversationController::DismissNotification(
+    mojom::AssistantNotificationPtr notification) {
+  // |assistant_manager_internal()| may not exist if we are dismissing
+  // notifications as part of a shutdown sequence.
+  if (!assistant_manager_internal())
+    return;
+
+  const std::string dismissed_interaction =
+      assistant::SerializeNotificationDismissedInteraction(
+          notification->server_id, notification->consistency_token,
+          notification->opaque_token, {notification->grouping_key});
+
+  assistant_client::VoicelessOptions options;
+  options.obfuscated_gaia_id = notification->obfuscated_gaia_id;
+
+  assistant_manager_internal()->SendVoicelessInteraction(
+      dismissed_interaction, /*description=*/"DismissNotification", options,
+      [](auto) {});
+}
+
+void ConversationController::SendAssistantFeedback(
+    mojom::AssistantFeedbackPtr feedback) {
+  std::string raw_image_data(feedback->screenshot_png.begin(),
+                             feedback->screenshot_png.end());
+  const std::string interaction = assistant::CreateSendFeedbackInteraction(
+      feedback->assistant_debug_info_allowed, feedback->description,
+      raw_image_data);
+
+  SendVoicelessInteraction(interaction,
+                           /*description=*/"send feedback with details",
+                           /*is_user_initiated=*/false);
+}
+
+void ConversationController::SendVoicelessInteraction(
+    const std::string& interaction,
+    const std::string& description,
+    bool is_user_initiated) {
+  assistant_client::VoicelessOptions voiceless_options;
+  voiceless_options.is_user_initiated = is_user_initiated;
+
+  assistant_manager_internal()->SendVoicelessInteraction(
+      interaction, description, voiceless_options, [](auto) {});
+}
+
 assistant_client::AssistantManagerInternal*
 ConversationController::assistant_manager_internal() {
   return service_controller_->assistant_manager_internal();

chromeos/services/libassistant/conversation_controller.h

@@ -7,6 +7,7 @@
 
 #include "base/component_export.h"
 #include "base/optional.h"
+#include "chromeos/services/assistant/public/cpp/assistant_notification.h"
 #include "chromeos/services/libassistant/public/mojom/conversation_controller.mojom.h"
 #include "mojo/public/cpp/bindings/receiver.h"
 
@@ -34,8 +35,17 @@ class COMPONENT_EXPORT(LIBASSISTANT_SERVICE) ConversationController
       const std::string& query,
       bool allow_tts,
       const base::Optional<std::string>& conversation_id) override;
+  void StartEditReminderInteraction(const std::string& client_id) override;
+  void RetrieveNotification(mojom::AssistantNotificationPtr notification,
+                            int32_t action_index) override;
+  void DismissNotification(mojom::AssistantNotificationPtr) override;
+  void SendAssistantFeedback(mojom::AssistantFeedbackPtr feedback) override;
 
  private:
+  void SendVoicelessInteraction(const std::string& interaction,
+                                const std::string& description,
+                                bool is_user_initiated);
+
   assistant_client::AssistantManagerInternal* assistant_manager_internal();
 
   mojo::Receiver<mojom::ConversationController> receiver_;

chromeos/services/libassistant/public/mojom/conversation_controller.mojom

@@ -4,11 +4,65 @@
 
 module chromeos.libassistant.mojom;
 
-// Interface for controller in charge of Assistant conversations.
+// Interface for controller supporting conversation related functionalities.
 interface ConversationController {
   // Sends the specific text query to Libassistant. |conversation_id| is an
   // identifier that will be propagated through the speech processor event
   // pipeline to uniquely identify a conversation turn. If omitted, a unique
   // identifier will be automatically generated.
   SendTextQuery(string query, bool allow_tts, string? conversation_id);
+
+  // Starts an interaction to edit the reminder uniquely identified by
+  // |client_id|. In response to the request, LibAssistant will initiate
+  // a user facing interaction with the context pre-populated with details
+  // to edit the specified reminder.
+  StartEditReminderInteraction(string client_id);
+
+  // Retrieves a notification. A voiceless interaction will be sent to server to
+  // retrieve the notification of |action_index|, which can trigger other
+  // Assistant events such as OnTextResponse to show the result in the UI. The
+  // retrieved notification will be removed from the UI.
+  // |action_index| is the index of the tapped action. The main UI in the
+  // notification contains the top level action, which index is 0. The buttons
+  // have the additional actions, which are indexed starting from 1.
+  RetrieveNotification(AssistantNotification notification,
+                       int32 action_index);
+
+  // Dismisses a notification.
+  DismissNotification(AssistantNotification notification);
+
+  // Sends Assistant feedback to Assistant server.
+  SendAssistantFeedback(AssistantFeedback feedback);
+};
+
+// Models an Assistant notification. This should be kept in sync with
+// |chromeos::assistant::AssistantNotification|. For now we only include
+// entries that are currently in use.
+struct AssistantNotification {
+  // An id that uniquely identifies a notification on the server.
+  string server_id;
+
+  // Used to fetch notification contents.
+  string consistency_token;
+  string opaque_token;
+
+  // Key that can be used to group multiple notifications together.
+  string grouping_key;
+
+  // Obfuscated Gaia id of the intended recipient of the user.
+  string obfuscated_gaia_id;
+};
+
+// Models an Assistant feedback. This should be kept in sync with
+// |chromeos::assistant::AssistantFeedback|.
+struct AssistantFeedback {
+  // User input to be sent with the feedback report.
+  string description;
+
+  // Whether user consent to send debug info.
+  bool assistant_debug_info_allowed = false;
+
+  // Screenshot if allowed by user.
+  // Raw data (non-encoded binary octets)
+  array<uint8> screenshot_png;
 };