Send Client Discourse Context for text queries

To enable pronoun resolutions using the screen context, made changes to
send screen context information. An example of a query that requires
pronoun resolution is "Who is he?". For such a query, the content of
the topmost app can be used to resolve pronoun
("he" as in this example).

When the Assistant UI is launched, we currently cache the screen
context. This screen context is currently cleared when sending it to the
backend. To allow re-sending the screen context multiple times in a
single UI session, the screen context is no longer cleared when sent to
the backend. The screen context is cleared when the Assistant UI is
shown again.

There is a TODO that will be done is a subsequent CL that will set a
flag to true if the first query is being sent or false otherwise.

Bug: b:120736468
Change-Id: Ib26b4da041cf46094c20511f6820c3b4a551c13c
Reviewed-on: https://chromium-review.googlesource.com/c/1369062
Commit-Queue: Eyor Alemayehu <eyor@google.com>
Reviewed-by: Xiaohui Chen <xiaohuic@chromium.org>
Cr-Commit-Position: refs/heads/master@{#622082}

chromeos/services/assistant/assistant_manager_service_impl.cc

@@ -243,6 +243,8 @@ void AssistantManagerServiceImpl::StartWarmerWelcomeInteraction(
       [](auto) {});
 }
 
+// TODO(eyor): Add a method that can be called to clear the cached interaction
+// when the UI is hidden/closed.
 void AssistantManagerServiceImpl::StartCachedScreenContextInteraction() {
   if (!IsScreenContextAllowed(service_->assistant_state()))
     return;
@@ -253,8 +255,8 @@ void AssistantManagerServiceImpl::StartCachedScreenContextInteraction() {
   DCHECK(assistant_tree_);
   DCHECK(!assistant_screenshot_.empty());
 
-  SendScreenContextRequest(std::move(assistant_extra_),
-                           std::move(assistant_tree_), assistant_screenshot_);
+  SendScreenContextRequest(assistant_extra_.get(), assistant_tree_.get(),
+                           assistant_screenshot_);
 }
 
 void AssistantManagerServiceImpl::StartMetalayerInteraction(
@@ -279,9 +281,23 @@ void AssistantManagerServiceImpl::StartTextInteraction(const std::string& query,
         assistant_client::VoicelessOptions::Modality::TYPING_MODALITY;
   }
 
-  std::string interaction = CreateTextQueryInteraction(query);
-  assistant_manager_internal_->SendVoicelessInteraction(
-      interaction, /*description=*/"text_query", options, [](auto) {});
+  if (base::FeatureList::IsEnabled(
+          assistant::features::kEnableTextQueriesWithClientDiscourseContext) &&
+      assistant_extra_ && assistant_tree_) {
+    // TODO(eyor): Replace hardcoded true with logic that is set to true
+    // if this is the first query after the UI is launched, or false if
+    // it is a query sent after the first query.
+    assistant_manager_internal_->SendTextQueryWithClientDiscourseContext(
+        query,
+        CreateContextProto(
+            AssistantBundle{assistant_extra_.get(), assistant_tree_.get()},
+            /*is_first_query=*/true),
+        options);
+  } else {
+    std::string interaction = CreateTextQueryInteraction(query);
+    assistant_manager_internal_->SendVoicelessInteraction(
+        interaction, /*description=*/"text_query", options, [](auto) {});
+  }
 }
 
 void AssistantManagerServiceImpl::AddAssistantInteractionSubscriber(
@@ -1094,8 +1110,8 @@ void AssistantManagerServiceImpl::CacheAssistantScreenshot(
 }
 
 void AssistantManagerServiceImpl::SendScreenContextRequest(
-    ax::mojom::AssistantExtraPtr assistant_extra,
-    std::unique_ptr<ui::AssistantTree> assistant_tree,
+    ax::mojom::AssistantExtra* assistant_extra,
+    ui::AssistantTree* assistant_tree,
     const std::vector<uint8_t>& assistant_screenshot) {
   std::vector<std::string> context_protos;
 
@@ -1104,13 +1120,20 @@ void AssistantManagerServiceImpl::SendScreenContextRequest(
   // the metalayer. For this scenario, we don't create a context proto for the
   // AssistantBundle that consists of the assistant_extra and assistant_tree.
   if (assistant_extra && assistant_tree) {
-    context_protos.emplace_back(CreateContextProto(AssistantBundle{
-        std::move(assistant_extra), std::move(assistant_tree)}));
+    // TODO(eyor): Replace hardcoded true with logic that is set to true
+    // if this is the first query after the UI is launched, or false if
+    // it is a query sent after the first query.
+    context_protos.emplace_back(
+        CreateContextProto(AssistantBundle{assistant_extra, assistant_tree},
+                           /*is_first_query=*/true));
   }
 
-  context_protos.emplace_back(CreateContextProto(assistant_screenshot));
+  // TODO(eyor): Replace hardcoded true with logic that is set to true
+  // if this is the first query after the UI is launched, or false if
+  // it is a query sent after the first query.
+  context_protos.emplace_back(
+      CreateContextProto(assistant_screenshot, /*is_first_query=*/true));
   assistant_manager_internal_->SendScreenContextRequest(context_protos);
-  assistant_screenshot_.clear();
 }
 
 std::string AssistantManagerServiceImpl::GetLastSearchSource() {

chromeos/services/assistant/assistant_manager_service_impl.h

@@ -222,8 +222,8 @@ class AssistantManagerServiceImpl
       const std::vector<uint8_t>& assistant_screenshot);
 
   void SendScreenContextRequest(
-      ax::mojom::AssistantExtraPtr assistant_extra,
-      std::unique_ptr<ui::AssistantTree> assistant_tree,
+      ax::mojom::AssistantExtra* assistant_extra,
+      ui::AssistantTree* assistant_tree,
       const std::vector<uint8_t>& assistant_screenshot);
 
   void FillServerExperimentIds(std::vector<std::string>* server_experiment_ids);

chromeos/services/assistant/public/features.cc

@@ -27,6 +27,9 @@ const base::Feature kEnableStereoAudioInput{"AssistantEnableStereoAudioInput",
 
 const base::Feature kTimerNotification{"ChromeOSAssistantTimerNotification",
                                        base::FEATURE_ENABLED_BY_DEFAULT};
+const base::Feature kEnableTextQueriesWithClientDiscourseContext{
+    "AssistantEnableTextQueriesWithClientDiscourseContext",
+    base::FEATURE_DISABLED_BY_DEFAULT};
 
 const base::Feature kTimerTicks{"ChromeOSAssistantTimerTicks",
                                 base::FEATURE_DISABLED_BY_DEFAULT};

chromeos/services/assistant/public/features.h

@@ -54,6 +54,10 @@ COMPONENT_EXPORT(ASSISTANT_SERVICE_PUBLIC) bool IsTimerTicksEnabled();
 
 COMPONENT_EXPORT(ASSISTANT_SERVICE_PUBLIC) bool IsWarmerWelcomeEnabled();
 
+// Enables sending the client discourse context with text queries.
+COMPONENT_EXPORT(ASSISTANT_SERVICE_PUBLIC)
+extern const base::Feature kEnableTextQueriesWithClientDiscourseContext;
+
 }  // namespace features
 }  // namespace assistant
 }  // namespace chromeos