Send screen context as a text query

This is a flag guarded feature for sending the screen context as a text
query. For more information, see go/screen-context-dialog.

Bug: b:123037261
Change-Id: Ibf6315450aad2f7ab89b04c37c47684e76be82f9
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1493371
Commit-Queue: Eyor Alemayehu <eyor@google.com>
Reviewed-by: Xiaohui Chen <xiaohuic@chromium.org>
Cr-Commit-Position: refs/heads/master@{#638184}

chromeos/services/assistant/assistant_manager_service_impl.cc

@@ -70,6 +70,10 @@ constexpr base::Feature kChromeOSAssistantDogfood{
 constexpr char kServersideDogfoodExperimentId[] = "20347368";
 constexpr char kServersideOpenAppExperimentId[] = "39651593";
 
+// The screen context query is locale independent. That is the same query
+// applies to all locales.
+constexpr char kScreenContextQuery[] = "screen context";
+
 constexpr float kDefaultSliderStep = 0.1f;
 
 bool IsScreenContextAllowed(ash::AssistantStateBase* assistant_state) {
@@ -305,6 +309,8 @@ void AssistantManagerServiceImpl::StartTextInteraction(const std::string& query,
   if (base::FeatureList::IsEnabled(
           assistant::features::kEnableTextQueriesWithClientDiscourseContext) &&
       assistant_extra_ && assistant_tree_) {
+    // We don't send the screenshot, because the backend only needs the
+    // view hierarchy to resolve contextual queries such as "Who is he?".
     assistant_manager_internal_->SendTextQueryWithClientDiscourseContext(
         query,
         CreateContextProto(
@@ -1160,6 +1166,19 @@ void AssistantManagerServiceImpl::SendScreenContextRequest(
     ax::mojom::AssistantExtra* assistant_extra,
     ui::AssistantTree* assistant_tree,
     const std::vector<uint8_t>& assistant_screenshot) {
+  if (assistant::features::IsScreenContextQueryEnabled()) {
+    assistant_client::VoicelessOptions options;
+    options.is_user_initiated = true;
+
+    assistant_manager_internal_->SendTextQueryWithClientDiscourseContext(
+        kScreenContextQuery,
+        CreateContextProto(
+            AssistantBundle{assistant_extra_.get(), assistant_tree_.get()},
+            assistant_screenshot),
+        options);
+    return;
+  }
+
   std::vector<std::string> context_protos;
 
   // Screen context can have the assistant_extra and assistant_tree set to

chromeos/services/assistant/public/features.cc

@@ -53,6 +53,12 @@ const base::Feature kEnablePowerManager{"ChromeOSAssistantEnablePowerManager",
 const base::Feature kAssistantKeyRemapping{"AssistantKeyRemapping",
                                            base::FEATURE_DISABLED_BY_DEFAULT};
 
+// Enables sending a screen context request ("What's on my screen?" and
+// metalayer selection) as a text query. This is as opposed to sending
+// the request as a contextual cards request.
+const base::Feature kScreenContextQuery{"ChromeOSAssistantScreenContextQuery",
+                                        base::FEATURE_DISABLED_BY_DEFAULT};
+
 bool IsAppSupportEnabled() {
   return base::FeatureList::IsEnabled(
       assistant::features::kAssistantAppSupport);
@@ -102,6 +108,10 @@ bool IsKeyRemappingEnabled() {
   return base::FeatureList::IsEnabled(kAssistantKeyRemapping);
 }
 
+bool IsScreenContextQueryEnabled() {
+  return base::FeatureList::IsEnabled(kScreenContextQuery);
+}
+
 }  // namespace features
 }  // namespace assistant
 }  // namespace chromeos

chromeos/services/assistant/public/features.h

@@ -56,6 +56,10 @@ extern const base::Feature kTimerNotification;
 COMPONENT_EXPORT(ASSISTANT_SERVICE_PUBLIC)
 extern const base::Feature kEnablePowerManager;
 
+// Enables sending the client discourse context with text queries.
+COMPONENT_EXPORT(ASSISTANT_SERVICE_PUBLIC)
+extern const base::Feature kEnableTextQueriesWithClientDiscourseContext;
+
 // Enables timer ticks. This feature causes alarms/timers tracked by the
 // AssistantAlarmTimerController to tick at a fixed interval, delivering updates
 // to AssistantAlarmTimerModelObservers of time remaining/elapsed since expiry.
@@ -91,9 +95,7 @@ COMPONENT_EXPORT(ASSISTANT_SERVICE_PUBLIC) bool IsWarmerWelcomeEnabled();
 
 COMPONENT_EXPORT(ASSISTANT_SERVICE_PUBLIC) bool IsKeyRemappingEnabled();
 
-// Enables sending the client discourse context with text queries.
-COMPONENT_EXPORT(ASSISTANT_SERVICE_PUBLIC)
-extern const base::Feature kEnableTextQueriesWithClientDiscourseContext;
+COMPONENT_EXPORT(ASSISTANT_SERVICE_PUBLIC) bool IsScreenContextQueryEnabled();
 
 COMPONENT_EXPORT(ASSISTANT_SERVICE_PUBLIC) bool IsPowerManagerEnabled();