Assistant: add autotest api to send text query.

Bug: b/120549692
Test: run autotest in chromeos chroot
Change-Id: I68111f28c73536226ad601ae63bba3459d0eb902
Reviewed-on: https://chromium-review.googlesource.com/c/1383040
Commit-Queue: Meilin Wang <meilinw@chromium.org>
Reviewed-by: Toni Baržić <tbarzic@chromium.org>
Reviewed-by: Xiyuan Xia <xiyuan@chromium.org>
Reviewed-by: Xiaohui Chen <xiaohuic@chromium.org>
Cr-Commit-Position: refs/heads/master@{#618667}

ash/public/cpp/assistant/assistant_state_proxy.h

@@ -23,7 +23,7 @@ class Connector;
 namespace ash {
 
 // Provides a convenient client to access various Assistant states. The state
-// infomration can be accessed through direct accessors which returns
+// information can be accessed through direct accessors which returns
 // |base::Optional<>| or observers. When adding an observer, all change events
 // will fire if this client already have data.
 class ASH_PUBLIC_EXPORT AssistantStateProxy

chrome/browser/chromeos/extensions/autotest_private/autotest_private_api.cc

@@ -48,6 +48,7 @@
 #include "chromeos/dbus/dbus_thread_manager.h"
 #include "chromeos/dbus/session_manager_client.h"
 #include "chromeos/printing/printer_configuration.h"
+#include "chromeos/services/assistant/public/mojom/constants.mojom.h"
 #include "chromeos/services/machine_learning/public/cpp/service_connection.h"
 #include "components/arc/arc_prefs.h"
 #include "components/arc/metrics/arc_metrics_constants.h"
@@ -1201,6 +1202,84 @@ void AutotestPrivateSetAssistantEnabledFunction::Timeout() {
   Respond(Error("Assistant service timed out"));
 }
 
+///////////////////////////////////////////////////////////////////////////////
+// AutotestPrivateSendAssistantTextQueryFunction
+///////////////////////////////////////////////////////////////////////////////
+AutotestPrivateSendAssistantTextQueryFunction::
+    AutotestPrivateSendAssistantTextQueryFunction()
+    : assistant_interaction_subscriber_binding_(this),
+      result_(std::make_unique<base::DictionaryValue>()) {}
+
+AutotestPrivateSendAssistantTextQueryFunction::
+    ~AutotestPrivateSendAssistantTextQueryFunction() = default;
+
+ExtensionFunction::ResponseAction
+AutotestPrivateSendAssistantTextQueryFunction::Run() {
+  DVLOG(1) << "AutotestPrivateSendAssistantTextQueryFunction";
+
+  std::unique_ptr<api::autotest_private::SendAssistantTextQuery::Params> params(
+      api::autotest_private::SendAssistantTextQuery::Params::Create(*args_));
+  EXTENSION_FUNCTION_VALIDATE(params);
+
+  Profile* profile = Profile::FromBrowserContext(browser_context());
+  if (!profile || arc::IsAssistantAllowedForProfile(profile) !=
+                      ash::mojom::AssistantAllowedState::ALLOWED) {
+    return RespondNow(Error("Assistant is not available for the current user"));
+  }
+
+  // Bind to Assistant service interface.
+  service_manager::Connector* connector =
+      content::BrowserContext::GetConnectorFor(profile);
+  connector->BindInterface(chromeos::assistant::mojom::kServiceName,
+                           &assistant_);
+
+  // Subscribe to Assistant interaction events.
+  chromeos::assistant::mojom::AssistantInteractionSubscriberPtr ptr;
+  assistant_interaction_subscriber_binding_.Bind(mojo::MakeRequest(&ptr));
+  assistant_->AddAssistantInteractionSubscriber(std::move(ptr));
+
+  // Start text interaction with Assistant server.
+  assistant_->StartTextInteraction(params->query, true);
+
+  // Set up a delayed timer to wait for the query response and hold a reference
+  // to |this| to avoid being destructed. Also make sure we stop and respond
+  // when timeout.
+  timeout_timer_.Start(
+      FROM_HERE, base::TimeDelta::FromMilliseconds(params->timeout_ms),
+      base::BindOnce(&AutotestPrivateSendAssistantTextQueryFunction::Timeout,
+                     this));
+
+  return RespondLater();
+}
+
+void AutotestPrivateSendAssistantTextQueryFunction::OnTextResponse(
+    const std::string& response) {
+  result_->SetKey("text", base::Value(response));
+}
+
+void AutotestPrivateSendAssistantTextQueryFunction::OnHtmlResponse(
+    const std::string& response,
+    const std::string& fallback) {
+  result_->SetKey("htmlResponse", base::Value(response));
+  result_->SetKey("htmlFallback", base::Value(fallback));
+}
+
+void AutotestPrivateSendAssistantTextQueryFunction::OnInteractionFinished(
+    AssistantInteractionResolution resolution) {
+  if (resolution != AssistantInteractionResolution::kNormal) {
+    Respond(Error("Interaction ends abnormally."));
+    timeout_timer_.AbandonAndStop();
+    return;
+  }
+
+  Respond(OneArgument(std::move(result_)));
+  timeout_timer_.AbandonAndStop();
+}
+
+void AutotestPrivateSendAssistantTextQueryFunction::Timeout() {
+  Respond(Error("Assistant response timeout."));
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 // AutotestPrivateAPI
 ///////////////////////////////////////////////////////////////////////////////

chrome/browser/chromeos/extensions/autotest_private/autotest_private_api.h

@@ -16,6 +16,7 @@
 #include "base/timer/timer.h"
 #include "chrome/browser/chromeos/printing/cups_printers_manager.h"
 #include "chrome/browser/extensions/chrome_extension_function.h"
+#include "chromeos/services/assistant/public/mojom/assistant.mojom.h"
 #include "chromeos/services/machine_learning/public/mojom/machine_learning_service.mojom.h"
 #include "extensions/browser/browser_context_keyed_api_factory.h"
 #include "ui/message_center/public/cpp/notification_types.h"
@@ -443,6 +444,54 @@ class AutotestPrivateSetAssistantEnabledFunction
   base::OneShotTimer timeout_timer_;
 };
 
+class AutotestPrivateSendAssistantTextQueryFunction
+    : public UIThreadExtensionFunction,
+      public chromeos::assistant::mojom::AssistantInteractionSubscriber {
+ public:
+  AutotestPrivateSendAssistantTextQueryFunction();
+  DECLARE_EXTENSION_FUNCTION("autotestPrivate.sendAssistantTextQuery",
+                             AUTOTESTPRIVATE_SENDASSISTANTTEXTQUERY)
+
+ private:
+  ~AutotestPrivateSendAssistantTextQueryFunction() override;
+  ResponseAction Run() override;
+
+  using AssistantSuggestionPtr =
+      chromeos::assistant::mojom::AssistantSuggestionPtr;
+  using AssistantInteractionResolution =
+      chromeos::assistant::mojom::AssistantInteractionResolution;
+
+  // chromeos::assistant::mojom::AssistantInteractionSubscriber:
+  void OnHtmlResponse(const std::string& response,
+                      const std::string& fallback) override;
+  void OnTextResponse(const std::string& response) override;
+  void OnInteractionFinished(
+      AssistantInteractionResolution resolution) override;
+  void OnInteractionStarted(bool is_voice_interaction) override {}
+  void OnSuggestionsResponse(
+      std::vector<AssistantSuggestionPtr> response) override {}
+  void OnOpenUrlResponse(const GURL& url) override {}
+  void OnSpeechRecognitionStarted() override {}
+  void OnSpeechRecognitionIntermediateResult(
+      const std::string& high_confidence_text,
+      const std::string& low_confidence_text) override {}
+  void OnSpeechRecognitionEndOfUtterance() override {}
+  void OnSpeechRecognitionFinalResult(
+      const std::string& final_result) override {}
+  void OnSpeechLevelUpdated(float speech_level) override {}
+  void OnTtsStarted(bool due_to_error) override {}
+
+  // Called when Assistant service fails to respond in a certain amount of
+  // time. We will respond with an error.
+  void Timeout();
+
+  chromeos::assistant::mojom::AssistantPtr assistant_;
+  mojo::Binding<chromeos::assistant::mojom::AssistantInteractionSubscriber>
+      assistant_interaction_subscriber_binding_;
+  base::OneShotTimer timeout_timer_;
+  std::unique_ptr<base::DictionaryValue> result_;
+};
+
 // The profile-keyed service that manages the autotestPrivate extension API.
 class AutotestPrivateAPI : public BrowserContextKeyedAPI {
  public:

chrome/common/extensions/api/autotest_private.idl

@@ -102,6 +102,13 @@ namespace autotestPrivate {
   };
   callback HistogramCallback = void (Histogram histogram);
 
+  dictionary AssistantQueryResponse {
+    DOMString? text;
+    DOMString? htmlResponse;
+    DOMString? htmlFallback;
+  };
+  callback AssistantQueryResponseCallback = void (AssistantQueryResponse response);
+
   callback IsAppShownCallback = void (boolean appShown);
 
   callback IsArcProvisionedCallback = void (boolean arcProvisioned);
@@ -272,5 +279,10 @@ namespace autotestPrivate {
     // Enable/disable the Google Assistant
     // |callback|: Called when the operation has completed.
     static void setAssistantEnabled(boolean enabled, long timeout_ms, VoidCallback callback);
+
+    // Send a text query via Google Assistant.
+    // |callback|: Called when response has been received.
+    static void sendAssistantTextQuery(DOMString query, long timeout_ms,
+                                       AssistantQueryResponseCallback callback);
   };
 };

chrome/test/data/extensions/api_test/autotest_private/test.js

@@ -217,6 +217,13 @@ var defaultTests = [
         chrome.test.callbackFail(
             'Assistant is not available for the current user'));
   },
+  function sendAssistantTextQuery() {
+    chrome.autotestPrivate.sendAssistantTextQuery(
+        'what time is it?' /* query */,
+        1000 /* timeout_ms */,
+        chrome.test.callbackFail(
+          'Assistant is not available for the current user'));
+  },
   // This test verifies that ARC is not provisioned by default.
   function isArcProvisioned() {
     chrome.autotestPrivate.isArcProvisioned(

extensions/browser/extension_function_histogram_value.h

@@ -1365,6 +1365,7 @@ enum HistogramValue {
   AUTOTESTPRIVATE_LAUNCHARCAPP = 1302,
   AUTOTESTPRIVATE_CLOSEAPP = 1303,
   ACCESSIBILITY_PRIVATE_SETSWITCHACCESSMENUSTATE = 1304,
+  AUTOTESTPRIVATE_SENDASSISTANTTEXTQUERY = 1305,
   // Last entry: Add new entries above, then run:
   // python tools/metrics/histograms/update_extension_histograms.py
   ENUM_BOUNDARY

tools/metrics/histograms/enums.xml

@@ -17543,6 +17543,7 @@ Called by update_net_error_codes.py.-->
   <int value="1302" label="AUTOTESTPRIVATE_LAUNCHARCAPP"/>
   <int value="1303" label="AUTOTESTPRIVATE_CLOSEAPP"/>
   <int value="1304" label="ACCESSIBILITY_PRIVATE_SETSWITCHACCESSMENUSTATE"/>
+  <int value="1305" label="AUTOTESTPRIVATE_SENDASSISTANTTEXTQUERY"/>
 </enum>
 
 <enum name="ExtensionIconState">