Wires up recognition state change events to AshAssistantController.

Bug: b:77282340
Change-Id: I88ea25128043ec69ec88357f36babfae36fe88fd
Reviewed-on: https://chromium-review.googlesource.com/986921Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Xiyuan Xia <xiyuan@chromium.org>
Reviewed-by: Xiaohui Chen <xiaohuic@chromium.org>
Commit-Queue: David Black <dmblack@google.com>
Cr-Commit-Position: refs/heads/master@{#547752}

ash/assistant/ash_assistant_controller.cc

@@ -47,6 +47,29 @@ void AshAssistantController::OnTextResponse(const std::string& response) {
   NOTIMPLEMENTED();
 }
 
+void AshAssistantController::OnSpeechRecognitionStarted() {
+  // TODO(dmblack): Handle.
+  NOTIMPLEMENTED();
+}
+
+void AshAssistantController::OnSpeechRecognitionIntermediateResult(
+    const std::string& high_confidence_text,
+    const std::string& low_confidence_text) {
+  // TODO(dmblack): Handle.
+  NOTIMPLEMENTED();
+}
+
+void AshAssistantController::OnSpeechRecognitionEndOfUtterance() {
+  // TODO(dmblack): Handle.
+  NOTIMPLEMENTED();
+}
+
+void AshAssistantController::OnSpeechRecognitionFinalResult(
+    const std::string& final_result) {
+  // TODO(dmblack): Handle.
+  NOTIMPLEMENTED();
+}
+
 void AshAssistantController::OnSpeechLevelUpdated(float speech_level) {
   // TODO(dmblack): Handle.
   NOTIMPLEMENTED();

ash/assistant/ash_assistant_controller.h

@@ -26,6 +26,12 @@ class AshAssistantController
   void OnSuggestionsResponse(const std::vector<std::string>& response) override;
   void OnTextResponse(const std::string& response) override;
   void OnOpenUrlResponse(const GURL& url) override;
+  void OnSpeechRecognitionStarted() override;
+  void OnSpeechRecognitionIntermediateResult(
+      const std::string& high_confidence_text,
+      const std::string& low_confidence_text) override;
+  void OnSpeechRecognitionEndOfUtterance() override;
+  void OnSpeechRecognitionFinalResult(const std::string& final_result) override;
 
   // Assistant got a speech level update in dB.
   void OnSpeechLevelUpdated(float speech_level) override;

chromeos/services/assistant/assistant_manager_service_impl.cc

@@ -32,7 +32,9 @@ AssistantManagerServiceImpl::AssistantManagerServiceImpl(
           UnwrapAssistantManagerInternal(assistant_manager_.get())),
       display_connection_(std::make_unique<CrosDisplayConnection>(this)),
       main_thread_task_runner_(base::ThreadTaskRunnerHandle::Get()),
-      weak_factory_(this) {}
+      weak_factory_(this) {
+  assistant_manager_->AddConversationStateListener(this);
+}
 
 AssistantManagerServiceImpl::~AssistantManagerServiceImpl() {}
 
@@ -106,6 +108,17 @@ void AssistantManagerServiceImpl::OnOpenUrl(const std::string& url) {
                      weak_factory_.GetWeakPtr(), url));
 }
 
+void AssistantManagerServiceImpl::OnRecognitionStateChanged(
+    assistant_client::ConversationStateListener::RecognitionState state,
+    const assistant_client::ConversationStateListener::RecognitionResult&
+        recognition_result) {
+  main_thread_task_runner_->PostTask(
+      FROM_HERE,
+      base::BindOnce(
+          &AssistantManagerServiceImpl::OnRecognitionStateChangedOnMainThread,
+          weak_factory_.GetWeakPtr(), state, recognition_result));
+}
+
 void AssistantManagerServiceImpl::OnSpeechLevelUpdated(
     const float speech_level) {
   main_thread_task_runner_->PostTask(
@@ -173,6 +186,38 @@ void AssistantManagerServiceImpl::OnOpenUrlOnMainThread(
       [&url](auto* ptr) { ptr->OnOpenUrlResponse(GURL(url)); });
 }
 
+void AssistantManagerServiceImpl::OnRecognitionStateChangedOnMainThread(
+    assistant_client::ConversationStateListener::RecognitionState state,
+    const assistant_client::ConversationStateListener::RecognitionResult&
+        recognition_result) {
+  switch (state) {
+    case assistant_client::ConversationStateListener::RecognitionState::STARTED:
+      subscribers_.ForAllPtrs(
+          [](auto* ptr) { ptr->OnSpeechRecognitionStarted(); });
+      break;
+    case assistant_client::ConversationStateListener::RecognitionState::
+        INTERMEDIATE_RESULT:
+      subscribers_.ForAllPtrs([&recognition_result](auto* ptr) {
+        ptr->OnSpeechRecognitionIntermediateResult(
+            recognition_result.high_confidence_text,
+            recognition_result.low_confidence_text);
+      });
+      break;
+    case assistant_client::ConversationStateListener::RecognitionState::
+        END_OF_UTTERANCE:
+      subscribers_.ForAllPtrs(
+          [](auto* ptr) { ptr->OnSpeechRecognitionEndOfUtterance(); });
+      break;
+    case assistant_client::ConversationStateListener::RecognitionState::
+        FINAL_RESULT:
+      subscribers_.ForAllPtrs([&recognition_result](auto* ptr) {
+        ptr->OnSpeechRecognitionFinalResult(
+            recognition_result.recognized_speech);
+      });
+      break;
+  }
+}
+
 void AssistantManagerServiceImpl::OnSpeechLevelUpdatedOnMainThread(
     const float speech_level) {
   subscribers_.ForAllPtrs(

chromeos/services/assistant/assistant_manager_service_impl.h

@@ -16,6 +16,7 @@
 #include "chromeos/services/assistant/platform_api_impl.h"
 #include "chromeos/services/assistant/public/mojom/assistant.mojom.h"
 #include "libassistant/contrib/core/macros.h"
+#include "libassistant/shared/public/conversation_state_listener.h"
 #include "mojo/public/cpp/bindings/interface_ptr_set.h"
 
 namespace assistant_client {
@@ -30,7 +31,8 @@ namespace assistant {
 class AssistantManagerServiceImpl
     : public AssistantManagerService,
       public ::chromeos::assistant::action::AssistantActionObserver,
-      public AssistantEventObserver {
+      public AssistantEventObserver,
+      public assistant_client::ConversationStateListener {
  public:
   explicit AssistantManagerServiceImpl(mojom::AudioInputPtr audio_input);
   ~AssistantManagerServiceImpl() override;
@@ -55,6 +57,12 @@ class AssistantManagerServiceImpl
   // AssistantEventObserver overrides:
   void OnSpeechLevelUpdated(float speech_level) override;
 
+  // assistant_client::ConversationStateListener overrides:
+  void OnRecognitionStateChanged(
+      assistant_client::ConversationStateListener::RecognitionState state,
+      const assistant_client::ConversationStateListener::RecognitionResult&
+          recognition_result) override;
+
  private:
   void StartAssistantInternal(const std::string& access_token,
                               const std::string& arc_version);
@@ -65,6 +73,10 @@ class AssistantManagerServiceImpl
       const std::vector<std::string>& suggestions);
   void OnShowTextOnMainThread(const std::string& text);
   void OnOpenUrlOnMainThread(const std::string& url);
+  void OnRecognitionStateChangedOnMainThread(
+      assistant_client::ConversationStateListener::RecognitionState state,
+      const assistant_client::ConversationStateListener::RecognitionResult&
+          recognition_result);
   void OnSpeechLevelUpdatedOnMainThread(const float speech_level);
 
   bool running_ = false;

chromeos/services/assistant/public/mojom/assistant.mojom

@@ -34,6 +34,19 @@ interface AssistantEventSubscriber {
   // Assistant got open URL response from server.
   OnOpenUrlResponse(url.mojom.Url url);
 
+  // Assistant speech recognition has started.
+  OnSpeechRecognitionStarted();
+
+  // Assistant speech recognition intermediate result is available.
+  OnSpeechRecognitionIntermediateResult(string high_confidence_text,
+                                        string low_confidence_text);
+
+  // Assistant speech recognition detected end of utterance.
+  OnSpeechRecognitionEndOfUtterance();
+
+  // Assistant speech recognition final result is available.
+  OnSpeechRecognitionFinalResult(string final_result);
+
   // Assistant got an instantaneous speech level update in dB.
   OnSpeechLevelUpdated(float speech_level);
 };