Adds AssistantResponse.

Previously, Assistant response text/cards/suggestions were streamed in
and rendered immediately. We actually want to cache them until all
renderable elements for an interaction are received, and then show them
all at once.

This is a prerequisite for motion specs which require animating the
response in as a whole and resizing the container smoothly.

To accomplish this, this CL adds:
- AssistantResponse object to hold the renderable elements.
- A cached AssistantResponse in AssistantInteractionController that is
  not added to the model until it is finalized.
- Notification of TTS started event to AssistantInteractionController.
  This is used as an early indication that we have received all
  renderable elements for an interaction.

Bug: b:111932420
Change-Id: I0cc84079183fc30b8c949491e5e2bc4d0b4365b1
Reviewed-on: https://chromium-review.googlesource.com/1153626Reviewed-by: Xiaohui Chen <xiaohuic@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Commit-Queue: David Black <dmblack@google.com>
Cr-Commit-Position: refs/heads/master@{#579859}

ash/BUILD.gn

@@ -113,6 +113,8 @@ component("ash") {
     "assistant/model/assistant_interaction_model_observer.h",
     "assistant/model/assistant_query.cc",
     "assistant/model/assistant_query.h",
+    "assistant/model/assistant_response.cc",
+    "assistant/model/assistant_response.h",
     "assistant/model/assistant_screen_context_model.cc",
     "assistant/model/assistant_screen_context_model.h",
     "assistant/model/assistant_screen_context_model_observer.h",

ash/assistant/assistant_interaction_controller.cc

@@ -8,6 +8,7 @@
 #include "ash/assistant/assistant_ui_controller.h"
 #include "ash/assistant/model/assistant_interaction_model_observer.h"
 #include "ash/assistant/model/assistant_query.h"
+#include "ash/assistant/model/assistant_response.h"
 #include "ash/assistant/model/assistant_ui_element.h"
 #include "ash/assistant/util/deep_link_util.h"
 #include "ash/shell.h"
@@ -172,19 +173,24 @@ void AssistantInteractionController::OnInteractionStarted(
 
     // Clear the interaction to wipe the stage.
     assistant_interaction_model_.ClearInteraction(
-        /*retain_committed_query=*/true);
+        /*retain_committed_query=*/true,
+        /*retain_pending_query=*/false);
   }
+
+  // Start caching a new Assistant response for the interaction.
+  assistant_interaction_model_.SetPendingResponse(
+      std::make_unique<AssistantResponse>());
 }
 
 void AssistantInteractionController::OnInteractionFinished(
     AssistantInteractionResolution resolution) {
   assistant_interaction_model_.SetInteractionState(InteractionState::kInactive);
+  assistant_interaction_model_.SetMicState(MicState::kClosed);
 
-  // When a voice query is interrupted we do not receive any follow up speech
-  // recognition events but the mic is closed.
-  if (resolution == AssistantInteractionResolution::kInterruption) {
-    assistant_interaction_model_.SetMicState(MicState::kClosed);
-  }
+  // The interaction has finished, so we finalize the pending response if it
+  // hasn't already been finalized.
+  if (assistant_interaction_model_.pending_response())
+    assistant_interaction_model_.FinalizePendingResponse();
 }
 
 void AssistantInteractionController::OnHtmlResponse(
@@ -194,13 +200,13 @@ void AssistantInteractionController::OnHtmlResponse(
     return;
   }
 
-  assistant_interaction_model_.AddUiElement(
+  assistant_interaction_model_.pending_response()->AddUiElement(
       std::make_unique<AssistantCardElement>(response));
 }
 
 void AssistantInteractionController::OnSuggestionChipPressed(int id) {
   const AssistantSuggestion* suggestion =
-      assistant_interaction_model_.GetSuggestionById(id);
+      assistant_interaction_model_.response()->GetSuggestionById(id);
 
   // If the suggestion contains a non-empty action url, we will handle the
   // suggestion chip pressed event by launching the action url in the browser.
@@ -220,7 +226,8 @@ void AssistantInteractionController::OnSuggestionsResponse(
     return;
   }
 
-  assistant_interaction_model_.AddSuggestions(std::move(response));
+  assistant_interaction_model_.pending_response()->AddSuggestions(
+      std::move(response));
 }
 
 void AssistantInteractionController::OnTextResponse(
@@ -230,7 +237,7 @@ void AssistantInteractionController::OnTextResponse(
     return;
   }
 
-  assistant_interaction_model_.AddUiElement(
+  assistant_interaction_model_.pending_response()->AddUiElement(
       std::make_unique<AssistantTextElement>(response));
 }
 
@@ -261,13 +268,27 @@ void AssistantInteractionController::OnSpeechRecognitionFinalResult(
 
   // Clear the interaction to wipe the stage.
   assistant_interaction_model_.ClearInteraction(
-      /*retain_committed_query=*/true);
+      /*retain_committed_query=*/true,
+      /*retain_pending_response=*/true);
 }
 
 void AssistantInteractionController::OnSpeechLevelUpdated(float speech_level) {
   assistant_interaction_model_.SetSpeechLevel(speech_level);
 }
 
+void AssistantInteractionController::OnTtsStarted() {
+  if (assistant_interaction_model_.interaction_state() !=
+      InteractionState::kActive) {
+    return;
+  }
+
+  // We have an agreement with the server that TTS will always be the last part
+  // of an interaction to be processed. To be timely in updating UI, we use
+  // this as an opportunity to finalize the Assistant response and update the
+  // interaction model.
+  assistant_interaction_model_.FinalizePendingResponse();
+}
+
 void AssistantInteractionController::OnOpenUrlResponse(const GURL& url) {
   if (assistant_interaction_model_.interaction_state() !=
       InteractionState::kActive) {
@@ -329,6 +350,11 @@ void AssistantInteractionController::StopActiveInteraction() {
   assistant_interaction_model_.SetInteractionState(InteractionState::kInactive);
   assistant_interaction_model_.ClearPendingQuery();
   assistant_->StopActiveInteraction();
+
+  // Because we are stopping an interaction in progress, we discard any pending
+  // response for it that is cached to prevent it from being finalized when the
+  // interaction is finished.
+  assistant_interaction_model_.ClearPendingResponse();
 }
 
 }  // namespace ash

ash/assistant/assistant_interaction_controller.h

@@ -87,6 +87,7 @@ class AssistantInteractionController
   void OnSpeechRecognitionEndOfUtterance() override;
   void OnSpeechRecognitionFinalResult(const std::string& final_result) override;
   void OnSpeechLevelUpdated(float speech_level) override;
+  void OnTtsStarted() override;
 
   // DialogPlateObserver:
   void OnDialogPlateButtonPressed(DialogPlateButtonId id) override;

ash/assistant/model/assistant_interaction_model.cc

@@ -6,7 +6,7 @@
 
 #include "ash/assistant/model/assistant_interaction_model_observer.h"
 #include "ash/assistant/model/assistant_query.h"
-#include "ash/assistant/model/assistant_ui_element.h"
+#include "ash/assistant/model/assistant_response.h"
 
 namespace ash {
 
@@ -26,13 +26,22 @@ void AssistantInteractionModel::RemoveObserver(
   observers_.RemoveObserver(observer);
 }
 
-void AssistantInteractionModel::ClearInteraction(bool retain_committed_query) {
+void AssistantInteractionModel::ClearInteraction() {
+  ClearInteraction(/*retain_committed_query=*/false,
+                   /*retain_pending_response=*/false);
+}
+
+void AssistantInteractionModel::ClearInteraction(bool retain_committed_query,
+                                                 bool retain_pending_response) {
   if (!retain_committed_query)
     ClearCommittedQuery();
 
   ClearPendingQuery();
-  ClearUiElements();
-  ClearSuggestions();
+
+  if (!retain_pending_response)
+    ClearPendingResponse();
+
+  ClearResponse();
 }
 
 void AssistantInteractionModel::SetInteractionState(
@@ -93,44 +102,24 @@ void AssistantInteractionModel::ClearPendingQuery() {
   NotifyPendingQueryCleared();
 }
 
-void AssistantInteractionModel::AddUiElement(
-    std::unique_ptr<AssistantUiElement> ui_element) {
-  AssistantUiElement* ptr = ui_element.get();
-  ui_element_list_.push_back(std::move(ui_element));
-  NotifyUiElementAdded(ptr);
+void AssistantInteractionModel::SetPendingResponse(
+    std::unique_ptr<AssistantResponse> pending_response) {
+  pending_response_ = std::move(pending_response);
 }
 
-void AssistantInteractionModel::ClearUiElements() {
-  ui_element_list_.clear();
-  NotifyUiElementsCleared();
-}
-
-void AssistantInteractionModel::AddSuggestions(
-    std::vector<AssistantSuggestionPtr> suggestions) {
-  std::map<int, AssistantSuggestion*> ptrs;
-
-  // We use vector index to uniquely identify a given suggestion. This means
-  // that suggestion ids will reset with each call to |ClearSuggestions|, but
-  // that is acceptable.
-  for (AssistantSuggestionPtr& suggestion : suggestions) {
-    int id = suggestions_.size();
-    suggestions_.push_back(std::move(suggestion));
-    ptrs[id] = suggestions_.back().get();
-  }
-
-  NotifySuggestionsAdded(ptrs);
+void AssistantInteractionModel::FinalizePendingResponse() {
+  DCHECK(pending_response_);
+  response_ = std::move(pending_response_);
+  NotifyResponseChanged();
 }
 
-const AssistantInteractionModel::AssistantSuggestion*
-AssistantInteractionModel::GetSuggestionById(int id) const {
-  return id >= 0 && id < static_cast<int>(suggestions_.size())
-             ? suggestions_.at(id).get()
-             : nullptr;
+void AssistantInteractionModel::ClearPendingResponse() {
+  pending_response_.reset();
 }
 
-void AssistantInteractionModel::ClearSuggestions() {
-  suggestions_.clear();
-  NotifySuggestionsCleared();
+void AssistantInteractionModel::ClearResponse() {
+  response_.reset();
+  NotifyResponseCleared();
 }
 
 void AssistantInteractionModel::SetSpeechLevel(float speech_level_db) {
@@ -172,26 +161,14 @@ void AssistantInteractionModel::NotifyPendingQueryCleared() {
     observer.OnPendingQueryCleared();
 }
 
-void AssistantInteractionModel::NotifyUiElementAdded(
-    const AssistantUiElement* ui_element) {
-  for (AssistantInteractionModelObserver& observer : observers_)
-    observer.OnUiElementAdded(ui_element);
-}
-
-void AssistantInteractionModel::NotifyUiElementsCleared() {
-  for (AssistantInteractionModelObserver& observer : observers_)
-    observer.OnUiElementsCleared();
-}
-
-void AssistantInteractionModel::NotifySuggestionsAdded(
-    const std::map<int, AssistantSuggestion*>& suggestions) {
+void AssistantInteractionModel::NotifyResponseChanged() {
   for (AssistantInteractionModelObserver& observer : observers_)
-    observer.OnSuggestionsAdded(suggestions);
+    observer.OnResponseChanged(*response_);
 }
 
-void AssistantInteractionModel::NotifySuggestionsCleared() {
+void AssistantInteractionModel::NotifyResponseCleared() {
   for (AssistantInteractionModelObserver& observer : observers_)
-    observer.OnSuggestionsCleared();
+    observer.OnResponseCleared();
 }
 
 void AssistantInteractionModel::NotifySpeechLevelChanged(

ash/assistant/model/assistant_interaction_model.h

@@ -11,13 +11,12 @@
 
 #include "base/macros.h"
 #include "base/observer_list.h"
-#include "chromeos/services/assistant/public/mojom/assistant.mojom.h"
 
 namespace ash {
 
 class AssistantInteractionModelObserver;
 class AssistantQuery;
-class AssistantUiElement;
+class AssistantResponse;
 
 // Enumeration of interaction input modalities.
 enum class InputModality {
@@ -42,13 +41,9 @@ enum class MicState {
 };
 
 // Models the Assistant interaction. This includes query state, state of speech
-// recognition, as well as renderable AssistantUiElements and suggestions.
+// recognition, as well as a renderable AssistantResponse.
 class AssistantInteractionModel {
  public:
-  using AssistantSuggestion = chromeos::assistant::mojom::AssistantSuggestion;
-  using AssistantSuggestionPtr =
-      chromeos::assistant::mojom::AssistantSuggestionPtr;
-
   AssistantInteractionModel();
   ~AssistantInteractionModel();
 
@@ -56,10 +51,16 @@ class AssistantInteractionModel {
   void AddObserver(AssistantInteractionModelObserver* observer);
   void RemoveObserver(AssistantInteractionModelObserver* observer);
 
+  // Resets the interaction to its initial state.
+  void ClearInteraction();
+
   // Resets the interaction to its initial state. There are instances in which
-  // we wish to clear the interaction but retain the committed query. As such,
-  // |retain_committed_query| is provided but defaults to |false|.
-  void ClearInteraction(bool retain_committed_query = false);
+  // we wish to clear the interaction but retain the committed query. Similarly,
+  // there are instances in which we wish to retain the pending response that is
+  // currently cached. For such instances, use |retain_committed_query| and
+  // |retain_pending_response| respectively.
+  void ClearInteraction(bool retain_committed_query,
+                        bool retain_pending_response);
 
   // Sets the interaction state.
   void SetInteractionState(InteractionState interaction_state);
@@ -97,23 +98,23 @@ class AssistantInteractionModel {
   // Clears the pending query for the interaction.
   void ClearPendingQuery();
 
-  // Adds the specified |ui_element| that should be rendered for the
-  // interaction.
-  void AddUiElement(std::unique_ptr<AssistantUiElement> ui_element);
+  // Sets the pending response for the interaction.
+  void SetPendingResponse(std::unique_ptr<AssistantResponse> response);
+
+  // Returns the pending response for the interaction.
+  AssistantResponse* pending_response() { return pending_response_.get(); }
 
-  // Clears all UI elements for the interaction.
-  void ClearUiElements();
+  // Finalizes the pending response for the interaction.
+  void FinalizePendingResponse();
 
-  // Adds the specified |suggestions| that should be rendered for the
-  // interaction.
-  void AddSuggestions(std::vector<AssistantSuggestionPtr> suggestions);
+  // Clears the pending response for the interaction.
+  void ClearPendingResponse();
 
-  // Returns the suggestion uniquely identified by the specified |id|, or
-  // |nullptr| if no matching suggestion is found.
-  const AssistantSuggestion* GetSuggestionById(int id) const;
+  // Returns the finalized response for the interaction.
+  const AssistantResponse* response() const { return response_.get(); }
 
-  // Clears all suggestions for the interaction.
-  void ClearSuggestions();
+  // Clears the finalized response for the interaction.
+  void ClearResponse();
 
   // Updates the speech level in dB.
   void SetSpeechLevel(float speech_level_db);
@@ -126,11 +127,8 @@ class AssistantInteractionModel {
   void NotifyCommittedQueryCleared();
   void NotifyPendingQueryChanged();
   void NotifyPendingQueryCleared();
-  void NotifyUiElementAdded(const AssistantUiElement* ui_element);
-  void NotifyUiElementsCleared();
-  void NotifySuggestionsAdded(
-      const std::map<int, AssistantSuggestion*>& suggestions);
-  void NotifySuggestionsCleared();
+  void NotifyResponseChanged();
+  void NotifyResponseCleared();
   void NotifySpeechLevelChanged(float speech_level_db);
 
   InteractionState interaction_state_ = InteractionState::kInactive;
@@ -138,8 +136,8 @@ class AssistantInteractionModel {
   MicState mic_state_ = MicState::kClosed;
   std::unique_ptr<AssistantQuery> committed_query_;
   std::unique_ptr<AssistantQuery> pending_query_;
-  std::vector<AssistantSuggestionPtr> suggestions_;
-  std::vector<std::unique_ptr<AssistantUiElement>> ui_element_list_;
+  std::unique_ptr<AssistantResponse> pending_response_;
+  std::unique_ptr<AssistantResponse> response_;
 
   base::ObserverList<AssistantInteractionModelObserver> observers_;
 

ash/assistant/model/assistant_interaction_model_observer.h

@@ -11,18 +11,10 @@
 
 #include "base/macros.h"
 
-namespace chromeos {
-namespace assistant {
-namespace mojom {
-class AssistantSuggestion;
-}  // namespace mojom
-}  // namespace assistant
-}  // namespace chromeos
-
 namespace ash {
 
 class AssistantQuery;
-class AssistantUiElement;
+class AssistantResponse;
 enum class InputModality;
 enum class InteractionState;
 enum class MicState;
@@ -31,8 +23,6 @@ enum class MicState;
 // interaction.
 class AssistantInteractionModelObserver {
  public:
-  using AssistantSuggestion = chromeos::assistant::mojom::AssistantSuggestion;
-
   // Invoked when the interaction state is changed.
   virtual void OnInteractionStateChanged(InteractionState interaction_state) {}
 
@@ -56,21 +46,11 @@ class AssistantInteractionModelObserver {
   // Invoked when the pending query associated with the interaction is cleared.
   virtual void OnPendingQueryCleared() {}
 
-  // Invoked when a UI element associated with the interaction is added.
-  virtual void OnUiElementAdded(const AssistantUiElement* ui_element) {}
-
-  // Invoked when all UI elements associated with the interaction are cleared.
-  virtual void OnUiElementsCleared() {}
-
-  // Invoked when the specified |suggestions| are added to the associated
-  // interaction. The key for the map is the unique identifier by which the
-  // interaction model identifies each suggestion before the next
-  // |OnSuggestionsCleared| call.
-  virtual void OnSuggestionsAdded(
-      const std::map<int, AssistantSuggestion*>& suggestions) {}
+  // Invoked when the response associated with the interaction is changed.
+  virtual void OnResponseChanged(const AssistantResponse& response) {}
 
-  // Invoked when all suggestions associated with the interaction are cleared.
-  virtual void OnSuggestionsCleared() {}
+  // Invoked when the response associated with the interaction is cleared.
+  virtual void OnResponseCleared() {}
 
   // Invoked when the speech level is changed.
   virtual void OnSpeechLevelChanged(float speech_level_db) {}

ash/assistant/model/assistant_response.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "ash/assistant/model/assistant_response.h"
+
+#include "ash/assistant/model/assistant_ui_element.h"
+
+namespace ash {
+
+AssistantResponse::AssistantResponse() = default;
+
+AssistantResponse::~AssistantResponse() = default;
+
+void AssistantResponse::AddUiElement(
+    std::unique_ptr<AssistantUiElement> ui_element) {
+  ui_elements_.push_back(std::move(ui_element));
+}
+
+const std::vector<std::unique_ptr<AssistantUiElement>>&
+AssistantResponse::GetUiElements() const {
+  return ui_elements_;
+}
+
+void AssistantResponse::AddSuggestions(
+    std::vector<AssistantSuggestionPtr> suggestions) {
+  for (AssistantSuggestionPtr& suggestion : suggestions)
+    suggestions_.push_back(std::move(suggestion));
+}
+
+const chromeos::assistant::mojom::AssistantSuggestion*
+AssistantResponse::GetSuggestionById(int id) const {
+  // We consider the index of a suggestion within our backing vector to be its
+  // unique identifier.
+  DCHECK_GE(id, 0);
+  DCHECK_LT(id, static_cast<int>(suggestions_.size()));
+  return suggestions_.at(id).get();
+}
+
+std::map<int, const chromeos::assistant::mojom::AssistantSuggestion*>
+AssistantResponse::GetSuggestions() const {
+  std::map<int, const AssistantSuggestion*> suggestions;
+
+  // We use index within our backing vector to represent the unique identifier
+  // for a suggestion.
+  int id = 0;
+  for (const AssistantSuggestionPtr& suggestion : suggestions_)
+    suggestions[id++] = suggestion.get();
+
+  return suggestions;
+}
+
+}  // namespace ash
\ No newline at end of file

ash/assistant/model/assistant_response.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef ASH_ASSISTANT_MODEL_ASSISTANT_RESPONSE_H_
+#define ASH_ASSISTANT_MODEL_ASSISTANT_RESPONSE_H_
+
+#include <map>
+#include <memory>
+#include <vector>
+
+#include "base/macros.h"
+#include "chromeos/services/assistant/public/mojom/assistant.mojom.h"
+
+namespace ash {
+
+class AssistantUiElement;
+
+// Models a renderable Assistant response.
+class AssistantResponse {
+ public:
+  using AssistantSuggestion = chromeos::assistant::mojom::AssistantSuggestion;
+  using AssistantSuggestionPtr =
+      chromeos::assistant::mojom::AssistantSuggestionPtr;
+
+  AssistantResponse();
+  ~AssistantResponse();
+
+  // Adds the specified |ui_element| that should be rendered for the
+  // interaction.
+  void AddUiElement(std::unique_ptr<AssistantUiElement> ui_element);
+
+  // Returns all UI elements belonging to the response.
+  const std::vector<std::unique_ptr<AssistantUiElement>>& GetUiElements() const;
+
+  // Adds the specified |suggestions| that should be rendered for the
+  // interaction.
+  void AddSuggestions(std::vector<AssistantSuggestionPtr> suggestions);
+
+  // Returns the suggestion uniquely identified by |id|, or |nullptr| if no
+  // matching suggestion exists.
+  const AssistantSuggestion* GetSuggestionById(int id) const;
+
+  // Returns all suggestions belongs to the response, mapped to a unique id.
+  std::map<int, const AssistantSuggestion*> GetSuggestions() const;
+
+ private:
+  std::vector<std::unique_ptr<AssistantUiElement>> ui_elements_;
+  std::vector<AssistantSuggestionPtr> suggestions_;
+
+  DISALLOW_COPY_AND_ASSIGN(AssistantResponse);
+};
+
+}  // namespace ash
+
+#endif  // ASH_ASSISTANT_MODEL_ASSISTANT_RESPONSE_H_

ash/assistant/ui/dialog_plate/dialog_plate.cc

@@ -114,13 +114,13 @@ void DialogPlate::InitLayout() {
   input_modality_layout_container->SetPaintToLayer();
   input_modality_layout_container->layer()->SetFillsBoundsOpaquely(false);
   input_modality_layout_container->layer()->SetMasksToBounds(true);
+  AddChildView(input_modality_layout_container);
+
+  layout_manager->SetFlexForView(input_modality_layout_container, 1);
 
   InitKeyboardLayoutContainer(input_modality_layout_container);
   InitVoiceLayoutContainer(input_modality_layout_container);
 
-  layout_manager->SetFlexForView(input_modality_layout_container, 1);
-  AddChildView(input_modality_layout_container);
-
   // Settings.
   views::ImageButton* settings_button = new views::ImageButton(this);
   settings_button->set_id(static_cast<int>(DialogPlateButtonId::kSettings));

ash/assistant/ui/main_stage/assistant_header_view.cc

@@ -107,8 +107,7 @@ void AssistantHeaderView::OnCommittedQueryChanged(
   progress_indicator_->SetVisible(true);
 }
 
-void AssistantHeaderView::OnUiElementAdded(
-    const AssistantUiElement* ui_element) {
+void AssistantHeaderView::OnResponseChanged(const AssistantResponse& response) {
   progress_indicator_->SetVisible(false);
 }
 

ash/assistant/ui/main_stage/assistant_header_view.h

@@ -36,7 +36,7 @@ class AssistantHeaderView : public views::View,
 
   // AssistantInteractionModelObserver:
   void OnCommittedQueryChanged(const AssistantQuery& committed_query) override;
-  void OnUiElementAdded(const AssistantUiElement* ui_element) override;
+  void OnResponseChanged(const AssistantResponse& response) override;
 
   // AssistantUiModelObserver:
   void OnUiVisibilityChanged(bool visible, AssistantSource source) override;

ash/assistant/ui/main_stage/suggestion_container_view.cc

@@ -8,6 +8,7 @@
 
 #include "ash/assistant/assistant_controller.h"
 #include "ash/assistant/assistant_interaction_controller.h"
+#include "ash/assistant/model/assistant_response.h"
 #include "ash/assistant/ui/assistant_ui_constants.h"
 #include "base/strings/utf_string_conversions.h"
 #include "ui/views/controls/scrollbar/overlay_scroll_bar.h"
@@ -81,9 +82,11 @@ void SuggestionContainerView::InitLayout() {
   SetVerticalScrollBar(new InvisibleScrollBar(/*horizontal=*/false));
 }
 
-void SuggestionContainerView::OnSuggestionsAdded(
-    const std::map<int, AssistantSuggestion*>& suggestions) {
-  for (const std::pair<int, AssistantSuggestion*>& suggestion : suggestions) {
+void SuggestionContainerView::OnResponseChanged(
+    const AssistantResponse& response) {
+  using AssistantSuggestion = chromeos::assistant::mojom::AssistantSuggestion;
+  for (const std::pair<int, const AssistantSuggestion*>& suggestion :
+       response.GetSuggestions()) {
     // We will use the same identifier by which the Assistant interaction model
     // uniquely identifies a suggestion to uniquely identify its corresponding
     // suggestion chip view.
@@ -120,10 +123,11 @@ void SuggestionContainerView::OnSuggestionsAdded(
 
     contents_view_->AddChildView(suggestion_chip_view);
   }
+
   UpdateContentsBounds();
 }
 
-void SuggestionContainerView::OnSuggestionsCleared() {
+void SuggestionContainerView::OnResponseCleared() {
   // Abort any download requests in progress.
   download_request_weak_factory_.InvalidateWeakPtrs();
 

ash/assistant/ui/main_stage/suggestion_container_view.h

@@ -10,6 +10,7 @@
 #include "ash/app_list/views/suggestion_chip_view.h"
 #include "ash/assistant/model/assistant_interaction_model_observer.h"
 #include "base/macros.h"
+#include "chromeos/services/assistant/public/mojom/assistant.mojom.h"
 #include "ui/views/controls/scroll_view.h"
 
 namespace ash {
@@ -33,9 +34,8 @@ class SuggestionContainerView : public views::ScrollView,
   int GetHeightForWidth(int width) const override;
 
   // AssistantInteractionModelObserver:
-  void OnSuggestionsAdded(
-      const std::map<int, AssistantSuggestion*>& suggestions) override;
-  void OnSuggestionsCleared() override;
+  void OnResponseChanged(const AssistantResponse& response) override;
+  void OnResponseCleared() override;
 
   // views::ButtonListener:
   void ButtonPressed(views::Button* sender, const ui::Event& event) override;

ash/assistant/ui/main_stage/ui_element_container_view.cc

@@ -6,6 +6,7 @@
 
 #include "ash/assistant/assistant_controller.h"
 #include "ash/assistant/assistant_interaction_controller.h"
+#include "ash/assistant/model/assistant_response.h"
 #include "ash/assistant/model/assistant_ui_element.h"
 #include "ash/assistant/ui/assistant_ui_constants.h"
 #include "ash/assistant/ui/main_stage/assistant_header_view.h"
@@ -63,26 +64,24 @@ void UiElementContainerView::InitLayout() {
   AddChildView(assistant_header_view_.get());
 }
 
-void UiElementContainerView::OnUiElementAdded(
-    const AssistantUiElement* ui_element) {
-  // If we are processing a UI element we need to pend the incoming element
-  // instead of handling it immediately.
-  if (is_processing_ui_element_) {
-    pending_ui_element_list_.push_back(ui_element);
-    return;
-  }
+void UiElementContainerView::OnResponseChanged(
+    const AssistantResponse& response) {
+  OnResponseCleared();
 
-  switch (ui_element->GetType()) {
-    case AssistantUiElementType::kCard:
-      OnCardElementAdded(static_cast<const AssistantCardElement*>(ui_element));
-      break;
-    case AssistantUiElementType::kText:
-      OnTextElementAdded(static_cast<const AssistantTextElement*>(ui_element));
-      break;
+  for (const std::unique_ptr<AssistantUiElement>& ui_element :
+       response.GetUiElements()) {
+    // If we are processing a UI element we need to pend the incoming elements
+    // instead of handling them immediately.
+    if (is_processing_ui_element_) {
+      pending_ui_element_list_.push_back(ui_element.get());
+      continue;
+    }
+
+    OnUiElementAdded(ui_element.get());
   }
 }
 
-void UiElementContainerView::OnUiElementsCleared() {
+void UiElementContainerView::OnResponseCleared() {
   // Prevent any in-flight card rendering requests from returning.
   render_request_weak_factory_.InvalidateWeakPtrs();
 
@@ -98,6 +97,18 @@ void UiElementContainerView::OnUiElementsCleared() {
   SetProcessingUiElement(false);
 }
 
+void UiElementContainerView::OnUiElementAdded(
+    const AssistantUiElement* ui_element) {
+  switch (ui_element->GetType()) {
+    case AssistantUiElementType::kCard:
+      OnCardElementAdded(static_cast<const AssistantCardElement*>(ui_element));
+      break;
+    case AssistantUiElementType::kText:
+      OnTextElementAdded(static_cast<const AssistantTextElement*>(ui_element));
+      break;
+  }
+}
+
 void UiElementContainerView::OnCardElementAdded(
     const AssistantCardElement* card_element) {
   DCHECK(!is_processing_ui_element_);

ash/assistant/ui/main_stage/ui_element_container_view.h

@@ -18,6 +18,7 @@ namespace ash {
 class AssistantCardElement;
 class AssistantController;
 class AssistantHeaderView;
+class AssistantResponse;
 class AssistantTextElement;
 class AssistantUiElement;
 
@@ -34,12 +35,13 @@ class UiElementContainerView : public views::View,
   void ChildPreferredSizeChanged(views::View* child) override;
 
   // AssistantInteractionModelObserver:
-  void OnUiElementAdded(const AssistantUiElement* ui_element) override;
-  void OnUiElementsCleared() override;
+  void OnResponseChanged(const AssistantResponse& response) override;
+  void OnResponseCleared() override;
 
  private:
   void InitLayout();
 
+  void OnUiElementAdded(const AssistantUiElement* ui_element);
   void OnCardElementAdded(const AssistantCardElement* card_element);
   void OnCardReady(const base::Optional<base::UnguessableToken>& embed_token);
   void OnTextElementAdded(const AssistantTextElement* text_element);

chromeos/services/assistant/assistant_manager_service_impl.cc

@@ -371,6 +371,14 @@ void AssistantManagerServiceImpl::OnRecognitionStateChanged(
           weak_factory_.GetWeakPtr(), state, recognition_result));
 }
 
+void AssistantManagerServiceImpl::OnRespondingStarted(bool is_error_response) {
+  main_thread_task_runner_->PostTask(
+      FROM_HERE,
+      base::BindOnce(
+          &AssistantManagerServiceImpl::OnRespondingStartedOnMainThread,
+          weak_factory_.GetWeakPtr()));
+}
+
 void AssistantManagerServiceImpl::OnSpeechLevelUpdated(
     const float speech_level) {
   main_thread_task_runner_->PostTask(
@@ -715,6 +723,10 @@ void AssistantManagerServiceImpl::OnRecognitionStateChangedOnMainThread(
   }
 }
 
+void AssistantManagerServiceImpl::OnRespondingStartedOnMainThread() {
+  interaction_subscribers_.ForAllPtrs([](auto* ptr) { ptr->OnTtsStarted(); });
+}
+
 void AssistantManagerServiceImpl::OnSpeechLevelUpdatedOnMainThread(
     const float speech_level) {
   interaction_subscribers_.ForAllPtrs(

chromeos/services/assistant/assistant_manager_service_impl.h

@@ -119,6 +119,7 @@ class AssistantManagerServiceImpl
       assistant_client::ConversationStateListener::RecognitionState state,
       const assistant_client::ConversationStateListener::RecognitionResult&
           recognition_result) override;
+  void OnRespondingStarted(bool is_error_response) override;
 
   // AssistantManagerDelegate overrides
   assistant_client::ActionModule::Result HandleModifySettingClientOp(
@@ -177,6 +178,7 @@ class AssistantManagerServiceImpl
       assistant_client::ConversationStateListener::RecognitionState state,
       const assistant_client::ConversationStateListener::RecognitionResult&
           recognition_result);
+  void OnRespondingStartedOnMainThread();
   void OnSpeechLevelUpdatedOnMainThread(const float speech_level);
   void OnModifySettingsAction(const std::string& modify_setting_args_proto);
 

chromeos/services/assistant/public/mojom/assistant.mojom

@@ -90,6 +90,9 @@ interface AssistantInteractionSubscriber {
 
   // Assistant got an instantaneous speech level update in dB.
   OnSpeechLevelUpdated(float speech_level);
+
+  // Assistant has started speaking.
+  OnTtsStarted();
 };
 
 // Subscribes to assistant's notification event. These events are server