Implement automatic reading of Assistant card responses.

https://docs.google.com/document/d/1dzjfTfiTf_wVq_cwH85i2VQWDjKWt3ZsiqokZg21jjw/edit?usp=sharing

Bug: b/117274233
Change-Id: I9ac16364a62cb8f55925dc8d5d04fdc5ce425ae9
Reviewed-on: https://chromium-review.googlesource.com/c/1277844Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Xiaohui Chen <xiaohuic@chromium.org>
Commit-Queue: Leo Liu <luciferleo@google.com>
Cr-Commit-Position: refs/heads/master@{#600128}

ash/assistant/assistant_interaction_controller.cc

@@ -90,8 +90,8 @@ void AssistantInteractionController::OnAssistantControllerDestroying() {
 void AssistantInteractionController::OnDeepLinkReceived(
     assistant::util::DeepLinkType type,
     const std::map<std::string, std::string>& params) {
-  using assistant::util::DeepLinkType;
   using assistant::util::DeepLinkParam;
+  using assistant::util::DeepLinkType;
 
   if (type == DeepLinkType::kWhatsOnMyScreen) {
     StartScreenContextInteraction();
@@ -352,7 +352,8 @@ void AssistantInteractionController::OnInteractionFinished(
 }
 
 void AssistantInteractionController::OnHtmlResponse(
-    const std::string& response) {
+    const std::string& response,
+    const std::string& fallback) {
   if (model_.interaction_state() != InteractionState::kActive) {
     return;
   }
@@ -365,7 +366,7 @@ void AssistantInteractionController::OnHtmlResponse(
   }
 
   model_.pending_response()->AddUiElement(
-      std::make_unique<AssistantCardElement>(response));
+      std::make_unique<AssistantCardElement>(response, fallback));
 }
 
 void AssistantInteractionController::OnSuggestionChipPressed(

ash/assistant/assistant_interaction_controller.h

@@ -88,7 +88,8 @@ class AssistantInteractionController
   void OnInteractionStarted(bool is_voice_interaction) override;
   void OnInteractionFinished(
       AssistantInteractionResolution resolution) override;
-  void OnHtmlResponse(const std::string& response) override;
+  void OnHtmlResponse(const std::string& response,
+                      const std::string& fallback) override;
   void OnSuggestionsResponse(
       std::vector<AssistantSuggestionPtr> response) override;
   void OnTextResponse(const std::string& response) override;

ash/assistant/model/assistant_ui_element.cc

@@ -8,9 +8,11 @@ namespace ash {
 
 // AssistantCardElement --------------------------------------------------------
 
-AssistantCardElement::AssistantCardElement(const std::string& html)
+AssistantCardElement::AssistantCardElement(const std::string& html,
+                                           const std::string& fallback)
     : AssistantUiElement(AssistantUiElementType::kCard),
       html_(html),
+      fallback_(fallback),
       id_token_(base::UnguessableToken::Create()) {}
 
 AssistantCardElement::~AssistantCardElement() = default;

ash/assistant/model/assistant_ui_element.h

@@ -44,11 +44,14 @@ class AssistantUiElement {
 // An Assistant UI element that will be rendered as an HTML card.
 class AssistantCardElement : public AssistantUiElement {
  public:
-  explicit AssistantCardElement(const std::string& html);
+  explicit AssistantCardElement(const std::string& html,
+                                const std::string& fallback);
   ~AssistantCardElement() override;
 
   const std::string& html() const { return html_; }
 
+  const std::string& fallback() const { return fallback_; }
+
   const base::UnguessableToken& id_token() const { return id_token_; }
 
   const base::Optional<base::UnguessableToken>& embed_token() const {
@@ -62,6 +65,7 @@ class AssistantCardElement : public AssistantUiElement {
 
  private:
   const std::string html_;
+  const std::string fallback_;
   base::UnguessableToken id_token_;
   base::Optional<base::UnguessableToken> embed_token_ = base::nullopt;
 

ash/assistant/ui/main_stage/ui_element_container_view.cc

@@ -27,6 +27,7 @@
 #include "ui/events/event.h"
 #include "ui/events/event_sink.h"
 #include "ui/events/event_utils.h"
+#include "ui/views/accessibility/view_accessibility.h"
 #include "ui/views/border.h"
 #include "ui/views/controls/native/native_view_host.h"
 #include "ui/views/layout/box_layout.h"
@@ -90,8 +91,9 @@ void CreateAndSendMouseClick(aura::WindowTreeHost* host,
 class CardElementViewHolder : public views::NativeViewHost,
                               public views::ViewObserver {
  public:
-  explicit CardElementViewHolder(views::View* card_element_view)
-      : card_element_view_(card_element_view) {
+  explicit CardElementViewHolder(const AssistantCardElement* card_element)
+      : card_element_view_(app_list::AnswerCardContentsRegistry::Get()->GetView(
+            card_element->embed_token().value())) {
     views::Widget::InitParams params(views::Widget::InitParams::TYPE_CONTROL);
 
     params.name = GetClassName();
@@ -107,6 +109,9 @@ class CardElementViewHolder : public views::NativeViewHost,
     contents_view_->AddChildView(card_element_view_);
 
     card_element_view_->AddObserver(this);
+
+    // OverrideDescription() doesn't work. Only names are read automatically.
+    GetViewAccessibility().OverrideName(card_element->fallback());
   }
 
   ~CardElementViewHolder() override {
@@ -398,9 +403,7 @@ void UiElementContainerView::OnCardElementAdded(
   // When the card has been rendered in the same process, its view is
   // available in the AnswerCardContentsRegistry's token-to-view map.
   if (app_list::AnswerCardContentsRegistry::Get()) {
-    CardElementViewHolder* view_holder = new CardElementViewHolder(
-        app_list::AnswerCardContentsRegistry::Get()->GetView(
-            card_element->embed_token().value()));
+    auto* view_holder = new CardElementViewHolder(card_element);
 
     if (is_first_card_) {
       is_first_card_ = false;
@@ -482,9 +485,9 @@ void UiElementContainerView::OnAllUiElementsAdded() {
             CreateOpacityElement(1.f, kUiElementAnimationFadeInDuration)));
   }
 
-  // TODO(luciferleo): Add ChromeVox description for WebView.
-  // Let screen reader read the query result. We don't read when there is TTS to
-  // avoid speaking over the server response.
+  // Let screen reader read the query result. This includes the text response
+  // and the card fallback text, but webview result is not included.
+  // We don't read when there is TTS to avoid speaking over the server response.
   const AssistantResponse* response =
       assistant_controller_->interaction_controller()->model()->response();
   if (!response->has_tts())

chromeos/services/assistant/assistant_manager_service_impl.cc

@@ -385,14 +385,15 @@ void AssistantManagerServiceImpl::OnShowContextualQueryFallback() {
   main_thread_task_runner_->PostTask(
       FROM_HERE,
       base::BindOnce(&AssistantManagerServiceImpl::OnShowHtmlOnMainThread,
-                     weak_factory_.GetWeakPtr(), html.str()));
+                     weak_factory_.GetWeakPtr(), html.str(), /*fallback=*/""));
 }
 
-void AssistantManagerServiceImpl::OnShowHtml(const std::string& html) {
+void AssistantManagerServiceImpl::OnShowHtml(const std::string& html,
+                                             const std::string& fallback) {
   main_thread_task_runner_->PostTask(
       FROM_HERE,
       base::BindOnce(&AssistantManagerServiceImpl::OnShowHtmlOnMainThread,
-                     weak_factory_.GetWeakPtr(), html));
+                     weak_factory_.GetWeakPtr(), html, fallback));
 }
 
 void AssistantManagerServiceImpl::OnShowSuggestions(
@@ -898,9 +899,10 @@ void AssistantManagerServiceImpl::OnConversationTurnFinishedOnMainThread(
 }
 
 void AssistantManagerServiceImpl::OnShowHtmlOnMainThread(
-    const std::string& html) {
+    const std::string& html,
+    const std::string& fallback) {
   interaction_subscribers_.ForAllPtrs(
-      [&html](auto* ptr) { ptr->OnHtmlResponse(html); });
+      [&html, &fallback](auto* ptr) { ptr->OnHtmlResponse(html, fallback); });
 }
 
 void AssistantManagerServiceImpl::OnShowSuggestionsOnMainThread(

chromeos/services/assistant/assistant_manager_service_impl.h

@@ -102,7 +102,8 @@ class AssistantManagerServiceImpl
 
   // AssistantActionObserver overrides:
   void OnShowContextualQueryFallback() override;
-  void OnShowHtml(const std::string& html) override;
+  void OnShowHtml(const std::string& html,
+                  const std::string& fallback) override;
   void OnShowSuggestions(
       const std::vector<action::Suggestion>& suggestions) override;
   void OnShowText(const std::string& text) override;
@@ -164,7 +165,8 @@ class AssistantManagerServiceImpl
   void OnConversationTurnStartedOnMainThread(bool is_mic_open);
   void OnConversationTurnFinishedOnMainThread(
       assistant_client::ConversationStateListener::Resolution resolution);
-  void OnShowHtmlOnMainThread(const std::string& html);
+  void OnShowHtmlOnMainThread(const std::string& html,
+                              const std::string& fallback);
   void OnShowSuggestionsOnMainThread(
       const std::vector<mojom::AssistantSuggestionPtr>& suggestions);
   void OnShowTextOnMainThread(const std::string& text);

chromeos/services/assistant/public/mojom/assistant.mojom

@@ -76,8 +76,8 @@ interface AssistantInteractionSubscriber {
   // Assistant interaction has ended with the specified |resolution|.
   OnInteractionFinished(AssistantInteractionResolution resolution);
 
-  // Assistant got Html response from server.
-  OnHtmlResponse(string response);
+  // Assistant got Html response with fallback text from server.
+  OnHtmlResponse(string response, string fallback);
 
   // Assistant got suggestions response from server.
   OnSuggestionsResponse(array<AssistantSuggestion> response);