[Live Caption] Remove "Detecting speech..." message.

The UI showed a "Detecting speech..." message in the caption bubble as
soon as audio started playing to alert the user that Live Caption was
beginning to detect speech. This caused a bug in which the caption
bubble appeared when audio that does not contain speech began to play,
such as short pings. The team decided to remove the "Detecting
speech..." message from the UI as a result.

This CL removes the text from the UI and removes the
OnSpeechRecognitionReady message sent from the
ChromeSpeechRecognitionClient to the caption UI, which is no longer
necessary. It also introduces a new accessible label for the caption
bubble, which is necessary so that the bubble can set the AXNodeData
name when the bubble label does not contain any text.

Bug: 1055150, 1114232
Change-Id: I85fdab48a34176e67733f4540ec534c6e7e0109f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2368464Reviewed-by: Elly Fong-Jones <ellyjones@chromium.org>
Reviewed-by: Avi Drissman <avi@chromium.org>
Reviewed-by: Tom Sepez <tsepez@chromium.org>
Commit-Queue: Abigail Klein <abigailbklein@google.com>
Cr-Commit-Position: refs/heads/master@{#801097}

chrome/app/generated_resources.grd

@@ -9319,9 +9319,6 @@ Please help our engineers fix this problem. Tell us what happened right before y
       </message>
 
       <!-- Live caption. -->
-      <message name="IDS_LIVE_CAPTION_BUBBLE_WAIT_TEXT" desc="Text displayed in the Live Caption bubble when it is waiting for captions from the speech service">
-        Detecting speech...
-      </message>
       <message name="IDS_LIVE_CAPTION_BUBBLE_CLOSE" desc="Tooltip for the Live Caption close button">
         Turn off Live Caption for now
       </message>
@@ -9334,6 +9331,9 @@ Please help our engineers fix this problem. Tell us what happened right before y
       <message name="IDS_LIVE_CAPTION_BUBBLE_ERROR" desc="Error message for the Live Caption bubble when Live Captions are unavailable.">
         Live Caption is not available right now
       </message>
+      <message name="IDS_LIVE_CAPTION_BUBBLE_ACCESSIBLE_NAME" desc="Label of the Live Caption bubble for accessibility.">
+        Live Caption
+      </message>
 
       <!-- Caret browsing. -->
       <if expr="not chromeos">

chrome/app/generated_resources_grd/IDS_LIVE_CAPTION_BUBBLE_ACCESSIBLE_NAME.png.sha1

+5230947232906708b0262c0f093cbe3e7617b65e
\ No newline at end of file

chrome/app/generated_resources_grd/IDS_LIVE_CAPTION_BUBBLE_WAIT_TEXT.png.sha1

-879ad09cc2870b9ee0ebac0bfc2e31338adf2144
\ No newline at end of file

chrome/browser/accessibility/caption_controller.cc

@@ -182,15 +182,6 @@ void CaptionController::OnBrowserRemoved(Browser* browser) {
   caption_bubble_controllers_.erase(browser);
 }
 
-bool CaptionController::OnSpeechRecognitionReady(
-    content::WebContents* web_contents) {
-  Browser* browser = chrome::FindBrowserWithWebContents(web_contents);
-  if (!browser || !caption_bubble_controllers_.count(browser))
-    return false;
-  return caption_bubble_controllers_[browser]->OnSpeechRecognitionReady(
-      web_contents);
-}
-
 bool CaptionController::DispatchTranscription(
     content::WebContents* web_contents,
     const chrome::mojom::TranscriptionResultPtr& transcription_result) {

chrome/browser/accessibility/caption_controller.h

@@ -64,12 +64,6 @@ class CaptionController : public BrowserListObserver, public KeyedService {
 
   void Init();
 
-  // Alerts the CaptionBubbleController that belongs to the appropriate browser
-  // that speech recognition is ready to start for the given web contents.
-  // Returns whether this message was routed successfully. Transcriptions will
-  // not proceed if this returns false.
-  bool OnSpeechRecognitionReady(content::WebContents* web_contents);
-
   // Routes a transcription to the CaptionBubbleController that belongs to the
   // appropriate browser. Returns whether the transcription result was routed
   // successfully. Transcriptions will halt if this returns false.

chrome/browser/accessibility/caption_host_impl.cc

@@ -34,22 +34,6 @@ CaptionHostImpl::CaptionHostImpl(content::RenderFrameHost* frame_host)
 
 CaptionHostImpl::~CaptionHostImpl() = default;
 
-void CaptionHostImpl::OnSpeechRecognitionReady(
-    OnSpeechRecognitionReadyCallback reply) {
-  content::WebContents* web_contents = GetWebContents();
-  if (!web_contents) {
-    std::move(reply).Run(false);
-    return;
-  }
-  CaptionController* caption_controller = GetCaptionController(web_contents);
-  if (!caption_controller) {
-    std::move(reply).Run(false);
-    return;
-  }
-  std::move(reply).Run(
-      caption_controller->OnSpeechRecognitionReady(web_contents));
-}
-
 void CaptionHostImpl::OnTranscription(
     chrome::mojom::TranscriptionResultPtr transcription_result,
     OnTranscriptionCallback reply) {

chrome/browser/accessibility/caption_host_impl.h

@@ -39,8 +39,6 @@ class CaptionHostImpl : public chrome::mojom::CaptionHost,
       mojo::PendingReceiver<chrome::mojom::CaptionHost> receiver);
 
   // chrome::mojom::CaptionHost:
-  void OnSpeechRecognitionReady(
-      OnSpeechRecognitionReadyCallback reply) override;
   void OnTranscription(
       chrome::mojom::TranscriptionResultPtr transcription_result,
       OnTranscriptionCallback reply) override;

chrome/browser/ui/caption_bubble_controller.h

@@ -37,10 +37,6 @@ class CaptionBubbleController {
 
   static std::unique_ptr<CaptionBubbleController> Create(Browser* browser);
 
-  // Called when speech recognition is ready to start for the given web
-  // contents.
-  virtual bool OnSpeechRecognitionReady(content::WebContents* web_contents) = 0;
-
   // Called when a transcription is received from the service. Returns whether
   // the transcription result was set on the caption bubble successfully.
   // Transcriptions will halt if this returns false.

chrome/browser/ui/views/accessibility/caption_bubble.cc

@@ -324,13 +324,6 @@ void CaptionBubble::Init() {
   // set the truncate_length to 0 to ensure that it never truncates.
   label->SetTruncateLength(0);
 
-  auto wait_text = std::make_unique<views::Label>();
-  wait_text->SetEnabledColor(gfx::kGoogleGrey500);
-  wait_text->SetBackgroundColor(SK_ColorTRANSPARENT);
-  wait_text->SetHorizontalAlignment(gfx::HorizontalAlignment::ALIGN_LEFT);
-  wait_text->SetText(
-      l10n_util::GetStringUTF16(IDS_LIVE_CAPTION_BUBBLE_WAIT_TEXT));
-
   auto error_text = std::make_unique<views::Label>();
   error_text->SetEnabledColor(SK_ColorWHITE);
   error_text->SetBackgroundColor(SK_ColorTRANSPARENT);
@@ -360,7 +353,6 @@ void CaptionBubble::Init() {
   auto close_button = BuildImageButton(vector_icons::kCloseRoundedIcon,
                                        IDS_LIVE_CAPTION_BUBBLE_CLOSE);
 
-  wait_text_ = content_container->AddChildView(std::move(wait_text));
   label_ = content_container->AddChildView(std::move(label));
 
   error_icon_ = error_message->AddChildView(std::move(error_icon));
@@ -470,14 +462,16 @@ void CaptionBubble::GetAccessibleNodeData(ui::AXNodeData* node_data) {
   if (model_ && model_->HasError()) {
     node_data->SetName(error_text_->GetText());
     node_data->SetNameFrom(ax::mojom::NameFrom::kContents);
-  } else if (label_->GetText().size()) {
-    node_data->SetName(label_->GetText());
+  } else if (model_ && !model_->GetFullText().empty()) {
+    node_data->SetName(model_->GetFullText());
     node_data->SetNameFrom(ax::mojom::NameFrom::kContents);
   } else {
-    node_data->SetName(wait_text_->GetText());
+    node_data->SetName(
+        l10n_util::GetStringUTF16(IDS_LIVE_CAPTION_BUBBLE_ACCESSIBLE_NAME));
     node_data->SetNameFrom(ax::mojom::NameFrom::kContents);
   }
-  node_data->SetDescription(wait_text_->GetText());
+  node_data->SetDescription(
+      l10n_util::GetStringUTF16(IDS_LIVE_CAPTION_BUBBLE_ACCESSIBLE_NAME));
   node_data->role = ax::mojom::Role::kCaption;
 }
 
@@ -526,7 +520,7 @@ void CaptionBubble::SetModel(CaptionBubbleModel* model) {
 void CaptionBubble::OnTextChanged() {
   DCHECK(model_);
   label_->SetText(base::UTF8ToUTF16(model_->GetFullText()));
-  UpdateBubbleAndWaitTextVisibility();
+  UpdateBubbleVisibility();
 }
 
 void CaptionBubble::OnErrorChanged() {
@@ -539,19 +533,6 @@ void CaptionBubble::OnErrorChanged() {
   Redraw();
 }
 
-void CaptionBubble::OnReadyChanged() {
-  DCHECK(model_);
-  // There is a bug in RenderText in which the label text must not be empty when
-  // it is displayed, or otherwise subsequent calculation of the number of lines
-  // (CaptionBubble::GetNumLinesInLabel) will be incorrect. The label text here
-  // is set to a space character.
-  // TODO(1055150): Fix the bug in RenderText and then remove this workaround.
-  if (model_->IsReady()) {
-    label_->SetText(base::ASCIIToUTF16("\u0020"));
-    UpdateBubbleAndWaitTextVisibility();
-  }
-}
-
 void CaptionBubble::OnIsExpandedChanged() {
   expand_button_->SetVisible(!is_expanded_);
   collapse_button_->SetVisible(is_expanded_);
@@ -561,14 +542,6 @@ void CaptionBubble::OnIsExpandedChanged() {
   Redraw();
 }
 
-void CaptionBubble::UpdateBubbleAndWaitTextVisibility() {
-  // Show the wait text if there is room for it and no error.
-  wait_text_->SetVisible(model_ && !model_->HasError() &&
-                         GetNumLinesInLabel() <
-                             static_cast<size_t>(GetNumLinesVisible()));
-  UpdateBubbleVisibility();
-}
-
 void CaptionBubble::UpdateBubbleVisibility() {
   DCHECK(GetWidget());
   if (!model_) {
@@ -579,14 +552,14 @@ void CaptionBubble::UpdateBubbleVisibility() {
     // Hide the widget if there is no room for it or the model is closed.
     if (GetWidget()->IsVisible())
       GetWidget()->Hide();
-  } else if (model_->IsReady() || model_->HasError()) {
-    // Show the widget if it is ready to receive transcriptions or it has an
-    // error to display. Only show the widget if it isn't already visible.
-    // Always calling Widget::Show() will mean the widget gets focus each time.
+  } else if (!model_->GetFullText().empty() || model_->HasError()) {
+    // Show the widget if it has text or an error to display. Only show the
+    // widget if it isn't already visible. Always calling Widget::Show() will
+    // mean the widget gets focus each time.
     if (!GetWidget()->IsVisible())
       GetWidget()->Show();
   } else if (GetWidget()->IsVisible()) {
-    // Not ready and no error. Hide it.
+    // No text and no error. Hide it.
     GetWidget()->Hide();
   }
 }
@@ -630,11 +603,9 @@ void CaptionBubble::UpdateTextSize() {
                     gfx::Font::FontStyle::NORMAL, kFontSizePx * textScaleFactor,
                     gfx::Font::Weight::NORMAL);
   label_->SetFontList(font_list);
-  wait_text_->SetFontList(font_list);
   error_text_->SetFontList(font_list);
 
   label_->SetLineHeight(kLineHeightDip * textScaleFactor);
-  wait_text_->SetLineHeight(kLineHeightDip * textScaleFactor);
   error_text_->SetLineHeight(kLineHeightDip * textScaleFactor);
   error_icon_->SetImageSize(gfx::Size(kErrorImageSizeDip * textScaleFactor,
                                       kErrorImageSizeDip * textScaleFactor));
@@ -646,12 +617,8 @@ void CaptionBubble::UpdateContentSize() {
       (model_ && model_->HasError())
           ? kLineHeightDip * text_scale_factor
           : kLineHeightDip * GetNumLinesVisible() * text_scale_factor;
-  // The wait text takes up 1 line.
-  int label_height = wait_text_->GetVisible()
-                         ? content_height - kLineHeightDip * text_scale_factor
-                         : content_height;
   label_->SetPreferredSize(
-      gfx::Size(kMaxWidthDip - kSidePaddingDip, label_height));
+      gfx::Size(kMaxWidthDip - kSidePaddingDip, content_height));
   content_container_->SetPreferredSize(gfx::Size(kMaxWidthDip, content_height));
   SetPreferredSize(
       gfx::Size(kMaxWidthDip, content_height +
@@ -660,7 +627,7 @@ void CaptionBubble::UpdateContentSize() {
 }
 
 void CaptionBubble::Redraw() {
-  UpdateBubbleAndWaitTextVisibility();
+  UpdateBubbleVisibility();
   UpdateContentSize();
   SizeToContents();
 }

chrome/browser/ui/views/accessibility/caption_bubble.h

@@ -105,16 +105,10 @@ class CaptionBubble : public views::BubbleDialogDelegateView,
   // the model has an error, otherwise displays the latest text.
   void OnErrorChanged();
 
-  // Called by the CaptionBubbleModel to notify this object that the model's
-  // on ready state has changed. Makes the caption bubble become visible and
-  // show the wait text.
-  void OnReadyChanged();
-
   // Called when the caption bubble expanded state has changed. Changes the
   // number of lines displayed.
   void OnIsExpandedChanged();
 
-  void UpdateBubbleAndWaitTextVisibility();
   // The caption bubble manages its own visibility based on whether there's
   // space for it to be shown, and if it has an error or text to display.
   void UpdateBubbleVisibility();
@@ -129,7 +123,6 @@ class CaptionBubble : public views::BubbleDialogDelegateView,
 
   // Unowned. Owned by views hierarchy.
   views::Label* label_;
-  views::Label* wait_text_;
   views::Label* error_text_;
   views::ImageView* error_icon_;
   views::View* error_message_;

chrome/browser/ui/views/accessibility/caption_bubble_controller_views.cc

@@ -69,18 +69,6 @@ void CaptionBubbleControllerViews::OnCaptionBubbleDestroyed() {
   browser_ = nullptr;
 }
 
-bool CaptionBubbleControllerViews::OnSpeechRecognitionReady(
-    content::WebContents* web_contents) {
-  if (!caption_bubble_ || !caption_bubble_models_.count(web_contents) ||
-      caption_bubble_models_[web_contents]->IsClosed())
-    return false;
-
-  CaptionBubbleModel* caption_bubble_model =
-      caption_bubble_models_[web_contents].get();
-  caption_bubble_model->OnReady();
-  return true;
-}
-
 bool CaptionBubbleControllerViews::OnTranscription(
     const chrome::mojom::TranscriptionResultPtr& transcription_result,
     content::WebContents* web_contents) {

chrome/browser/ui/views/accessibility/caption_bubble_controller_views.h

@@ -38,10 +38,6 @@ class CaptionBubbleControllerViews : public CaptionBubbleController,
   CaptionBubbleControllerViews& operator=(const CaptionBubbleControllerViews&) =
       delete;
 
-  // Called when speech recognition is ready to start for the given web
-  // contents.
-  bool OnSpeechRecognitionReady(content::WebContents* web_contents) override;
-
   // Called when a transcription is received from the service. Returns whether
   // the transcription result was set on the caption bubble successfully.
   // Transcriptions will halt if this returns false.

chrome/browser/ui/views/accessibility/caption_bubble_model.cc

@@ -29,7 +29,6 @@ void CaptionBubbleModel::SetObserver(CaptionBubble* observer) {
     return;
   observer_ = observer;
   if (observer_) {
-    observer_->OnReadyChanged();
     observer_->OnTextChanged();
     observer_->OnErrorChanged();
   }
@@ -53,23 +52,9 @@ void CaptionBubbleModel::Close() {
   final_text_.clear();
   partial_text_.clear();
   is_closed_ = true;
-  is_ready_ = false;
   OnTextChanged();
 }
 
-void CaptionBubbleModel::OnReady() {
-  final_text_.clear();
-  partial_text_.clear();
-  is_ready_ = true;
-  // The label text must not be empty when it is displayed, so there is a
-  // special OnReadyChanged() function in the CaptionBubble that handles the
-  // on_ready state change.
-  // TODO(1055150): Fix the bug in RenderText and then change this to
-  // OnTextChanged().
-  if (observer_)
-    observer_->OnReadyChanged();
-}
-
 void CaptionBubbleModel::OnError() {
   has_error_ = true;
   if (observer_)
@@ -85,10 +70,8 @@ void CaptionBubbleModel::DidFinishNavigation(
   final_text_.clear();
   partial_text_.clear();
   is_closed_ = false;
-  is_ready_ = false;
   has_error_ = false;
   if (observer_) {
-    observer_->OnReadyChanged();
     observer_->OnTextChanged();
     observer_->OnErrorChanged();
   }

chrome/browser/ui/views/accessibility/caption_bubble_model.h

@@ -57,10 +57,7 @@ class CaptionBubbleModel : public content::WebContentsObserver {
   // observer.
   void Close();
 
-  void OnReady();
-
   bool IsClosed() const { return is_closed_; }
-  bool IsReady() const { return is_ready_; }
   bool HasError() const { return has_error_; }
   std::string GetFullText() const { return final_text_ + partial_text_; }
 
@@ -79,9 +76,6 @@ class CaptionBubbleModel : public content::WebContentsObserver {
   // Whether the bubble has been closed by the user.
   bool is_closed_ = false;
 
-  // Whether bubble is ready to receive transcriptions.
-  bool is_ready_ = false;
-
   // Whether an error should be displayed one the bubble.
   bool has_error_ = false;
 

chrome/browser/ui/views/frame/browser_view_browsertest.cc

@@ -397,11 +397,9 @@ IN_PROC_BROWSER_TEST_F(BrowserViewTest, F6CyclesThroughCaptionBubbleToo) {
           caption_controller->GetCaptionBubbleControllerForBrowser(browser()));
   EXPECT_FALSE(bubble_controller->GetFocusableCaptionBubble());
 
-  content::WebContents* contents =
-      browser()->tab_strip_model()->GetActiveWebContents();
-  caption_controller->OnSpeechRecognitionReady(contents);
   caption_controller->DispatchTranscription(
-      contents, chrome::mojom::TranscriptionResult::New("Hello, world", false));
+      browser()->tab_strip_model()->GetActiveWebContents(),
+      chrome::mojom::TranscriptionResult::New("Hello, world", false));
   // Now the caption bubble exists but is not focused.
   views::View* bubble = bubble_controller->GetFocusableCaptionBubble();
   EXPECT_TRUE(bubble);

chrome/common/caption.mojom

@@ -7,9 +7,6 @@ module chrome.mojom;
 // Browser process interface exposed to the renderer for communication about
 // the Live Caption feature.
 interface CaptionHost {
-  // Called when speech recognition service is ready to send transcriptions.
-  OnSpeechRecognitionReady() => (bool success);
-
   // Called when the speech recognition client receives a transcription from the
   // speech service. Returns whether the transcription result was received
   // successfully. Transcriptions will halt if this returns false.

chrome/renderer/media/chrome_speech_recognition_client.cc

@@ -5,7 +5,6 @@
 #include "chrome/renderer/media/chrome_speech_recognition_client.h"
 
 #include <utility>
-#include <vector>
 
 #include "base/metrics/field_trial_params.h"
 #include "content/public/renderer/render_frame.h"
@@ -65,6 +64,7 @@ void ChromeSpeechRecognitionClient::OnRecognizerBound(
     bool is_multichannel_supported) {
   is_multichannel_supported_ = is_multichannel_supported;
   is_recognizer_bound_ = true;
+
   if (on_ready_callback_)
     std::move(on_ready_callback_).Run();
 }
@@ -118,11 +118,11 @@ void ChromeSpeechRecognitionClient::OnSpeechRecognitionRecognitionEvent(
   caption_host_->OnTranscription(
       chrome::mojom::TranscriptionResult::New(result->transcription,
                                               result->is_final),
-      base::BindOnce(&ChromeSpeechRecognitionClient::OnBrowserCallback,
+      base::BindOnce(&ChromeSpeechRecognitionClient::OnTranscriptionCallback,
                      base::Unretained(this)));
 }
 
-void ChromeSpeechRecognitionClient::OnBrowserCallback(bool success) {
+void ChromeSpeechRecognitionClient::OnTranscriptionCallback(bool success) {
   is_browser_requesting_transcription_ = success;
 }
 
@@ -161,17 +161,6 @@ void ChromeSpeechRecognitionClient::SendAudioToSpeechRecognitionService(
   if (IsSpeechRecognitionAvailable()) {
     speech_recognition_recognizer_->SendAudioToSpeechRecognitionService(
         std::move(audio_data));
-
-    // When the speech recognition client receives speech, it alerts the
-    // live caption host in the browser that it is ready so that the UI can
-    // display a message. This happens at the time of playing the video and not
-    // at the time of construction of this object.
-    if (!on_ready_message_sent_to_caption_host_) {
-      caption_host_->OnSpeechRecognitionReady(
-          base::BindOnce(&ChromeSpeechRecognitionClient::OnBrowserCallback,
-                         base::Unretained(this)));
-      on_ready_message_sent_to_caption_host_ = true;
-    }
   }
 }
 

chrome/renderer/media/chrome_speech_recognition_client.h

@@ -65,8 +65,8 @@ class ChromeSpeechRecognitionClient
   media::mojom::AudioDataS16Ptr ConvertToAudioDataS16(
       scoped_refptr<media::AudioBuffer> buffer);
 
-  // Called as a response to sending a message to the browser.
-  void OnBrowserCallback(bool success);
+  // Called as a response to sending a transcription to the browser.
+  void OnTranscriptionCallback(bool success);
 
   media::mojom::AudioDataS16Ptr ConvertToAudioDataS16(
       std::unique_ptr<media::AudioBus> audio_bus,
@@ -116,9 +116,6 @@ class ChromeSpeechRecognitionClient
 
   bool is_recognizer_bound_ = false;
 
-  // Whether or not the on ready message has been sent to the caption host.
-  bool on_ready_message_sent_to_caption_host_ = false;
-
   // The temporary audio bus used to mix multichannel audio into a single
   // channel.
   std::unique_ptr<media::AudioBus> monaural_audio_bus_;