[Live Caption] Stop speech recognition when caption bubble is closed.

Introduce a reply to the OnTranscription function of the live caption
mojom interface. The reply is a boolean of success which denotes whether
the browser successfully received, routed, and set the transcription
result. If there was not success, because the caption bubble was closed,
the browser was closed, or the associated web contents did not match an
existing browser with a caption bubble, then the OnTranscription mojo
function returns false, causing speech recognition to stop.

Bug: 1055150
Change-Id: I8c8e0b443c8c90afa78e85e81e3ca7cda8a49920
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2225878Reviewed-by: Avi Drissman <avi@chromium.org>
Reviewed-by: Katie Dektar <katie@chromium.org>
Reviewed-by: Dominic Mazzoni <dmazzoni@chromium.org>
Reviewed-by: Alex Gough <ajgo@chromium.org>
Commit-Queue: Abigail Klein <abigailbklein@google.com>
Cr-Commit-Position: refs/heads/master@{#776086}

chrome/browser/accessibility/caption_controller.cc

@@ -163,14 +163,14 @@ void CaptionController::OnBrowserRemoved(Browser* browser) {
   caption_bubble_controllers_.erase(browser);
 }
 
-void CaptionController::DispatchTranscription(
+bool CaptionController::DispatchTranscription(
     content::WebContents* web_contents,
     const chrome::mojom::TranscriptionResultPtr& transcription_result) {
   Browser* browser = chrome::FindBrowserWithWebContents(web_contents);
   if (!browser || !caption_bubble_controllers_.count(browser))
-    return;
-  caption_bubble_controllers_[browser]->OnTranscription(transcription_result,
-                                                        web_contents);
+    return false;
+  return caption_bubble_controllers_[browser]->OnTranscription(
+      transcription_result, web_contents);
 }
 
 CaptionBubbleController*

chrome/browser/accessibility/caption_controller.h

@@ -68,8 +68,9 @@ class CaptionController : public BrowserListObserver, public KeyedService {
   void Init();
 
   // Routes a transcription to the CaptionBubbleController that belongs to the
-  // appropriate browser.
-  void DispatchTranscription(
+  // appropriate browser. Returns whether the transcription result was routed
+  // successfully. Transcriptions will halt if this returns false.
+  bool DispatchTranscription(
       content::WebContents* web_contents,
       const chrome::mojom::TranscriptionResultPtr& transcription_result);
 

chrome/browser/accessibility/caption_host_impl.cc

@@ -40,21 +40,28 @@ CaptionHostImpl::CaptionHostImpl(content::RenderFrameHost* frame_host)
 CaptionHostImpl::~CaptionHostImpl() = default;
 
 void CaptionHostImpl::OnTranscription(
-    chrome::mojom::TranscriptionResultPtr transcription_result) {
-  if (!frame_host_)
+    chrome::mojom::TranscriptionResultPtr transcription_result,
+    OnTranscriptionCallback reply) {
+  if (!frame_host_) {
+    std::move(reply).Run(false);
     return;
+  }
   content::WebContents* web_contents =
       content::WebContents::FromRenderFrameHost(frame_host_);
   if (!web_contents) {
     frame_host_ = nullptr;
+    std::move(reply).Run(false);
     return;
   }
   Profile* profile =
       Profile::FromBrowserContext(web_contents->GetBrowserContext());
-  if (!profile)
+  if (!profile) {
+    std::move(reply).Run(false);
     return;
-  CaptionControllerFactory::GetForProfile(profile)->DispatchTranscription(
-      web_contents, transcription_result);
+  }
+  std::move(reply).Run(
+      CaptionControllerFactory::GetForProfile(profile)->DispatchTranscription(
+          web_contents, transcription_result));
 }
 
 void CaptionHostImpl::RenderFrameDeleted(content::RenderFrameHost* frame_host) {

chrome/browser/accessibility/caption_host_impl.h

@@ -38,7 +38,8 @@ class CaptionHostImpl : public chrome::mojom::CaptionHost,
 
   // chrome::mojom::CaptionHost:
   void OnTranscription(
-      chrome::mojom::TranscriptionResultPtr transcription_result) override;
+      chrome::mojom::TranscriptionResultPtr transcription_result,
+      OnTranscriptionCallback reply) override;
 
   // content::WebContentsObserver:
   void RenderFrameDeleted(content::RenderFrameHost* frame_host) override;

chrome/browser/ui/caption_bubble_controller.h

@@ -37,14 +37,16 @@ class CaptionBubbleController {
 
   static std::unique_ptr<CaptionBubbleController> Create(Browser* browser);
 
-  // Called when a transcription is received from the service.
-  virtual void OnTranscription(
+  // Called when a transcription is received from the service. Returns whether
+  // the transcription result was set on the caption bubble successfully.
+  // Transcriptions will halt if this returns false.
+  virtual bool OnTranscription(
       const chrome::mojom::TranscriptionResultPtr& transcription_result,
-      content::WebContents* web_contents) {}
+      content::WebContents* web_contents) = 0;
 
   // Called when the caption style changes.
   virtual void UpdateCaptionStyle(
-      base::Optional<ui::CaptionStyle> caption_style) {}
+      base::Optional<ui::CaptionStyle> caption_style) = 0;
 };
 
 }  // namespace captions

chrome/browser/ui/views/accessibility/caption_bubble_controller_views.cc

@@ -69,7 +69,7 @@ CaptionBubbleControllerViews::~CaptionBubbleControllerViews() {
 
 void CaptionBubbleControllerViews::OnCaptionBubbleCloseClicked() {
   // Hide the caption bubble on the active tab.
-  caption_texts_[active_contents_].clear();
+  caption_bubble_models_[active_contents_].close();
   // TODO(crbug.com/1051150): Ensure that caption bubble disappears on the tab
   // if it is currently displaying an error message.
   SetCaptionBubbleText();
@@ -84,13 +84,14 @@ void CaptionBubbleControllerViews::OnCaptionBubbleDestroyed() {
   browser_ = nullptr;
 }
 
-void CaptionBubbleControllerViews::OnTranscription(
+bool CaptionBubbleControllerViews::OnTranscription(
     const chrome::mojom::TranscriptionResultPtr& transcription_result,
     content::WebContents* web_contents) {
-  if (!caption_bubble_)
-    return;
-  std::string& partial_text = caption_texts_[web_contents].partial_text;
-  std::string& final_text = caption_texts_[web_contents].final_text;
+  if (!caption_bubble_ || caption_bubble_models_[web_contents].is_closed)
+    return false;
+
+  std::string& partial_text = caption_bubble_models_[web_contents].partial_text;
+  std::string& final_text = caption_bubble_models_[web_contents].final_text;
 
   partial_text = transcription_result->transcription;
   SetCaptionBubbleText();
@@ -117,6 +118,7 @@ void CaptionBubbleControllerViews::OnTranscription(
       SetCaptionBubbleText();
     }
   }
+  return true;
 }
 
 void CaptionBubbleControllerViews::OnTabStripModelChanged(
@@ -128,7 +130,7 @@ void CaptionBubbleControllerViews::OnTabStripModelChanged(
   if (!selection.active_tab_changed())
     return;
   if (selection.selected_tabs_were_removed)
-    caption_texts_.erase(selection.old_contents);
+    caption_bubble_models_.erase(selection.old_contents);
 
   active_contents_ = selection.new_contents;
   SetCaptionBubbleText();
@@ -136,8 +138,8 @@ void CaptionBubbleControllerViews::OnTabStripModelChanged(
 
 void CaptionBubbleControllerViews::SetCaptionBubbleText() {
   std::string text;
-  if (active_contents_ && caption_texts_.count(active_contents_))
-    text = caption_texts_[active_contents_].full_text();
+  if (active_contents_ && caption_bubble_models_.count(active_contents_))
+    text = caption_bubble_models_[active_contents_].full_text();
   caption_bubble_->SetText(text);
 }
 

chrome/browser/ui/views/accessibility/caption_bubble_controller_views.h

@@ -20,13 +20,15 @@ namespace captions {
 
 class CaptionBubble;
 
-struct CaptionText {
+struct CaptionBubbleModel {
   std::string final_text;
   std::string partial_text;
+  bool is_closed = false;
 
-  void clear() {
+  void close() {
     final_text.clear();
     partial_text.clear();
+    is_closed = true;
   }
 
   std::string full_text() { return final_text + partial_text; }
@@ -48,8 +50,10 @@ class CaptionBubbleControllerViews : public CaptionBubbleController,
   CaptionBubbleControllerViews& operator=(const CaptionBubbleControllerViews&) =
       delete;
 
-  // Called when a transcription is received from the service.
-  void OnTranscription(
+  // Called when a transcription is received from the service. Returns whether
+  // the transcription result was set on the caption bubble successfully.
+  // Transcriptions will halt if this returns false.
+  bool OnTranscription(
       const chrome::mojom::TranscriptionResultPtr& transcription_result,
       content::WebContents* web_contents) override;
 
@@ -93,7 +97,8 @@ class CaptionBubbleControllerViews : public CaptionBubbleController,
   // final texts in order to show the latest partial text to a user when they
   // switch back to the tab in case the speech service has not sent a final
   // transcription in a while.
-  std::unordered_map<content::WebContents*, CaptionText> caption_texts_;
+  std::unordered_map<content::WebContents*, CaptionBubbleModel>
+      caption_bubble_models_;
 };
 }  // namespace captions
 

chrome/browser/ui/views/accessibility/caption_bubble_controller_views_browsertest.cc

@@ -106,14 +106,14 @@ class CaptionBubbleControllerViewsTest : public InProcessBrowserTest {
     EXPECT_EQ(bubble_bounds.bottom(), anchor_bounds.bottom() - 48);
   }
 
-  void OnPartialTranscription(std::string text, int tab_index = 0) {
-    GetController()->OnTranscription(
+  bool OnPartialTranscription(std::string text, int tab_index = 0) {
+    return GetController()->OnTranscription(
         chrome::mojom::TranscriptionResult::New(text, false),
         browser()->tab_strip_model()->GetWebContentsAt(tab_index));
   }
 
-  void OnFinalTranscription(std::string text, int tab_index = 0) {
-    GetController()->OnTranscription(
+  bool OnFinalTranscription(std::string text, int tab_index = 0) {
+    return GetController()->OnTranscription(
         chrome::mojom::TranscriptionResult::New(text, true),
         browser()->tab_strip_model()->GetWebContentsAt(tab_index));
   }
@@ -343,12 +343,21 @@ IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest, ShowsAndHidesError) {
 }
 
 IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest, CloseButtonCloses) {
-  OnPartialTranscription("Elephants have 3-4 toenails per foot");
+  bool success = OnFinalTranscription("Elephants have 3-4 toenails per foot");
+  EXPECT_TRUE(success);
   EXPECT_TRUE(GetCaptionWidget());
   EXPECT_TRUE(GetCaptionWidget()->IsVisible());
+  EXPECT_EQ("Elephants have 3-4 toenails per foot", GetLabelText());
   ClickCloseButton();
   EXPECT_TRUE(GetCaptionWidget());
   EXPECT_FALSE(GetCaptionWidget()->IsVisible());
+  success = OnFinalTranscription(
+      "Elephants wander 35 miles a day in search of water");
+  EXPECT_FALSE(success);
+  EXPECT_EQ("", GetLabelText());
+
+  // TODO(crbug.com/1055150): The caption bubble should reappear when the tab
+  // refreshes.
 }
 
 IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest,

chrome/common/caption.mojom

@@ -8,8 +8,9 @@ module chrome.mojom;
 // the Live Caption feature.
 interface CaptionHost {
   // Called when the speech recognition client receives a transcription from the
-  // speech service.
-  OnTranscription(TranscriptionResult transcription_result);
+  // speech service. Returns whether the transcription result was received
+  // successfully. Transcriptions will halt if this returns false.
+  OnTranscription(TranscriptionResult transcription_result) => (bool success);
 };
 
 // A transcription result created by the speech recognition client in the

chrome/renderer/media/chrome_speech_recognition_client.cc

@@ -36,14 +36,22 @@ void ChromeSpeechRecognitionClient::AddAudio(
 }
 
 bool ChromeSpeechRecognitionClient::IsSpeechRecognitionAvailable() {
-  return speech_recognition_recognizer_.is_bound() &&
+  return is_browser_requesting_transcription_ &&
+         speech_recognition_recognizer_.is_bound() &&
          speech_recognition_recognizer_.is_connected();
 }
 
 void ChromeSpeechRecognitionClient::OnSpeechRecognitionRecognitionEvent(
     media::mojom::SpeechRecognitionResultPtr result) {
-  caption_host_->OnTranscription(chrome::mojom::TranscriptionResult::New(
-      result->transcription, result->is_final));
+  caption_host_->OnTranscription(
+      chrome::mojom::TranscriptionResult::New(result->transcription,
+                                              result->is_final),
+      base::BindOnce(&ChromeSpeechRecognitionClient::OnTranscriptionCallback,
+                     base::Unretained(this)));
+}
+
+void ChromeSpeechRecognitionClient::OnTranscriptionCallback(bool success) {
+  is_browser_requesting_transcription_ = success;
 }
 
 media::mojom::AudioDataS16Ptr

chrome/renderer/media/chrome_speech_recognition_client.h

@@ -41,6 +41,9 @@ class ChromeSpeechRecognitionClient
   media::mojom::AudioDataS16Ptr ConvertToAudioDataS16(
       scoped_refptr<media::AudioBuffer> buffer);
 
+  // Called as a response to sending a transcription to the browser.
+  void OnTranscriptionCallback(bool success);
+
   mojo::Remote<media::mojom::SpeechRecognitionContext>
       speech_recognition_context_;
   mojo::Remote<media::mojom::SpeechRecognitionRecognizer>
@@ -52,6 +55,9 @@ class ChromeSpeechRecognitionClient
   // The temporary audio bus used to convert the raw audio to the appropriate
   // format.
   std::unique_ptr<media::AudioBus> temp_audio_bus_;
+
+  // Whether the browser is still requesting transcriptions.
+  bool is_browser_requesting_transcription_ = true;
 };
 
 #endif  // CHROME_RENDERER_MEDIA_CHROME_SPEECH_RECOGNITION_CLIENT_H_