[Live Caption] Clear caption bubble text after inactivity timeout.

Clears caption bubble text after the inactivity timeout. Also, prevents
a final transcription received after the bubble has no activity from
setting text. This fixes a bug where the bubble disappeared after
inactivity, and then the speech service sent a final transcription
after several seconds of no audio, which caused the bubble to reappear.

Bug: 1055150
Change-Id: I3a0df495c73df0273e925847671f351b61afcd51
AX-Relnotes: N/A (feature not launched)
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2598622
Commit-Queue: Abigail Klein <abigailbklein@google.com>
Reviewed-by: Josiah Krutz <josiahk@google.com>
Cr-Commit-Position: refs/heads/master@{#839812}

chrome/browser/accessibility/caption_controller_browsertest.cc

@@ -107,7 +107,7 @@ class CaptionControllerTest : public InProcessBrowserTest {
                                                 Profile* profile) {
     return GetControllerForProfile(profile)->DispatchTranscription(
         browser->tab_strip_model()->GetActiveWebContents(),
-        chrome::mojom::TranscriptionResult::New(text, true /* is_final */));
+        chrome::mojom::TranscriptionResult::New(text, false /* is_final */));
   }
 
   void OnError() { OnErrorOnBrowser(browser()); }

chrome/browser/ui/views/accessibility/caption_bubble.cc

@@ -678,26 +678,37 @@ void CaptionBubble::UpdateBubbleAndTitleVisibility() {
 
 void CaptionBubble::UpdateBubbleVisibility() {
   DCHECK(GetWidget());
-  if (!model_) {
   // If there is no model set, do not show the bubble.
+  if (!model_) {
     if (GetWidget()->IsVisible())
       GetWidget()->Hide();
-  } else if (!can_layout_ || model_->IsClosed()) {
-    // Hide the widget if there is no room for it or the model is closed.
+    return;
+  }
+
+  // Hide the widget if there is no room for it, the model is closed. or the
+  // bubble has no activity. Activity is defined as transcription received from
+  // the speech service or user interacting with the bubble through focus,
+  // pressing buttons, or dragging.
+  if (!can_layout_ || model_->IsClosed() || !HasActivity()) {
     if (GetWidget()->IsVisible())
       GetWidget()->Hide();
-  } else if (!model_->GetFullText().empty() || model_->HasError()) {
+    return;
+  }
+
   // Show the widget if it has text or an error to display.
+  if (!model_->GetFullText().empty() || model_->HasError()) {
     if (!GetWidget()->IsVisible()) {
       GetWidget()->ShowInactive();
       GetViewAccessibility().AnnounceText(l10n_util::GetStringUTF16(
           IDS_LIVE_CAPTION_BUBBLE_APPEAR_SCREENREADER_ANNOUNCEMENT));
       LogSessionEvent(SessionEvent::kStreamStarted);
     }
-  } else if (GetWidget()->IsVisible()) {
+    return;
+  }
+
   // No text and no error. Hide it.
+  if (GetWidget()->IsVisible())
     GetWidget()->Hide();
-  }
 }
 
 void CaptionBubble::OnWidgetVisibilityChanged(views::Widget* widget,
@@ -791,6 +802,20 @@ void CaptionBubble::OnInactivityTimeout() {
   LogSessionEvent(SessionEvent::kStreamEnded);
   if (GetWidget()->IsVisible())
     GetWidget()->Hide();
+
+  // Clear the partial and final text in the caption bubble model and the label.
+  // Does not affect the speech service. The speech service will emit a final
+  // result after ~10-15 seconds of no audio which the caption bubble will
+  // receive but will not display. If the speech service is in the middle of a
+  // recognition phrase, and the caption bubble regains activity (such as if the
+  // audio stream restarts), the speech service will emit partial results that
+  // contain text cleared by the UI.
+  model_->ClearText();
+}
+
+bool CaptionBubble::HasActivity() {
+  return model_ && (inactivity_timer_->IsRunning() || HasFocus() ||
+                    !model_->GetFullText().empty() || model_->HasError());
 }
 
 views::Label* CaptionBubble::GetLabelForTesting() {

chrome/browser/ui/views/accessibility/caption_bubble.h

@@ -67,15 +67,9 @@ class CaptionBubble : public views::BubbleDialogDelegateView {
   // the caption text size.
   void UpdateCaptionStyle(base::Optional<ui::CaptionStyle> caption_style);
 
-  // For the provided line index, gets the corresponding rendered line in the
-  // label and returns the text position of the first character of that line.
-  // Returns the same value regardless of whether the label is visible or not.
-  // TODO(crbug.com/1055150): This feature is launching for English first.
-  // Make sure this is correct for all languages.
-  size_t GetTextIndexOfLineInLabel(size_t line) const;
-
-  // Returns the number of lines in the caption bubble label that are rendered.
-  size_t GetNumLinesInLabel() const;
+  // Returns whether the bubble has activity, with the above definition of
+  // activity.
+  bool HasActivity();
 
   views::Label* GetLabelForTesting();
   base::RetainingOneShotTimer* GetInactivityTimerForTesting();
@@ -126,6 +120,16 @@ class CaptionBubble : public views::BubbleDialogDelegateView {
   void UpdateBubbleVisibility();
   void UpdateBubbleAndTitleVisibility();
 
+  // For the provided line index, gets the corresponding rendered line in the
+  // label and returns the text position of the first character of that line.
+  // Returns the same value regardless of whether the label is visible or not.
+  // TODO(crbug.com/1055150): This feature is launching for English first.
+  // Make sure this is correct for all languages.
+  size_t GetTextIndexOfLineInLabel(size_t line) const;
+
+  // Returns the number of lines in the caption bubble label that are rendered.
+  size_t GetNumLinesInLabel() const;
+
   double GetTextScaleFactor();
   int GetNumLinesVisible();
   void UpdateTextSize();

chrome/browser/ui/views/accessibility/caption_bubble_controller_views.cc

@@ -78,8 +78,16 @@ bool CaptionBubbleControllerViews::OnTranscription(
 
   CaptionBubbleModel* caption_bubble_model =
       caption_bubble_models_[web_contents].get();
-  caption_bubble_model->SetPartialText(transcription_result->transcription);
 
+  // If the caption bubble has no activity and it receives a final
+  // transcription, don't set text. The speech service sends a final
+  // transcription after several seconds of no audio. This prevents the bubble
+  // reappearing with a final transcription after it had disappeared due to no
+  // activity.
+  if (!caption_bubble_->HasActivity() && transcription_result->is_final)
+    return true;
+
+  caption_bubble_model->SetPartialText(transcription_result->transcription);
   if (transcription_result->is_final)
     caption_bubble_model->CommitPartialText();
 

chrome/browser/ui/views/accessibility/caption_bubble_controller_views_browsertest.cc

@@ -104,6 +104,10 @@ class CaptionBubbleControllerViewsTest : public InProcessBrowserTest {
     return controller_ ? controller_->GetBubbleLabelTextForTesting() : "";
   }
 
+  size_t GetNumLinesInLabel() {
+    return controller_ ? controller_->caption_bubble_->GetNumLinesInLabel() : 0;
+  }
+
   views::Widget* GetCaptionWidget() {
     return controller_ ? controller_->caption_widget_ : nullptr;
   }
@@ -441,7 +445,7 @@ IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest, ShowsAndHidesError) {
 }
 
 IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest, CloseButtonCloses) {
-  bool success = OnFinalTranscription("Elephants have 3-4 toenails per foot");
+  bool success = OnPartialTranscription("Elephants have 3-4 toenails per foot");
   EXPECT_TRUE(success);
   EXPECT_TRUE(GetCaptionWidget());
   EXPECT_TRUE(IsWidgetVisible());
@@ -449,7 +453,7 @@ IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest, CloseButtonCloses) {
   ClickButton(GetCloseButton());
   EXPECT_TRUE(GetCaptionWidget());
   EXPECT_FALSE(IsWidgetVisible());
-  success = OnFinalTranscription(
+  success = OnPartialTranscription(
       "Elephants wander 35 miles a day in search of water");
   EXPECT_FALSE(success);
   EXPECT_EQ("", GetLabelText());
@@ -774,21 +778,23 @@ IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest, TruncatesFinalText) {
   for (int i = 10; i < 40; i++) {
     text += base::NumberToString(i) + line + " ";
   }
+  OnPartialTranscription(text);
   OnFinalTranscription(text);
   EXPECT_EQ(text.substr(10500, 15000), GetLabelText());
-  EXPECT_EQ(9u, GetBubble()->GetNumLinesInLabel());
+  EXPECT_EQ(9u, GetNumLinesInLabel());
   OnPartialTranscription(text);
   EXPECT_EQ(text.substr(10500, 15000) + text, GetLabelText());
-  EXPECT_EQ(39u, GetBubble()->GetNumLinesInLabel());
+  EXPECT_EQ(39u, GetNumLinesInLabel());
   OnFinalTranscription("a ");
   EXPECT_EQ(text.substr(11000, 15000) + "a ", GetLabelText());
-  EXPECT_EQ(9u, GetBubble()->GetNumLinesInLabel());
+  EXPECT_EQ(9u, GetNumLinesInLabel());
 }
 
 IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest, TabNavigation) {
   ui_test_utils::NavigateToURL(browser(), GURL("http://www.google.com"));
   content::WaitForLoadStop(
       browser()->tab_strip_model()->GetActiveWebContents());
+  OnPartialTranscription("Elephant calves");
   OnFinalTranscription("Elephant calves can stand within 20 minutes of birth");
   EXPECT_TRUE(IsWidgetVisible());
   EXPECT_EQ("Elephant calves can stand within 20 minutes of birth",
@@ -924,6 +930,7 @@ IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest,
                        AccessibleTextComputedWhenAccessibilityModeEnabled) {
   // If accessibility is disabled, virtual children aren't computed.
   content::BrowserAccessibilityState::GetInstance()->DisableAccessibility();
+  OnPartialTranscription("A");
   OnFinalTranscription("A dog's nose print");
   EXPECT_EQ(0u, GetAXLineText().size());
 
@@ -963,7 +970,7 @@ IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest,
 IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest,
                        AccessibleTextClearsWhenBubbleCloses) {
   content::BrowserAccessibilityState::GetInstance()->EnableAccessibility();
-  OnFinalTranscription("Dogs' noses are wet to help them smell.");
+  OnPartialTranscription("Dogs' noses are wet to help them smell.");
   EXPECT_EQ(1u, GetAXLineText().size());
   EXPECT_EQ("Dogs' noses are wet to help them smell.", GetAXLineText()[0]);
   ClickButton(GetCloseButton());
@@ -973,7 +980,7 @@ IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest,
 IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest,
                        AccessibleTextClearsWhenTabRefreshes) {
   content::BrowserAccessibilityState::GetInstance()->EnableAccessibility();
-  OnFinalTranscription("Newfoundlands are amazing lifeguards.");
+  OnPartialTranscription("Newfoundlands are amazing lifeguards.");
   EXPECT_EQ(1u, GetAXLineText().size());
   EXPECT_EQ("Newfoundlands are amazing lifeguards.", GetAXLineText()[0]);
   chrome::Reload(browser(), WindowOpenDisposition::CURRENT_TAB);
@@ -985,7 +992,7 @@ IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest,
 IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest,
                        AccessibleTextChangesWhenTabChanges) {
   content::BrowserAccessibilityState::GetInstance()->EnableAccessibility();
-  OnFinalTranscription("3 dogs survived the Titanic sinking.");
+  OnPartialTranscription("3 dogs survived the Titanic sinking.");
   EXPECT_EQ(1u, GetAXLineText().size());
   EXPECT_EQ("3 dogs survived the Titanic sinking.", GetAXLineText()[0]);
 
@@ -997,7 +1004,7 @@ IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest,
 
   ActivateTabAt(0);
   EXPECT_EQ(1u, GetAXLineText().size());
-  EXPECT_EQ("3 dogs survived the Titanic sinking. ", GetAXLineText()[0]);
+  EXPECT_EQ("3 dogs survived the Titanic sinking.", GetAXLineText()[0]);
 }
 
 IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest,
@@ -1009,6 +1016,7 @@ IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest,
     text += base::NumberToString(i) + line + " ";
   }
   content::BrowserAccessibilityState::GetInstance()->EnableAccessibility();
+  OnPartialTranscription(text);
   OnFinalTranscription(text);
   EXPECT_EQ(9u, GetAXLineText().size());
   for (int i = 0; i < 9; i++) {
@@ -1062,24 +1070,32 @@ IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest, HidesAfterInactivity) {
   // Caption bubble hides after 5 seconds without receiving a transcription.
   OnPartialTranscription("Bowhead whales can live for over 200 years.");
   EXPECT_TRUE(IsWidgetVisible());
+  EXPECT_EQ("Bowhead whales can live for over 200 years.", GetLabelText());
   ASSERT_TRUE(GetBubble()->GetInactivityTimerForTesting()->IsRunning());
   test_task_runner->FastForwardBy(base::TimeDelta::FromSeconds(5));
   EXPECT_FALSE(IsWidgetVisible());
+  EXPECT_EQ("", GetLabelText());
 
   // Caption bubble becomes visible when transcription is received, and stays
   // visible if transcriptions are received before 5 seconds have passed.
   OnPartialTranscription("Killer whales");
   EXPECT_TRUE(IsWidgetVisible());
+  EXPECT_EQ("Killer whales", GetLabelText());
   test_task_runner->FastForwardBy(base::TimeDelta::FromSeconds(4));
   EXPECT_TRUE(IsWidgetVisible());
   OnPartialTranscription("Killer whales travel in matrifocal groups");
   EXPECT_TRUE(IsWidgetVisible());
+  EXPECT_EQ("Killer whales travel in matrifocal groups", GetLabelText());
   test_task_runner->FastForwardBy(base::TimeDelta::FromSeconds(4));
   EXPECT_TRUE(IsWidgetVisible());
   OnFinalTranscription(
       "Killer whales travel in matrifocal groups--a family unit centered on "
       "the mother.");
   EXPECT_TRUE(IsWidgetVisible());
+  EXPECT_EQ(
+      "Killer whales travel in matrifocal groups--a family unit centered on "
+      "the mother.",
+      GetLabelText());
   test_task_runner->FastForwardBy(base::TimeDelta::FromSeconds(4));
   EXPECT_TRUE(IsWidgetVisible());
 
@@ -1090,12 +1106,48 @@ IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest, HidesAfterInactivity) {
   EXPECT_TRUE(IsWidgetVisible());
   test_task_runner->FastForwardBy(base::TimeDelta::FromSeconds(10));
   EXPECT_TRUE(IsWidgetVisible());
+  EXPECT_EQ(
+      "Killer whales travel in matrifocal groups--a family unit centered on "
+      "the mother.",
+      GetLabelText());
 
   UnfocusCaptionWidget();
   EXPECT_FALSE(GetBubble()->HasFocus());
+  EXPECT_EQ(
+      "Killer whales travel in matrifocal groups--a family unit centered on "
+      "the mother.",
+      GetLabelText());
   EXPECT_TRUE(IsWidgetVisible());
   test_task_runner->FastForwardBy(base::TimeDelta::FromSeconds(5));
   EXPECT_FALSE(IsWidgetVisible());
+  EXPECT_EQ("", GetLabelText());
+}
+
+IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest,
+                       ClearsTextAfterInactivity) {
+  // Use a ScopedMockTimeMessageLoopTaskRunner to test the inactivity timer with
+  // a mock tick clock that replaces the default tick clock with mock time.
+  base::ScopedMockTimeMessageLoopTaskRunner test_task_runner;
+  SetTickClockForTesting(test_task_runner->GetMockTickClock());
+
+  // Caption bubble hides after 5 seconds without receiving a transcription.
+  OnPartialTranscription("Bowhead whales can live for over 200 years.");
+  EXPECT_TRUE(IsWidgetVisible());
+  EXPECT_EQ("Bowhead whales can live for over 200 years.", GetLabelText());
+  ASSERT_TRUE(GetBubble()->GetInactivityTimerForTesting()->IsRunning());
+  test_task_runner->FastForwardBy(base::TimeDelta::FromSeconds(5));
+  EXPECT_FALSE(IsWidgetVisible());
+  EXPECT_EQ("", GetLabelText());
+
+  // Caption bubble stays hidden when receiving a final transcription.
+  OnFinalTranscription("Bowhead whales can live for over 200 years.");
+  EXPECT_FALSE(IsWidgetVisible());
+  EXPECT_EQ("", GetLabelText());
+
+  // Caption bubble reappears when receiving a partial transcription.
+  OnPartialTranscription("Killer whales");
+  EXPECT_TRUE(IsWidgetVisible());
+  EXPECT_EQ("Killer whales", GetLabelText());
 }
 
 IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest,

chrome/browser/ui/views/accessibility/caption_bubble_model.cc

@@ -54,10 +54,8 @@ void CaptionBubbleModel::SetPartialText(const std::string& partial_text) {
 }
 
 void CaptionBubbleModel::Close() {
-  final_text_.clear();
-  partial_text_.clear();
   is_closed_ = true;
-  OnTextChanged();
+  ClearText();
 }
 
 void CaptionBubbleModel::OnError() {
@@ -66,20 +64,23 @@ void CaptionBubbleModel::OnError() {
     observer_->OnErrorChanged();
 }
 
+void CaptionBubbleModel::ClearText() {
+  partial_text_.clear();
+  final_text_.clear();
+  OnTextChanged();
+}
+
 void CaptionBubbleModel::DidFinishNavigation(
     content::NavigationHandle* navigation_handle) {
   if (!navigation_handle->IsInMainFrame())
     return;
 
   // Reset caption bubble to it's starting state.
-  final_text_.clear();
-  partial_text_.clear();
   is_closed_ = false;
   has_error_ = false;
-  if (observer_) {
-    observer_->OnTextChanged();
+  ClearText();
+  if (observer_)
     observer_->OnErrorChanged();
-  }
 }
 
 void CaptionBubbleModel::CommitPartialText() {

chrome/browser/ui/views/accessibility/caption_bubble_model.h

@@ -57,6 +57,9 @@ class CaptionBubbleModel : public content::WebContentsObserver {
   // observer.
   void Close();
 
+  // Clears the partial and final text and alerts the observer.
+  void ClearText();
+
   bool IsClosed() const { return is_closed_; }
   bool HasError() const { return has_error_; }
   std::string GetFullText() const { return final_text_ + partial_text_; }