Commit 8dc47fe6 authored by Abigail Klein's avatar Abigail Klein Committed by Commit Bot

[Live Caption] Inform the UI of an error when the speech service

disconnects.

Add disconnect handlers to the speech service. If the UI in the browser
disonnects from the service in the renderer, have the speech service stop
transcribing captions. If the service in the renderer disconnects from
the utility process which connects to SODA or the cloud, send a message
to the UI that there was an error and have the UI display the error
message.

Bug: 1055150
Change-Id: I3940cae2cd12503353e105fd612b064908d9951c
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2353024Reviewed-by: default avatarAvi Drissman <avi@chromium.org>
Reviewed-by: default avatarAlex Gough <ajgo@chromium.org>
Reviewed-by: default avatarEvan Liu <evliu@google.com>
Commit-Queue: Abigail Klein <abigailbklein@google.com>
Cr-Commit-Position: refs/heads/master@{#797773}
parent 2c172aae
......@@ -180,6 +180,13 @@ bool CaptionController::DispatchTranscription(
transcription_result, web_contents);
}
void CaptionController::OnError(content::WebContents* web_contents) {
Browser* browser = chrome::FindBrowserWithWebContents(web_contents);
if (!browser || !caption_bubble_controllers_.count(browser))
return;
return caption_bubble_controllers_[browser]->OnError(web_contents);
}
CaptionBubbleController*
CaptionController::GetCaptionBubbleControllerForBrowser(Browser* browser) {
if (!browser || !caption_bubble_controllers_.count(browser))
......
......@@ -77,6 +77,10 @@ class CaptionController : public BrowserListObserver, public KeyedService {
content::WebContents* web_contents,
const chrome::mojom::TranscriptionResultPtr& transcription_result);
// Alerts the CaptionBubbleController that belongs to the appropriate browser
// that there is an error in the speech recognition service.
void OnError(content::WebContents* web_contents);
CaptionBubbleController* GetCaptionBubbleControllerForBrowser(
Browser* browser);
......
......@@ -115,6 +115,17 @@ class CaptionControllerTest : public InProcessBrowserTest {
chrome::mojom::TranscriptionResult::New(text, true /* is_final */));
}
void OnError() { OnErrorOnBrowser(browser()); }
void OnErrorOnBrowser(Browser* browser) {
OnErrorOnBrowserForProfile(browser, browser->profile());
}
void OnErrorOnBrowserForProfile(Browser* browser, Profile* profile) {
GetControllerForProfile(profile)->OnError(
browser->tab_strip_model()->GetActiveWebContents());
}
int NumBubbleControllers() {
return NumBubbleControllersForProfile(browser()->profile());
}
......@@ -484,6 +495,62 @@ IN_PROC_BROWSER_TEST_F(CaptionControllerTest,
#endif
}
IN_PROC_BROWSER_TEST_F(CaptionControllerTest, OnError) {
OnError();
EXPECT_EQ(0, NumBubbleControllers());
SetLiveCaptionEnabled(true);
OnError();
// The CaptionBubbleController is currently only implemented in Views.
#if defined(TOOLKIT_VIEWS)
EXPECT_TRUE(IsWidgetVisible());
#else
EXPECT_FALSE(IsWidgetVisible());
#endif
SetLiveCaptionEnabled(false);
OnError();
EXPECT_EQ(0, NumBubbleControllers());
}
IN_PROC_BROWSER_TEST_F(CaptionControllerTest, OnError_MultipleBrowsers) {
Browser* browser1 = browser();
Browser* browser2 = CreateBrowser(browser()->profile());
Browser* incognito_browser = CreateIncognitoBrowser();
SetLiveCaptionEnabled(true);
// OnError routes to the right browser.
OnErrorOnBrowser(browser1);
// The CaptionBubbleController is currently only implemented in Views.
#if defined(TOOLKIT_VIEWS)
EXPECT_TRUE(IsWidgetVisibleOnBrowser(browser1));
EXPECT_FALSE(IsWidgetVisibleOnBrowser(browser2));
EXPECT_FALSE(IsWidgetVisibleOnBrowser(incognito_browser));
#else
EXPECT_FALSE(IsWidgetVisibleOnBrowser(browser1));
#endif
OnErrorOnBrowser(browser2);
// The CaptionBubbleController is currently only implemented in Views.
#if defined(TOOLKIT_VIEWS)
EXPECT_TRUE(IsWidgetVisibleOnBrowser(browser1));
EXPECT_TRUE(IsWidgetVisibleOnBrowser(browser2));
EXPECT_FALSE(IsWidgetVisibleOnBrowser(incognito_browser));
#else
EXPECT_FALSE(IsWidgetVisibleOnBrowser(browser2));
#endif
OnErrorOnBrowser(incognito_browser);
// The CaptionBubbleController is currently only implemented in Views.
#if defined(TOOLKIT_VIEWS)
EXPECT_TRUE(IsWidgetVisibleOnBrowser(browser1));
EXPECT_TRUE(IsWidgetVisibleOnBrowser(browser2));
EXPECT_TRUE(IsWidgetVisibleOnBrowser(incognito_browser));
#else
EXPECT_FALSE(IsWidgetVisibleOnBrowser(incognito_browser));
#endif
}
#if !defined(OS_CHROMEOS) // No multi-profile on ChromeOS.
IN_PROC_BROWSER_TEST_F(CaptionControllerTest,
......@@ -704,6 +771,47 @@ IN_PROC_BROWSER_TEST_F(CaptionControllerTest,
#endif
}
IN_PROC_BROWSER_TEST_F(CaptionControllerTest, OnError_MultipleProfiles) {
Profile* profile1 = browser()->profile();
Profile* profile2 = CreateProfile();
Browser* browser1 = browser();
Browser* browser2 = CreateBrowser(profile2);
// Enable live caption on both profiles.
SetLiveCaptionEnabled(true);
profile2->GetPrefs()->SetBoolean(prefs::kLiveCaptionEnabled, true);
// OnError routes to the right browser on the right profile.
OnErrorOnBrowserForProfile(browser1, profile1);
// The CaptionBubbleController is currently only implemented in Views.
#if defined(TOOLKIT_VIEWS)
EXPECT_TRUE(IsWidgetVisibleOnBrowser(browser1));
EXPECT_FALSE(IsWidgetVisibleOnBrowser(browser2));
#else
EXPECT_FALSE(IsWidgetVisibleOnBrowser(browser1));
#endif
OnErrorOnBrowserForProfile(browser2, profile2);
// The CaptionBubbleController is currently only implemented in Views.
#if defined(TOOLKIT_VIEWS)
EXPECT_TRUE(IsWidgetVisibleOnBrowser(browser1));
EXPECT_TRUE(IsWidgetVisibleOnBrowser(browser2));
#else
EXPECT_FALSE(IsWidgetVisibleOnBrowser(browser1));
#endif
// OnError does nothing when sent to browsers on different profiles.
OnErrorOnBrowserForProfile(browser1, profile2);
// The CaptionBubbleController is currently only implemented in Views.
#if defined(TOOLKIT_VIEWS)
EXPECT_TRUE(IsWidgetVisibleOnBrowser(browser1));
EXPECT_TRUE(IsWidgetVisibleOnBrowser(browser2));
#else
EXPECT_FALSE(IsWidgetVisibleOnBrowser(browser1));
EXPECT_FALSE(IsWidgetVisibleOnBrowser(browser2));
#endif
}
#endif // !defined (OS_CHROMEOS)
} // namespace captions
......@@ -26,14 +26,9 @@ void CaptionHostImpl::Create(
CaptionHostImpl::CaptionHostImpl(content::RenderFrameHost* frame_host)
: frame_host_(frame_host) {
if (!frame_host_)
return;
content::WebContents* web_contents =
content::WebContents::FromRenderFrameHost(frame_host_);
if (!web_contents) {
frame_host_ = nullptr;
content::WebContents* web_contents = GetWebContents();
if (!web_contents)
return;
}
Observe(web_contents);
}
......@@ -41,50 +36,44 @@ CaptionHostImpl::~CaptionHostImpl() = default;
void CaptionHostImpl::OnSpeechRecognitionReady(
OnSpeechRecognitionReadyCallback reply) {
if (!frame_host_) {
std::move(reply).Run(false);
return;
}
content::WebContents* web_contents =
content::WebContents::FromRenderFrameHost(frame_host_);
content::WebContents* web_contents = GetWebContents();
if (!web_contents) {
frame_host_ = nullptr;
std::move(reply).Run(false);
return;
}
Profile* profile =
Profile::FromBrowserContext(web_contents->GetBrowserContext());
if (!profile) {
CaptionController* caption_controller = GetCaptionController(web_contents);
if (!caption_controller) {
std::move(reply).Run(false);
return;
}
std::move(reply).Run(CaptionControllerFactory::GetForProfile(profile)
->OnSpeechRecognitionReady(web_contents));
std::move(reply).Run(
caption_controller->OnSpeechRecognitionReady(web_contents));
}
void CaptionHostImpl::OnTranscription(
chrome::mojom::TranscriptionResultPtr transcription_result,
OnTranscriptionCallback reply) {
if (!frame_host_) {
std::move(reply).Run(false);
return;
}
content::WebContents* web_contents =
content::WebContents::FromRenderFrameHost(frame_host_);
content::WebContents* web_contents = GetWebContents();
if (!web_contents) {
frame_host_ = nullptr;
std::move(reply).Run(false);
return;
}
Profile* profile =
Profile::FromBrowserContext(web_contents->GetBrowserContext());
if (!profile) {
CaptionController* caption_controller = GetCaptionController(web_contents);
if (!caption_controller) {
std::move(reply).Run(false);
return;
}
std::move(reply).Run(
CaptionControllerFactory::GetForProfile(profile)->DispatchTranscription(
web_contents, transcription_result));
std::move(reply).Run(caption_controller->DispatchTranscription(
web_contents, transcription_result));
}
void CaptionHostImpl::OnError() {
content::WebContents* web_contents = GetWebContents();
if (!web_contents)
return;
CaptionController* caption_controller = GetCaptionController(web_contents);
if (caption_controller)
caption_controller->OnError(web_contents);
}
void CaptionHostImpl::RenderFrameDeleted(content::RenderFrameHost* frame_host) {
......@@ -92,4 +81,23 @@ void CaptionHostImpl::RenderFrameDeleted(content::RenderFrameHost* frame_host) {
frame_host_ = nullptr;
}
content::WebContents* CaptionHostImpl::GetWebContents() {
if (!frame_host_)
return nullptr;
content::WebContents* web_contents =
content::WebContents::FromRenderFrameHost(frame_host_);
if (!web_contents)
frame_host_ = nullptr;
return web_contents;
}
CaptionController* CaptionHostImpl::GetCaptionController(
content::WebContents* web_contents) {
Profile* profile =
Profile::FromBrowserContext(web_contents->GetBrowserContext());
if (!profile)
return nullptr;
return CaptionControllerFactory::GetForProfile(profile);
}
} // namespace captions
......@@ -17,6 +17,8 @@ class RenderFrameHost;
namespace captions {
class CaptionController;
///////////////////////////////////////////////////////////////////////////////
// Caption Host Impl
//
......@@ -42,11 +44,20 @@ class CaptionHostImpl : public chrome::mojom::CaptionHost,
void OnTranscription(
chrome::mojom::TranscriptionResultPtr transcription_result,
OnTranscriptionCallback reply) override;
void OnError() override;
// content::WebContentsObserver:
void RenderFrameDeleted(content::RenderFrameHost* frame_host) override;
private:
// Returns the WebContents if it exists. If it does not exist, sets the
// RenderFrameHost reference to nullptr and returns nullptr.
content::WebContents* GetWebContents();
// Returns the CaptionController for this WebContents. Returns nullptr if
// it does not exist.
CaptionController* GetCaptionController(content::WebContents*);
content::RenderFrameHost* frame_host_;
};
......
......@@ -37,6 +37,8 @@ class CaptionBubbleController {
static std::unique_ptr<CaptionBubbleController> Create(Browser* browser);
// Called when speech recognition is ready to start for the given web
// contents.
virtual bool OnSpeechRecognitionReady(content::WebContents* web_contents) = 0;
// Called when a transcription is received from the service. Returns whether
......@@ -46,6 +48,9 @@ class CaptionBubbleController {
const chrome::mojom::TranscriptionResultPtr& transcription_result,
content::WebContents* web_contents) = 0;
// Called when the speech service has an error.
virtual void OnError(content::WebContents* web_contents) = 0;
// Called when the caption style changes.
virtual void UpdateCaptionStyle(
base::Optional<ui::CaptionStyle> caption_style) = 0;
......
......@@ -540,13 +540,16 @@ void CaptionBubble::OnErrorChanged() {
}
void CaptionBubble::OnReadyChanged() {
DCHECK(model_);
// There is a bug in RenderText in which the label text must not be empty when
// it is displayed, or otherwise subsequent calculation of the number of lines
// (CaptionBubble::GetNumLinesInLabel) will be incorrect. The label text here
// is set to a space character.
// TODO(1055150): Fix the bug in RenderText and then remove this workaround.
label_->SetText(base::ASCIIToUTF16("\u0020"));
UpdateBubbleAndWaitTextVisibility();
if (model_->IsReady()) {
label_->SetText(base::ASCIIToUTF16("\u0020"));
UpdateBubbleAndWaitTextVisibility();
}
}
void CaptionBubble::OnIsExpandedChanged() {
......
......@@ -98,6 +98,16 @@ bool CaptionBubbleControllerViews::OnTranscription(
return true;
}
void CaptionBubbleControllerViews::OnError(content::WebContents* web_contents) {
if (!caption_bubble_ || !caption_bubble_models_.count(web_contents) ||
caption_bubble_models_[web_contents]->IsClosed())
return;
CaptionBubbleModel* caption_bubble_model =
caption_bubble_models_[web_contents].get();
caption_bubble_model->OnError();
}
void CaptionBubbleControllerViews::OnTabStripModelChanged(
TabStripModel* tab_strip_model,
const TabStripModelChange& change,
......
......@@ -38,6 +38,8 @@ class CaptionBubbleControllerViews : public CaptionBubbleController,
CaptionBubbleControllerViews& operator=(const CaptionBubbleControllerViews&) =
delete;
// Called when speech recognition is ready to start for the given web
// contents.
bool OnSpeechRecognitionReady(content::WebContents* web_contents) override;
// Called when a transcription is received from the service. Returns whether
......@@ -47,6 +49,9 @@ class CaptionBubbleControllerViews : public CaptionBubbleController,
const chrome::mojom::TranscriptionResultPtr& transcription_result,
content::WebContents* web_contents) override;
// Called when the speech service has an error.
void OnError(content::WebContents* web_contents) override;
// Called when the caption style changes.
void UpdateCaptionStyle(
base::Optional<ui::CaptionStyle> caption_style) override;
......
......@@ -152,14 +152,9 @@ class CaptionBubbleControllerViewsTest : public InProcessBrowserTest {
TabStripModel::CLOSE_NONE);
}
void SetHasError(bool has_error) {
// TODO(crbug.com/1055150): Use a public function on the
// CaptionBubbleController to set an error once error messages are wired up
// from the speech service to the CaptionBubbleController.
GetController()
->caption_bubble_models_
[browser()->tab_strip_model()->GetActiveWebContents()]
->SetHasError(has_error);
void OnError(int tab_index = 0) {
GetController()->OnError(
browser()->tab_strip_model()->GetWebContentsAt(tab_index));
}
private:
......@@ -380,7 +375,7 @@ IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest, ShowsAndHidesError) {
EXPECT_TRUE(GetLabel()->GetVisible());
EXPECT_FALSE(GetErrorMessage()->GetVisible());
SetHasError(true);
OnError(0);
EXPECT_FALSE(GetWaitText()->GetVisible());
EXPECT_FALSE(GetLabel()->GetVisible());
EXPECT_TRUE(GetErrorMessage()->GetVisible());
......@@ -391,8 +386,27 @@ IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest, ShowsAndHidesError) {
EXPECT_FALSE(GetLabel()->GetVisible());
EXPECT_TRUE(GetErrorMessage()->GetVisible());
// Clear the error and everything should be visible again.
SetHasError(false);
// The error should not be visible on a new tab.
InsertNewTab();
ActivateTabAt(1);
OnSpeechRecognitionReady();
OnPartialTranscription("Elephants are vegetarians.");
EXPECT_TRUE(GetWaitText()->GetVisible());
EXPECT_TRUE(GetLabel()->GetVisible());
EXPECT_FALSE(GetErrorMessage()->GetVisible());
// The error should still be visible when switching back to the tab.
ActivateTabAt(0);
EXPECT_FALSE(GetWaitText()->GetVisible());
EXPECT_FALSE(GetLabel()->GetVisible());
EXPECT_TRUE(GetErrorMessage()->GetVisible());
// The error should disappear when the tab refreshes.
chrome::Reload(browser(), WindowOpenDisposition::CURRENT_TAB);
content::WaitForLoadStop(
browser()->tab_strip_model()->GetActiveWebContents());
OnSpeechRecognitionReady();
OnPartialTranscription("Elephants can communicate through seismic signals.");
EXPECT_TRUE(GetWaitText()->GetVisible());
EXPECT_TRUE(GetLabel()->GetVisible());
EXPECT_FALSE(GetErrorMessage()->GetVisible());
......@@ -593,7 +607,7 @@ IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest,
// Set the error message.
caption_style.text_size = "50%";
GetController()->UpdateCaptionStyle(caption_style);
SetHasError(true);
OnError();
EXPECT_EQ(lineHeight / 2, GetErrorText()->GetLineHeight());
EXPECT_EQ(errorIconHeight / 2, GetErrorIcon()->GetImageBounds().height());
EXPECT_GT(GetBubble()->GetPreferredSize().height(), lineHeight / 2);
......@@ -623,10 +637,13 @@ IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest, ShowsAndHidesBubble) {
GetController();
EXPECT_FALSE(IsWidgetVisible());
// It is shown if there is an error, and hidden when that error goes away.
SetHasError(true);
// It is shown if there is an error, and hidden when the page refreshes and
// that error goes away.
OnError();
EXPECT_TRUE(IsWidgetVisible());
SetHasError(false);
chrome::Reload(browser(), WindowOpenDisposition::CURRENT_TAB);
content::WaitForLoadStop(
browser()->tab_strip_model()->GetActiveWebContents());
EXPECT_FALSE(IsWidgetVisible());
// It is shown if the bubble is ready and should not show if it is not.
......@@ -662,7 +679,7 @@ IN_PROC_BROWSER_TEST_F(CaptionBubbleControllerViewsTest, ShowsAndHidesBubble) {
// Close the bubble. It should not show, even when it has an error.
ClickButton(GetCloseButton());
EXPECT_FALSE(IsWidgetVisible());
SetHasError(true);
OnError();
EXPECT_FALSE(IsWidgetVisible());
}
......
......@@ -70,8 +70,8 @@ void CaptionBubbleModel::OnReady() {
observer_->OnReadyChanged();
}
void CaptionBubbleModel::SetHasError(bool has_error) {
has_error_ = has_error;
void CaptionBubbleModel::OnError() {
has_error_ = true;
if (observer_)
observer_->OnErrorChanged();
}
......@@ -87,7 +87,11 @@ void CaptionBubbleModel::DidFinishNavigation(
is_closed_ = false;
is_ready_ = false;
has_error_ = false;
OnTextChanged();
if (observer_) {
observer_->OnReadyChanged();
observer_->OnTextChanged();
observer_->OnErrorChanged();
}
}
void CaptionBubbleModel::CommitPartialText() {
......
......@@ -50,8 +50,8 @@ class CaptionBubbleModel : public content::WebContentsObserver {
// Commits the partial text as final text.
void CommitPartialText();
// Set whether the bubble has an error and alert the observer.
void SetHasError(bool has_error);
// Set that the bubble has an error and alert the observer.
void OnError();
// Mark the bubble as closed, clear the partial and final text, and alert the
// observer.
......
......@@ -14,6 +14,10 @@ interface CaptionHost {
// speech service. Returns whether the transcription result was received
// successfully. Transcriptions will halt if this returns false.
OnTranscription(TranscriptionResult transcription_result) => (bool success);
// Called when there is an error in the speech recognition service. Informs
// the UI that it should show an error message to the user.
OnError();
};
// A transcription result created by the speech recognition client in the
......
......@@ -52,6 +52,16 @@ ChromeSpeechRecognitionClient::ChromeSpeechRecognitionClient(
send_audio_callback_ = media::BindToCurrentLoop(base::BindRepeating(
&ChromeSpeechRecognitionClient::SendAudioToSpeechRecognitionService,
weak_factory_.GetWeakPtr()));
speech_recognition_context_.set_disconnect_handler(
base::BindOnce(&ChromeSpeechRecognitionClient::OnRecognizerDisconnected,
base::Unretained(this)));
speech_recognition_recognizer_.set_disconnect_handler(
base::BindOnce(&ChromeSpeechRecognitionClient::OnRecognizerDisconnected,
base::Unretained(this)));
caption_host_.set_disconnect_handler(
base::BindOnce(&ChromeSpeechRecognitionClient::OnCaptionHostDisconnected,
base::Unretained(this)));
}
void ChromeSpeechRecognitionClient::OnRecognizerBound(
......@@ -62,6 +72,15 @@ void ChromeSpeechRecognitionClient::OnRecognizerBound(
std::move(on_ready_callback_).Run();
}
void ChromeSpeechRecognitionClient::OnRecognizerDisconnected() {
is_recognizer_bound_ = false;
caption_host_->OnError();
}
void ChromeSpeechRecognitionClient::OnCaptionHostDisconnected() {
is_browser_requesting_transcription_ = false;
}
ChromeSpeechRecognitionClient::~ChromeSpeechRecognitionClient() = default;
void ChromeSpeechRecognitionClient::AddAudio(
......
......@@ -83,6 +83,14 @@ class ChromeSpeechRecognitionClient
bool IsUrlBlocked(const std::string& url) const;
// Called when the speech recognition context or the speech recognition
// recognizer is disconnected. Sends an error message to the UI and halts
// future transcriptions.
void OnRecognizerDisconnected();
// Called when the caption host is disconnected. Halts future transcriptions.
void OnCaptionHostDisconnected();
media::SpeechRecognitionClient::OnReadyCallback on_ready_callback_;
// Sends audio to the speech recognition thread on the renderer thread.
......@@ -103,7 +111,7 @@ class ChromeSpeechRecognitionClient
// format.
std::unique_ptr<media::AudioBus> temp_audio_bus_;
// Whether the browser is still requesting transcriptions.
// Whether the UI in the browser is still requesting transcriptions.
bool is_browser_requesting_transcription_ = true;
bool is_recognizer_bound_ = false;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment