Skip to content

GitLab

T
tangled
Commit 04bba6df authored by Roberto Carrillo's avatar Roberto Carrillo Committed by Commit Bot
Browse files
Options

Revert "Add speech recognition API wrapper in VR" 

This reverts commit 37d5fc56.

Reason for revert: Speculative revert for SOLINK failure in https://build.chromium.org/p/chromium.linux/builders/Android%20Arm64%20Builder%20%28dbg%29/builds/46752

Original change's description:
> Add speech recognition API wrapper in VR
> 
> This CL adds an invisible button which should trigger voice search.
> Note that the UI will change significantly. The main purpose of this
> CL is to add speech recognition API wrapper.
> 
> Bug: 773767
> Change-Id: If760a88cc950b231d3b2c14b94403afd8f23c56b
> Reviewed-on: https://chromium-review.googlesource.com/701174
> Commit-Queue: Biao She <bshe@chromium.org>
> Reviewed-by: Ian Vollick <vollick@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#510182}

TBR=vollick@chromium.org,bshe@chromium.org

Change-Id: I2943d46e3d7d160852371a33f16e44ba1d62fbb6
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: 773767
Reviewed-on: https://chromium-review.googlesource.com/729319Reviewed-by: default avatarRoberto Carrillo <robertocn@chromium.org>
Commit-Queue: Roberto Carrillo <robertocn@chromium.org>
Cr-Commit-Position: refs/heads/master@{#510219}
parent 2bdaf0af
Hide whitespace changes
Inline Side-by-side
Showing with 9 additions and 868 deletions
chrome/android/java/src/org/chromium/chrome/browser/vr_shell/VrShellImpl.java
View file @ 04bba6df
......@@ -49,7 +49,6 @@ import org.chromium.chrome.browser.tabmodel.TabModelSelectorObserver;
import org.chromium.chrome.browser.tabmodel.TabModelSelectorTabObserver;
import org.chromium.chrome.browser.tabmodel.TabModelUtils;
import org.chromium.content.browser.ContentViewCore;
import org.chromium.content_public.browser.LoadUrlParams;
import org.chromium.content_public.browser.WebContents;
import org.chromium.content_public.common.BrowserControlsState;
import org.chromium.ui.UiUtils;
......@@ -735,13 +734,6 @@ public class VrShellImpl
UrlConstants.NTP_URL, TabLaunchType.FROM_CHROME_UI);
}
@CalledByNative
private void loadUrl(String url) {
// TODO(bshe): reuse voice suggestion provider if possible.
assert mTab != null;
mTab.loadUrl(new LoadUrlParams(url));
}
@VisibleForTesting
@Override
@CalledByNative
......
chrome/browser/android/vr_shell/vr_gl_thread.cc
View file @ 04bba6df
......@@ -146,13 +146,6 @@ void VrGLThread::OnContentScreenBoundsChanged(const gfx::SizeF& bounds) {
weak_vr_shell_, bounds));
}
void VrGLThread::SetVoiceSearchActivate(bool activate) {
DCHECK(OnGlThread());
main_thread_task_runner_->PostTask(
FROM_HERE,
base::Bind(&VrShell::SetVoiceSearchActivate, weak_vr_shell_, activate));
}
void VrGLThread::SetFullscreen(bool enabled) {
DCHECK(OnMainThread());
task_runner()->PostTask(
......
chrome/browser/android/vr_shell/vr_gl_thread.h
View file @ 04bba6df
......@@ -58,7 +58,6 @@ class VrGLThread : public base::android::JavaHandlerThread,
void OnExitVrPromptResult(vr::UiUnsupportedMode reason,
vr::ExitVrPromptChoice choice) override;
void OnContentScreenBoundsChanged(const gfx::SizeF& bounds) override;
void SetVoiceSearchActivate(bool activate) override;
// vr::BrowserUiInterface implementation (Browser calling to UI).
void SetWebVrMode(bool enabled, bool show_toast) override;
......
chrome/browser/android/vr_shell/vr_shell.cc
View file @ 04bba6df
......@@ -27,19 +27,14 @@
#include "chrome/browser/android/vr_shell/vr_shell_gl.h"
#include "chrome/browser/android/vr_shell/vr_usage_monitor.h"
#include "chrome/browser/android/vr_shell/vr_web_contents_observer.h"
#include "chrome/browser/browser_process.h"
#include "chrome/browser/media/webrtc/media_capture_devices_dispatcher.h"
#include "chrome/browser/media/webrtc/media_stream_capture_indicator.h"
#include "chrome/browser/profiles/profile_manager.h"
#include "chrome/browser/search_engines/template_url_service_factory.h"
#include "chrome/browser/vr/toolbar_helper.h"
#include "chrome/browser/vr/ui_interface.h"
#include "chrome/browser/vr/ui_scene_manager.h"
#include "chrome/browser/vr/vr_tab_helper.h"
#include "chrome/browser/vr/web_contents_event_forwarder.h"
#include "chrome/common/url_constants.h"
#include "components/search_engines/template_url_service.h"
#include "components/search_engines/util.h"
#include "content/public/browser/browser_context.h"
#include "content/public/browser/browser_thread.h"
#include "content/public/browser/navigation_controller.h"
......@@ -721,23 +716,6 @@ void VrShell::OnContentScreenBoundsChanged(const gfx::SizeF& bounds) {
window_size.height(), dpr);
}
void VrShell::SetVoiceSearchActivate(bool activate) {
if (!activate && !speech_recognizer_)
return;
if (!speech_recognizer_) {
Profile* profile = ProfileManager::GetActiveUserProfile();
std::string profile_locale = g_browser_process->GetApplicationLocale();
speech_recognizer_.reset(new vr::SpeechRecognizer(
this, profile->GetRequestContext(), profile_locale));
}
if (activate) {
speech_recognizer_->Start();
} else {
speech_recognizer_->Stop();
}
}
void VrShell::PollMediaAccessFlag() {
poll_capturing_media_task_.Cancel();
......@@ -879,17 +857,6 @@ bool VrShell::ShouldDisplayURL() const {
return ChromeToolbarModelDelegate::ShouldDisplayURL();
}
void VrShell::OnVoiceResults(const base::string16& result) {
TemplateURLService* template_url_service =
TemplateURLServiceFactory::GetForProfile(
ProfileManager::GetActiveUserProfile());
GURL url(GetDefaultSearchURLForSearchTerms(template_url_service, result));
JNIEnv* env = base::android::AttachCurrentThread();
Java_VrShellImpl_loadUrl(
env, j_vr_shell_,
base::android::ConvertUTF8ToJavaString(env, url.spec()));
}
// ----------------------------------------------------------------------------
// Native JNI methods
// ----------------------------------------------------------------------------
......
chrome/browser/android/vr_shell/vr_shell.h
View file @ 04bba6df
......@@ -13,10 +13,8 @@
#include "base/macros.h"
#include "base/memory/weak_ptr.h"
#include "base/single_thread_task_runner.h"
#include "base/strings/string16.h"
#include "chrome/browser/ui/toolbar/chrome_toolbar_model_delegate.h"
#include "chrome/browser/vr/exit_vr_prompt_choice.h"
#include "chrome/browser/vr/speech_recognizer.h"
#include "chrome/browser/vr/ui.h"
#include "chrome/browser/vr/ui_unsupported_mode.h"
#include "content/public/browser/web_contents_observer.h"
......@@ -62,11 +60,12 @@ enum UiAction {
EXIT_PRESENT,
};
class VrMetricsHelper;
// The native instance of the Java VrShell. This class is not threadsafe and
// must only be used on the UI thread.
class VrShell : device::GvrGamepadDataProvider,
device::CardboardGamepadDataProvider,
vr::VoiceResultDelegate,
public ChromeToolbarModelDelegate {
public:
VrShell(JNIEnv* env,
......@@ -181,7 +180,6 @@ class VrShell : device::GvrGamepadDataProvider,
void OnExitVrPromptResult(vr::UiUnsupportedMode reason,
vr::ExitVrPromptChoice choice);
void OnContentScreenBoundsChanged(const gfx::SizeF& bounds);
void SetVoiceSearchActivate(bool activate);
void ProcessContentGesture(std::unique_ptr<blink::WebInputEvent> event,
int content_id);
......@@ -203,8 +201,6 @@ class VrShell : device::GvrGamepadDataProvider,
content::WebContents* GetActiveWebContents() const override;
bool ShouldDisplayURL() const override;
void OnVoiceResults(const base::string16& result) override;
private:
~VrShell() override;
void PostToGlThread(const base::Location& from_here,
......@@ -239,7 +235,6 @@ class VrShell : device::GvrGamepadDataProvider,
std::unique_ptr<vr::WebContentsEventForwarder> web_contents_event_forwarder_;
std::unique_ptr<AndroidUiGestureTarget> android_ui_gesture_target_;
std::unique_ptr<VrMetricsHelper> metrics_helper_;
std::unique_ptr<vr::SpeechRecognizer> speech_recognizer_;
scoped_refptr<base::SingleThreadTaskRunner> main_thread_task_runner_;
std::unique_ptr<VrGLThread> gl_thread_;
......
chrome/browser/vr/BUILD.gn
View file @ 04bba6df
......@@ -107,8 +107,6 @@ source_set("vr_common_source") {
"service/vr_display_host.h",
"service/vr_service_impl.cc",
"service/vr_service_impl.h",
"speech_recognizer.cc",
"speech_recognizer.h",
"target_property.cc",
"target_property.h",
"toolbar_helper.cc",
......@@ -147,12 +145,12 @@ source_set("vr_common_source") {
"//cc/paint",
"//chrome/app:generated_resources",
"//chrome/browser/resources:vr_shell_resources",
"//chrome/common:constants",
"//components/security_state/core",
"//components/strings",
"//components/toolbar",
"//components/url_formatter",
"//components/vector_icons",
"//components/vector_icons",
"//content/public/browser",
"//content/public/common",
"//skia",
......@@ -210,7 +208,6 @@ test("vr_common_unittests") {
"fps_meter_unittest.cc",
"gltf_parser_unittest.cc",
"service/vr_device_manager_unittest.cc",
"speech_recognizer_unittest.cc",
"test/animation_utils.cc",
"test/animation_utils.h",
"test/fake_ui_element_renderer.cc",
......@@ -243,7 +240,6 @@ test("vr_common_unittests") {
deps = [
":vr_test_support",
"//cc:test_support",
"//content/test:test_support",
"//mojo/common",
"//mojo/public/cpp/bindings",
"//testing/gmock",
......@@ -305,6 +301,11 @@ source_set("vr_test_support") {
# TODO(mthiesse, crbug.com/769373): Remove dependency on device/vr:fakes.
"//device/vr:fakes",
# TODO(mthiesse): Figure out why these webrtc deps are necessary for compiling these tests.
# Is a dependency missing from some other target?
"//third_party/webrtc/api:video_frame_api",
"//third_party/webrtc/system_wrappers:metrics_default",
]
if (is_win) {
......
chrome/browser/vr/elements/ui_element_name.h
View file @ 04bba6df
......@@ -31,7 +31,6 @@ enum UiElementName {
kBluetoothConnectedIndicator,
kLoadingIndicator,
kCloseButton,
kVoiceSearchButton,
kScreenDimmer,
kExitWarning,
kExitPrompt,
......
chrome/browser/vr/speech_recognizer.cc deleted 100644 → 0
View file @ 2bdaf0af
// Copyright 2017 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "chrome/browser/vr/speech_recognizer.h"
#include "base/bind.h"
#include "base/strings/string16.h"
#include "content/public/browser/browser_thread.h"
#include "content/public/browser/speech_recognition_event_listener.h"
#include "content/public/browser/speech_recognition_manager.h"
#include "content/public/browser/speech_recognition_session_config.h"
#include "content/public/common/child_process_host.h"
#include "content/public/common/speech_recognition_error.h"
#include "net/url_request/url_request_context_getter.h"
namespace vr {
namespace {
// Length of timeout to cancel recognition if there's no speech heard.
static const int kNoSpeechTimeoutInSeconds = 5;
// Length of timeout to cancel recognition if no different results are received.
static const int kNoNewSpeechTimeoutInSeconds = 2;
// Invalid speech session.
static const int kInvalidSessionId = -1;
static content::SpeechRecognitionManager* g_manager_for_test = nullptr;
content::SpeechRecognitionManager* GetSpeechRecognitionManager() {
if (g_manager_for_test)
return g_manager_for_test;
return content::SpeechRecognitionManager::GetInstance();
}
} // namespace
// Not thread safe. This object can be created on any thread but must only be
// destroyed on IO thread. All of its functions are also IO thread only.
// Notes on the life time of this object:
// The life time of this object is tied to SpeechRecognizer object on browser UI
// thread. However, it doesn't assume SpeechRecognizer on Browser UI thread
// lives longer. So it is safe to delete SpeechRecognizer at any time on UI
// thread to cancel a speech recognition session.
class SpeechRecognizerOnIO : public content::SpeechRecognitionEventListener {
public:
explicit SpeechRecognizerOnIO(
const base::WeakPtr<IOBrowserUIInterface>& browser_ui);
~SpeechRecognizerOnIO() override;
void Start(
scoped_refptr<net::URLRequestContextGetter> url_request_context_getter,
const std::string& locale,
const std::string& auth_scope,
const std::string& auth_token);
void Stop();
// Starts a timer for |timeout_seconds|. When the timer expires, will stop
// capturing audio and get a final utterance from the recognition manager.
void StartSpeechTimeout(int timeout_seconds);
void SpeechTimeout();
// Overidden from content::SpeechRecognitionEventListener:
// These are always called on the IO thread.
void OnRecognitionStart(int session_id) override;
void OnRecognitionEnd(int session_id) override;
void OnRecognitionResults(
int session_id,
const content::SpeechRecognitionResults& results) override;
void OnRecognitionError(
int session_id,
const content::SpeechRecognitionError& error) override;
void OnSoundStart(int session_id) override;
void OnSoundEnd(int session_id) override;
void OnAudioLevelsChange(int session_id,
float volume,
float noise_volume) override;
void OnEnvironmentEstimationComplete(int session_id) override;
void OnAudioStart(int session_id) override;
void OnAudioEnd(int session_id) override;
void SetTimerForTest(std::unique_ptr<base::Timer> speech_timer);
private:
void NotifyRecognitionStateChanged(SpeechRecognitionState new_state);
// Only dereferenced from the UI thread, but copied on IO thread.
base::WeakPtr<IOBrowserUIInterface> browser_ui_;
// All remaining members only accessed from the IO thread.
scoped_refptr<net::URLRequestContextGetter> url_request_context_getter_;
std::string locale_;
std::unique_ptr<base::Timer> speech_timeout_;
int session_;
base::string16 last_result_str_;
base::WeakPtrFactory<SpeechRecognizerOnIO> weak_factory_;
DISALLOW_COPY_AND_ASSIGN(SpeechRecognizerOnIO);
};
SpeechRecognizerOnIO::SpeechRecognizerOnIO(
const base::WeakPtr<IOBrowserUIInterface>& browser_ui)
: browser_ui_(browser_ui),
speech_timeout_(new base::Timer(false, false)),
session_(kInvalidSessionId),
weak_factory_(this) {
}
SpeechRecognizerOnIO::~SpeechRecognizerOnIO() {
DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
if (GetSpeechRecognitionManager())
GetSpeechRecognitionManager()->StopAudioCaptureForSession(session_);
}
void SpeechRecognizerOnIO::Start(
scoped_refptr<net::URLRequestContextGetter> url_request_context_getter,
const std::string& locale,
const std::string& auth_scope,
const std::string& auth_token) {
DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
DCHECK(session_ == kInvalidSessionId)
<< "stop previous session before start new one";
content::SpeechRecognitionSessionConfig config;
config.language = locale;
config.continuous = true;
config.interim_results = true;
config.max_hypotheses = 1;
config.filter_profanities = true;
config.url_request_context_getter = url_request_context_getter;
config.event_listener = weak_factory_.GetWeakPtr();
// kInvalidUniqueID is not a valid render process, so the speech permission
// check allows the request through.
config.initial_context.render_process_id =
content::ChildProcessHost::kInvalidUniqueID;
config.auth_scope = auth_scope;
config.auth_token = auth_token;
auto* speech_instance = GetSpeechRecognitionManager();
if (!speech_instance)
return;
session_ = speech_instance->CreateSession(config);
speech_instance->StartSession(session_);
}
void SpeechRecognizerOnIO::Stop() {
DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
if (session_ == kInvalidSessionId)
return;
if (GetSpeechRecognitionManager())
GetSpeechRecognitionManager()->StopAudioCaptureForSession(session_);
session_ = kInvalidSessionId;
speech_timeout_->Stop();
weak_factory_.InvalidateWeakPtrs();
}
void SpeechRecognizerOnIO::NotifyRecognitionStateChanged(
SpeechRecognitionState new_state) {
content::BrowserThread::PostTask(
content::BrowserThread::UI, FROM_HERE,
base::Bind(&IOBrowserUIInterface::OnSpeechRecognitionStateChanged,
browser_ui_, new_state));
}
void SpeechRecognizerOnIO::StartSpeechTimeout(int timeout_seconds) {
DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
speech_timeout_->Start(FROM_HERE,
base::TimeDelta::FromSeconds(timeout_seconds),
base::Bind(&SpeechRecognizerOnIO::SpeechTimeout,
weak_factory_.GetWeakPtr()));
}
void SpeechRecognizerOnIO::SpeechTimeout() {
DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
NotifyRecognitionStateChanged(SPEECH_RECOGNITION_READY);
Stop();
}
void SpeechRecognizerOnIO::OnRecognitionStart(int session_id) {
NotifyRecognitionStateChanged(SPEECH_RECOGNITION_RECOGNIZING);
}
void SpeechRecognizerOnIO::OnRecognitionEnd(int session_id) {
NotifyRecognitionStateChanged(SPEECH_RECOGNITION_READY);
Stop();
}
void SpeechRecognizerOnIO::OnRecognitionResults(
int session_id,
const content::SpeechRecognitionResults& results) {
base::string16 result_str;
size_t final_count = 0;
// The number of results with |is_provisional| false. If |final_count| ==
// results.size(), then all results are non-provisional and the recognition is
// complete.
for (const auto& result : results) {
if (!result.is_provisional)
final_count++;
result_str += result.hypotheses[0].utterance;
}
content::BrowserThread::PostTask(
content::BrowserThread::UI, FROM_HERE,
base::Bind(&IOBrowserUIInterface::OnSpeechResult, browser_ui_, result_str,
final_count == results.size()));
if (result_str != last_result_str_) {
StartSpeechTimeout(kNoNewSpeechTimeoutInSeconds);
}
last_result_str_ = result_str;
}
void SpeechRecognizerOnIO::OnRecognitionError(
int session_id,
const content::SpeechRecognitionError& error) {
if (error.code == content::SPEECH_RECOGNITION_ERROR_NETWORK) {
NotifyRecognitionStateChanged(SPEECH_RECOGNITION_NETWORK_ERROR);
}
}
void SpeechRecognizerOnIO::OnSoundStart(int session_id) {
StartSpeechTimeout(kNoSpeechTimeoutInSeconds);
NotifyRecognitionStateChanged(SPEECH_RECOGNITION_IN_SPEECH);
}
void SpeechRecognizerOnIO::OnSoundEnd(int session_id) {}
void SpeechRecognizerOnIO::OnAudioLevelsChange(int session_id,
float volume,
float noise_volume) {
// Both |volume| and |noise_volume| are defined to be in the range [0.0, 1.0].
DCHECK_LE(0.0, volume);
DCHECK_GE(1.0, volume);
DCHECK_LE(0.0, noise_volume);
DCHECK_GE(1.0, noise_volume);
volume = std::max(0.0f, volume - noise_volume);
content::BrowserThread::PostTask(
content::BrowserThread::UI, FROM_HERE,
base::Bind(&IOBrowserUIInterface::OnSpeechSoundLevelChanged, browser_ui_,
volume));
}
void SpeechRecognizerOnIO::OnEnvironmentEstimationComplete(int session_id) {}
void SpeechRecognizerOnIO::OnAudioStart(int session_id) {}
void SpeechRecognizerOnIO::OnAudioEnd(int session_id) {}
void SpeechRecognizerOnIO::SetTimerForTest(
std::unique_ptr<base::Timer> speech_timer) {
speech_timeout_ = std::move(speech_timer);
}
SpeechRecognizer::SpeechRecognizer(
VoiceResultDelegate* delegate,
net::URLRequestContextGetter* url_request_context_getter,
const std::string& locale)
: delegate_(delegate),
url_request_context_getter_(url_request_context_getter),
locale_(locale),
weak_factory_(this) {
DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
speech_recognizer_on_io_ = base::MakeUnique<SpeechRecognizerOnIO>(
weak_factory_.GetWeakPtr());
}
SpeechRecognizer::~SpeechRecognizer() {
DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
if (speech_recognizer_on_io_) {
content::BrowserThread::DeleteSoon(content::BrowserThread::IO, FROM_HERE,
speech_recognizer_on_io_.release());
}
}
void SpeechRecognizer::Start() {
DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
std::string auth_scope;
std::string auth_token;
GetSpeechAuthParameters(&auth_scope, &auth_token);
// It is safe to use unretained because speech_recognizer_on_io_ only gets
// deleted on IO thread when SpeechRecognizer is deleted.
content::BrowserThread::PostTask(
content::BrowserThread::IO, FROM_HERE,
base::BindOnce(&SpeechRecognizerOnIO::Start,
base::Unretained(speech_recognizer_on_io_.get()),
url_request_context_getter_, locale_, auth_scope,
auth_token));
}
void SpeechRecognizer::Stop() {
DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
weak_factory_.InvalidateWeakPtrs();
// It is safe to use unretained because speech_recognizer_on_io_ only gets
// deleted on IO thread when SpeechRecognizer is deleted.
content::BrowserThread::PostTask(
content::BrowserThread::IO, FROM_HERE,
base::BindOnce(&SpeechRecognizerOnIO::Stop,
base::Unretained(speech_recognizer_on_io_.get())));
}
void SpeechRecognizer::OnSpeechResult(const base::string16& query,
bool is_final) {
DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
if (!is_final)
return;
if (delegate_)
delegate_->OnVoiceResults(query);
}
void SpeechRecognizer::OnSpeechSoundLevelChanged(float level) {
DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
// TODO(bshe): notify VR UI thread to draw UI.
}
void SpeechRecognizer::OnSpeechRecognitionStateChanged(
vr::SpeechRecognitionState new_state) {
DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
// TODO(bshe): notify VR UI thread state change.
}
void SpeechRecognizer::GetSpeechAuthParameters(std::string* auth_scope,
std::string* auth_token) {
// TODO(bshe): audio history requires auth_scope and auto_token. Get real
// value if we need to save audio history.
}
// static
void SpeechRecognizer::SetManagerForTest(
content::SpeechRecognitionManager* manager) {
g_manager_for_test = manager;
}
// static
void SpeechRecognizer::SetSpeechTimerForTest(
std::unique_ptr<base::Timer> speech_timer) {
if (!speech_recognizer_on_io_)
return;
speech_recognizer_on_io_->SetTimerForTest(std::move(speech_timer));
}
} // namespace vr
chrome/browser/vr/speech_recognizer.h deleted 100644 → 0
View file @ 2bdaf0af
// Copyright 2017 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef CHROME_BROWSER_VR_SPEECH_RECOGNIZER_H_
#define CHROME_BROWSER_VR_SPEECH_RECOGNIZER_H_
#include <string>
#include "base/macros.h"
#include "base/memory/weak_ptr.h"
#include "base/timer/timer.h"
namespace content {
class SpeechRecognitionManager;
}
namespace net {
class URLRequestContextGetter;
}
namespace vr {
// Note that speech recognition is activated on VR UI thread. This means it
// usually involves 3 threads. In the simplest case, the thread communication
// looks like the following:
// VR UI thread Browser thread IO thread
// | | |
// |----ActivateVS----->| |
// | |------Start------> |
// | | |
// | |<-NotifyStateChange-|
// |<--OnSRStateChanged-| |
// | | |
// | |<--OnSpeechResult---|
// |<--OnSRStateChanged-| |
// | navigate |
// | | |
// VS = voice search, SR = speech recognition
enum SpeechRecognitionState {
SPEECH_RECOGNITION_OFF = 0,
SPEECH_RECOGNITION_READY,
SPEECH_RECOGNITION_RECOGNIZING,
SPEECH_RECOGNITION_IN_SPEECH,
SPEECH_RECOGNITION_NETWORK_ERROR,
};
class VoiceResultDelegate {
public:
virtual void OnVoiceResults(const base::string16& result) = 0;
};
class SpeechRecognizerOnIO;
// An interface for IO to communicate with browser UI thread.
// This is used by SpeechRecognizerOnIO class who lives on IO thread.
class IOBrowserUIInterface {
public:
// Receive a speech recognition result. |is_final| indicated whether the
// result is an intermediate or final result. If |is_final| is true, then the
// recognizer stops and no more results will be returned.
virtual void OnSpeechResult(const base::string16& query, bool is_final) = 0;
// Invoked regularly to indicate the average sound volume.
virtual void OnSpeechSoundLevelChanged(float level) = 0;
// Invoked when the state of speech recognition is changed.
virtual void OnSpeechRecognitionStateChanged(
SpeechRecognitionState new_state) = 0;
protected:
virtual ~IOBrowserUIInterface() {}
};
// SpeechRecognizer is a wrapper around the speech recognition engine that
// simplifies its use from the UI thread. This class handles all setup/shutdown,
// collection of results, error cases, and threading.
class SpeechRecognizer : public IOBrowserUIInterface {
public:
SpeechRecognizer(VoiceResultDelegate* delegate,
net::URLRequestContextGetter* url_request_context_getter,
const std::string& locale);
~SpeechRecognizer() override;
// Start/stop the speech recognizer.
// Must be called on the UI thread.
void Start();
void Stop();
// Overridden from vr::IOBrowserUIInterface:
void OnSpeechResult(const base::string16& query, bool is_final) override;
void OnSpeechSoundLevelChanged(float level) override;
void OnSpeechRecognitionStateChanged(
vr::SpeechRecognitionState new_state) override;
void GetSpeechAuthParameters(std::string* auth_scope,
std::string* auth_token);
static void SetManagerForTest(content::SpeechRecognitionManager* manager);
void SetSpeechTimerForTest(std::unique_ptr<base::Timer> speech_timer);
private:
VoiceResultDelegate* delegate_;
scoped_refptr<net::URLRequestContextGetter> url_request_context_getter_;
std::string locale_;
// Note that this object is destroyed on IO thread.
std::unique_ptr<SpeechRecognizerOnIO> speech_recognizer_on_io_;
base::WeakPtrFactory<SpeechRecognizer> weak_factory_;
DISALLOW_COPY_AND_ASSIGN(SpeechRecognizer);
};
} // namespace vr
#endif // CHROME_BROWSER_VR_SPEECH_RECOGNIZER_H_
chrome/browser/vr/speech_recognizer_unittest.cc deleted 100644 → 0
View file @ 2bdaf0af
// Copyright 2017 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "chrome/browser/vr/speech_recognizer.h"
#include "base/logging.h"
#include "base/memory/ptr_util.h"
#include "base/memory/ref_counted.h"
#include "base/message_loop/message_loop.h"
#include "base/strings/utf_string_conversions.h"
#include "base/threading/thread_task_runner_handle.h"
#include "base/timer/mock_timer.h"
#include "content/public/browser/browser_thread.h"
#include "content/public/browser/speech_recognition_event_listener.h"
#include "content/public/browser/speech_recognition_manager.h"
#include "content/public/browser/speech_recognition_session_config.h"
#include "content/public/browser/speech_recognition_session_context.h"
#include "content/public/common/speech_recognition_error.h"
#include "content/public/common/speech_recognition_result.h"
#include "content/public/test/test_browser_thread_bundle.h"
#include "testing/gmock/include/gmock/gmock.h"
#include "testing/gtest/include/gtest/gtest.h"
namespace vr {
static const int kTestSessionId = 1;
const char kTestInterimResult[] = "kitten";
const char kTestResult[] = "cat";
enum FakeRecognitionEvent {
SPEECH_RECOGNITION_START = 0,
SPEECH_RECOGNITION_END,
NETWORK_ERROR,
SOUND_START,
SOUND_END,
AUDIO_START,
AUDIO_END,
INTERIM_RESULT,
FINAL_RESULT,
};
class FakeSpeechRecognitionManager : public content::SpeechRecognitionManager {
public:
FakeSpeechRecognitionManager() {}
~FakeSpeechRecognitionManager() override {}
// SpeechRecognitionManager methods.
int CreateSession(
const content::SpeechRecognitionSessionConfig& config) override {
session_ctx_ = config.initial_context;
session_config_ = config;
session_id_ = kTestSessionId;
return session_id_;
}
void StartSession(int session_id) override {}
void AbortSession(int session_id) override {
DCHECK(session_id_ == session_id);
session_id_ = 0;
}
void AbortAllSessionsForRenderProcess(int render_process_id) override {}
void AbortAllSessionsForRenderView(int render_process_id,
int render_view_id) override {}
void StopAudioCaptureForSession(int session_id) override {}
const content::SpeechRecognitionSessionConfig& GetSessionConfig(
int session_id) const override {
DCHECK(session_id_ == session_id);
return session_config_;
}
content::SpeechRecognitionSessionContext GetSessionContext(
int session_id) const override {
DCHECK(session_id_ == session_id);
return session_ctx_;
}
int GetSession(int render_process_id,
int render_view_id,
int request_id) const override {
return session_id_;
}
void FakeSpeechRecognitionEvent(FakeRecognitionEvent event) {
if (!content::BrowserThread::CurrentlyOn(content::BrowserThread::IO)) {
content::BrowserThread::PostTask(
content::BrowserThread::IO, FROM_HERE,
base::BindOnce(
&FakeSpeechRecognitionManager::FakeSpeechRecognitionEvent,
base::Unretained(this), event));
return;
}
DCHECK(GetActiveListener());
content::SpeechRecognitionError error(
content::SPEECH_RECOGNITION_ERROR_NETWORK);
switch (event) {
case SPEECH_RECOGNITION_START:
GetActiveListener()->OnRecognitionStart(kTestSessionId);
break;
case SPEECH_RECOGNITION_END:
GetActiveListener()->OnRecognitionEnd(kTestSessionId);
break;
case NETWORK_ERROR:
GetActiveListener()->OnRecognitionError(kTestSessionId, error);
break;
case SOUND_START:
GetActiveListener()->OnSoundStart(kTestSessionId);
break;
case INTERIM_RESULT:
SendFakeInterimResults();
break;
case FINAL_RESULT:
SendFakeFinalResults();
break;
default:
NOTREACHED();
}
}
void SendFakeInterimResults() {
if (!session_id_)
return;
content::SpeechRecognitionEventListener* listener = GetActiveListener();
if (!listener)
return;
listener->OnAudioEnd(session_id_);
content::SpeechRecognitionResult result;
result.is_provisional = true;
result.hypotheses.push_back(content::SpeechRecognitionHypothesis(
base::ASCIIToUTF16(kTestInterimResult), 1.0));
content::SpeechRecognitionResults results;
results.push_back(result);
listener->OnRecognitionResults(session_id_, results);
}
void SendFakeFinalResults() {
if (!session_id_)
return;
content::SpeechRecognitionEventListener* listener = GetActiveListener();
if (!listener)
return;
listener->OnAudioEnd(session_id_);
content::SpeechRecognitionResult result;
result.hypotheses.push_back(content::SpeechRecognitionHypothesis(
base::ASCIIToUTF16(kTestResult), 1.0));
content::SpeechRecognitionResults results;
results.push_back(result);
listener->OnRecognitionResults(session_id_, results);
listener->OnRecognitionEnd(session_id_);
session_id_ = 0;
}
private:
content::SpeechRecognitionEventListener* GetActiveListener() {
DCHECK(session_id_ != 0);
return session_config_.event_listener.get();
}
int session_id_ = 0;
content::SpeechRecognitionSessionContext session_ctx_;
content::SpeechRecognitionSessionConfig session_config_;
DISALLOW_COPY_AND_ASSIGN(FakeSpeechRecognitionManager);
};
class SpeechRecognizerTestWrapper : public SpeechRecognizer {
public:
SpeechRecognizerTestWrapper() : SpeechRecognizer(nullptr, nullptr, "en") {}
~SpeechRecognizerTestWrapper() override {}
MOCK_METHOD2(OnSpeechResult,
void(const base::string16& query, bool is_final));
MOCK_METHOD1(OnSpeechRecognitionStateChanged,
void(SpeechRecognitionState new_state));
private:
DISALLOW_COPY_AND_ASSIGN(SpeechRecognizerTestWrapper);
};
class SpeechRecognizerTest : public testing::Test {
public:
SpeechRecognizerTest()
: fake_speech_recognition_manager_(new FakeSpeechRecognitionManager()),
speech_recognizer_(new SpeechRecognizerTestWrapper) {
SpeechRecognizer::SetManagerForTest(fake_speech_recognition_manager_.get());
}
~SpeechRecognizerTest() override {
SpeechRecognizer::SetManagerForTest(nullptr);
}
protected:
content::TestBrowserThreadBundle thread_bundle_;
std::unique_ptr<FakeSpeechRecognitionManager>
fake_speech_recognition_manager_;
std::unique_ptr<SpeechRecognizerTestWrapper> speech_recognizer_;
private:
DISALLOW_COPY_AND_ASSIGN(SpeechRecognizerTest);
};
TEST_F(SpeechRecognizerTest, ReceiveSpeechResult) {
speech_recognizer_->Start();
base::RunLoop().RunUntilIdle();
testing::Sequence s;
EXPECT_CALL(*speech_recognizer_,
OnSpeechResult(base::ASCIIToUTF16("kitten"), false))
.InSequence(s);
EXPECT_CALL(*speech_recognizer_,
OnSpeechResult(base::ASCIIToUTF16("cat"), true))
.InSequence(s);
// After receiving final speech result, speech recognition should send a reset
// request.
EXPECT_CALL(*speech_recognizer_,
OnSpeechRecognitionStateChanged(SPEECH_RECOGNITION_READY))
.InSequence(s);
fake_speech_recognition_manager_->FakeSpeechRecognitionEvent(INTERIM_RESULT);
base::RunLoop().RunUntilIdle();
fake_speech_recognition_manager_->FakeSpeechRecognitionEvent(FINAL_RESULT);
base::RunLoop().RunUntilIdle();
}
TEST_F(SpeechRecognizerTest, ReceivedSpeechRecognitionStates) {
speech_recognizer_->Start();
base::RunLoop().RunUntilIdle();
testing::Sequence s;
EXPECT_CALL(*speech_recognizer_,
OnSpeechRecognitionStateChanged(SPEECH_RECOGNITION_RECOGNIZING))
.InSequence(s);
EXPECT_CALL(*speech_recognizer_,
OnSpeechRecognitionStateChanged(SPEECH_RECOGNITION_NETWORK_ERROR))
.InSequence(s);
EXPECT_CALL(*speech_recognizer_,
OnSpeechRecognitionStateChanged(SPEECH_RECOGNITION_READY))
.InSequence(s);
fake_speech_recognition_manager_->FakeSpeechRecognitionEvent(
SPEECH_RECOGNITION_START);
base::RunLoop().RunUntilIdle();
fake_speech_recognition_manager_->FakeSpeechRecognitionEvent(NETWORK_ERROR);
base::RunLoop().RunUntilIdle();
fake_speech_recognition_manager_->FakeSpeechRecognitionEvent(
SPEECH_RECOGNITION_END);
base::RunLoop().RunUntilIdle();
}
TEST_F(SpeechRecognizerTest, NoSoundTimeout) {
speech_recognizer_->Start();
base::RunLoop().RunUntilIdle();
auto mock_timer = std::make_unique<base::MockTimer>(false, false);
base::MockTimer* timer_ptr = mock_timer.get();
speech_recognizer_->SetSpeechTimerForTest(std::move(mock_timer));
testing::Sequence s;
EXPECT_CALL(*speech_recognizer_,
OnSpeechRecognitionStateChanged(SPEECH_RECOGNITION_IN_SPEECH))
.InSequence(s);
EXPECT_CALL(*speech_recognizer_,
OnSpeechRecognitionStateChanged(SPEECH_RECOGNITION_READY))
.InSequence(s);
fake_speech_recognition_manager_->FakeSpeechRecognitionEvent(SOUND_START);
base::RunLoop().RunUntilIdle();
// This should trigger a SPEECH_RECOGNITION_READY state notification.
timer_ptr->Fire();
base::RunLoop().RunUntilIdle();
}
// This test that it is safe to reset speech_recognizer_ on UI thread after post
// a task to start speech recognition on IO thread.
TEST_F(SpeechRecognizerTest, SafeToResetAfterStart) {
speech_recognizer_->Start();
base::RunLoop().RunUntilIdle();
EXPECT_CALL(*speech_recognizer_,
OnSpeechRecognitionStateChanged(SPEECH_RECOGNITION_RECOGNIZING));
EXPECT_CALL(*speech_recognizer_,
OnSpeechResult(base::ASCIIToUTF16("cat"), true))
.Times(0);
fake_speech_recognition_manager_->FakeSpeechRecognitionEvent(
SPEECH_RECOGNITION_START);
base::RunLoop().RunUntilIdle();
fake_speech_recognition_manager_->FakeSpeechRecognitionEvent(FINAL_RESULT);
// Reset shouldn't crash the test.
speech_recognizer_.reset(nullptr);
base::RunLoop().RunUntilIdle();
}
} // namespace vr
chrome/browser/vr/test/mock_browser_interface.h
View file @ 04bba6df
......@@ -24,7 +24,6 @@ class MockBrowserInterface : public UiBrowserInterface {
MOCK_METHOD2(OnExitVrPromptResult,
void(UiUnsupportedMode reason, ExitVrPromptChoice choice));
MOCK_METHOD1(OnContentScreenBoundsChanged, void(const gfx::SizeF& bounds));
MOCK_METHOD1(SetVoiceSearchActivate, void(bool activate));
private:
DISALLOW_COPY_AND_ASSIGN(MockBrowserInterface);
......
chrome/browser/vr/ui_browser_interface.h
View file @ 04bba6df
......@@ -11,7 +11,7 @@
namespace vr {
// An interface for the VR UI to communicate with VrShell. Many of the functions
// An interface for the UI to communicate with VrShell. Many of the functions
// in this interface are proxies to methods on VrShell.
class UiBrowserInterface {
public:
......@@ -25,7 +25,6 @@ class UiBrowserInterface {
virtual void OnExitVrPromptResult(UiUnsupportedMode reason,
ExitVrPromptChoice choice) = 0;
virtual void OnContentScreenBoundsChanged(const gfx::SizeF& bounds) = 0;
virtual void SetVoiceSearchActivate(bool activate) = 0;
};
} // namespace vr
......
chrome/browser/vr/ui_scene_manager.cc
View file @ 04bba6df
......@@ -38,7 +38,6 @@
#include "chrome/browser/vr/ui_scene.h"
#include "chrome/browser/vr/ui_scene_constants.h"
#include "chrome/browser/vr/vr_gl_util.h"
#include "chrome/common/chrome_features.h"
#include "chrome/grit/generated_resources.h"
#include "components/vector_icons/vector_icons.h"
#include "ui/base/l10n/l10n_util.h"
......@@ -101,7 +100,6 @@ UiSceneManager::UiSceneManager(UiBrowserInterface* browser,
CreateToasts();
CreateSplashScreen();
CreateUnderDevelopmentNotice();
CreateVoiceSearchButton();
ConfigureScene();
}
......@@ -394,21 +392,6 @@ void UiSceneManager::CreateViewportAwareRoot() {
scene_->AddUiElement(k2dBrowsingRoot, std::move(element));
}
void UiSceneManager::CreateVoiceSearchButton() {
// TODO(bshe): Use a proper microphone button and update size and translation
// according to UX.
auto element = base::MakeUnique<Button>(
base::Bind(&UiSceneManager::OnVoiceSearchButtonClicked,
base::Unretained(this)),
base::MakeUnique<CloseButtonTexture>());
element->set_name(kVoiceSearchButton);
element->set_draw_phase(kPhaseForeground);
element->SetTranslate(0.f, 0.f, -kCloseButtonDistance);
element->SetSize(kCloseButtonWidth, kCloseButtonHeight);
voice_search_button_ = element.get();
scene_->AddUiElement(k2dBrowsingForeground, std::move(element));
}
void UiSceneManager::CreateUrlBar(Model* model) {
auto url_bar = base::MakeUnique<UrlBar>(
512,
......@@ -696,9 +679,6 @@ void UiSceneManager::ConfigureScene() {
// Close button is a special control element that needs to be hidden when in
// WebVR, but it needs to be visible when in cct or fullscreen.
close_button_->SetVisible(browsing_mode && (fullscreen_ || in_cct_));
voice_search_button_->SetVisible(
browsing_mode &&
base::FeatureList::IsEnabled(features::kExperimentalVRFeatures));
// Content elements.
for (UiElement* element : content_elements_) {
......@@ -927,11 +907,6 @@ void UiSceneManager::OnCloseButtonClicked() {
}
}
void UiSceneManager::OnVoiceSearchButtonClicked() {
recognizing_speech_ = !recognizing_speech_;
browser_->SetVoiceSearchActivate(recognizing_speech_);
}
void UiSceneManager::OnUnsupportedMode(UiUnsupportedMode mode) {
browser_->OnUnsupportedMode(mode);
}
......
chrome/browser/vr/ui_scene_manager.h
View file @ 04bba6df
......@@ -62,7 +62,6 @@ struct UiInitialState;
// kUrlBar
// kLoadingIndicator
// kExitButton
// kUnderDevelopmentNotice
// kFullscreenToast
// kScreenDimmer
// k2dBrowsingViewportAwareRoot
......@@ -140,7 +139,6 @@ class UiSceneManager {
void CreateCloseButton();
void CreateExitPrompt();
void CreateToasts();
void CreateVoiceSearchButton();
void ConfigureScene();
void ConfigureExclusiveScreenToast();
......@@ -152,7 +150,6 @@ class UiSceneManager {
void OnExitPromptBackplaneClicked();
void OnCloseButtonClicked();
void OnUnsupportedMode(UiUnsupportedMode mode);
void OnVoiceSearchButtonClicked();
ColorScheme::Mode mode() const;
TransientElement* AddTransientParent(UiElementName name,
......@@ -181,7 +178,6 @@ class UiSceneManager {
Rect* ceiling_ = nullptr;
Grid* floor_ = nullptr;
UiElement* close_button_ = nullptr;
UiElement* voice_search_button_ = nullptr;
UrlBar* url_bar_ = nullptr;
TransientElement* webvr_url_toast_transient_parent_ = nullptr;
WebVrUrlToast* webvr_url_toast_ = nullptr;
......@@ -207,7 +203,6 @@ class UiSceneManager {
bool screen_capturing_ = false;
bool location_access_ = false;
bool bluetooth_connected_ = false;
bool recognizing_speech_ = false;
UiUnsupportedMode exit_vr_prompt_reason_ = UiUnsupportedMode::kCount;
std::vector<Rect*> background_panels_;
......
chrome/browser/vr/ui_scene_manager_unittest.cc
View file @ 04bba6df
......@@ -52,7 +52,6 @@ const std::set<UiElementName> kHitTestableElements = {
kUrlBar,
kLoadingIndicator,
kCloseButton,
kVoiceSearchButton,
};
const std::set<UiElementName> kElementsVisibleWithExitWarning = {
kScreenDimmer, kExitWarning,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment