Implement native speech recognition for the launcher.

BUG=397019

Review URL: https://codereview.chromium.org/676593003

Cr-Commit-Position: refs/heads/master@{#302767}

chrome/browser/ui/app_list/speech_recognizer.h

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_UI_APP_LIST_SPEECH_RECOGNIZER_H_
+#define CHROME_BROWSER_UI_APP_LIST_SPEECH_RECOGNIZER_H_
+
+#include <string>
+
+#include "base/memory/weak_ptr.h"
+
+namespace net {
+class URLRequestContextGetter;
+}
+
+namespace app_list {
+
+class SpeechRecognizerDelegate;
+
+// SpeechRecognizer is a wrapper around the speech recognition engine that
+// simplifies its use from the UI thread. This class handles all setup/shutdown,
+// collection of results, error cases, and threading.
+class SpeechRecognizer {
+ public:
+  SpeechRecognizer(const base::WeakPtr<SpeechRecognizerDelegate>& delegate,
+                   net::URLRequestContextGetter* url_request_context_getter,
+                   const std::string& locale);
+  ~SpeechRecognizer();
+
+  // Start/stop the speech recognizer. Must be called on the UI thread.
+  void Start();
+  void Stop();
+
+ private:
+  class EventListener;
+
+  base::WeakPtr<SpeechRecognizerDelegate> delegate_;
+  scoped_refptr<EventListener> speech_event_listener_;
+
+  DISALLOW_COPY_AND_ASSIGN(SpeechRecognizer);
+};
+
+}  // namespace app_list
+
+#endif  // CHROME_BROWSER_UI_APP_LIST_SPEECH_RECOGNIZER_H_

chrome/browser/ui/app_list/speech_recognizer_browsertest.cc

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/ui/app_list/speech_recognizer.h"
+
+#include "base/command_line.h"
+#include "base/memory/weak_ptr.h"
+#include "base/run_loop.h"
+#include "base/strings/utf_string_conversions.h"
+#include "base/threading/platform_thread.h"
+#include "chrome/browser/ui/app_list/speech_recognizer_delegate.h"
+#include "chrome/browser/ui/browser.h"
+#include "chrome/common/chrome_switches.h"
+#include "chrome/test/base/in_process_browser_test.h"
+#include "chrome/test/base/testing_profile.h"
+#include "content/public/browser/browser_thread.h"
+#include "content/public/test/fake_speech_recognition_manager.h"
+#include "content/public/test/web_contents_tester.h"
+#include "testing/gmock/include/gmock/gmock.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+using ::testing::InvokeWithoutArgs;
+using ::testing::Return;
+
+namespace app_list {
+
+class MockSpeechRecognizerDelegate : public SpeechRecognizerDelegate {
+ public:
+  MockSpeechRecognizerDelegate() : weak_factory_(this) {}
+
+  base::WeakPtr<MockSpeechRecognizerDelegate> GetWeakPtr() {
+    return weak_factory_.GetWeakPtr();
+  }
+
+  MOCK_METHOD2(OnSpeechResult, void(const base::string16&, bool));
+  MOCK_METHOD1(OnSpeechSoundLevelChanged, void(int16_t));
+  MOCK_METHOD1(OnSpeechRecognitionStateChanged, void(SpeechRecognitionState));
+  MOCK_METHOD0(GetSpeechContents, content::WebContents*());
+
+ private:
+  base::WeakPtrFactory<MockSpeechRecognizerDelegate> weak_factory_;
+};
+
+class AppListSpeechRecognizerBrowserTest : public InProcessBrowserTest {
+ public:
+  AppListSpeechRecognizerBrowserTest() {}
+
+  void SetUpOnMainThread() override {
+    InProcessBrowserTest::SetUpOnMainThread();
+
+    fake_speech_recognition_manager_.reset(
+        new content::FakeSpeechRecognitionManager());
+    fake_speech_recognition_manager_->set_should_send_fake_response(true);
+    content::SpeechRecognitionManager::SetManagerForTesting(
+        fake_speech_recognition_manager_.get());
+    mock_speech_delegate_.reset(new MockSpeechRecognizerDelegate());
+    test_contents_.reset(content::WebContentsTester::CreateTestWebContents(
+        browser()->profile(), NULL));
+  }
+
+  void SetUpCommandLine(base::CommandLine* command_line) override {
+    command_line->AppendSwitch(switches::kEnableExperimentalHotwording);
+  }
+
+  void TearDownOnMainThread() override {
+    // This puts stuff on the IO loop which needs to be executed.
+    test_contents_.reset();
+
+    // Poor-person's way of ensuring IO loop is idle.
+    auto io_loop = content::BrowserThread::UnsafeGetMessageLoopForThread(
+        content::BrowserThread::IO);
+    ASSERT_TRUE(io_loop);
+    while (!io_loop->IsIdleForTesting()) {
+      // Sleep for a little bit, allowing the IO thread to obtain any locks
+      // taken by IsIdleForTesting(). Without this sleep, this loop may livelock
+      // the message loop causing the test to fail.
+      base::PlatformThread::Sleep(base::TimeDelta::FromMilliseconds(10));
+    }
+  }
+
+ protected:
+  scoped_ptr<content::FakeSpeechRecognitionManager>
+      fake_speech_recognition_manager_;
+  scoped_ptr<MockSpeechRecognizerDelegate> mock_speech_delegate_;
+  scoped_ptr<content::WebContents> test_contents_;
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(AppListSpeechRecognizerBrowserTest);
+};
+
+IN_PROC_BROWSER_TEST_F(AppListSpeechRecognizerBrowserTest, RecognizeSpeech) {
+  SpeechRecognizer recognizer(mock_speech_delegate_->GetWeakPtr(),
+                              browser()->profile()->GetRequestContext(),
+                              "en");
+
+  base::RunLoop run_loop;
+  EXPECT_CALL(*mock_speech_delegate_, GetSpeechContents())
+      .WillOnce(Return(test_contents_.get()));
+  EXPECT_CALL(*mock_speech_delegate_,
+              OnSpeechResult(base::ASCIIToUTF16("Pictures of the moon"), true));
+  EXPECT_CALL(*mock_speech_delegate_,
+              OnSpeechRecognitionStateChanged(SPEECH_RECOGNITION_READY))
+      .WillOnce(InvokeWithoutArgs(&run_loop, &base::RunLoop::Quit));
+  recognizer.Start();
+  run_loop.Run();
+}
+
+}  // namespace app_list
+
+

chrome/browser/ui/app_list/speech_recognizer_delegate.h

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_UI_APP_LIST_SPEECH_RECOGNIZER_DELEGATE_H_
+#define CHROME_BROWSER_UI_APP_LIST_SPEECH_RECOGNIZER_DELEGATE_H_
+
+#include <stdint.h>
+
+#include "base/strings/string16.h"
+#include "ui/app_list/speech_ui_model_observer.h"
+
+namespace content {
+class WebContents;
+}
+
+namespace app_list {
+
+// Delegate for the app list speech recognizer. All methods are called from the
+// UI thread.
+class SpeechRecognizerDelegate {
+ public:
+  // Receive a speech recognition result. |is_final| indicated whether the
+  // result is an intermediate or final result. If |is_final| is true, then the
+  // recognizer stops and no more results will be returned.
+  virtual void OnSpeechResult(const base::string16& query, bool is_final) = 0;
+
+  // Invoked regularly to indicate the average sound volume.
+  virtual void OnSpeechSoundLevelChanged(int16_t level) = 0;
+
+  // Invoked when the state of speech recognition is changed.
+  virtual void OnSpeechRecognitionStateChanged(
+      SpeechRecognitionState new_state) = 0;
+
+  // Return a WebContents that is whitelisted to use the speech recognizer.
+  // TODO(amistry): This is an implementation detail that shouldn't be
+  // necessary. Somehow, eliminate this dependency.
+  virtual content::WebContents* GetSpeechContents() = 0;
+
+ protected:
+  virtual ~SpeechRecognizerDelegate() {}
+};
+
+}  // namespace app_list
+
+#endif  // CHROME_BROWSER_UI_APP_LIST_SPEECH_RECOGNIZER_DELEGATE_H_

chrome/browser/ui/app_list/start_page_service.cc

@@ -11,12 +11,14 @@
 #include "base/memory/singleton.h"
 #include "base/metrics/user_metrics.h"
 #include "base/prefs/pref_service.h"
+#include "chrome/browser/browser_process.h"
 #include "chrome/browser/chrome_notification_types.h"
 #include "chrome/browser/media/media_stream_infobar_delegate.h"
 #include "chrome/browser/profiles/profile.h"
 #include "chrome/browser/search/hotword_service.h"
 #include "chrome/browser/search/hotword_service_factory.h"
 #include "chrome/browser/ui/app_list/recommended_apps.h"
+#include "chrome/browser/ui/app_list/speech_recognizer.h"
 #include "chrome/browser/ui/app_list/start_page_observer.h"
 #include "chrome/browser/ui/app_list/start_page_service_factory.h"
 #include "chrome/common/chrome_switches.h"
@@ -112,7 +114,8 @@ StartPageService::StartPageService(Profile* profile)
       state_(app_list::SPEECH_RECOGNITION_OFF),
       speech_button_toggled_manually_(false),
       speech_result_obtained_(false),
-      webui_finished_loading_(false) {
+      webui_finished_loading_(false),
+      weak_factory_(this) {
   // If experimental hotwording is enabled, then we're always "ready".
   // Transitioning into the "hotword recognizing" state is handled by the
   // hotword extension.
@@ -136,14 +139,14 @@ void StartPageService::RemoveObserver(StartPageObserver* observer) {
 void StartPageService::AppListShown() {
   if (!contents_) {
     LoadContents();
-  } else if (contents_->GetWebUI()) {
-    // If experimental hotwording is enabled, don't enable hotwording in the
-    // start page, since the hotword extension is taking care of this.
-    bool hotword_enabled = HotwordEnabled() &&
-        !HotwordService::IsExperimentalHotwordingEnabled();
+  } else if (contents_->GetWebUI() &&
+             !HotwordService::IsExperimentalHotwordingEnabled()) {
+    // If experimental hotwording is enabled, don't call onAppListShown.
+    // onAppListShown() initializes the web speech API, which is not used with
+    // experimental hotwording.
     contents_->GetWebUI()->CallJavascriptFunction(
         "appList.startPage.onAppListShown",
-        base::FundamentalValue(hotword_enabled));
+        base::FundamentalValue(HotwordEnabled()));
   }
 }
 
@@ -154,6 +157,11 @@ void StartPageService::AppListHidden() {
   }
   if (!app_list::switches::IsExperimentalAppListEnabled())
     UnloadContents();
+
+  if (HotwordService::IsExperimentalHotwordingEnabled() &&
+      speech_recognizer_) {
+    speech_recognizer_->Stop();
+  }
 }
 
 void StartPageService::ToggleSpeechRecognition() {
@@ -162,14 +170,35 @@ void StartPageService::ToggleSpeechRecognition() {
   if (!contents_->GetWebUI())
     return;
 
-  if (webui_finished_loading_) {
-    contents_->GetWebUI()->CallJavascriptFunction(
-        "appList.startPage.toggleSpeechRecognition");
-  } else {
+  if (!webui_finished_loading_) {
     pending_webui_callbacks_.push_back(
         base::Bind(&StartPageService::ToggleSpeechRecognition,
                    base::Unretained(this)));
+    return;
   }
+
+  if (HotwordService::IsExperimentalHotwordingEnabled()) {
+    if (!speech_recognizer_) {
+      std::string profile_locale;
+#if defined(OS_CHROMEOS)
+      profile_locale = profile_->GetPrefs()->GetString(
+          prefs::kApplicationLocale);
+#endif
+      if (profile_locale.empty())
+        profile_locale = g_browser_process->GetApplicationLocale();
+
+      speech_recognizer_.reset(
+          new SpeechRecognizer(weak_factory_.GetWeakPtr(),
+                               profile_->GetRequestContext(),
+                               profile_locale));
+    }
+
+    speech_recognizer_->Start();
+    return;
+  }
+
+  contents_->GetWebUI()->CallJavascriptFunction(
+      "appList.startPage.toggleSpeechRecognition");
 }
 
 bool StartPageService::HotwordEnabled() {
@@ -212,7 +241,7 @@ void StartPageService::OnSpeechResult(
                     OnSpeechResult(query, is_final));
 }
 
-void StartPageService::OnSpeechSoundLevelChanged(int16 level) {
+void StartPageService::OnSpeechSoundLevelChanged(int16_t level) {
   FOR_EACH_OBSERVER(StartPageObserver,
                     observers_,
                     OnSpeechSoundLevelChanged(level));
@@ -220,6 +249,13 @@ void StartPageService::OnSpeechSoundLevelChanged(int16 level) {
 
 void StartPageService::OnSpeechRecognitionStateChanged(
     SpeechRecognitionState new_state) {
+
+  if (HotwordService::IsExperimentalHotwordingEnabled() &&
+      new_state == SPEECH_RECOGNITION_READY &&
+      speech_recognizer_) {
+    speech_recognizer_->Stop();
+  }
+
   if (!InSpeechRecognition(state_) && InSpeechRecognition(new_state)) {
     if (!speech_button_toggled_manually_ &&
         state_ == SPEECH_RECOGNITION_HOTWORD_LISTENING) {
@@ -239,6 +275,10 @@ void StartPageService::OnSpeechRecognitionStateChanged(
                     OnSpeechRecognitionStateChanged(new_state));
 }
 
+content::WebContents* StartPageService::GetSpeechContents() {
+  return GetSpeechRecognitionContents();
+}
+
 void StartPageService::Shutdown() {
   UnloadContents();
 }

chrome/browser/ui/app_list/start_page_service.h

@@ -5,14 +5,17 @@
 #ifndef CHROME_BROWSER_UI_APP_LIST_START_PAGE_SERVICE_H_
 #define CHROME_BROWSER_UI_APP_LIST_START_PAGE_SERVICE_H_
 
+#include <stdint.h>
 #include <vector>
 
 #include "base/basictypes.h"
 #include "base/callback.h"
 #include "base/memory/ref_counted.h"
 #include "base/memory/scoped_ptr.h"
+#include "base/memory/weak_ptr.h"
 #include "base/observer_list.h"
 #include "base/strings/string16.h"
+#include "chrome/browser/ui/app_list/speech_recognizer_delegate.h"
 #include "components/keyed_service/core/keyed_service.h"
 #include "content/public/browser/web_contents.h"
 #include "ui/app_list/speech_ui_model_observer.h"
@@ -26,11 +29,13 @@ class Profile;
 namespace app_list {
 
 class RecommendedApps;
+class SpeechRecognizer;
 class StartPageObserver;
 
 // StartPageService collects data to be displayed in app list's start page
 // and hosts the start page contents.
-class StartPageService : public KeyedService {
+class StartPageService : public KeyedService,
+                         public SpeechRecognizerDelegate {
  public:
   typedef std::vector<scoped_refptr<const extensions::Extension> >
       ExtensionList;
@@ -58,9 +63,18 @@ class StartPageService : public KeyedService {
   RecommendedApps* recommended_apps() { return recommended_apps_.get(); }
   Profile* profile() { return profile_; }
   SpeechRecognitionState state() { return state_; }
-  void OnSpeechResult(const base::string16& query, bool is_final);
-  void OnSpeechSoundLevelChanged(int16 level);
-  void OnSpeechRecognitionStateChanged(SpeechRecognitionState new_state);
+
+  // Overridden from app_list::SpeechRecognizerDelegate:
+  void OnSpeechResult(const base::string16& query, bool is_final) override;
+  void OnSpeechSoundLevelChanged(int16_t level) override;
+  void OnSpeechRecognitionStateChanged(
+      SpeechRecognitionState new_state) override;
+  content::WebContents* GetSpeechContents() override;
+
+ protected:
+  // Protected for testing.
+  explicit StartPageService(Profile* profile);
+  ~StartPageService() override;
 
  private:
   friend class StartPageServiceFactory;
@@ -74,9 +88,6 @@ class StartPageService : public KeyedService {
   // getUserMedia() request from the web contents.
   class StartPageWebContentsDelegate;
 
-  explicit StartPageService(Profile* profile);
-  ~StartPageService() override;
-
   void LoadContents();
   void UnloadContents();
 
@@ -96,6 +107,10 @@ class StartPageService : public KeyedService {
   bool webui_finished_loading_;
   std::vector<base::Closure> pending_webui_callbacks_;
 
+  scoped_ptr<SpeechRecognizer> speech_recognizer_;
+
+  base::WeakPtrFactory<StartPageService> weak_factory_;
+
   DISALLOW_COPY_AND_ASSIGN(StartPageService);
 };
 

chrome/chrome_browser_ui.gypi

@@ -1195,6 +1195,9 @@
       'browser/ui/app_list/search/webstore/webstore_provider.h',
       'browser/ui/app_list/search/webstore/webstore_result.cc',
       'browser/ui/app_list/search/webstore/webstore_result.h',
+      'browser/ui/app_list/speech_recognizer.cc',
+      'browser/ui/app_list/speech_recognizer.h',
+      'browser/ui/app_list/speech_recognizer_delegate.h',
       'browser/ui/app_list/start_page_observer.h',
       'browser/ui/app_list/start_page_service.cc',
       'browser/ui/app_list/start_page_service.h',

chrome/chrome_tests.gypi

@@ -557,6 +557,7 @@
       'browser/ui/app_list/app_list_service_views_browsertest.cc',
       'browser/ui/app_list/search/people/people_provider_browsertest.cc',
       'browser/ui/app_list/search/webstore/webstore_provider_browsertest.cc',
+      'browser/ui/app_list/speech_recognizer_browsertest.cc',
       'browser/ui/ash/accelerator_commands_browsertest.cc',
       'browser/ui/ash/launcher/chrome_launcher_controller_browsertest.cc',
       'browser/ui/ash/launcher/launcher_favicon_loader_browsertest.cc',