add platform audio input to assistant.

Bug: 820255
Test: Manual

Change-Id: Ib0a2227350acceb2214e0ed77195473a546f245a
Reviewed-on: https://chromium-review.googlesource.com/923179
Commit-Queue: Muyuan Li <muyuanli@chromium.org>
Reviewed-by: Xiyuan Xia <xiyuan@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Xiaohui Chen <xiaohuic@chromium.org>
Cr-Commit-Position: refs/heads/master@{#543899}

chrome/browser/chrome_content_browser_manifest_overlay.json

@@ -29,6 +29,7 @@
         "ash": [ "system_ui", "test", "display" ],
         // Only used in classic ash case.
         "ash_pref_connector": [ "pref_connector" ],
+        "assistant": [ "assistant" ],
         // Only used in classic ash case.
         "chrome": [ "input_device_controller" ],
         "chrome_printing": [ "converter" ],

chrome/browser/chromeos/BUILD.gn

@@ -1698,6 +1698,13 @@ source_set("chromeos") {
       "//chromeos/services/assistant:lib",
       "//chromeos/services/assistant/public/mojom",
     ]
+
+    sources += [
+      "assistant/assistant_client.cc",
+      "assistant/assistant_client.h",
+      "assistant/platform_audio_input_host.cc",
+      "assistant/platform_audio_input_host.h",
+    ]
   }
 
   if (use_cups) {

chrome/browser/chromeos/assistant/DEPS

+include_rules = [
+  # Add browser dependencies explicitly.
+  "-chrome",
+  "+chrome/browser/chromeos/assistant",
+  "-chromeos/services",
+  "+chromeos/services/assistant/public",
+]

chrome/browser/chromeos/assistant/assistant_client.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/chromeos/assistant/assistant_client.h"
+
+#include <utility>
+
+#include "chromeos/services/assistant/public/mojom/constants.mojom.h"
+#include "services/service_manager/public/cpp/connector.h"
+
+namespace chromeos {
+namespace assistant {
+
+namespace {
+// Owned by ChromeBrowserMainChromeOS:
+AssistantClient* g_instance = nullptr;
+}  // namespace
+
+// static
+AssistantClient* AssistantClient::Get() {
+  DCHECK(g_instance);
+  return g_instance;
+}
+
+AssistantClient::AssistantClient() : audio_input_binding_(&audio_input_) {
+  DCHECK_EQ(nullptr, g_instance);
+  g_instance = this;
+}
+
+AssistantClient::~AssistantClient() {
+  DCHECK(g_instance);
+  g_instance = nullptr;
+}
+
+void AssistantClient::Start(service_manager::Connector* connector) {
+  connector->BindInterface(mojom::kServiceName, &assistant_connection_);
+  mojom::AudioInputPtr audio_input_ptr;
+  audio_input_binding_.Bind(mojo::MakeRequest(&audio_input_ptr));
+
+  assistant_connection_->Init(std::move(audio_input_ptr));
+}
+
+}  // namespace assistant
+}  // namespace chromeos

chrome/browser/chromeos/assistant/assistant_client.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_CHROMEOS_ASSISTANT_ASSISTANT_CLIENT_H_
+#define CHROME_BROWSER_CHROMEOS_ASSISTANT_ASSISTANT_CLIENT_H_
+
+#include "base/macros.h"
+#include "chrome/browser/chromeos/assistant/platform_audio_input_host.h"
+#include "chromeos/services/assistant/public/mojom/assistant.mojom.h"
+#include "mojo/public/cpp/bindings/binding.h"
+
+namespace service_manager {
+class Connector;
+}  // namespace service_manager
+
+namespace chromeos {
+namespace assistant {
+
+// Class to handle all assistant in-browser-process functionalities.
+class AssistantClient {
+ public:
+  static AssistantClient* Get();
+
+  AssistantClient();
+  ~AssistantClient();
+
+  void Start(service_manager::Connector* connector);
+
+ private:
+  mojom::AssistantPlatformPtr assistant_connection_;
+  mojo::Binding<mojom::AudioInput> audio_input_binding_;
+
+  PlatformAudioInputHost audio_input_;
+
+  DISALLOW_COPY_AND_ASSIGN(AssistantClient);
+};
+
+}  // namespace assistant
+}  // namespace chromeos
+
+#endif  // CHROME_BROWSER_CHROMEOS_ASSISTANT_ASSISTANT_CLIENT_H_

chrome/browser/chromeos/assistant/platform_audio_input_host.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/chromeos/assistant/platform_audio_input_host.h"
+
+#include <utility>
+
+#include "base/bind.h"
+#include "base/bind_helpers.h"
+#include "base/logging.h"
+#include "media/audio/audio_input_controller.h"
+#include "media/audio/audio_manager.h"
+#include "media/base/audio_bus.h"
+#include "media/base/audio_parameters.h"
+#include "media/base/audio_sample_types.h"
+#include "media/base/channel_layout.h"
+
+namespace chromeos {
+namespace assistant {
+
+class PlatformAudioInputHost::Writer
+    : public media::AudioInputController::SyncWriter {
+ public:
+  explicit Writer(base::WeakPtr<PlatformAudioInputHost> host)
+      : host_(std::move(host)) {}
+  ~Writer() override = default;
+
+  // media::AudioInputController::SyncWriter overrides:
+  void Write(const media::AudioBus* data,
+             double volume,
+             bool key_pressed,
+             base::TimeTicks capture_time) override {
+    if (!host_)
+      return;
+    // 2 channels * # of frames.
+    std::vector<int32_t> buffer(2 * data->frames());
+    data->ToInterleaved<media::SignedInt32SampleTypeTraits>(data->frames(),
+                                                            buffer.data());
+    host_->NotifyDataAvailable(std::move(buffer), data->frames(), capture_time);
+  }
+
+  void Close() override {
+    if (host_)
+      host_->NotifyAudioClosed();
+  }
+
+ private:
+  base::WeakPtr<PlatformAudioInputHost> host_;
+
+  DISALLOW_COPY_AND_ASSIGN(Writer);
+};
+
+class PlatformAudioInputHost::EventHandler
+    : public media::AudioInputController::EventHandler {
+ public:
+  EventHandler() = default;
+  ~EventHandler() override = default;
+
+  // media::AudioInputController::EventHandler overrides:
+  void OnCreated(bool initially_muted) override {}
+  void OnError(media::AudioInputController::ErrorCode error_code) override {}
+  void OnLog(base::StringPiece log) override {}
+  void OnMuted(bool is_muted) override {}
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(EventHandler);
+};
+
+PlatformAudioInputHost::PlatformAudioInputHost() : weak_factory_(this) {
+  sync_writer_ = std::make_unique<Writer>(weak_factory_.GetWeakPtr());
+  event_handler_ = std::make_unique<EventHandler>();
+  audio_input_controller_ = media::AudioInputController::Create(
+      media::AudioManager::Get(), event_handler_.get(), sync_writer_.get(),
+      nullptr,
+      media::AudioParameters(media::AudioParameters::AUDIO_PCM_LINEAR,
+                             media::CHANNEL_LAYOUT_STEREO,
+                             16000 /* 16000 frames per second */,
+                             32 /* 32 bits per frame */,
+                             1600 /* 1600 (16000 / 10) frames per buffer */),
+      "default" /* device_id */, false /* agc_is_enabled */);
+}
+
+PlatformAudioInputHost::~PlatformAudioInputHost() {
+  // Bind |sync_writer_| and |event_handler_| to the callback closure to ensure
+  // they live longer than the |Close| call, which is async.
+  audio_input_controller_->Close(
+      base::BindOnce([](std::unique_ptr<EventHandler> event_handler,
+                        std::unique_ptr<Writer> writer) {},
+                     std::move(event_handler_), std::move(sync_writer_)));
+}
+
+void PlatformAudioInputHost::NotifyDataAvailable(
+    const std::vector<int32_t>& data,
+    int32_t frames,
+    base::TimeTicks capture_time) {
+  observers_.ForAllPtrs([&data, frames, capture_time](auto* observer) {
+    observer->OnAudioInputFramesAvailable(data, frames, capture_time);
+  });
+
+  if (observers_.empty() && recording_) {
+    recording_ = false;
+    audio_input_controller_->Close(base::DoNothing());
+  }
+}
+
+void PlatformAudioInputHost::NotifyAudioClosed() {
+  recording_ = false;
+  observers_.ForAllPtrs([](auto* observer) { observer->OnAudioInputClosed(); });
+  observers_.CloseAll();
+}
+
+void PlatformAudioInputHost::AddObserver(
+    mojom::AudioInputObserverPtr observer) {
+  observers_.AddPtr(std::move(observer));
+  if (!recording_) {
+    audio_input_controller_->Record();
+    recording_ = true;
+  }
+}
+
+}  // namespace assistant
+}  // namespace chromeos

chrome/browser/chromeos/assistant/platform_audio_input_host.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_CHROMEOS_ASSISTANT_PLATFORM_AUDIO_INPUT_HOST_H_
+#define CHROME_BROWSER_CHROMEOS_ASSISTANT_PLATFORM_AUDIO_INPUT_HOST_H_
+
+#include <memory>
+#include <vector>
+
+#include "base/memory/scoped_refptr.h"
+#include "base/memory/weak_ptr.h"
+#include "base/strings/string_piece_forward.h"
+#include "base/time/time.h"
+#include "chromeos/services/assistant/public/mojom/assistant.mojom.h"
+#include "mojo/public/cpp/bindings/interface_ptr_set.h"
+
+namespace media {
+class AudioInputController;
+}  // namespace media
+
+namespace chromeos {
+namespace assistant {
+
+// Interacts with AudioController and forwards audio input stream to assistant.
+class PlatformAudioInputHost : public mojom::AudioInput {
+ public:
+  PlatformAudioInputHost();
+  ~PlatformAudioInputHost() override;
+
+  // mojom::AudioInput overrides:
+  void AddObserver(mojom::AudioInputObserverPtr observer) override;
+
+  void NotifyDataAvailable(const std::vector<int32_t>& data,
+                           int32_t frames,
+                           base::TimeTicks capture_time);
+  void NotifyAudioClosed();
+
+ private:
+  class Writer;
+  class EventHandler;
+
+  scoped_refptr<media::AudioInputController> audio_input_controller_;
+  std::unique_ptr<Writer> sync_writer_;
+  std::unique_ptr<EventHandler> event_handler_;
+  mojo::InterfacePtrSet<mojom::AudioInputObserver> observers_;
+
+  bool recording_ = false;
+
+  base::WeakPtrFactory<PlatformAudioInputHost> weak_factory_;
+
+  DISALLOW_COPY_AND_ASSIGN(PlatformAudioInputHost);
+};
+
+}  // namespace assistant
+}  // namespace chromeos
+
+#endif  // CHROME_BROWSER_CHROMEOS_ASSISTANT_PLATFORM_AUDIO_INPUT_HOST_H_

chrome/browser/chromeos/chrome_browser_main_chromeos.cc

@@ -189,6 +189,10 @@
 #include "components/rlz/rlz_tracker.h"
 #endif
 
+#if BUILDFLAG(ENABLE_CROS_ASSISTANT)
+#include "chrome/browser/chromeos/assistant/assistant_client.h"
+#endif
+
 namespace chromeos {
 
 namespace {
@@ -702,6 +706,11 @@ void ChromeBrowserMainPartsChromeos::PreMainMessageLoopRun() {
   arc_voice_interaction_controller_client_ =
       std::make_unique<arc::VoiceInteractionControllerClient>();
 
+#if BUILDFLAG(ENABLE_CROS_ASSISTANT)
+  if (chromeos::switches::IsAssistantEnabled())
+    assistant_client_ = std::make_unique<assistant::AssistantClient>();
+#endif
+
   chromeos::ResourceReporter::GetInstance()->StartMonitoring(
       task_manager::TaskManagerInterface::GetTaskManager());
 
@@ -1071,6 +1080,10 @@ void ChromeBrowserMainPartsChromeos::PostMainMessageLoopRun() {
 
   arc_voice_interaction_controller_client_.reset();
 
+#if BUILDFLAG(ENABLE_CROS_ASSISTANT)
+  assistant_client_.reset();
+#endif
+
   // Unregister CrosSettings observers before CrosSettings is destroyed.
   shutdown_policy_forwarder_.reset();
 

chrome/browser/chromeos/chrome_browser_main_chromeos.h

@@ -12,6 +12,7 @@
 #include "chrome/browser/chrome_browser_main_linux.h"
 #include "chrome/browser/chromeos/external_metrics.h"
 #include "chrome/browser/memory/memory_kills_monitor.h"
+#include "chromeos/assistant/buildflags.h"
 #include "chromeos/system/version_loader.h"
 
 class NotificationPlatformBridge;
@@ -45,6 +46,12 @@ namespace default_app_order {
 class ExternalLoader;
 }
 
+#if BUILDFLAG(ENABLE_CROS_ASSISTANT)
+namespace assistant {
+class AssistantClient;
+}  // namespace assistant
+#endif
+
 namespace internal {
 class DBusPreEarlyInit;
 class DBusServices;
@@ -110,6 +117,10 @@ class ChromeBrowserMainPartsChromeos : public ChromeBrowserMainPartsLinux {
 
   std::unique_ptr<arc::ArcServiceLauncher> arc_service_launcher_;
 
+#if BUILDFLAG(ENABLE_CROS_ASSISTANT)
+  std::unique_ptr<assistant::AssistantClient> assistant_client_;
+#endif
+
   std::unique_ptr<arc::VoiceInteractionControllerClient>
       arc_voice_interaction_controller_client_;
 

chrome/browser/chromeos/login/session/chrome_session_manager.cc

@@ -49,6 +49,7 @@
 #include "services/service_manager/public/cpp/connector.h"
 
 #if BUILDFLAG(ENABLE_CROS_ASSISTANT)
+#include "chrome/browser/chromeos/assistant/assistant_client.h"
 #include "chromeos/services/assistant/public/mojom/constants.mojom.h"
 #endif
 
@@ -131,8 +132,8 @@ void StartUserSession(Profile* user_profile, const std::string& login_user_id) {
 
 #if BUILDFLAG(ENABLE_CROS_ASSISTANT)
     if (chromeos::switches::IsAssistantEnabled()) {
-      content::BrowserContext::GetConnectorFor(user_profile)
-          ->StartService(chromeos::assistant::mojom::kServiceName);
+      assistant::AssistantClient::Get()->Start(
+          content::BrowserContext::GetConnectorFor(user_profile));
     }
 #endif
 

chrome/browser/chromeos/login/session/user_session_manager.cc

@@ -119,16 +119,13 @@
 #include "components/user_manager/user_names.h"
 #include "components/user_manager/user_type.h"
 #include "components/version_info/version_info.h"
-#include "content/public/browser/browser_context.h"
 #include "content/public/browser/browser_thread.h"
 #include "content/public/browser/notification_service.h"
 #include "content/public/browser/storage_partition.h"
 #include "content/public/common/content_switches.h"
-#include "content/public/common/service_manager_connection.h"
 #include "extensions/common/features/feature_session_type.h"
 #include "net/cert/sth_distributor.h"
 #include "rlz/features/features.h"
-#include "services/service_manager/public/cpp/connector.h"
 #include "third_party/cros_system_api/switches/chrome_switches.h"
 #include "ui/base/ime/chromeos/input_method_descriptor.h"
 #include "ui/base/ime/chromeos/input_method_manager.h"
@@ -141,7 +138,7 @@
 #endif
 
 #if BUILDFLAG(ENABLE_CROS_ASSISTANT)
-#include "chromeos/services/assistant/public/mojom/constants.mojom.h"
+#include "chrome/browser/chromeos/assistant/assistant_client.h"
 #endif
 
 namespace chromeos {
@@ -1379,8 +1376,8 @@ void UserSessionManager::FinalizePrepareProfile(Profile* profile) {
 
 #if BUILDFLAG(ENABLE_CROS_ASSISTANT)
     if (chromeos::switches::IsAssistantEnabled()) {
-      content::BrowserContext::GetConnectorFor(profile)->StartService(
-          chromeos::assistant::mojom::kServiceName);
+      assistant::AssistantClient::Get()->Start(
+          content::BrowserContext::GetConnectorFor(profile));
     }
 #endif
 

chromeos/services/assistant/BUILD.gn

@@ -37,6 +37,8 @@ source_set("lib") {
     sources += [
       "assistant_manager_service_impl.cc",
       "assistant_manager_service_impl.h",
+      "platform/audio_input_provider_impl.cc",
+      "platform/audio_input_provider_impl.h",
       "platform/system_provider_impl.cc",
       "platform/system_provider_impl.h",
       "platform_api_impl.cc",

chromeos/services/assistant/assistant_manager_service.h

@@ -21,6 +21,9 @@ class AssistantManagerService : public mojom::Assistant {
   // Start the assistant in the background with |token|.
   virtual void Start(const std::string& access_token) = 0;
 
+  // Returns whether assistant is running.
+  virtual bool IsRunning() const = 0;
+
   // Set access token for assistant.
   virtual void SetAccessToken(const std::string& access_token) = 0;
 

chromeos/services/assistant/assistant_manager_service_impl.cc

@@ -9,13 +9,15 @@
 #include "base/logging.h"
 #include "chromeos/assistant/internal/internal_constants.h"
 #include "chromeos/assistant/internal/internal_util.h"
+#include "chromeos/services/assistant/service.h"
 #include "libassistant/shared/internal_api/assistant_manager_internal.h"
 
 namespace chromeos {
 namespace assistant {
 
-AssistantManagerServiceImpl::AssistantManagerServiceImpl()
-    : platform_api_(kDefaultConfigStr),
+AssistantManagerServiceImpl::AssistantManagerServiceImpl(
+    mojom::AudioInputPtr audio_input)
+    : platform_api_(kDefaultConfigStr, std::move(audio_input)),
       action_module_(std::make_unique<action::CrosActionModule>(this)),
       assistant_manager_(
           assistant_client::AssistantManager::Create(&platform_api_,
@@ -37,6 +39,11 @@ void AssistantManagerServiceImpl::Start(const std::string& access_token) {
 
   SetAccessToken(access_token);
   assistant_manager_->Start();
+  running_ = true;
+}
+
+bool AssistantManagerServiceImpl::IsRunning() const {
+  return running_;
 }
 
 void AssistantManagerServiceImpl::SetAccessToken(

chromeos/services/assistant/assistant_manager_service_impl.h

@@ -30,11 +30,12 @@ class AssistantManagerServiceImpl
     : public AssistantManagerService,
       public ::chromeos::assistant::action::AssistantActionObserver {
  public:
-  AssistantManagerServiceImpl();
+  explicit AssistantManagerServiceImpl(mojom::AudioInputPtr audio_input);
   ~AssistantManagerServiceImpl() override;
 
   // assistant::AssistantManagerService overrides
   void Start(const std::string& access_token) override;
+  bool IsRunning() const override;
   void SetAccessToken(const std::string& access_token) override;
   void EnableListening(bool enable) override;
 
@@ -49,6 +50,7 @@ class AssistantManagerServiceImpl
   void OnOpenUrl(const std::string& url) override;
 
  private:
+  bool running_ = false;
   PlatformApiImpl platform_api_;
   std::unique_ptr<action::CrosActionModule> action_module_;
   std::unique_ptr<assistant_client::AssistantManager> assistant_manager_;

chromeos/services/assistant/fake_assistant_manager_service_impl.cc

@@ -11,13 +11,19 @@ FakeAssistantManagerServiceImpl::FakeAssistantManagerServiceImpl() = default;
 
 FakeAssistantManagerServiceImpl::~FakeAssistantManagerServiceImpl() = default;
 
-void FakeAssistantManagerServiceImpl::Start(const std::string& access_token) {}
+void FakeAssistantManagerServiceImpl::Start(const std::string& access_token) {
+  running_ = true;
+}
 
 void FakeAssistantManagerServiceImpl::SetAccessToken(
     const std::string& access_token) {}
 
 void FakeAssistantManagerServiceImpl::EnableListening(bool enable) {}
 
+bool FakeAssistantManagerServiceImpl::IsRunning() const {
+  return running_;
+}
+
 void FakeAssistantManagerServiceImpl::SendTextQuery(const std::string& query) {}
 
 void FakeAssistantManagerServiceImpl::AddAssistantEventSubscriber(

chromeos/services/assistant/fake_assistant_manager_service_impl.h

@@ -26,6 +26,7 @@ class FakeAssistantManagerServiceImpl : public AssistantManagerService {
   void Start(const std::string& access_token) override;
   void SetAccessToken(const std::string& access_token) override;
   void EnableListening(bool enable) override;
+  bool IsRunning() const override;
 
   // mojom::AssistantEvent overrides:
   void SendTextQuery(const std::string& query) override;
@@ -33,6 +34,7 @@ class FakeAssistantManagerServiceImpl : public AssistantManagerService {
       mojom::AssistantEventSubscriberPtr subscriber) override;
 
  private:
+  bool running_ = false;
   DISALLOW_COPY_AND_ASSIGN(FakeAssistantManagerServiceImpl);
 };
 

chromeos/services/assistant/manifest.json

@@ -8,7 +8,10 @@
         "identity": [ "identity_manager" ]
       },
       "provides": {
-        "assistant": [ "chromeos::assistant::mojom::Assistant" ]
+        "assistant": [
+            "chromeos::assistant::mojom::Assistant",
+            "chromeos::assistant::mojom::AssistantPlatform"
+        ]
       }
     }
   }

chromeos/services/assistant/platform/audio_input_provider_impl.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chromeos/services/assistant/platform/audio_input_provider_impl.h"
+
+#include "base/logging.h"
+#include "base/stl_util.h"
+#include "libassistant/shared/public/platform_audio_buffer.h"
+
+namespace chromeos {
+namespace assistant {
+
+namespace {
+
+// This format should match //c/b/c/assistant/platform_audio_input_host.cc.
+constexpr assistant_client::BufferFormat kFormat{
+    16000 /* sample_rate */, assistant_client::INTERLEAVED_S32, 2 /* channels */
+};
+
+}  // namespace
+
+AudioInputConfigImpl::AudioInputConfigImpl() = default;
+
+AudioInputConfigImpl::~AudioInputConfigImpl() = default;
+
+std::vector<int> AudioInputConfigImpl::GetSelectedChannels() const {
+  return std::vector<int>{};
+}
+
+float AudioInputConfigImpl::GetMicSensitivity() const {
+  return 0.f;
+}
+
+assistant_client::AudioInputConfig::InputType
+AudioInputConfigImpl::GetInputType() const {
+  return assistant_client::AudioInputConfig::MICROPHONE_NEAR_FIELD;
+}
+
+std::string AudioInputConfigImpl::GetMicManufacturer() const {
+  return "default";
+}
+
+std::string AudioInputConfigImpl::GetMicModel() const {
+  return "default";
+}
+
+std::string AudioInputConfigImpl::GetMicVersion() const {
+  return "0";
+}
+
+assistant_client::AudioInputConfig::MicState AudioInputConfigImpl::GetMicState()
+    const {
+  return assistant_client::AudioInputConfig::ENABLED;
+}
+
+class AudioInputBufferImpl : public assistant_client::AudioBuffer {
+ public:
+  AudioInputBufferImpl(const void* data, uint32_t frame_count)
+      : data_(data), frame_count_(frame_count) {}
+  ~AudioInputBufferImpl() override = default;
+
+  // assistant_client::AudioBuffer overrides:
+  assistant_client::BufferFormat GetFormat() const override { return kFormat; }
+  const void* GetData() const override { return data_; }
+  void* GetWritableData() override {
+    NOTREACHED();
+    return nullptr;
+  }
+  int GetFrameCount() const override { return frame_count_; }
+
+ private:
+  const void* data_;
+  int frame_count_;
+  DISALLOW_COPY_AND_ASSIGN(AudioInputBufferImpl);
+};
+
+AudioInputImpl::AudioInputImpl(mojom::AudioInputPtr audio_input)
+    : binding_(this) {
+  mojom::AudioInputObserverPtr observer;
+  binding_.Bind(mojo::MakeRequest(&observer));
+  audio_input->AddAudioInputObserver(std::move(observer));
+}
+
+AudioInputImpl::~AudioInputImpl() = default;
+
+void AudioInputImpl::OnAudioInputFramesAvailable(
+    const std::vector<int32_t>& buffer,
+    uint32_t frame_count,
+    base::TimeTicks timestamp) {
+  AudioInputBufferImpl input_buffer(buffer.data(), frame_count);
+  int64_t time = timestamp.since_origin().InMilliseconds();
+  {
+    base::AutoLock auto_lock(lock_);
+    for (auto* observer : observers_)
+      observer->OnBufferAvailable(input_buffer, time);
+  }
+}
+
+void AudioInputImpl::OnAudioInputClosed() {
+  base::AutoLock auto_lock(lock_);
+  for (auto* observer : observers_)
+    observer->OnStopped();
+}
+
+assistant_client::BufferFormat AudioInputImpl::GetFormat() const {
+  return kFormat;
+}
+
+void AudioInputImpl::AddObserver(
+    assistant_client::AudioInput::Observer* observer) {
+  base::AutoLock auto_lock(lock_);
+  observers_.push_back(observer);
+}
+
+void AudioInputImpl::RemoveObserver(
+    assistant_client::AudioInput::Observer* observer) {
+  base::AutoLock auto_lock(lock_);
+  base::Erase(observers_, observer);
+}
+
+AudioInputProviderImpl::AudioInputProviderImpl(mojom::AudioInputPtr audio_input)
+    : audio_input_(std::move(audio_input)) {}
+
+AudioInputProviderImpl::~AudioInputProviderImpl() = default;
+
+assistant_client::AudioInputConfig&
+AudioInputProviderImpl::GetAudioInputConfig() {
+  return audio_input_config_;
+}
+
+assistant_client::AudioInput& AudioInputProviderImpl::GetAudioInput() {
+  return audio_input_;
+}
+
+}  // namespace assistant
+}  // namespace chromeos

chromeos/services/assistant/platform/audio_input_provider_impl.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROMEOS_SERVICES_ASSISTANT_PLATFORM_AUDIO_INPUT_PROVIDER_IMPL_H_
+#define CHROMEOS_SERVICES_ASSISTANT_PLATFORM_AUDIO_INPUT_PROVIDER_IMPL_H_
+
+#include <memory>
+#include <vector>
+
+#include "base/macros.h"
+#include "base/synchronization/lock.h"
+#include "base/time/time.h"
+#include "chromeos/services/assistant/public/mojom/assistant.mojom.h"
+#include "libassistant/shared/public/platform_audio_input.h"
+#include "mojo/public/cpp/bindings/binding.h"
+
+namespace chromeos {
+namespace assistant {
+
+// TODO(muyuanli): This class is currently a stub. It will need to be wired up
+// to use the real HW parameters.
+class AudioInputConfigImpl : public assistant_client::AudioInputConfig {
+ public:
+  AudioInputConfigImpl();
+  ~AudioInputConfigImpl() override;
+
+  // assistant_client::AudioInputConfigImpl overrides:
+  std::vector<int> GetSelectedChannels() const override;
+  float GetMicSensitivity() const override;
+  assistant_client::AudioInputConfig::InputType GetInputType() const override;
+  std::string GetMicManufacturer() const override;
+  std::string GetMicModel() const override;
+  std::string GetMicVersion() const override;
+  assistant_client::AudioInputConfig::MicState GetMicState() const override;
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(AudioInputConfigImpl);
+};
+
+class AudioInputImpl : public assistant_client::AudioInput,
+                       public mojom::AudioInputObserver {
+ public:
+  explicit AudioInputImpl(mojom::AudioInputPtr audio_input);
+  ~AudioInputImpl() override;
+
+  // mojom::AudioInputObserver overrides:
+  void OnAudioInputFramesAvailable(const std::vector<int32_t>& buffer,
+                                   uint32_t frame_count,
+                                   base::TimeTicks timestamp) override;
+  void OnAudioInputClosed() override;
+
+  // assistant_client::AudioInput overrides:
+  assistant_client::BufferFormat GetFormat() const override;
+  void AddObserver(assistant_client::AudioInput::Observer* observer) override;
+  void RemoveObserver(
+      assistant_client::AudioInput::Observer* observer) override;
+
+ private:
+  // Protects |observers_| access becase AssistantManager calls
+  // Add/RemoveObserver on its own thread.
+  base::Lock lock_;
+  std::vector<assistant_client::AudioInput::Observer*> observers_;
+  mojo::Binding<mojom::AudioInputObserver> binding_;
+  DISALLOW_COPY_AND_ASSIGN(AudioInputImpl);
+};
+
+class AudioInputProviderImpl : public assistant_client::AudioInputProvider {
+ public:
+  explicit AudioInputProviderImpl(mojom::AudioInputPtr audio_input);
+  ~AudioInputProviderImpl() override;
+
+  // assistant_client::AudioInputProvider overrides:
+  assistant_client::AudioInputConfig& GetAudioInputConfig() override;
+  assistant_client::AudioInput& GetAudioInput() override;
+
+  void RegisterConfigChangeCallback(ConfigChangeCallback callback) override {}
+
+ private:
+  AudioInputConfigImpl audio_input_config_;
+  AudioInputImpl audio_input_;
+
+  DISALLOW_COPY_AND_ASSIGN(AudioInputProviderImpl);
+};
+
+}  // namespace assistant
+}  // namespace chromeos
+
+#endif  // CHROMEOS_SERVICES_ASSISTANT_PLATFORM_AUDIO_INPUT_PROVIDER_IMPL_H_

chromeos/services/assistant/platform_api_impl.cc

@@ -72,8 +72,9 @@ void PlatformApiImpl::DummyAuthProvider::Reset() {}
 // PlatformApiImpl
 ////////////////////////////////////////////////////////////////////////////////
 
-PlatformApiImpl::PlatformApiImpl(const std::string& config)
-    : audio_input_provider_(config),
+PlatformApiImpl::PlatformApiImpl(const std::string& config,
+                                 mojom::AudioInputPtr audio_input)
+    : audio_input_provider_(std::move(audio_input)),
       audio_output_provider_(config, this),
       auth_provider_(),
       file_provider_(config),

chromeos/services/assistant/platform_api_impl.h

@@ -10,11 +10,12 @@
 #include <utility>
 #include <vector>
 
+#include "chromeos/services/assistant/platform/audio_input_provider_impl.h"
 #include "chromeos/services/assistant/platform/system_provider_impl.h"
+#include "chromeos/services/assistant/public/mojom/assistant.mojom.h"
 // TODO(xiaohuic): replace with "base/macros.h" once we remove
 // libassistant/contrib dependency.
 #include "libassistant/contrib/core/macros.h"
-#include "libassistant/contrib/platform/audio/input/audio_input_provider_impl.h"
 #include "libassistant/contrib/platform/audio/output/audio_output_provider_impl.h"
 #include "libassistant/contrib/platform/auth/auth_provider_impl.h"
 #include "libassistant/contrib/platform/file/file_provider_impl.h"
@@ -28,7 +29,8 @@ namespace assistant {
 // Platform API required by the voice assistant.
 class PlatformApiImpl : public assistant_client::PlatformApi {
  public:
-  explicit PlatformApiImpl(const std::string& config);
+  explicit PlatformApiImpl(const std::string& config,
+                           mojom::AudioInputPtr audio_input);
   ~PlatformApiImpl() override;
 
   // assistant_client::PlatformApi overrides
@@ -73,7 +75,7 @@ class PlatformApiImpl : public assistant_client::PlatformApi {
     void Reset() override;
   };
 
-  assistant_contrib::AudioInputProviderImpl audio_input_provider_;
+  AudioInputProviderImpl audio_input_provider_;
   assistant_contrib::AudioOutputProviderImpl audio_output_provider_;
   DummyAuthProvider auth_provider_;
   assistant_contrib::FileProviderImpl file_provider_;

chromeos/services/assistant/public/mojom/assistant.mojom

@@ -5,6 +5,7 @@
 module chromeos.assistant.mojom;
 
 import "url/mojom/url.mojom";
+import "mojo/common/time.mojom";
 
 // Interface to communicate with assistant backend.
 interface Assistant {
@@ -28,3 +29,30 @@ interface AssistantEventSubscriber {
   // Assistant's got open URL response from server.
   OnOpenUrlResponse(url.mojom.Url url);
 };
+
+// Platform connection to assistant.
+interface AssistantPlatform {
+  // Initiates assistant and provides interfaces for assistant to call into the
+  // browser.
+  Init(AudioInput audio_input_interface);
+};
+
+// Interface for assistant to call into browser for audio input related
+// functions.
+interface AudioInput {
+  // Registers audio input observer. We assume dual-channel, 16kHz and signed
+  // 32-bit int interleaved format. Audio stream starts recording as soon as
+  // the first observer is added. And will stop recording if last observer is
+  // closed.
+  AddObserver(AudioInputObserver observer);
+};
+
+// Observer for assistant to receive audio input data.
+interface AudioInputObserver {
+  // Notifies that audio frames are available.
+  // TODO(muyuanli): Use data pipe.
+  OnAudioInputFramesAvailable(array<int32> buffer, uint32 frame_count,
+                              mojo.common.mojom.TimeTicks timestamp);
+  // Notifies that audio input stream is closed.
+  OnAudioInputClosed();
+};

chromeos/services/assistant/service.cc

@@ -20,6 +20,7 @@
 #include "services/service_manager/public/cpp/service_context.h"
 
 #if BUILDFLAG(ENABLE_CROS_LIBASSISTANT)
+#include "chromeos/assistant/internal/internal_constants.h"
 #include "chromeos/services/assistant/assistant_manager_service_impl.h"
 #else
 #include "chromeos/services/assistant/fake_assistant_manager_service_impl.h"
@@ -37,8 +38,12 @@ constexpr char kScopeAssistant[] =
 }  // namespace
 
 Service::Service()
-    : session_observer_binding_(this),
-      token_refresh_timer_(std::make_unique<base::OneShotTimer>()) {}
+    : platform_binding_(this),
+      session_observer_binding_(this),
+      token_refresh_timer_(std::make_unique<base::OneShotTimer>()) {
+  registry_.AddInterface<mojom::AssistantPlatform>(base::BindRepeating(
+      &Service::BindAssistantPlatformConnection, base::Unretained(this)));
+}
 
 Service::~Service() = default;
 
@@ -56,9 +61,7 @@ void Service::SetTimerForTesting(std::unique_ptr<base::OneShotTimer> timer) {
   token_refresh_timer_ = std::move(timer);
 }
 
-void Service::OnStart() {
-  RequestAccessToken();
-}
+void Service::OnStart() {}
 
 void Service::OnBindInterface(
     const service_manager::BindSourceInfo& source_info,
@@ -74,6 +77,11 @@ void Service::BindAssistantConnection(mojom::AssistantRequest request) {
   bindings_.AddBinding(assistant_manager_service_.get(), std::move(request));
 }
 
+void Service::BindAssistantPlatformConnection(
+    mojom::AssistantPlatformRequest request) {
+  platform_binding_.Bind(std::move(request));
+}
+
 void Service::OnSessionActivated(bool activated) {
   assistant_manager_service_->EnableListening(activated);
 }
@@ -91,6 +99,17 @@ identity::mojom::IdentityManager* Service::GetIdentityManager() {
   return identity_manager_.get();
 }
 
+void Service::Init(mojom::AudioInputPtr audio_input) {
+#if BUILDFLAG(ENABLE_CROS_LIBASSISTANT)
+  assistant_manager_service_ =
+      std::make_unique<AssistantManagerServiceImpl>(std::move(audio_input));
+#else
+  assistant_manager_service_ =
+      std::make_unique<FakeAssistantManagerServiceImpl>();
+#endif
+  RequestAccessToken();
+}
+
 void Service::GetPrimaryAccountInfoCallback(
     const base::Optional<AccountInfo>& account_info,
     const identity::AccountState& account_state) {
@@ -116,18 +135,13 @@ void Service::GetAccessTokenCallback(const base::Optional<std::string>& token,
     return;
   }
 
-  if (!assistant_manager_service_) {
-#if BUILDFLAG(ENABLE_CROS_LIBASSISTANT)
-    assistant_manager_service_ =
-        std::make_unique<AssistantManagerServiceImpl>();
-#else
-    assistant_manager_service_ =
-        std::make_unique<FakeAssistantManagerServiceImpl>();
-#endif
+  DCHECK(assistant_manager_service_);
+  if (!assistant_manager_service_->IsRunning()) {
     assistant_manager_service_->Start(token.value());
     AddAshSessionObserver();
     registry_.AddInterface<mojom::Assistant>(base::BindRepeating(
         &Service::BindAssistantConnection, base::Unretained(this)));
+    DVLOG(1) << "Assistant started";
   } else {
     assistant_manager_service_->SetAccessToken(token.value());
   }

chromeos/services/assistant/service.h

@@ -34,7 +34,8 @@ namespace assistant {
 class AssistantManagerService;
 
 class Service : public service_manager::Service,
-                public ash::mojom::SessionActivationObserver {
+                public ash::mojom::SessionActivationObserver,
+                public mojom::AssistantPlatform {
  public:
   Service();
   ~Service() override;
@@ -55,6 +56,10 @@ class Service : public service_manager::Service,
                        const std::string& interface_name,
                        mojo::ScopedMessagePipeHandle interface_pipe) override;
   void BindAssistantConnection(mojom::AssistantRequest request);
+  void BindAssistantPlatformConnection(mojom::AssistantPlatformRequest request);
+
+  // mojom::AssistantPlatform overrides:
+  void Init(mojom::AudioInputPtr audio_input) override;
 
   // ash::mojom::SessionActivationObserver overrides:
   void OnSessionActivated(bool activated) override;
@@ -73,19 +78,17 @@ class Service : public service_manager::Service,
 
   void AddAshSessionObserver();
 
-  AccountId account_id_;
-
   service_manager::BinderRegistry registry_;
 
-  identity::mojom::IdentityManagerPtr identity_manager_;
-
   mojo::BindingSet<mojom::Assistant> bindings_;
-
+  mojo::Binding<mojom::AssistantPlatform> platform_binding_;
   mojo::Binding<ash::mojom::SessionActivationObserver>
       session_observer_binding_;
 
-  std::unique_ptr<AssistantManagerService> assistant_manager_service_;
+  identity::mojom::IdentityManagerPtr identity_manager_;
 
+  AccountId account_id_;
+  std::unique_ptr<AssistantManagerService> assistant_manager_service_;
   std::unique_ptr<base::OneShotTimer> token_refresh_timer_;
 
   DISALLOW_COPY_AND_ASSIGN(Service);