Implement AudioInputController in Libassistant mojom service

This CL migrates the implementation of |AudioInputProviderImpl| and
|AudioInputImpl| to the new Libassistant mojom service.

To reduce the size of the CL this migration has been split in 3 CLs:
   1. The mojom APIs
   2. The implementation on the Libassistant mojom service (this CL).
   3. Changing the Browser side to use the new mojom service.

This does mean that this CL has to copy |AudioInputImpl|,
|AudioInputProviderImpl| and |AudioInputStream|, rather than move them.
In a follow up CL these classes will be removed from
//chromeos/services/assistant, when they are unused there.

These classes are mostly unchanged, except:
   - Renamed |AudioStream| to |AudioInputStream|.
   - Replaced |AudioStreamFactoryDelegate| with
     |mojom::AudioStreamFactoryDelegate|.
   - Stripped support for the gn flag
     `enable_fake_assistant_microphone`, as that would have required
     some extra files to be moved (and this CL is big enough as-is).
     I will re-add this in a follow up CL.
     Note that the fake microphone still works, as this new version is
     not used anyway.
   - An extra VLOG in UpdateRecordingState().

Bug: b/171748795
Test: chromeos_unittests --gtest_filter="Audio*"
Cq-Include-Trybots: luci.chrome.try:linux-chromeos-chrome
Change-Id: I80965e5de31ea48c19363966bae384414b16bcc8
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2605706
Commit-Queue: Jeroen Dhollander <jeroendh@chromium.org>
Auto-Submit: Jeroen Dhollander <jeroendh@chromium.org>
Reviewed-by: Xiaohui Chen <xiaohuic@chromium.org>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Cr-Commit-Position: refs/heads/master@{#844206}

chromeos/services/libassistant/BUILD.gn

@@ -33,6 +33,14 @@ source_set("internal") {
 
   sources = [
     "assistant_manager_observer.h",
+    "audio/audio_input_impl.cc",
+    "audio/audio_input_impl.h",
+    "audio/audio_input_provider_impl.cc",
+    "audio/audio_input_provider_impl.h",
+    "audio/audio_input_stream.cc",
+    "audio/audio_input_stream.h",
+    "audio_input_controller.cc",
+    "audio_input_controller.h",
     "conversation_controller.cc",
     "conversation_controller.h",
     "platform_api.cc",
@@ -52,7 +60,12 @@ source_set("internal") {
     "//chromeos/services/assistant/public/cpp/migration",
     "//chromeos/services/libassistant/public/mojom",
     "//libassistant/shared/internal_api:assistant_manager_internal",
+    "//libassistant/shared/internal_api:fuchsia_api_helper",
+    "//libassistant/shared/internal_api/c:api_wrappers_entrypoint",
     "//libassistant/shared/public",
+    "//libassistant/shared/public:export",
+    "//media",
+    "//services/audio/public/cpp",
   ]
 
   defines = [ "IS_LIBASSISTANT_SERVICE_IMPL" ]
@@ -60,7 +73,10 @@ source_set("internal") {
 
 source_set("unit_tests") {
   testonly = true
-  sources = [ "service_controller_unittest.cc" ]
+  sources = [
+    "audio_input_controller_unittest.cc",
+    "service_controller_unittest.cc",
+  ]
 
   deps = [
     ":internal",
@@ -73,6 +89,7 @@ source_set("unit_tests") {
     "//chromeos/services/assistant/public/cpp/migration:test_support",
     "//chromeos/services/libassistant/public/mojom",
     "//libassistant/shared/internal_api:assistant_manager_internal",
+    "//services/audio/public/cpp:test_support",
     "//testing/gmock",
     "//testing/gtest",
   ]

chromeos/services/libassistant/DEPS

 include_rules = [
   "+libassistant",
+  "+media/audio",
+  "+media/base",
+  "+services/audio/public",
 ]

chromeos/services/libassistant/audio/audio_input_impl.cc

+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chromeos/services/libassistant/audio/audio_input_impl.h"
+
+#include <utility>
+
+#include "base/bind.h"
+#include "base/logging.h"
+#include "base/memory/weak_ptr.h"
+#include "base/metrics/histogram_functions.h"
+#include "base/optional.h"
+#include "base/stl_util.h"
+#include "base/strings/string_util.h"
+#include "base/timer/timer.h"
+#include "chromeos/services/assistant/public/cpp/assistant_client.h"
+#include "chromeos/services/assistant/public/cpp/features.h"
+#include "chromeos/services/libassistant/audio/audio_input_stream.h"
+#include "libassistant/shared/public/platform_audio_buffer.h"
+#include "media/audio/audio_device_description.h"
+#include "media/base/audio_parameters.h"
+#include "media/base/audio_sample_types.h"
+#include "media/base/channel_layout.h"
+#include "services/audio/public/cpp/device_factory.h"
+#include "services/audio/public/mojom/stream_factory.mojom.h"
+
+namespace chromeos {
+namespace libassistant {
+
+namespace {
+
+constexpr assistant_client::BufferFormat kFormatMono{
+    16000 /* sample_rate */, assistant_client::INTERLEAVED_S16, 1 /* channels */
+};
+
+constexpr assistant_client::BufferFormat kFormatStereo{
+    44100 /* sample_rate */, assistant_client::INTERLEAVED_S16, 2 /* channels */
+};
+
+assistant_client::BufferFormat g_current_format = kFormatMono;
+
+class DspHotwordStateManager : public AudioInputImpl::HotwordStateManager {
+ public:
+  DspHotwordStateManager(AudioInputImpl* input,
+                         scoped_refptr<base::SequencedTaskRunner> task_runner)
+      : AudioInputImpl::HotwordStateManager(input), task_runner_(task_runner) {
+    DCHECK(task_runner_->RunsTasksInCurrentSequence());
+  }
+
+  DspHotwordStateManager(const DspHotwordStateManager&) = delete;
+  DspHotwordStateManager& operator=(const DspHotwordStateManager&) = delete;
+
+  // HotwordStateManager overrides:
+  // Runs on main thread.
+  void OnConversationTurnStarted() override {
+    DCHECK(task_runner_->RunsTasksInCurrentSequence());
+    if (second_phase_timer_.IsRunning()) {
+      DCHECK(stream_state_ == StreamState::HOTWORD);
+      second_phase_timer_.Stop();
+    } else {
+      // Handles user click on mic button.
+      input_->RecreateAudioInputStream(false /* use_dsp */);
+    }
+    stream_state_ = StreamState::NORMAL;
+  }
+
+  // Runs on main thread.
+  void OnConversationTurnFinished() override {
+    DCHECK(task_runner_->RunsTasksInCurrentSequence());
+    input_->RecreateAudioInputStream(true /* use_dsp */);
+    if (stream_state_ == StreamState::HOTWORD) {
+      // If |stream_state_| remains unchanged, that indicates the first stage
+      // DSP hotword detection was rejected by Libassistant.
+      RecordDspHotwordDetection(DspHotwordDetectionStatus::SOFTWARE_REJECTED);
+    }
+    stream_state_ = StreamState::HOTWORD;
+  }
+
+  // Runs on audio service thread
+  void OnCaptureDataArrived() override {
+    // Posting to main thread to avoid timer's sequence check error.
+    task_runner_->PostTask(
+        FROM_HERE,
+        base::BindOnce(&DspHotwordStateManager::OnCaptureDataArrivedMainThread,
+                       weak_factory_.GetWeakPtr()));
+  }
+
+  void RecreateAudioInputStream() override {
+    DCHECK(task_runner_->RunsTasksInCurrentSequence());
+    input_->RecreateAudioInputStream(stream_state_ == StreamState::HOTWORD);
+  }
+
+  // Runs on main thread.
+  void OnCaptureDataArrivedMainThread() {
+    DCHECK(task_runner_->RunsTasksInCurrentSequence());
+    if (stream_state_ == StreamState::HOTWORD &&
+        !second_phase_timer_.IsRunning()) {
+      RecordDspHotwordDetection(DspHotwordDetectionStatus::HARDWARE_ACCEPTED);
+      // 1s from now, if OnConversationTurnStarted is not called, we assume that
+      // libassistant has rejected the hotword supplied by DSP. Thus, we reset
+      // and reopen the device on hotword state.
+      second_phase_timer_.Start(
+          FROM_HERE, base::TimeDelta::FromSeconds(1),
+          base::BindRepeating(
+              &DspHotwordStateManager::OnConversationTurnFinished,
+              base::Unretained(this)));
+    }
+  }
+
+ private:
+  enum class StreamState {
+    HOTWORD,
+    NORMAL,
+  };
+
+  // Defines possible detection states of Dsp hotword. These values are
+  // persisted to logs. Entries should not be renumbered and numeric values
+  // should never be reused. Only append to this enum is allowed if the possible
+  // source grows.
+  enum class DspHotwordDetectionStatus {
+    HARDWARE_ACCEPTED = 0,
+    SOFTWARE_REJECTED = 1,
+    kMaxValue = SOFTWARE_REJECTED
+  };
+
+  // Helper function to record UMA metrics for Dsp hotword detection.
+  void RecordDspHotwordDetection(DspHotwordDetectionStatus status) {
+    base::UmaHistogramEnumeration("Assistant.DspHotwordDetection", status);
+  }
+
+  scoped_refptr<base::SequencedTaskRunner> task_runner_;
+  StreamState stream_state_ = StreamState::HOTWORD;
+  base::OneShotTimer second_phase_timer_;
+  base::WeakPtrFactory<DspHotwordStateManager> weak_factory_{this};
+};
+
+class AudioInputBufferImpl : public assistant_client::AudioBuffer {
+ public:
+  AudioInputBufferImpl(const void* data, uint32_t frame_count)
+      : data_(data), frame_count_(frame_count) {}
+  AudioInputBufferImpl(const AudioInputBufferImpl&) = delete;
+  AudioInputBufferImpl& operator=(const AudioInputBufferImpl&) = delete;
+  ~AudioInputBufferImpl() override = default;
+
+  // assistant_client::AudioBuffer overrides:
+  assistant_client::BufferFormat GetFormat() const override {
+    return g_current_format;
+  }
+  const void* GetData() const override { return data_; }
+  void* GetWritableData() override {
+    NOTREACHED();
+    return nullptr;
+  }
+  int GetFrameCount() const override { return frame_count_; }
+
+ private:
+  const void* data_;
+  int frame_count_;
+};
+
+}  // namespace
+
+AudioInputImpl::HotwordStateManager::HotwordStateManager(
+    AudioInputImpl* audio_input)
+    : input_(audio_input) {}
+
+void AudioInputImpl::HotwordStateManager::RecreateAudioInputStream() {
+  input_->RecreateAudioInputStream(/*use_dsp=*/false);
+}
+
+AudioInputImpl::AudioInputImpl(const base::Optional<std::string>& device_id)
+    : task_runner_(base::SequencedTaskRunnerHandle::Get()),
+      preferred_device_id_(device_id),
+      weak_factory_(this) {
+  DETACH_FROM_SEQUENCE(observer_sequence_checker_);
+
+  RecreateStateManager();
+  if (assistant::features::IsStereoAudioInputEnabled())
+    g_current_format = kFormatStereo;
+  else
+    g_current_format = kFormatMono;
+}
+
+AudioInputImpl::~AudioInputImpl() {
+  DCHECK(task_runner_->RunsTasksInCurrentSequence());
+  StopRecording();
+}
+
+void AudioInputImpl::RecreateStateManager() {
+  if (IsHotwordAvailable()) {
+    state_manager_ =
+        std::make_unique<DspHotwordStateManager>(this, task_runner_);
+  } else {
+    state_manager_ = std::make_unique<HotwordStateManager>(this);
+  }
+}
+
+void AudioInputImpl::Bind(
+    mojo::PendingRemote<mojom::AudioStreamFactoryDelegate> delegate) {
+  audio_stream_factory_delegate_.Bind(std::move(delegate));
+  UpdateRecordingState();
+}
+
+// Runs on audio service thread.
+void AudioInputImpl::Capture(const media::AudioBus* audio_source,
+                             base::TimeTicks audio_capture_time,
+                             double volume,
+                             bool key_pressed) {
+  DCHECK_EQ(g_current_format.num_channels, audio_source->channels());
+
+  state_manager_->OnCaptureDataArrived();
+
+  std::vector<int16_t> buffer(audio_source->channels() *
+                              audio_source->frames());
+  audio_source->ToInterleaved<media::SignedInt16SampleTypeTraits>(
+      audio_source->frames(), buffer.data());
+  int64_t time = 0;
+  // Only provide accurate timestamp when eraser is enabled, otherwise it seems
+  // break normal libassistant voice recognition.
+  if (assistant::features::IsAudioEraserEnabled())
+    time = audio_capture_time.since_origin().InMicroseconds();
+  AudioInputBufferImpl input_buffer(buffer.data(), audio_source->frames());
+  {
+    base::AutoLock lock(lock_);
+    for (auto* observer : observers_)
+      observer->OnAudioBufferAvailable(input_buffer, time);
+  }
+
+  captured_frames_count_ += audio_source->frames();
+  if (VLOG_IS_ON(1)) {
+    auto now = base::TimeTicks::Now();
+    if ((now - last_frame_count_report_time_) >
+        base::TimeDelta::FromMinutes(2)) {
+      VLOG(1) << open_audio_stream_->device_id()
+              << " captured frames: " << captured_frames_count_;
+      last_frame_count_report_time_ = now;
+    }
+  }
+}
+
+// Runs on audio service thread.
+void AudioInputImpl::OnCaptureError(const std::string& message) {
+  LOG(ERROR) << open_audio_stream_->device_id() << " capture error " << message;
+  base::AutoLock lock(lock_);
+  for (auto* observer : observers_)
+    observer->OnAudioError(AudioInput::Error::FATAL_ERROR);
+}
+
+// Runs on audio service thread.
+void AudioInputImpl::OnCaptureMuted(bool is_muted) {}
+
+// Run on LibAssistant thread.
+assistant_client::BufferFormat AudioInputImpl::GetFormat() const {
+  return g_current_format;
+}
+
+// Run on LibAssistant thread.
+void AudioInputImpl::AddObserver(
+    assistant_client::AudioInput::Observer* observer) {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(observer_sequence_checker_);
+  VLOG(1) << " add observer";
+
+  bool have_first_observer = false;
+  {
+    base::AutoLock lock(lock_);
+    observers_.push_back(observer);
+    have_first_observer = observers_.size() == 1;
+  }
+
+  if (have_first_observer) {
+    // Post to main thread runner to start audio recording. Assistant thread
+    // does not have thread context defined in //base and will fail sequence
+    // check in AudioCapturerSource::Start().
+    task_runner_->PostTask(FROM_HERE,
+                           base::BindOnce(&AudioInputImpl::UpdateRecordingState,
+                                          weak_factory_.GetWeakPtr()));
+  }
+}
+
+// Run on LibAssistant thread.
+void AudioInputImpl::RemoveObserver(
+    assistant_client::AudioInput::Observer* observer) {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(observer_sequence_checker_);
+  VLOG(1) << open_audio_stream_->device_id() << " remove observer";
+  bool have_no_observer = false;
+  {
+    base::AutoLock lock(lock_);
+    base::Erase(observers_, observer);
+    have_no_observer = observers_.size() == 0;
+  }
+
+  if (have_no_observer) {
+    task_runner_->PostTask(FROM_HERE,
+                           base::BindOnce(&AudioInputImpl::UpdateRecordingState,
+                                          weak_factory_.GetWeakPtr()));
+
+    // Reset the sequence checker since assistant may call from different thread
+    // after restart.
+    DETACH_FROM_SEQUENCE(observer_sequence_checker_);
+  }
+}
+
+void AudioInputImpl::SetMicState(bool mic_open) {
+  DCHECK(task_runner_->RunsTasksInCurrentSequence());
+  if (mic_open_ == mic_open)
+    return;
+
+  mic_open_ = mic_open;
+  UpdateRecordingState();
+}
+
+void AudioInputImpl::OnConversationTurnStarted() {
+  DCHECK(task_runner_->RunsTasksInCurrentSequence());
+  state_manager_->OnConversationTurnStarted();
+}
+
+void AudioInputImpl::OnConversationTurnFinished() {
+  DCHECK(task_runner_->RunsTasksInCurrentSequence());
+  state_manager_->OnConversationTurnFinished();
+}
+
+void AudioInputImpl::OnHotwordEnabled(bool enable) {
+  DCHECK(task_runner_->RunsTasksInCurrentSequence());
+
+  if (hotword_enabled_ == enable)
+    return;
+
+  hotword_enabled_ = enable;
+  UpdateRecordingState();
+}
+
+void AudioInputImpl::SetDeviceId(const base::Optional<std::string>& device_id) {
+  if (preferred_device_id_ == device_id)
+    return;
+
+  preferred_device_id_ = device_id;
+
+  UpdateRecordingState();
+  if (open_audio_stream_)
+    state_manager_->RecreateAudioInputStream();
+}
+
+void AudioInputImpl::SetHotwordDeviceId(
+    const base::Optional<std::string>& device_id) {
+  if (hotword_device_id_ == device_id)
+    return;
+
+  hotword_device_id_ = device_id;
+  RecreateStateManager();
+  if (open_audio_stream_)
+    state_manager_->RecreateAudioInputStream();
+}
+
+void AudioInputImpl::OnLidStateChanged(mojom::LidState new_state) {
+  // Lid switch event still gets fired during system suspend, which enables
+  // us to stop DSP recording correctly when user closes lid after the device
+  // goes to sleep.
+  if (new_state != lid_state_) {
+    lid_state_ = new_state;
+    UpdateRecordingState();
+  }
+}
+
+void AudioInputImpl::RecreateAudioInputStream(bool use_dsp) {
+  DCHECK(task_runner_->RunsTasksInCurrentSequence());
+  StopRecording();
+
+  open_audio_stream_ = std::make_unique<AudioInputStream>(
+      audio_stream_factory_delegate_.get(), GetDeviceId(use_dsp),
+      ShouldEnableDeadStreamDetection(use_dsp), GetFormat(),
+      /*capture_callback=*/this);
+
+  VLOG(1) << open_audio_stream_->device_id() << " start recording";
+}
+
+bool AudioInputImpl::IsHotwordAvailable() const {
+  return assistant::features::IsDspHotwordEnabled() &&
+         hotword_device_id_.has_value();
+}
+
+bool AudioInputImpl::IsRecordingForTesting() const {
+  return !!open_audio_stream_;
+}
+
+bool AudioInputImpl::IsUsingHotwordDeviceForTesting() const {
+  return IsRecordingForTesting()  // IN-TEST
+         && open_audio_stream_->device_id() == hotword_device_id_ &&
+         IsHotwordAvailable();
+}
+
+base::Optional<std::string> AudioInputImpl::GetOpenDeviceIdForTesting() const {
+  if (!open_audio_stream_)
+    return base::nullopt;
+  return open_audio_stream_->device_id();
+}
+
+base::Optional<bool> AudioInputImpl::IsUsingDeadStreamDetectionForTesting()
+    const {
+  if (!open_audio_stream_)
+    return base::nullopt;
+  return open_audio_stream_->has_dead_stream_detection();
+}
+
+void AudioInputImpl::StartRecording() {
+  DCHECK(task_runner_->RunsTasksInCurrentSequence());
+  DCHECK(!open_audio_stream_);
+  RecreateAudioInputStream(IsHotwordAvailable());
+}
+
+void AudioInputImpl::StopRecording() {
+  DCHECK(task_runner_->RunsTasksInCurrentSequence());
+  if (open_audio_stream_) {
+    VLOG(1) << open_audio_stream_->device_id() << " stop recording";
+    VLOG(1) << open_audio_stream_->device_id()
+            << " ending captured frames: " << captured_frames_count_;
+    open_audio_stream_.reset();
+  }
+}
+
+void AudioInputImpl::UpdateRecordingState() {
+  DCHECK(task_runner_->RunsTasksInCurrentSequence());
+
+  bool has_observers = false;
+  {
+    base::AutoLock lock(lock_);
+    has_observers = observers_.size() > 0;
+  }
+
+  bool is_lid_closed = (lid_state_ == mojom::LidState::kClosed);
+  bool should_enable_hotword =
+      hotword_enabled_ && preferred_device_id_.has_value();
+  bool has_delegate = audio_stream_factory_delegate_.is_bound();
+  bool should_start = !is_lid_closed && (should_enable_hotword || mic_open_) &&
+                      has_observers && has_delegate;
+
+  VLOG(1) << "UpdateRecordingState: "
+          << "    is_lid_closed: " << is_lid_closed << "\n"
+          << "    hotword_enabled: " << hotword_enabled_ << "\n"
+          << "    preferred_device_id: '"
+          << preferred_device_id_.value_or("<unset>") << "'\n"
+          << "    hotword_device_id: '"
+          << hotword_device_id_.value_or("<unset>") << "'\n"
+          << "    mic_open: " << mic_open_ << "\n"
+          << "    has_observers: " << has_observers << "\n"
+          << "    has_delegate: " << has_delegate << "\n"
+          << " => should_start: " << should_start;
+
+  if (!open_audio_stream_ && should_start)
+    StartRecording();
+  else if (open_audio_stream_ && !should_start)
+    StopRecording();
+}
+
+std::string AudioInputImpl::GetDeviceId(bool use_dsp) const {
+  if (use_dsp && hotword_device_id_.has_value())
+    return hotword_device_id_.value();
+  else if (preferred_device_id_.has_value())
+    return preferred_device_id_.value();
+  else
+    return media::AudioDeviceDescription::kDefaultDeviceId;
+}
+
+bool AudioInputImpl::ShouldEnableDeadStreamDetection(bool use_dsp) const {
+  if (use_dsp && hotword_device_id_.has_value()) {
+    // The DSP device won't provide data until it detects a hotword, so
+    // we disable its the dead stream detection.
+    return false;
+  }
+  return true;
+}
+
+}  // namespace libassistant
+}  // namespace chromeos

chromeos/services/libassistant/audio/audio_input_impl.h

+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROMEOS_SERVICES_LIBASSISTANT_AUDIO_AUDIO_INPUT_IMPL_H_
+#define CHROMEOS_SERVICES_LIBASSISTANT_AUDIO_AUDIO_INPUT_IMPL_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "base/component_export.h"
+#include "base/macros.h"
+#include "base/observer_list.h"
+#include "base/optional.h"
+#include "base/sequence_checker.h"
+#include "base/synchronization/lock.h"
+#include "base/time/time.h"
+#include "chromeos/services/assistant/public/cpp/assistant_service.h"
+#include "chromeos/services/libassistant/public/mojom/audio_input_controller.mojom.h"
+#include "libassistant/shared/public/platform_audio_input.h"
+#include "media/base/audio_capturer_source.h"
+#include "mojo/public/cpp/bindings/remote.h"
+
+namespace chromeos {
+namespace libassistant {
+
+class AudioInputStream;
+
+class COMPONENT_EXPORT(LIBASSISTANT_SERVICE) AudioInputImpl
+    : public assistant_client::AudioInput,
+      public media::AudioCapturerSource::CaptureCallback {
+ public:
+  explicit AudioInputImpl(const base::Optional<std::string>& device_id);
+  AudioInputImpl(const AudioInputImpl&) = delete;
+  AudioInputImpl& operator=(const AudioInputImpl&) = delete;
+  ~AudioInputImpl() override;
+
+  class HotwordStateManager {
+   public:
+    explicit HotwordStateManager(AudioInputImpl* audio_input_);
+    HotwordStateManager(const HotwordStateManager&) = delete;
+    HotwordStateManager& operator=(const HotwordStateManager&) = delete;
+    virtual ~HotwordStateManager() = default;
+
+    virtual void OnConversationTurnStarted() {}
+    virtual void OnConversationTurnFinished() {}
+    virtual void OnCaptureDataArrived() {}
+    virtual void RecreateAudioInputStream();
+
+   protected:
+    AudioInputImpl* input_;
+  };
+
+  void RecreateStateManager();
+
+  void Bind(mojo::PendingRemote<mojom::AudioStreamFactoryDelegate> delegate);
+
+  // media::AudioCapturerSource::CaptureCallback overrides:
+  void Capture(const media::AudioBus* audio_source,
+               base::TimeTicks audio_capture_time,
+               double volume,
+               bool key_pressed) override;
+  void OnCaptureError(const std::string& message) override;
+  void OnCaptureMuted(bool is_muted) override;
+
+  // assistant_client::AudioInput overrides. These function are called by
+  // assistant from assistant thread, for which we should not assume any
+  // //base related thread context to be in place.
+  assistant_client::BufferFormat GetFormat() const override;
+  void AddObserver(assistant_client::AudioInput::Observer* observer) override;
+  void RemoveObserver(
+      assistant_client::AudioInput::Observer* observer) override;
+
+  // Called when the mic state associated with the interaction is changed.
+  void SetMicState(bool mic_open);
+  void OnConversationTurnStarted();
+  void OnConversationTurnFinished();
+
+  // Called when hotword enabled status changed.
+  void OnHotwordEnabled(bool enable);
+
+  void SetDeviceId(const base::Optional<std::string>& device_id);
+  void SetHotwordDeviceId(const base::Optional<std::string>& device_id);
+
+  // Called when the user opens/closes the lid.
+  void OnLidStateChanged(mojom::LidState new_state);
+
+  void RecreateAudioInputStream(bool use_dsp);
+
+  bool IsHotwordAvailable() const;
+
+  // Returns the recording state used in unittests.
+  bool IsRecordingForTesting() const;
+  // Returns if the hotword device is used for recording now.
+  bool IsUsingHotwordDeviceForTesting() const;
+  // Returns the id of the device that is currently recording audio.
+  // Returns nullopt if no audio is being recorded.
+  base::Optional<std::string> GetOpenDeviceIdForTesting() const;
+  // Returns if dead stream detection is being used for the current audio
+  // recording. Returns nullopt if no audio is being recorded.
+  base::Optional<bool> IsUsingDeadStreamDetectionForTesting() const;
+
+ private:
+  void StartRecording();
+  void StopRecording();
+  void UpdateRecordingState();
+
+  std::string GetDeviceId(bool use_dsp) const;
+  bool ShouldEnableDeadStreamDetection(bool use_dsp) const;
+
+  // User explicitly requested to open microphone.
+  bool mic_open_ = false;
+
+  // Whether hotword is currently enabled.
+  bool hotword_enabled_ = true;
+
+  // Guards observers_;
+  base::Lock lock_;
+  std::vector<assistant_client::AudioInput::Observer*> observers_;
+
+  // This is the total number of frames captured during the life time of this
+  // object. We don't worry about overflow because this count is only used for
+  // logging purposes. If in the future this changes, we should re-evaluate.
+  int captured_frames_count_ = 0;
+  base::TimeTicks last_frame_count_report_time_;
+
+  // To be initialized on assistant thread the first call to AddObserver.
+  // It ensures that AddObserver / RemoveObserver are called on the same
+  // sequence.
+  SEQUENCE_CHECKER(observer_sequence_checker_);
+
+  scoped_refptr<base::SequencedTaskRunner> task_runner_;
+
+  std::unique_ptr<HotwordStateManager> state_manager_;
+
+  mojo::Remote<mojom::AudioStreamFactoryDelegate>
+      audio_stream_factory_delegate_;
+
+  // Preferred audio input device which will be used for capture.
+  base::Optional<std::string> preferred_device_id_;
+  // Hotword input device used for hardware based hotword detection.
+  base::Optional<std::string> hotword_device_id_;
+
+  // Currently open audio stream. nullptr if no audio stream is open.
+  std::unique_ptr<AudioInputStream> open_audio_stream_;
+
+  // Start with lidstate |kClosed| so we do not open the microphone before we
+  // know if the lid is open or closed.
+  mojom::LidState lid_state_ = mojom::LidState ::kClosed;
+
+  base::WeakPtrFactory<AudioInputImpl> weak_factory_;
+};
+
+}  // namespace libassistant
+}  // namespace chromeos
+
+#endif  // CHROMEOS_SERVICES_LIBASSISTANT_AUDIO_AUDIO_INPUT_IMPL_H_

chromeos/services/libassistant/audio/audio_input_provider_impl.cc

+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chromeos/services/libassistant/audio/audio_input_provider_impl.h"
+
+#include "base/time/time.h"
+#include "chromeos/services/assistant/public/cpp/features.h"
+
+namespace chromeos {
+namespace libassistant {
+
+AudioInputProviderImpl::AudioInputProviderImpl()
+    : audio_input_(/*device_id=*/base::nullopt) {}
+
+AudioInputProviderImpl::~AudioInputProviderImpl() = default;
+
+AudioInputImpl& AudioInputProviderImpl::GetAudioInput() {
+  return audio_input_;
+}
+
+int64_t AudioInputProviderImpl::GetCurrentAudioTime() {
+  if (chromeos::assistant::features::IsAudioEraserEnabled())
+    return base::TimeTicks::Now().since_origin().InMicroseconds();
+
+  return 0;
+}
+
+}  // namespace libassistant
+}  // namespace chromeos

chromeos/services/libassistant/audio/audio_input_provider_impl.h

+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROMEOS_SERVICES_LIBASSISTANT_AUDIO_AUDIO_INPUT_PROVIDER_IMPL_H_
+#define CHROMEOS_SERVICES_LIBASSISTANT_AUDIO_AUDIO_INPUT_PROVIDER_IMPL_H_
+
+#include <cstdint>
+#include <memory>
+
+#include "base/macros.h"
+#include "chromeos/services/libassistant/audio/audio_input_impl.h"
+#include "libassistant/shared/public/platform_audio_input.h"
+
+namespace chromeos {
+namespace libassistant {
+
+class AudioInputProviderImpl : public assistant_client::AudioInputProvider {
+ public:
+  AudioInputProviderImpl();
+  AudioInputProviderImpl(const AudioInputProviderImpl&) = delete;
+  AudioInputProviderImpl& operator=(const AudioInputProviderImpl&) = delete;
+  ~AudioInputProviderImpl() override;
+
+  // assistant_client::AudioInputProvider overrides:
+  AudioInputImpl& GetAudioInput() override;
+  int64_t GetCurrentAudioTime() override;
+
+ private:
+  AudioInputImpl audio_input_;
+};
+
+}  // namespace libassistant
+}  // namespace chromeos
+
+#endif  // CHROMEOS_SERVICES_LIBASSISTANT_AUDIO_AUDIO_INPUT_PROVIDER_IMPL_H_

chromeos/services/libassistant/audio/audio_input_stream.cc

+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chromeos/services/libassistant/audio/audio_input_stream.h"
+
+#include "base/notreached.h"
+#include "chromeos/services/libassistant/public/mojom/audio_input_controller.mojom.h"
+
+namespace chromeos {
+namespace libassistant {
+
+namespace {
+
+audio::DeadStreamDetection ToDeadStreamDetection(bool detect_dead_stream) {
+  return detect_dead_stream ? audio::DeadStreamDetection::kEnabled
+                            : audio::DeadStreamDetection::kDisabled;
+}
+
+}  // namespace
+
+AudioInputStream::AudioInputStream(
+    mojom::AudioStreamFactoryDelegate* delegate,
+    const std::string& device_id,
+    bool detect_dead_stream,
+    assistant_client::BufferFormat buffer_format,
+    media::AudioCapturerSource::CaptureCallback* capture_callback)
+    : device_id_(device_id),
+      detect_dead_stream_(detect_dead_stream),
+      buffer_format_(buffer_format),
+      delegate_(delegate),
+      capture_callback_(capture_callback) {
+  Start();
+}
+
+AudioInputStream::~AudioInputStream() {
+  Stop();
+}
+
+void AudioInputStream::Start() {
+  delegate_->GetAudioStreamFactory(
+      base::BindOnce(&AudioInputStream::OnAudioSteamFactoryReady,
+                     weak_ptr_factory_.GetWeakPtr()));
+}
+
+void AudioInputStream::OnAudioSteamFactoryReady(
+    mojo::PendingRemote<audio::mojom::StreamFactory> audio_stream_factory) {
+  if (!audio_stream_factory.is_valid())
+    return;
+
+  source_ =
+      audio::CreateInputDevice(std::move(audio_stream_factory), device_id(),
+                               ToDeadStreamDetection(detect_dead_stream_));
+
+  source_->Initialize(GetAudioParameters(), capture_callback_);
+  source_->Start();
+}
+
+void AudioInputStream::Stop() {
+  if (source_) {
+    source_->Stop();
+    source_.reset();
+  }
+}
+
+media::AudioParameters AudioInputStream::GetAudioParameters() const {
+  // AUDIO_PCM_LINEAR and AUDIO_PCM_LOW_LATENCY are the same on CRAS.
+  auto result = media::AudioParameters(
+      media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
+      media::GuessChannelLayout(buffer_format_.num_channels),
+      buffer_format_.sample_rate,
+      buffer_format_.sample_rate / 10 /* buffer size for 100 ms */);
+
+  // Set the HOTWORD mask so CRAS knows the device is used for HOTWORD purpose
+  // and is able to conduct the tuning specifically for the scenario. Whether
+  // the HOTWORD is conducted by a hotword device or other devices like
+  // internal mic will be determined by the device_id passed to CRAS.
+  result.set_effects(media::AudioParameters::PlatformEffectsMask::HOTWORD);
+
+  return result;
+}
+
+}  // namespace libassistant
+}  // namespace chromeos

chromeos/services/libassistant/audio/audio_input_stream.h

+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROMEOS_SERVICES_LIBASSISTANT_AUDIO_AUDIO_INPUT_STREAM_H_
+#define CHROMEOS_SERVICES_LIBASSISTANT_AUDIO_AUDIO_INPUT_STREAM_H_
+
+#include <string>
+
+#include "base/memory/scoped_refptr.h"
+#include "base/memory/weak_ptr.h"
+#include "chromeos/services/libassistant/public/mojom/audio_input_controller.mojom-forward.h"
+#include "libassistant/shared/public/platform_audio_buffer.h"
+#include "media/base/audio_capturer_source.h"
+#include "mojo/public/cpp/bindings/pending_remote.h"
+#include "services/audio/public/cpp/device_factory.h"
+#include "services/audio/public/mojom/stream_factory.mojom.h"
+
+namespace chromeos {
+namespace libassistant {
+
+// A single audio stream. All captured packets will be sent to the given
+// capture callback.
+// The audio stream will be opened as soon as this class is created, and
+// will be closed in the destructor.
+class AudioInputStream {
+ public:
+  AudioInputStream(
+      mojom::AudioStreamFactoryDelegate* delegate,
+      const std::string& device_id,
+      bool detect_dead_stream,
+      assistant_client::BufferFormat buffer_format,
+      media::AudioCapturerSource::CaptureCallback* capture_callback);
+  AudioInputStream(AudioInputStream&) = delete;
+  AudioInputStream& operator=(AudioInputStream&) = delete;
+  ~AudioInputStream();
+
+  const std::string& device_id() const { return device_id_; }
+
+  bool has_dead_stream_detection() const { return detect_dead_stream_; }
+
+ private:
+  void Start();
+
+  void OnAudioSteamFactoryReady(
+      mojo::PendingRemote<audio::mojom::StreamFactory> audio_stream_factory);
+
+  void Stop();
+
+  media::AudioParameters GetAudioParameters() const;
+
+  // Device used for recording.
+  std::string device_id_;
+  bool detect_dead_stream_;
+  assistant_client::BufferFormat buffer_format_;
+  mojom::AudioStreamFactoryDelegate* const delegate_;
+  media::AudioCapturerSource::CaptureCallback* const capture_callback_;
+  scoped_refptr<media::AudioCapturerSource> source_;
+  base::WeakPtrFactory<AudioInputStream> weak_ptr_factory_{this};
+};
+
+}  // namespace libassistant
+}  // namespace chromeos
+
+#endif  // CHROMEOS_SERVICES_LIBASSISTANT_AUDIO_AUDIO_INPUT_STREAM_H_

chromeos/services/libassistant/audio_input_controller.cc

+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chromeos/services/libassistant/audio_input_controller.h"
+
+#include "base/notreached.h"
+
+namespace chromeos {
+namespace libassistant {
+
+AudioInputController::AudioInputController() = default;
+
+AudioInputController::~AudioInputController() = default;
+
+void AudioInputController::Bind(
+    mojo::PendingReceiver<mojom::AudioInputController> receiver,
+    mojo::PendingRemote<mojom::AudioStreamFactoryDelegate> delegate) {
+  receiver_.Bind(std::move(receiver));
+  audio_input().Bind(std::move(delegate));
+}
+
+void AudioInputController::SetMicOpen(bool mic_open) {
+  audio_input().SetMicState(mic_open);
+}
+
+void AudioInputController::SetHotwordEnabled(bool enable) {
+  audio_input().OnHotwordEnabled(enable);
+}
+
+void AudioInputController::SetDeviceId(
+    const base::Optional<std::string>& device_id) {
+  DCHECK(device_id != "");
+  audio_input().SetDeviceId(device_id);
+}
+
+void AudioInputController::SetHotwordDeviceId(
+    const base::Optional<std::string>& device_id) {
+  DCHECK(device_id != "");
+  audio_input().SetHotwordDeviceId(device_id);
+}
+
+void AudioInputController::SetLidState(mojom::LidState new_state) {
+  audio_input().OnLidStateChanged(new_state);
+}
+
+void AudioInputController::OnConversationTurnStarted() {
+  audio_input().OnConversationTurnStarted();
+}
+
+void AudioInputController::OnConversationTurnFinished() {
+  audio_input().OnConversationTurnFinished();
+}
+
+AudioInputImpl& AudioInputController::audio_input() {
+  return audio_input_provider().GetAudioInput();
+}
+
+}  // namespace libassistant
+}  // namespace chromeos

chromeos/services/libassistant/audio_input_controller.h

+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROMEOS_SERVICES_LIBASSISTANT_AUDIO_INPUT_CONTROLLER_H_
+#define CHROMEOS_SERVICES_LIBASSISTANT_AUDIO_INPUT_CONTROLLER_H_
+
+#include "chromeos/services/libassistant/audio/audio_input_provider_impl.h"
+#include "chromeos/services/libassistant/public/mojom/audio_input_controller.mojom.h"
+#include "mojo/public/cpp/bindings/pending_receiver.h"
+#include "mojo/public/cpp/bindings/receiver.h"
+
+namespace chromeos {
+namespace libassistant {
+
+// Implementation of |mojom::AudioInputController| that will forward all calls
+// to a Libassistant V1 |assistant_client::AudioInputProvider| implementation.
+class COMPONENT_EXPORT(LIBASSISTANT_SERVICE) AudioInputController
+    : public mojom::AudioInputController {
+ public:
+  AudioInputController();
+  AudioInputController(AudioInputController&) = delete;
+  AudioInputController& operator=(AudioInputController&) = delete;
+  ~AudioInputController() override;
+
+  void Bind(mojo::PendingReceiver<mojom::AudioInputController> receiver,
+            mojo::PendingRemote<mojom::AudioStreamFactoryDelegate> delegate);
+
+  // mojom::AudioInputController implementation:
+  void SetMicOpen(bool mic_open) override;
+  void SetHotwordEnabled(bool enable) override;
+  void SetDeviceId(const base::Optional<std::string>& device_id) override;
+  void SetHotwordDeviceId(
+      const base::Optional<std::string>& device_id) override;
+  void SetLidState(mojom::LidState new_state) override;
+  void OnConversationTurnStarted() override;
+  void OnConversationTurnFinished() override;
+
+  AudioInputProviderImpl& audio_input_provider() {
+    return audio_input_provider_;
+  }
+
+ private:
+  AudioInputImpl& audio_input();
+
+  mojo::Receiver<mojom::AudioInputController> receiver_{this};
+  AudioInputProviderImpl audio_input_provider_;
+};
+
+}  // namespace libassistant
+}  // namespace chromeos
+
+#endif  // CHROMEOS_SERVICES_LIBASSISTANT_AUDIO_INPUT_CONTROLLER_H_

chromeos/services/libassistant/audio_input_controller_unittest.cc

+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chromeos/services/libassistant/audio_input_controller.h"
+
+#include "base/test/gtest_util.h"
+#include "base/test/scoped_feature_list.h"
+#include "base/test/task_environment.h"
+#include "chromeos/services/assistant/public/cpp/features.h"
+#include "chromeos/services/libassistant/audio/audio_input_impl.h"
+#include "chromeos/services/libassistant/public/mojom/audio_input_controller.mojom.h"
+#include "media/audio/audio_device_description.h"
+#include "mojo/public/cpp/bindings/receiver.h"
+#include "mojo/public/cpp/bindings/remote.h"
+#include "services/audio/public/cpp/fake_stream_factory.h"
+#include "testing/gmock/include/gmock/gmock.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace chromeos {
+namespace libassistant {
+
+namespace {
+using mojom::LidState;
+
+class FakeAudioInputObserver : public assistant_client::AudioInput::Observer {
+ public:
+  FakeAudioInputObserver() = default;
+  FakeAudioInputObserver(FakeAudioInputObserver&) = delete;
+  FakeAudioInputObserver& operator=(FakeAudioInputObserver&) = delete;
+  ~FakeAudioInputObserver() override = default;
+
+  // assistant_client::AudioInput::Observer implementation:
+  void OnAudioBufferAvailable(const assistant_client::AudioBuffer& buffer,
+                              int64_t timestamp) override {}
+  void OnAudioError(assistant_client::AudioInput::Error error) override {}
+  void OnAudioStopped() override {}
+};
+
+class FakeAudioStreamFactoryDelegate
+    : public mojom::AudioStreamFactoryDelegate {
+ public:
+  FakeAudioStreamFactoryDelegate() = default;
+  FakeAudioStreamFactoryDelegate(FakeAudioStreamFactoryDelegate&) = delete;
+  FakeAudioStreamFactoryDelegate& operator=(FakeAudioStreamFactoryDelegate&) =
+      delete;
+  ~FakeAudioStreamFactoryDelegate() override = default;
+
+  // mojom::AudioStreamFactoryDelegate implementation:
+  void GetAudioStreamFactory(GetAudioStreamFactoryCallback callback) override {
+    if (return_null_) {
+      // A null pending remote is simply an unbound pending remote.
+      std::move(callback).Run(
+          mojo::PendingRemote<audio::mojom::StreamFactory>());
+    } else {
+      std::move(callback).Run(stream_factory_.MakeRemote());
+      // Without resetting the receiver the test will abort because the code
+      // calls AudioStreamFactory::CreateInputStream() and expects an answer,
+      // but the |FakeStreamFactory| simply drops the call and never calls the
+      // callback.
+      // By resetting the callback the code is happy.
+      stream_factory_.ResetReceiver();
+    }
+  }
+
+  mojo::PendingRemote<mojom::AudioStreamFactoryDelegate>
+  BindNewPipeAndPassRemote() {
+    return delegate_.BindNewPipeAndPassRemote();
+  }
+
+  void StartReturningNull() { return_null_ = true; }
+
+ private:
+  mojo::Receiver<mojom::AudioStreamFactoryDelegate> delegate_{this};
+  audio::FakeStreamFactory stream_factory_;
+
+  // Causes |GetAudioStreamFactory| to return null to the callback instead of
+  // a valid stream factory.
+  bool return_null_ = false;
+};
+
+class AssistantAudioInputControllerTest : public testing::Test {
+ public:
+  AssistantAudioInputControllerTest() : controller_() {
+    controller_.Bind(client_.BindNewPipeAndPassReceiver(),
+                     audio_stream_factory_delegate_.BindNewPipeAndPassRemote());
+
+    // Enable DSP feature flag.
+    scoped_feature_list_.InitAndEnableFeature(
+        assistant::features::kEnableDspHotword);
+  }
+
+  // See |InitializeForTestOfType| for an explanation of this enum.
+  enum TestType {
+    kLidStateTest,
+    kAudioInputObserverTest,
+    kDeviceIdTest,
+    kHotwordDeviceIdTest,
+    kHotwordEnabledTest,
+  };
+
+  // To successfully start recording audio, a lot of requirements must be met
+  // (we need a device-id, an audio-input-observer, the lid must be open, and so
+  // on).
+  // This method will ensure all these requirements are met *except* the ones
+  // that we're testing. So for example if you call
+  // InitializeForTestOfType(kLidState) then this will ensure all requirements
+  // are set but not the lid state (which is left in its initial value).
+  void InitializeForTestOfType(TestType type) {
+    if (type != kLidStateTest)
+      SetLidState(LidState::kOpen);
+
+    if (type != kAudioInputObserverTest)
+      AddAudioInputObserver();
+
+    if (type != kDeviceIdTest)
+      SetDeviceId("fake-audio-device");
+
+    if (type != kHotwordEnabledTest)
+      SetHotwordEnabled(true);
+  }
+
+  mojo::Remote<mojom::AudioInputController>& client() { return client_; }
+
+  AudioInputController& controller() { return controller_; }
+
+  AudioInputImpl& audio_input() {
+    return controller().audio_input_provider().GetAudioInput();
+  }
+
+  FakeAudioStreamFactoryDelegate& audio_stream_factory_delegate() {
+    return audio_stream_factory_delegate_;
+  }
+
+  bool IsRecordingAudio() { return audio_input().IsRecordingForTesting(); }
+
+  bool IsUsingDeadStreamDetection() {
+    return audio_input().IsUsingDeadStreamDetectionForTesting().value_or(false);
+  }
+
+  std::string GetOpenDeviceId() {
+    return audio_input().GetOpenDeviceIdForTesting().value_or("<none>");
+  }
+
+  void SetLidState(LidState new_state) {
+    client()->SetLidState(new_state);
+    client().FlushForTesting();
+  }
+
+  void SetDeviceId(const base::Optional<std::string>& value) {
+    client()->SetDeviceId(value);
+    client().FlushForTesting();
+  }
+
+  void SetHotwordDeviceId(const base::Optional<std::string>& value) {
+    client()->SetHotwordDeviceId(value);
+    client().FlushForTesting();
+  }
+
+  void SetHotwordEnabled(bool value) {
+    client()->SetHotwordEnabled(value);
+    client().FlushForTesting();
+  }
+
+  void SetMicOpen(bool mic_open) {
+    client()->SetMicOpen(mic_open);
+    client().FlushForTesting();
+  }
+
+  void AddAudioInputObserver() {
+    audio_input().AddObserver(&audio_input_observer_);
+  }
+
+  void OnConversationTurnStarted() { controller().OnConversationTurnStarted(); }
+
+  void OnConversationTurnFinished() {
+    controller().OnConversationTurnFinished();
+  }
+
+ private:
+  base::test::SingleThreadTaskEnvironment environment_;
+  base::test::ScopedFeatureList scoped_feature_list_;
+  mojo::Remote<mojom::AudioInputController> client_;
+  AudioInputController controller_;
+  FakeAudioInputObserver audio_input_observer_;
+  FakeAudioStreamFactoryDelegate audio_stream_factory_delegate_;
+};
+
+}  // namespace
+
+TEST_F(AssistantAudioInputControllerTest, ShouldOnlyRecordWhenLidIsOpen) {
+  InitializeForTestOfType(kLidStateTest);
+
+  // Initially the lid is considered closed.
+  EXPECT_FALSE(IsRecordingAudio());
+
+  SetLidState(LidState::kOpen);
+  EXPECT_TRUE(IsRecordingAudio());
+
+  SetLidState(LidState::kClosed);
+  EXPECT_FALSE(IsRecordingAudio());
+}
+
+TEST_F(AssistantAudioInputControllerTest, ShouldOnlyRecordWhenDeviceIdIsSet) {
+  InitializeForTestOfType(kDeviceIdTest);
+
+  // Initially there is no device id.
+  EXPECT_FALSE(IsRecordingAudio());
+
+  SetDeviceId("device-id");
+  EXPECT_TRUE(IsRecordingAudio());
+
+  SetDeviceId(base::nullopt);
+  EXPECT_FALSE(IsRecordingAudio());
+}
+
+TEST_F(AssistantAudioInputControllerTest, StopOnlyRecordWhenHotwordIsEnabled) {
+  InitializeForTestOfType(kHotwordEnabledTest);
+
+  // Hotword is enabled by default.
+  EXPECT_TRUE(IsRecordingAudio());
+
+  SetHotwordEnabled(false);
+  EXPECT_FALSE(IsRecordingAudio());
+
+  SetHotwordEnabled(true);
+  EXPECT_TRUE(IsRecordingAudio());
+}
+
+TEST_F(AssistantAudioInputControllerTest,
+       StartRecordingWhenDisableHotwordAndForceOpenMic) {
+  InitializeForTestOfType(kHotwordEnabledTest);
+  SetHotwordEnabled(false);
+  EXPECT_FALSE(IsRecordingAudio());
+
+  // Force open mic should start recording.
+  SetMicOpen(true);
+  EXPECT_TRUE(IsRecordingAudio());
+
+  SetMicOpen(false);
+  EXPECT_FALSE(IsRecordingAudio());
+}
+
+TEST_F(AssistantAudioInputControllerTest, ShouldUseProvidedDeviceId) {
+  InitializeForTestOfType(kDeviceIdTest);
+  SetDeviceId("the-expected-device-id");
+
+  EXPECT_TRUE(IsRecordingAudio());
+  EXPECT_EQ("the-expected-device-id", GetOpenDeviceId());
+}
+
+TEST_F(AssistantAudioInputControllerTest,
+       ShouldSwitchToHotwordDeviceIdWhenSet) {
+  InitializeForTestOfType(kHotwordDeviceIdTest);
+
+  SetDeviceId("the-device-id");
+  EXPECT_TRUE(IsRecordingAudio());
+  EXPECT_EQ("the-device-id", GetOpenDeviceId());
+
+  SetHotwordDeviceId("the-hotword-device-id");
+  EXPECT_TRUE(IsRecordingAudio());
+  EXPECT_EQ("the-hotword-device-id", GetOpenDeviceId());
+}
+
+TEST_F(AssistantAudioInputControllerTest,
+       ShouldKeepUsingHotwordDeviceIdWhenDeviceIdChanges) {
+  InitializeForTestOfType(kHotwordDeviceIdTest);
+
+  SetDeviceId("the-original-device-id");
+  SetHotwordDeviceId("the-hotword-device-id");
+
+  EXPECT_TRUE(IsRecordingAudio());
+  EXPECT_EQ("the-hotword-device-id", GetOpenDeviceId());
+
+  SetDeviceId("the-new-device-id");
+  EXPECT_TRUE(IsRecordingAudio());
+  EXPECT_EQ("the-hotword-device-id", GetOpenDeviceId());
+}
+
+TEST_F(AssistantAudioInputControllerTest,
+       ShouldUseDefaultDeviceIdIfNoDeviceIdIsSet) {
+  InitializeForTestOfType(kDeviceIdTest);
+  // Mic must be open, otherwise we will not start recording audio if the
+  // device id is not set.
+  SetMicOpen(true);
+  SetDeviceId(base::nullopt);
+  SetHotwordDeviceId(base::nullopt);
+
+  EXPECT_TRUE(IsRecordingAudio());
+  EXPECT_EQ(media::AudioDeviceDescription::kDefaultDeviceId, GetOpenDeviceId());
+}
+
+TEST_F(AssistantAudioInputControllerTest,
+       DeadStreamDetectionShouldBeDisabledWhenUsingHotwordDevice) {
+  InitializeForTestOfType(kHotwordDeviceIdTest);
+
+  SetHotwordDeviceId(base::nullopt);
+  EXPECT_TRUE(IsUsingDeadStreamDetection());
+
+  SetHotwordDeviceId("fake-hotword-device");
+  EXPECT_FALSE(IsUsingDeadStreamDetection());
+}
+
+TEST_F(AssistantAudioInputControllerTest,
+       ShouldNotCrashWhenDelegateReturnsNull) {
+  InitializeForTestOfType(kDeviceIdTest);
+  audio_stream_factory_delegate().StartReturningNull();
+
+  SetDeviceId("device-id");
+
+  EXPECT_TRUE(IsRecordingAudio());
+}
+
+TEST_F(AssistantAudioInputControllerTest,
+       ShouldSwitchToNormalAudioDeviceWhenConversationTurnStarts) {
+  InitializeForTestOfType(kDeviceIdTest);
+  SetDeviceId("normal-device-id");
+  SetHotwordDeviceId("hotword-device-id");
+
+  // While checking for hotword we should be using the hotword device.
+  EXPECT_EQ("hotword-device-id", GetOpenDeviceId());
+
+  // But once the conversation starts we should be using the normal audio
+  // device.
+  OnConversationTurnStarted();
+  EXPECT_EQ("normal-device-id", GetOpenDeviceId());
+}
+
+TEST_F(AssistantAudioInputControllerTest,
+       ShouldSwitchToHotwordAudioDeviceWhenConversationIsFinished) {
+  InitializeForTestOfType(kDeviceIdTest);
+  SetDeviceId("normal-device-id");
+  SetHotwordDeviceId("hotword-device-id");
+
+  // During the conversation we should be using the normal audio device.
+  OnConversationTurnStarted();
+  EXPECT_EQ("normal-device-id", GetOpenDeviceId());
+
+  // But once the conversation finishes, we should check for hotwords using the
+  // hotword device.
+  OnConversationTurnFinished();
+  EXPECT_EQ("hotword-device-id", GetOpenDeviceId());
+}
+
+}  // namespace libassistant
+}  // namespace chromeos

chromeos/services/libassistant/libassistant_service.cc

@@ -10,6 +10,7 @@
 #include "base/check.h"
 #include "base/logging.h"
 #include "chromeos/services/assistant/public/cpp/migration/cros_platform_api.h"
+#include "chromeos/services/libassistant/audio_input_controller.h"
 #include "chromeos/services/libassistant/conversation_controller.h"
 #include "chromeos/services/libassistant/platform_api.h"
 #include "chromeos/services/libassistant/service_controller.h"
@@ -26,7 +27,8 @@ LibassistantService::LibassistantService(
       service_controller_(
           std::make_unique<ServiceController>(delegate, platform_api_.get())),
       conversation_controller_(
-          std::make_unique<ConversationController>(service_controller_.get())) {
+          std::make_unique<ConversationController>(service_controller_.get())),
+      audio_input_controller_(std::make_unique<AudioInputController>()) {
   platform_api_->SetAudioInputProvider(&platform_api->GetAudioInputProvider())
       .SetAudioOutputProvider(&platform_api->GetAudioOutputProvider())
       .SetAuthProvider(&platform_api->GetAuthProvider())
@@ -44,6 +46,8 @@ void LibassistantService::Bind(
     mojo::PendingReceiver<mojom::ConversationController>
         conversation_controller,
     mojo::PendingReceiver<mojom::ServiceController> service_controller) {
+  audio_input_controller_->Bind(std::move(audio_input_controller),
+                                std::move(audio_stream_factory_delegate));
   service_controller_->Bind(std::move(service_controller));
   conversation_controller_->Bind(std::move(conversation_controller));
 }

chromeos/services/libassistant/libassistant_service.h

@@ -29,6 +29,7 @@ class CrosPlatformApi;
 namespace chromeos {
 namespace libassistant {
 
+class AudioInputController;
 class ConversationController;
 class PlatformApi;
 class ServiceController;
@@ -67,6 +68,7 @@ class COMPONENT_EXPORT(LIBASSISTANT_SERVICE) LibassistantService
   std::unique_ptr<PlatformApi> platform_api_;
   std::unique_ptr<ServiceController> service_controller_;
   std::unique_ptr<ConversationController> conversation_controller_;
+  std::unique_ptr<AudioInputController> audio_input_controller_;
 };
 
 }  // namespace libassistant