Toggle audio input stream when mic is used by voice queries

Bug: b/110219351
Test: Manual Test
Change-Id: I3ec437d4c7526c7b1b9187b4835870605268c7a9
Reviewed-on: https://chromium-review.googlesource.com/1136878
Commit-Queue: Yue Li <updowndota@chromium.org>
Reviewed-by: Xiaohui Chen <xiaohuic@chromium.org>
Cr-Commit-Position: refs/heads/master@{#575495}

chromeos/services/assistant/assistant_manager_service_impl.cc

@@ -29,7 +29,8 @@
 #include "services/service_manager/public/cpp/connector.h"
 #include "url/gurl.h"
 
-using assistant_client::ActionModule;
+using ActionModule = assistant_client::ActionModule;
+using Resolution = assistant_client::ConversationStateListener::Resolution;
 
 namespace api = ::assistant::api;
 
@@ -197,10 +198,12 @@ void AssistantManagerServiceImpl::RequestScreenContext(
 }
 
 void AssistantManagerServiceImpl::StartVoiceInteraction() {
+  platform_api_.SetMicState(true);
   assistant_manager_->StartAssistantInteraction();
 }
 
 void AssistantManagerServiceImpl::StopActiveInteraction() {
+  platform_api_.SetMicState(false);
   assistant_manager_->StopAssistantInteraction();
 }
 
@@ -263,7 +266,13 @@ void AssistantManagerServiceImpl::OnConversationTurnStarted() {
 }
 
 void AssistantManagerServiceImpl::OnConversationTurnFinished(
-    assistant_client::ConversationStateListener::Resolution resolution) {
+    Resolution resolution) {
+  // TODO(updowndota): Find a better way to handle the edge cases.
+  if (resolution != Resolution::NORMAL_WITH_FOLLOW_ON &&
+      resolution != Resolution::CANCELLED &&
+      resolution != Resolution::BARGE_IN) {
+    platform_api_.SetMicState(false);
+  }
   main_thread_task_runner_->PostTask(
       FROM_HERE,
       base::BindOnce(
@@ -557,39 +566,37 @@ void AssistantManagerServiceImpl::OnConversationTurnStartedOnMainThread() {
 }
 
 void AssistantManagerServiceImpl::OnConversationTurnFinishedOnMainThread(
-    assistant_client::ConversationStateListener::Resolution resolution) {
+    Resolution resolution) {
   switch (resolution) {
     // Interaction ended normally.
     // Note that TIMEOUT here does not refer to server timeout, but rather mic
     // timeout due to speech inactivity. As this case does not require special
     // UI logic, it is treated here as a normal interaction completion.
-    case assistant_client::ConversationStateListener::Resolution::NORMAL:
-    case assistant_client::ConversationStateListener::Resolution::
-        NORMAL_WITH_FOLLOW_ON:
-    case assistant_client::ConversationStateListener::Resolution::TIMEOUT:
+    case Resolution::NORMAL:
+    case Resolution::NORMAL_WITH_FOLLOW_ON:
+    case Resolution::TIMEOUT:
       interaction_subscribers_.ForAllPtrs([](auto* ptr) {
         ptr->OnInteractionFinished(
             mojom::AssistantInteractionResolution::kNormal);
       });
       break;
     // Interaction ended due to interruption.
-    case assistant_client::ConversationStateListener::Resolution::BARGE_IN:
-    case assistant_client::ConversationStateListener::Resolution::CANCELLED:
+    case Resolution::BARGE_IN:
+    case Resolution::CANCELLED:
       interaction_subscribers_.ForAllPtrs([](auto* ptr) {
         ptr->OnInteractionFinished(
             mojom::AssistantInteractionResolution::kInterruption);
       });
       break;
     // Interaction ended due to multi-device hotword loss.
-    case assistant_client::ConversationStateListener::Resolution::NO_RESPONSE:
+    case Resolution::NO_RESPONSE:
       interaction_subscribers_.ForAllPtrs([](auto* ptr) {
         ptr->OnInteractionFinished(
             mojom::AssistantInteractionResolution::kMultiDeviceHotwordLoss);
       });
       break;
     // Interaction ended due to error.
-    case assistant_client::ConversationStateListener::Resolution::
-        COMMUNICATION_ERROR:
+    case Resolution::COMMUNICATION_ERROR:
       interaction_subscribers_.ForAllPtrs([](auto* ptr) {
         ptr->OnInteractionFinished(
             mojom::AssistantInteractionResolution::kError);

chromeos/services/assistant/platform/audio_input_provider_impl.cc

@@ -50,10 +50,12 @@ int AudioInputBufferImpl::GetFrameCount() const {
 }
 
 AudioInputImpl::AudioInputImpl(
-    std::unique_ptr<service_manager::Connector> connector)
+    std::unique_ptr<service_manager::Connector> connector,
+    bool default_on)
     : source_(audio::CreateInputDevice(
           std::move(connector),
           media::AudioDeviceDescription::kDefaultDeviceId)),
+      default_on_(default_on),
       task_runner_(base::ThreadTaskRunnerHandle::Get()),
       weak_factory_(this) {
   DETACH_FROM_SEQUENCE(observer_sequence_checker_);
@@ -112,7 +114,7 @@ void AudioInputImpl::AddObserver(
     should_start = observers_.size() == 1;
   }
 
-  if (should_start) {
+  if (default_on_ && should_start) {
     // Post to main thread runner to start audio recording. Assistant thread
     // does not have thread context defined in //base and will fail sequence
     // check in AudioCapturerSource::Start().
@@ -138,6 +140,16 @@ void AudioInputImpl::RemoveObserver(
   }
 }
 
+void AudioInputImpl::SetMicState(bool mic_open) {
+  DCHECK(task_runner_->RunsTasksInCurrentSequence());
+  if (!default_on_) {
+    if (mic_open)
+      source_->Start();
+    else
+      source_->Stop();
+  }
+}
+
 void AudioInputImpl::StartRecording() {
   DCHECK(task_runner_->RunsTasksInCurrentSequence());
   source_->Start();
@@ -149,8 +161,9 @@ void AudioInputImpl::StopRecording() {
 }
 
 AudioInputProviderImpl::AudioInputProviderImpl(
-    service_manager::Connector* connector)
-    : audio_input_(connector->Clone()) {}
+    service_manager::Connector* connector,
+    bool default_on)
+    : audio_input_(connector->Clone(), default_on) {}
 
 AudioInputProviderImpl::~AudioInputProviderImpl() = default;
 
@@ -163,5 +176,9 @@ int64_t AudioInputProviderImpl::GetCurrentAudioTime() {
   return 0;
 }
 
+void AudioInputProviderImpl::SetMicState(bool mic_open) {
+  audio_input_.SetMicState(mic_open);
+}
+
 }  // namespace assistant
 }  // namespace chromeos

chromeos/services/assistant/platform/audio_input_provider_impl.h

@@ -49,8 +49,8 @@ class AudioInputBufferImpl : public assistant_client::AudioBuffer {
 class AudioInputImpl : public assistant_client::AudioInput,
                        public media::AudioCapturerSource::CaptureCallback {
  public:
-  explicit AudioInputImpl(
-      std::unique_ptr<service_manager::Connector> connector);
+  AudioInputImpl(std::unique_ptr<service_manager::Connector> connector,
+                 bool default_on);
   ~AudioInputImpl() override;
 
   // media::AudioCapturerSource::CaptureCallback overrides:
@@ -69,12 +69,18 @@ class AudioInputImpl : public assistant_client::AudioInput,
   void RemoveObserver(
       assistant_client::AudioInput::Observer* observer) override;
 
+  // Called when the mic state associated with the interaction is changed.
+  void SetMicState(bool mic_open);
+
  private:
   void StartRecording();
   void StopRecording();
 
   scoped_refptr<media::AudioCapturerSource> source_;
 
+  // Should audio input always recording actively.
+  const bool default_on_;
+
   // Guards observers_;
   base::Lock lock_;
   std::vector<assistant_client::AudioInput::Observer*> observers_;
@@ -92,13 +98,17 @@ class AudioInputImpl : public assistant_client::AudioInput,
 
 class AudioInputProviderImpl : public assistant_client::AudioInputProvider {
  public:
-  explicit AudioInputProviderImpl(service_manager::Connector* connector);
+  explicit AudioInputProviderImpl(service_manager::Connector* connector,
+                                  bool default_on);
   ~AudioInputProviderImpl() override;
 
   // assistant_client::AudioInputProvider overrides:
   assistant_client::AudioInput& GetAudioInput() override;
   int64_t GetCurrentAudioTime() override;
 
+  // Called when the mic state associated with the interaction is changed.
+  void SetMicState(bool mic_open);
+
  private:
   AudioInputImpl audio_input_;
 

chromeos/services/assistant/platform/system_provider_impl.cc

@@ -16,12 +16,8 @@ namespace chromeos {
 namespace assistant {
 
 SystemProviderImpl::SystemProviderImpl(
-    device::mojom::BatteryMonitorPtr battery_monitor,
-    bool muted)
-    : battery_monitor_(std::move(battery_monitor)),
-      mic_mute_state_(
-          muted ? assistant_client::MicMuteState::MICROPHONE_OFF
-                : assistant_client::MicMuteState::MICROPHONE_ENABLED) {
+    device::mojom::BatteryMonitorPtr battery_monitor)
+    : battery_monitor_(std::move(battery_monitor)) {
   battery_monitor_->QueryNextStatus(base::BindOnce(
       &SystemProviderImpl::OnBatteryStatus, base::Unretained(this)));
 }
@@ -29,7 +25,8 @@ SystemProviderImpl::SystemProviderImpl(
 SystemProviderImpl::~SystemProviderImpl() = default;
 
 assistant_client::MicMuteState SystemProviderImpl::GetMicMuteState() {
-  return mic_mute_state_;
+  // CRAS input is never muted.
+  return assistant_client::MicMuteState::MICROPHONE_ENABLED;
 }
 
 void SystemProviderImpl::RegisterMicMuteChangeCallback(

chromeos/services/assistant/platform/system_provider_impl.h

@@ -16,8 +16,7 @@ namespace assistant {
 
 class SystemProviderImpl : public assistant_client::SystemProvider {
  public:
-  explicit SystemProviderImpl(device::mojom::BatteryMonitorPtr battery_monitor,
-                              bool muted);
+  explicit SystemProviderImpl(device::mojom::BatteryMonitorPtr battery_monitor);
   ~SystemProviderImpl() override;
 
   // assistant_client::SystemProvider implementation:
@@ -36,7 +35,6 @@ class SystemProviderImpl : public assistant_client::SystemProvider {
 
   device::mojom::BatteryMonitorPtr battery_monitor_;
   device::mojom::BatteryStatusPtr current_battery_status_;
-  const assistant_client::MicMuteState mic_mute_state_;
 
   DISALLOW_COPY_AND_ASSIGN(SystemProviderImpl);
 };

chromeos/services/assistant/platform_api_impl.cc

@@ -77,9 +77,9 @@ PlatformApiImpl::PlatformApiImpl(
     service_manager::Connector* connector,
     device::mojom::BatteryMonitorPtr battery_monitor,
     bool enable_hotword)
-    : audio_input_provider_(connector),
+    : audio_input_provider_(connector, enable_hotword),
       audio_output_provider_(CreateLibAssistantConfig(!enable_hotword), this),
-      system_provider_(std::move(battery_monitor), !enable_hotword) {}
+      system_provider_(std::move(battery_monitor)) {}
 
 PlatformApiImpl::~PlatformApiImpl() = default;
 
@@ -111,5 +111,9 @@ SystemProvider& PlatformApiImpl::GetSystemProvider() {
   return system_provider_;
 }
 
+void PlatformApiImpl::SetMicState(bool mic_open) {
+  audio_input_provider_.SetMicState(mic_open);
+}
+
 }  // namespace assistant
 }  // namespace chromeos

chromeos/services/assistant/platform_api_impl.h

@@ -48,6 +48,9 @@ class PlatformApiImpl : public assistant_client::PlatformApi {
   assistant_client::ResourceProvider& GetResourceProvider() override;
   assistant_client::SystemProvider& GetSystemProvider() override;
 
+  // Called when the mic state associated with the interaction is changed.
+  void SetMicState(bool mic_open);
+
  private:
   // ChromeOS does not use auth manager, so we don't yet need to implement a
   // real auth provider.