Commit 3553bb7b authored by Yue Li's avatar Yue Li Committed by Commit Bot

Toggle audio input stream when mic is used by voice queries

Bug: b/110219351
Test: Manual Test
Change-Id: I3ec437d4c7526c7b1b9187b4835870605268c7a9
Reviewed-on: https://chromium-review.googlesource.com/1136878
Commit-Queue: Yue Li <updowndota@chromium.org>
Reviewed-by: default avatarXiaohui Chen <xiaohuic@chromium.org>
Cr-Commit-Position: refs/heads/master@{#575495}
parent ce49be89
...@@ -29,7 +29,8 @@ ...@@ -29,7 +29,8 @@
#include "services/service_manager/public/cpp/connector.h" #include "services/service_manager/public/cpp/connector.h"
#include "url/gurl.h" #include "url/gurl.h"
using assistant_client::ActionModule; using ActionModule = assistant_client::ActionModule;
using Resolution = assistant_client::ConversationStateListener::Resolution;
namespace api = ::assistant::api; namespace api = ::assistant::api;
...@@ -197,10 +198,12 @@ void AssistantManagerServiceImpl::RequestScreenContext( ...@@ -197,10 +198,12 @@ void AssistantManagerServiceImpl::RequestScreenContext(
} }
void AssistantManagerServiceImpl::StartVoiceInteraction() { void AssistantManagerServiceImpl::StartVoiceInteraction() {
platform_api_.SetMicState(true);
assistant_manager_->StartAssistantInteraction(); assistant_manager_->StartAssistantInteraction();
} }
void AssistantManagerServiceImpl::StopActiveInteraction() { void AssistantManagerServiceImpl::StopActiveInteraction() {
platform_api_.SetMicState(false);
assistant_manager_->StopAssistantInteraction(); assistant_manager_->StopAssistantInteraction();
} }
...@@ -263,7 +266,13 @@ void AssistantManagerServiceImpl::OnConversationTurnStarted() { ...@@ -263,7 +266,13 @@ void AssistantManagerServiceImpl::OnConversationTurnStarted() {
} }
void AssistantManagerServiceImpl::OnConversationTurnFinished( void AssistantManagerServiceImpl::OnConversationTurnFinished(
assistant_client::ConversationStateListener::Resolution resolution) { Resolution resolution) {
// TODO(updowndota): Find a better way to handle the edge cases.
if (resolution != Resolution::NORMAL_WITH_FOLLOW_ON &&
resolution != Resolution::CANCELLED &&
resolution != Resolution::BARGE_IN) {
platform_api_.SetMicState(false);
}
main_thread_task_runner_->PostTask( main_thread_task_runner_->PostTask(
FROM_HERE, FROM_HERE,
base::BindOnce( base::BindOnce(
...@@ -557,39 +566,37 @@ void AssistantManagerServiceImpl::OnConversationTurnStartedOnMainThread() { ...@@ -557,39 +566,37 @@ void AssistantManagerServiceImpl::OnConversationTurnStartedOnMainThread() {
} }
void AssistantManagerServiceImpl::OnConversationTurnFinishedOnMainThread( void AssistantManagerServiceImpl::OnConversationTurnFinishedOnMainThread(
assistant_client::ConversationStateListener::Resolution resolution) { Resolution resolution) {
switch (resolution) { switch (resolution) {
// Interaction ended normally. // Interaction ended normally.
// Note that TIMEOUT here does not refer to server timeout, but rather mic // Note that TIMEOUT here does not refer to server timeout, but rather mic
// timeout due to speech inactivity. As this case does not require special // timeout due to speech inactivity. As this case does not require special
// UI logic, it is treated here as a normal interaction completion. // UI logic, it is treated here as a normal interaction completion.
case assistant_client::ConversationStateListener::Resolution::NORMAL: case Resolution::NORMAL:
case assistant_client::ConversationStateListener::Resolution:: case Resolution::NORMAL_WITH_FOLLOW_ON:
NORMAL_WITH_FOLLOW_ON: case Resolution::TIMEOUT:
case assistant_client::ConversationStateListener::Resolution::TIMEOUT:
interaction_subscribers_.ForAllPtrs([](auto* ptr) { interaction_subscribers_.ForAllPtrs([](auto* ptr) {
ptr->OnInteractionFinished( ptr->OnInteractionFinished(
mojom::AssistantInteractionResolution::kNormal); mojom::AssistantInteractionResolution::kNormal);
}); });
break; break;
// Interaction ended due to interruption. // Interaction ended due to interruption.
case assistant_client::ConversationStateListener::Resolution::BARGE_IN: case Resolution::BARGE_IN:
case assistant_client::ConversationStateListener::Resolution::CANCELLED: case Resolution::CANCELLED:
interaction_subscribers_.ForAllPtrs([](auto* ptr) { interaction_subscribers_.ForAllPtrs([](auto* ptr) {
ptr->OnInteractionFinished( ptr->OnInteractionFinished(
mojom::AssistantInteractionResolution::kInterruption); mojom::AssistantInteractionResolution::kInterruption);
}); });
break; break;
// Interaction ended due to multi-device hotword loss. // Interaction ended due to multi-device hotword loss.
case assistant_client::ConversationStateListener::Resolution::NO_RESPONSE: case Resolution::NO_RESPONSE:
interaction_subscribers_.ForAllPtrs([](auto* ptr) { interaction_subscribers_.ForAllPtrs([](auto* ptr) {
ptr->OnInteractionFinished( ptr->OnInteractionFinished(
mojom::AssistantInteractionResolution::kMultiDeviceHotwordLoss); mojom::AssistantInteractionResolution::kMultiDeviceHotwordLoss);
}); });
break; break;
// Interaction ended due to error. // Interaction ended due to error.
case assistant_client::ConversationStateListener::Resolution:: case Resolution::COMMUNICATION_ERROR:
COMMUNICATION_ERROR:
interaction_subscribers_.ForAllPtrs([](auto* ptr) { interaction_subscribers_.ForAllPtrs([](auto* ptr) {
ptr->OnInteractionFinished( ptr->OnInteractionFinished(
mojom::AssistantInteractionResolution::kError); mojom::AssistantInteractionResolution::kError);
......
...@@ -50,10 +50,12 @@ int AudioInputBufferImpl::GetFrameCount() const { ...@@ -50,10 +50,12 @@ int AudioInputBufferImpl::GetFrameCount() const {
} }
AudioInputImpl::AudioInputImpl( AudioInputImpl::AudioInputImpl(
std::unique_ptr<service_manager::Connector> connector) std::unique_ptr<service_manager::Connector> connector,
bool default_on)
: source_(audio::CreateInputDevice( : source_(audio::CreateInputDevice(
std::move(connector), std::move(connector),
media::AudioDeviceDescription::kDefaultDeviceId)), media::AudioDeviceDescription::kDefaultDeviceId)),
default_on_(default_on),
task_runner_(base::ThreadTaskRunnerHandle::Get()), task_runner_(base::ThreadTaskRunnerHandle::Get()),
weak_factory_(this) { weak_factory_(this) {
DETACH_FROM_SEQUENCE(observer_sequence_checker_); DETACH_FROM_SEQUENCE(observer_sequence_checker_);
...@@ -112,7 +114,7 @@ void AudioInputImpl::AddObserver( ...@@ -112,7 +114,7 @@ void AudioInputImpl::AddObserver(
should_start = observers_.size() == 1; should_start = observers_.size() == 1;
} }
if (should_start) { if (default_on_ && should_start) {
// Post to main thread runner to start audio recording. Assistant thread // Post to main thread runner to start audio recording. Assistant thread
// does not have thread context defined in //base and will fail sequence // does not have thread context defined in //base and will fail sequence
// check in AudioCapturerSource::Start(). // check in AudioCapturerSource::Start().
...@@ -138,6 +140,16 @@ void AudioInputImpl::RemoveObserver( ...@@ -138,6 +140,16 @@ void AudioInputImpl::RemoveObserver(
} }
} }
void AudioInputImpl::SetMicState(bool mic_open) {
DCHECK(task_runner_->RunsTasksInCurrentSequence());
if (!default_on_) {
if (mic_open)
source_->Start();
else
source_->Stop();
}
}
void AudioInputImpl::StartRecording() { void AudioInputImpl::StartRecording() {
DCHECK(task_runner_->RunsTasksInCurrentSequence()); DCHECK(task_runner_->RunsTasksInCurrentSequence());
source_->Start(); source_->Start();
...@@ -149,8 +161,9 @@ void AudioInputImpl::StopRecording() { ...@@ -149,8 +161,9 @@ void AudioInputImpl::StopRecording() {
} }
AudioInputProviderImpl::AudioInputProviderImpl( AudioInputProviderImpl::AudioInputProviderImpl(
service_manager::Connector* connector) service_manager::Connector* connector,
: audio_input_(connector->Clone()) {} bool default_on)
: audio_input_(connector->Clone(), default_on) {}
AudioInputProviderImpl::~AudioInputProviderImpl() = default; AudioInputProviderImpl::~AudioInputProviderImpl() = default;
...@@ -163,5 +176,9 @@ int64_t AudioInputProviderImpl::GetCurrentAudioTime() { ...@@ -163,5 +176,9 @@ int64_t AudioInputProviderImpl::GetCurrentAudioTime() {
return 0; return 0;
} }
void AudioInputProviderImpl::SetMicState(bool mic_open) {
audio_input_.SetMicState(mic_open);
}
} // namespace assistant } // namespace assistant
} // namespace chromeos } // namespace chromeos
...@@ -49,8 +49,8 @@ class AudioInputBufferImpl : public assistant_client::AudioBuffer { ...@@ -49,8 +49,8 @@ class AudioInputBufferImpl : public assistant_client::AudioBuffer {
class AudioInputImpl : public assistant_client::AudioInput, class AudioInputImpl : public assistant_client::AudioInput,
public media::AudioCapturerSource::CaptureCallback { public media::AudioCapturerSource::CaptureCallback {
public: public:
explicit AudioInputImpl( AudioInputImpl(std::unique_ptr<service_manager::Connector> connector,
std::unique_ptr<service_manager::Connector> connector); bool default_on);
~AudioInputImpl() override; ~AudioInputImpl() override;
// media::AudioCapturerSource::CaptureCallback overrides: // media::AudioCapturerSource::CaptureCallback overrides:
...@@ -69,12 +69,18 @@ class AudioInputImpl : public assistant_client::AudioInput, ...@@ -69,12 +69,18 @@ class AudioInputImpl : public assistant_client::AudioInput,
void RemoveObserver( void RemoveObserver(
assistant_client::AudioInput::Observer* observer) override; assistant_client::AudioInput::Observer* observer) override;
// Called when the mic state associated with the interaction is changed.
void SetMicState(bool mic_open);
private: private:
void StartRecording(); void StartRecording();
void StopRecording(); void StopRecording();
scoped_refptr<media::AudioCapturerSource> source_; scoped_refptr<media::AudioCapturerSource> source_;
// Should audio input always recording actively.
const bool default_on_;
// Guards observers_; // Guards observers_;
base::Lock lock_; base::Lock lock_;
std::vector<assistant_client::AudioInput::Observer*> observers_; std::vector<assistant_client::AudioInput::Observer*> observers_;
...@@ -92,13 +98,17 @@ class AudioInputImpl : public assistant_client::AudioInput, ...@@ -92,13 +98,17 @@ class AudioInputImpl : public assistant_client::AudioInput,
class AudioInputProviderImpl : public assistant_client::AudioInputProvider { class AudioInputProviderImpl : public assistant_client::AudioInputProvider {
public: public:
explicit AudioInputProviderImpl(service_manager::Connector* connector); explicit AudioInputProviderImpl(service_manager::Connector* connector,
bool default_on);
~AudioInputProviderImpl() override; ~AudioInputProviderImpl() override;
// assistant_client::AudioInputProvider overrides: // assistant_client::AudioInputProvider overrides:
assistant_client::AudioInput& GetAudioInput() override; assistant_client::AudioInput& GetAudioInput() override;
int64_t GetCurrentAudioTime() override; int64_t GetCurrentAudioTime() override;
// Called when the mic state associated with the interaction is changed.
void SetMicState(bool mic_open);
private: private:
AudioInputImpl audio_input_; AudioInputImpl audio_input_;
......
...@@ -16,12 +16,8 @@ namespace chromeos { ...@@ -16,12 +16,8 @@ namespace chromeos {
namespace assistant { namespace assistant {
SystemProviderImpl::SystemProviderImpl( SystemProviderImpl::SystemProviderImpl(
device::mojom::BatteryMonitorPtr battery_monitor, device::mojom::BatteryMonitorPtr battery_monitor)
bool muted) : battery_monitor_(std::move(battery_monitor)) {
: battery_monitor_(std::move(battery_monitor)),
mic_mute_state_(
muted ? assistant_client::MicMuteState::MICROPHONE_OFF
: assistant_client::MicMuteState::MICROPHONE_ENABLED) {
battery_monitor_->QueryNextStatus(base::BindOnce( battery_monitor_->QueryNextStatus(base::BindOnce(
&SystemProviderImpl::OnBatteryStatus, base::Unretained(this))); &SystemProviderImpl::OnBatteryStatus, base::Unretained(this)));
} }
...@@ -29,7 +25,8 @@ SystemProviderImpl::SystemProviderImpl( ...@@ -29,7 +25,8 @@ SystemProviderImpl::SystemProviderImpl(
SystemProviderImpl::~SystemProviderImpl() = default; SystemProviderImpl::~SystemProviderImpl() = default;
assistant_client::MicMuteState SystemProviderImpl::GetMicMuteState() { assistant_client::MicMuteState SystemProviderImpl::GetMicMuteState() {
return mic_mute_state_; // CRAS input is never muted.
return assistant_client::MicMuteState::MICROPHONE_ENABLED;
} }
void SystemProviderImpl::RegisterMicMuteChangeCallback( void SystemProviderImpl::RegisterMicMuteChangeCallback(
......
...@@ -16,8 +16,7 @@ namespace assistant { ...@@ -16,8 +16,7 @@ namespace assistant {
class SystemProviderImpl : public assistant_client::SystemProvider { class SystemProviderImpl : public assistant_client::SystemProvider {
public: public:
explicit SystemProviderImpl(device::mojom::BatteryMonitorPtr battery_monitor, explicit SystemProviderImpl(device::mojom::BatteryMonitorPtr battery_monitor);
bool muted);
~SystemProviderImpl() override; ~SystemProviderImpl() override;
// assistant_client::SystemProvider implementation: // assistant_client::SystemProvider implementation:
...@@ -36,7 +35,6 @@ class SystemProviderImpl : public assistant_client::SystemProvider { ...@@ -36,7 +35,6 @@ class SystemProviderImpl : public assistant_client::SystemProvider {
device::mojom::BatteryMonitorPtr battery_monitor_; device::mojom::BatteryMonitorPtr battery_monitor_;
device::mojom::BatteryStatusPtr current_battery_status_; device::mojom::BatteryStatusPtr current_battery_status_;
const assistant_client::MicMuteState mic_mute_state_;
DISALLOW_COPY_AND_ASSIGN(SystemProviderImpl); DISALLOW_COPY_AND_ASSIGN(SystemProviderImpl);
}; };
......
...@@ -77,9 +77,9 @@ PlatformApiImpl::PlatformApiImpl( ...@@ -77,9 +77,9 @@ PlatformApiImpl::PlatformApiImpl(
service_manager::Connector* connector, service_manager::Connector* connector,
device::mojom::BatteryMonitorPtr battery_monitor, device::mojom::BatteryMonitorPtr battery_monitor,
bool enable_hotword) bool enable_hotword)
: audio_input_provider_(connector), : audio_input_provider_(connector, enable_hotword),
audio_output_provider_(CreateLibAssistantConfig(!enable_hotword), this), audio_output_provider_(CreateLibAssistantConfig(!enable_hotword), this),
system_provider_(std::move(battery_monitor), !enable_hotword) {} system_provider_(std::move(battery_monitor)) {}
PlatformApiImpl::~PlatformApiImpl() = default; PlatformApiImpl::~PlatformApiImpl() = default;
...@@ -111,5 +111,9 @@ SystemProvider& PlatformApiImpl::GetSystemProvider() { ...@@ -111,5 +111,9 @@ SystemProvider& PlatformApiImpl::GetSystemProvider() {
return system_provider_; return system_provider_;
} }
void PlatformApiImpl::SetMicState(bool mic_open) {
audio_input_provider_.SetMicState(mic_open);
}
} // namespace assistant } // namespace assistant
} // namespace chromeos } // namespace chromeos
...@@ -48,6 +48,9 @@ class PlatformApiImpl : public assistant_client::PlatformApi { ...@@ -48,6 +48,9 @@ class PlatformApiImpl : public assistant_client::PlatformApi {
assistant_client::ResourceProvider& GetResourceProvider() override; assistant_client::ResourceProvider& GetResourceProvider() override;
assistant_client::SystemProvider& GetSystemProvider() override; assistant_client::SystemProvider& GetSystemProvider() override;
// Called when the mic state associated with the interaction is changed.
void SetMicState(bool mic_open);
private: private:
// ChromeOS does not use auth manager, so we don't yet need to implement a // ChromeOS does not use auth manager, so we don't yet need to implement a
// real auth provider. // real auth provider.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment