Commit 00f2282b authored by Max Morin's avatar Max Morin Committed by Commit Bot

Reland "Fork AudioInputController to the audio service."

This is a reland of 040f2ad0

Test was flaky even before landing this.

Original change's description:
> Fork AudioInputController to the audio service.
>
> Also migrate the audio service to it. We want to add audio processing
> capabilities to the audio service, and the media/ code cannot depend
> on services/audio/, so we need the AudioInputController code in the
> audio service. There's a very sketched design doc for processing in the
> audio service at
> https://docs.google.com/document/d/1HJnii4kuBXshVM202TVPJt_9iDmlaCX-VhatjRxRbC4/edit#.
>
> Bug: 851959
> Change-Id: Icabef727e6d309f86472d2fb401d7383051fa0ae
> Reviewed-on: https://chromium-review.googlesource.com/1104464
> Commit-Queue: Max Morin <maxmorin@chromium.org>
> Reviewed-by: Olga Sharonova <olka@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#570014}

Tbr: Olka
Bug: 851959, 856215
Change-Id: I2e99e92937f8ccd73fb2ea3bd09d4175ab21419d
Reviewed-on: https://chromium-review.googlesource.com/1114618Reviewed-by: default avatarMax Morin <maxmorin@chromium.org>
Commit-Queue: Max Morin <maxmorin@chromium.org>
Cr-Commit-Position: refs/heads/master@{#570366}
parent 85e0d0be
...@@ -41,8 +41,12 @@ source_set("lib") { ...@@ -41,8 +41,12 @@ source_set("lib") {
"group_member.h", "group_member.h",
"in_process_audio_manager_accessor.cc", "in_process_audio_manager_accessor.cc",
"in_process_audio_manager_accessor.h", "in_process_audio_manager_accessor.h",
"input_controller.cc",
"input_controller.h",
"input_stream.cc", "input_stream.cc",
"input_stream.h", "input_stream.h",
"input_sync_writer.cc",
"input_sync_writer.h",
"local_muter.cc", "local_muter.cc",
"local_muter.h", "local_muter.h",
"loopback_stream.cc", "loopback_stream.cc",
...@@ -89,7 +93,9 @@ source_set("tests") { ...@@ -89,7 +93,9 @@ source_set("tests") {
"delay_buffer_unittest.cc", "delay_buffer_unittest.cc",
"device_notifier_unittest.cc", "device_notifier_unittest.cc",
"group_coordinator_unittest.cc", "group_coordinator_unittest.cc",
"input_controller_unittest.cc",
"input_stream_unittest.cc", "input_stream_unittest.cc",
"input_sync_writer_unittest.cc",
"local_muter_unittest.cc", "local_muter_unittest.cc",
"loopback_stream_unittest.cc", "loopback_stream_unittest.cc",
"output_controller_unittest.cc", "output_controller_unittest.cc",
......
// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "services/audio/input_controller.h"
#include <inttypes.h>
#include <algorithm>
#include <limits>
#include <utility>
#include "base/bind.h"
#include "base/memory/ptr_util.h"
#include "base/metrics/histogram_macros.h"
#include "base/single_thread_task_runner.h"
#include "base/strings/string_number_conversions.h"
#include "base/strings/stringprintf.h"
#include "base/threading/thread_restrictions.h"
#include "base/threading/thread_task_runner_handle.h"
#include "base/time/time.h"
#include "base/trace_event/trace_event.h"
#include "media/audio/audio_io.h"
#include "media/audio/audio_manager.h"
#include "media/base/audio_bus.h"
#include "media/base/user_input_monitor.h"
namespace audio {
namespace {
const int kMaxInputChannels = 3;
constexpr int kCheckMutedStateIntervalSeconds = 1;
#if defined(AUDIO_POWER_MONITORING)
// Time in seconds between two successive measurements of audio power levels.
const int kPowerMonitorLogIntervalSeconds = 15;
// A warning will be logged when the microphone audio volume is below this
// threshold.
const int kLowLevelMicrophoneLevelPercent = 10;
// Logs if the user has enabled the microphone mute or not. This is normally
// done by marking a checkbox in an audio-settings UI which is unique for each
// platform. Elements in this enum should not be added, deleted or rearranged.
enum MicrophoneMuteResult {
MICROPHONE_IS_MUTED = 0,
MICROPHONE_IS_NOT_MUTED = 1,
MICROPHONE_MUTE_MAX = MICROPHONE_IS_NOT_MUTED
};
void LogMicrophoneMuteResult(MicrophoneMuteResult result) {
UMA_HISTOGRAM_ENUMERATION("Media.MicrophoneMuted", result,
MICROPHONE_MUTE_MAX + 1);
}
// Helper method which calculates the average power of an audio bus. Unit is in
// dBFS, where 0 dBFS corresponds to all channels and samples equal to 1.0.
float AveragePower(const media::AudioBus& buffer) {
const int frames = buffer.frames();
const int channels = buffer.channels();
if (frames <= 0 || channels <= 0)
return 0.0f;
// Scan all channels and accumulate the sum of squares for all samples.
float sum_power = 0.0f;
for (int ch = 0; ch < channels; ++ch) {
const float* channel_data = buffer.channel(ch);
for (int i = 0; i < frames; i++) {
const float sample = channel_data[i];
sum_power += sample * sample;
}
}
// Update accumulated average results, with clamping for sanity.
const float average_power =
std::max(0.0f, std::min(1.0f, sum_power / (frames * channels)));
// Convert average power level to dBFS units, and pin it down to zero if it
// is insignificantly small.
const float kInsignificantPower = 1.0e-10f; // -100 dBFS
const float power_dbfs = average_power < kInsignificantPower
? -std::numeric_limits<float>::infinity()
: 10.0f * log10f(average_power);
return power_dbfs;
}
#endif // AUDIO_POWER_MONITORING
} // namespace
// Private subclass of AIC that covers the state while capturing audio.
// This class implements the callback interface from the lower level audio
// layer and gets called back on the audio hw thread.
// We implement this in a sub class instead of directly in the AIC so that
// - The AIC itself is not an AudioInputCallback.
// - The lifetime of the AudioCallback is shorter than the AIC
// - How tasks are posted to the AIC from the hw callback thread, is different
// than how tasks are posted from the AIC to itself from the main thread.
// So, this difference is isolated to the subclass (see below).
// - The callback class can gather information on what happened during capture
// and store it in a state that can be fetched after stopping capture
// (received_callback, error_during_callback).
// The AIC itself must not be AddRef-ed on the hw callback thread so that we
// can be guaranteed to not receive callbacks generated by the hw callback
// thread after Close() has been called on the audio manager thread and
// the callback object deleted. To avoid AddRef-ing the AIC and to cancel
// potentially pending tasks, we use a weak pointer to the AIC instance
// when posting.
class InputController::AudioCallback
: public media::AudioInputStream::AudioInputCallback {
public:
explicit AudioCallback(InputController* controller)
: controller_(controller),
weak_controller_(controller->weak_ptr_factory_.GetWeakPtr()) {}
~AudioCallback() override = default;
bool received_callback() const { return received_callback_; }
bool error_during_callback() const { return error_during_callback_; }
private:
void OnData(const media::AudioBus* source,
base::TimeTicks capture_time,
double volume) override {
TRACE_EVENT1("audio", "InputController::OnData", "capture time (ms)",
(capture_time - base::TimeTicks()).InMillisecondsF());
received_callback_ = true;
DeliverDataToSyncWriter(source, capture_time, volume);
}
void OnError() override {
error_during_callback_ = true;
controller_->task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&InputController::DoReportError, weak_controller_));
}
void DeliverDataToSyncWriter(const media::AudioBus* source,
base::TimeTicks capture_time,
double volume) {
bool key_pressed = controller_->CheckForKeyboardInput();
controller_->sync_writer_->Write(source, volume, key_pressed, capture_time);
// The way the two classes interact here, could be done in a nicer way.
// As is, we call the AIC here to check the audio power, return and then
// post a task to the AIC based on what the AIC said.
// The reason for this is to keep all PostTask calls from the hw callback
// thread to the AIC, that use a weak pointer, in the same class.
float average_power_dbfs;
int mic_volume_percent;
if (controller_->CheckAudioPower(source, volume, &average_power_dbfs,
&mic_volume_percent)) {
// Use event handler on the audio thread to relay a message to the ARIH
// in content which does the actual logging on the IO thread.
controller_->task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&InputController::DoLogAudioLevels, weak_controller_,
average_power_dbfs, mic_volume_percent));
}
}
InputController* const controller_;
// We do not want any pending posted tasks generated from the callback class
// to keep the controller object alive longer than it should. So we use
// a weak ptr whenever we post, we use this weak pointer.
base::WeakPtr<InputController> weak_controller_;
bool received_callback_ = false;
bool error_during_callback_ = false;
};
InputController::InputController(
scoped_refptr<base::SingleThreadTaskRunner> task_runner,
EventHandler* handler,
SyncWriter* sync_writer,
media::UserInputMonitor* user_input_monitor,
const media::AudioParameters& params,
StreamType type)
: task_runner_(std::move(task_runner)),
handler_(handler),
stream_(nullptr),
sync_writer_(sync_writer),
type_(type),
user_input_monitor_(user_input_monitor),
weak_ptr_factory_(this) {
DCHECK(handler_);
DCHECK(sync_writer_);
}
InputController::~InputController() {
DCHECK(!audio_callback_);
DCHECK(!stream_);
DCHECK(!check_muted_state_timer_.IsRunning());
}
// static
scoped_refptr<InputController> InputController::Create(
media::AudioManager* audio_manager,
EventHandler* event_handler,
SyncWriter* sync_writer,
media::UserInputMonitor* user_input_monitor,
const media::AudioParameters& params,
const std::string& device_id,
bool enable_agc) {
DCHECK(audio_manager);
DCHECK(sync_writer);
DCHECK(event_handler);
// TODO(https://crbug.com/803102): remove check after switching to input
// stream factory.
if (!params.IsValid() || (params.channels() > kMaxInputChannels))
return nullptr;
// Create the InputController object and ensure that it runs on
// the audio-manager thread.
scoped_refptr<InputController> controller(new InputController(
audio_manager->GetTaskRunner(), event_handler, sync_writer,
user_input_monitor, params, ParamsToStreamType(params)));
if (controller->task_runner_->BelongsToCurrentThread()) {
controller->DoCreate(audio_manager, params, device_id, enable_agc);
return controller;
}
// Create and open a new audio input stream from the existing
// audio-device thread. Use the provided audio-input device.
controller->task_runner_->PostTask(
FROM_HERE, base::BindOnce(&InputController::DoCreate, controller,
base::Unretained(audio_manager), params,
device_id, enable_agc));
return controller;
}
void InputController::Record() {
DCHECK_CALLED_ON_VALID_SEQUENCE(owning_sequence_);
if (task_runner_->BelongsToCurrentThread()) {
DoRecord();
return;
}
task_runner_->PostTask(FROM_HERE,
base::BindOnce(&InputController::DoRecord, this));
}
void InputController::Close(base::OnceClosure closed_task) {
DCHECK_CALLED_ON_VALID_SEQUENCE(owning_sequence_);
if (task_runner_->BelongsToCurrentThread()) {
DCHECK(closed_task.is_null());
DoClose();
return;
}
task_runner_->PostTaskAndReply(
FROM_HERE, base::BindOnce(&InputController::DoClose, this),
std::move(closed_task));
}
void InputController::SetVolume(double volume) {
DCHECK_CALLED_ON_VALID_SEQUENCE(owning_sequence_);
if (task_runner_->BelongsToCurrentThread()) {
DoSetVolume(volume);
return;
}
task_runner_->PostTask(
FROM_HERE, base::BindOnce(&InputController::DoSetVolume, this, volume));
}
void InputController::SetOutputDeviceForAec(
const std::string& output_device_id) {
DCHECK_CALLED_ON_VALID_SEQUENCE(owning_sequence_);
if (task_runner_->BelongsToCurrentThread()) {
DoSetOutputDeviceForAec(output_device_id);
return;
}
task_runner_->PostTask(
FROM_HERE, base::BindOnce(&InputController::DoSetOutputDeviceForAec, this,
output_device_id));
}
void InputController::DoCreate(media::AudioManager* audio_manager,
const media::AudioParameters& params,
const std::string& device_id,
bool enable_agc) {
DCHECK(task_runner_->BelongsToCurrentThread());
DCHECK(!stream_);
SCOPED_UMA_HISTOGRAM_TIMER("Media.InputController.CreateTime");
handler_->OnLog("AIC::DoCreate");
#if defined(AUDIO_POWER_MONITORING)
// We only do power measurements for UMA stats for low latency streams, and
// only if agc is requested, to avoid adding logs and UMA for non-WebRTC
// clients.
power_measurement_is_enabled_ = (type_ == LOW_LATENCY && enable_agc);
last_audio_level_log_time_ = base::TimeTicks::Now();
#endif
// MakeAudioInputStream might fail and return nullptr. If so,
// DoCreateForStream will handle and report it.
auto* stream = audio_manager->MakeAudioInputStream(
params, device_id,
base::BindRepeating(&InputController::LogMessage, this));
DoCreateForStream(stream, enable_agc);
}
void InputController::DoCreateForStream(
media::AudioInputStream* stream_to_control,
bool enable_agc) {
DCHECK(task_runner_->BelongsToCurrentThread());
DCHECK(!stream_);
handler_->OnLog("AIC::DoCreateForStream");
if (!stream_to_control) {
LogCaptureStartupResult(CAPTURE_STARTUP_CREATE_STREAM_FAILED);
handler_->OnError(STREAM_CREATE_ERROR);
return;
}
if (!stream_to_control->Open()) {
stream_to_control->Close();
LogCaptureStartupResult(CAPTURE_STARTUP_OPEN_STREAM_FAILED);
handler_->OnError(STREAM_OPEN_ERROR);
return;
}
#if defined(AUDIO_POWER_MONITORING)
bool agc_is_supported =
stream_to_control->SetAutomaticGainControl(enable_agc);
// Disable power measurements on platforms that does not support AGC at a
// lower level. AGC can fail on platforms where we don't support the
// functionality to modify the input volume slider. One such example is
// Windows XP.
power_measurement_is_enabled_ &= agc_is_supported;
#else
stream_to_control->SetAutomaticGainControl(enable_agc);
#endif
// Finally, keep the stream pointer around, update the state and notify.
stream_ = stream_to_control;
// Send initial muted state along with OnCreated, to avoid races.
is_muted_ = stream_->IsMuted();
handler_->OnCreated(is_muted_);
check_muted_state_timer_.Start(
FROM_HERE, base::TimeDelta::FromSeconds(kCheckMutedStateIntervalSeconds),
this, &InputController::CheckMutedState);
DCHECK(check_muted_state_timer_.IsRunning());
}
void InputController::DoRecord() {
DCHECK(task_runner_->BelongsToCurrentThread());
SCOPED_UMA_HISTOGRAM_TIMER("Media.InputController.RecordTime");
if (!stream_ || audio_callback_)
return;
handler_->OnLog("AIC::DoRecord");
if (user_input_monitor_) {
user_input_monitor_->EnableKeyPressMonitoring();
prev_key_down_count_ = user_input_monitor_->GetKeyPressCount();
}
stream_create_time_ = base::TimeTicks::Now();
audio_callback_.reset(new AudioCallback(this));
stream_->Start(audio_callback_.get());
}
void InputController::DoClose() {
DCHECK(task_runner_->BelongsToCurrentThread());
SCOPED_UMA_HISTOGRAM_TIMER("Media.InputController.CloseTime");
if (!stream_)
return;
check_muted_state_timer_.AbandonAndStop();
std::string log_string;
static const char kLogStringPrefix[] = "AIC::DoClose:";
// Allow calling unconditionally and bail if we don't have a stream to close.
if (audio_callback_) {
stream_->Stop();
// Sometimes a stream (and accompanying audio track) is created and
// immediately closed or discarded. In this case they are registered as
// 'stopped early' rather than 'never got data'.
const base::TimeDelta duration =
base::TimeTicks::Now() - stream_create_time_;
CaptureStartupResult capture_startup_result =
audio_callback_->received_callback()
? CAPTURE_STARTUP_OK
: (duration.InMilliseconds() < 500
? CAPTURE_STARTUP_STOPPED_EARLY
: CAPTURE_STARTUP_NEVER_GOT_DATA);
LogCaptureStartupResult(capture_startup_result);
LogCallbackError();
log_string = base::StringPrintf(
"%s stream duration=%" PRId64 " seconds%s", kLogStringPrefix,
duration.InSeconds(),
audio_callback_->received_callback() ? "" : " (no callbacks received)");
if (type_ == LOW_LATENCY) {
if (audio_callback_->received_callback()) {
UMA_HISTOGRAM_LONG_TIMES("Media.InputStreamDuration", duration);
} else {
UMA_HISTOGRAM_LONG_TIMES("Media.InputStreamDurationWithoutCallback",
duration);
}
}
if (user_input_monitor_)
user_input_monitor_->DisableKeyPressMonitoring();
audio_callback_.reset();
} else {
log_string =
base::StringPrintf("%s recording never started", kLogStringPrefix);
}
handler_->OnLog(log_string);
stream_->Close();
stream_ = nullptr;
sync_writer_->Close();
#if defined(AUDIO_POWER_MONITORING)
// Send UMA stats if enabled.
if (power_measurement_is_enabled_)
LogSilenceState(silence_state_);
#endif
max_volume_ = 0.0;
weak_ptr_factory_.InvalidateWeakPtrs();
}
void InputController::DoReportError() {
DCHECK(task_runner_->BelongsToCurrentThread());
handler_->OnError(STREAM_ERROR);
}
void InputController::DoSetVolume(double volume) {
DCHECK(task_runner_->BelongsToCurrentThread());
DCHECK_GE(volume, 0);
DCHECK_LE(volume, 1.0);
if (!stream_)
return;
// Only ask for the maximum volume at first call and use cached value
// for remaining function calls.
if (!max_volume_) {
max_volume_ = stream_->GetMaxVolume();
}
if (max_volume_ == 0.0) {
DLOG(WARNING) << "Failed to access input volume control";
return;
}
// Set the stream volume and scale to a range matched to the platform.
stream_->SetVolume(max_volume_ * volume);
}
void InputController::DoSetOutputDeviceForAec(
const std::string& output_device_id) {
DCHECK(task_runner_->BelongsToCurrentThread());
if (stream_)
stream_->SetOutputDeviceForAec(output_device_id);
}
void InputController::DoLogAudioLevels(float level_dbfs,
int microphone_volume_percent) {
#if defined(AUDIO_POWER_MONITORING)
DCHECK(task_runner_->BelongsToCurrentThread());
if (!stream_)
return;
// Detect if the user has enabled hardware mute by pressing the mute
// button in audio settings for the selected microphone.
const bool microphone_is_muted = stream_->IsMuted();
if (microphone_is_muted) {
LogMicrophoneMuteResult(MICROPHONE_IS_MUTED);
handler_->OnLog("AIC::OnData: microphone is muted!");
// Return early if microphone is muted. No need to adding logs and UMA stats
// of audio levels if we know that the micropone is muted.
return;
}
LogMicrophoneMuteResult(MICROPHONE_IS_NOT_MUTED);
std::string log_string = base::StringPrintf(
"AIC::OnData: average audio level=%.2f dBFS", level_dbfs);
static const float kSilenceThresholdDBFS = -72.24719896f;
if (level_dbfs < kSilenceThresholdDBFS)
log_string += " <=> low audio input level!";
handler_->OnLog(log_string);
UpdateSilenceState(level_dbfs < kSilenceThresholdDBFS);
UMA_HISTOGRAM_PERCENTAGE("Media.MicrophoneVolume", microphone_volume_percent);
log_string = base::StringPrintf("AIC::OnData: microphone volume=%d%%",
microphone_volume_percent);
if (microphone_volume_percent < kLowLevelMicrophoneLevelPercent)
log_string += " <=> low microphone level!";
handler_->OnLog(log_string);
#endif
}
#if defined(AUDIO_POWER_MONITORING)
void InputController::UpdateSilenceState(bool silence) {
if (silence) {
if (silence_state_ == SILENCE_STATE_NO_MEASUREMENT) {
silence_state_ = SILENCE_STATE_ONLY_SILENCE;
} else if (silence_state_ == SILENCE_STATE_ONLY_AUDIO) {
silence_state_ = SILENCE_STATE_AUDIO_AND_SILENCE;
} else {
DCHECK(silence_state_ == SILENCE_STATE_ONLY_SILENCE ||
silence_state_ == SILENCE_STATE_AUDIO_AND_SILENCE);
}
} else {
if (silence_state_ == SILENCE_STATE_NO_MEASUREMENT) {
silence_state_ = SILENCE_STATE_ONLY_AUDIO;
} else if (silence_state_ == SILENCE_STATE_ONLY_SILENCE) {
silence_state_ = SILENCE_STATE_AUDIO_AND_SILENCE;
} else {
DCHECK(silence_state_ == SILENCE_STATE_ONLY_AUDIO ||
silence_state_ == SILENCE_STATE_AUDIO_AND_SILENCE);
}
}
}
void InputController::LogSilenceState(SilenceState value) {
UMA_HISTOGRAM_ENUMERATION("Media.AudioInputControllerSessionSilenceReport",
value, SILENCE_STATE_MAX + 1);
}
#endif
void InputController::LogCaptureStartupResult(CaptureStartupResult result) {
switch (type_) {
case LOW_LATENCY:
UMA_HISTOGRAM_ENUMERATION("Media.LowLatencyAudioCaptureStartupSuccess",
result, CAPTURE_STARTUP_RESULT_MAX + 1);
break;
case HIGH_LATENCY:
UMA_HISTOGRAM_ENUMERATION("Media.HighLatencyAudioCaptureStartupSuccess",
result, CAPTURE_STARTUP_RESULT_MAX + 1);
break;
case VIRTUAL:
UMA_HISTOGRAM_ENUMERATION("Media.VirtualAudioCaptureStartupSuccess",
result, CAPTURE_STARTUP_RESULT_MAX + 1);
break;
default:
break;
}
}
void InputController::LogCallbackError() {
bool error_during_callback = audio_callback_->error_during_callback();
switch (type_) {
case LOW_LATENCY:
UMA_HISTOGRAM_BOOLEAN("Media.Audio.Capture.LowLatencyCallbackError",
error_during_callback);
break;
case HIGH_LATENCY:
UMA_HISTOGRAM_BOOLEAN("Media.Audio.Capture.HighLatencyCallbackError",
error_during_callback);
break;
case VIRTUAL:
UMA_HISTOGRAM_BOOLEAN("Media.Audio.Capture.VirtualCallbackError",
error_during_callback);
break;
default:
break;
}
}
void InputController::LogMessage(const std::string& message) {
DCHECK(task_runner_->BelongsToCurrentThread());
handler_->OnLog(message);
}
bool InputController::CheckForKeyboardInput() {
if (!user_input_monitor_)
return false;
const size_t current_count = user_input_monitor_->GetKeyPressCount();
const bool key_pressed = current_count != prev_key_down_count_;
prev_key_down_count_ = current_count;
DVLOG_IF(6, key_pressed) << "Detected keypress.";
return key_pressed;
}
bool InputController::CheckAudioPower(const media::AudioBus* source,
double volume,
float* average_power_dbfs,
int* mic_volume_percent) {
#if defined(AUDIO_POWER_MONITORING)
// Only do power-level measurements if DoCreate() has been called. It will
// ensure that logging will mainly be done for WebRTC and WebSpeech
// clients.
if (!power_measurement_is_enabled_)
return false;
// Perform periodic audio (power) level measurements.
const auto now = base::TimeTicks::Now();
if ((now - last_audio_level_log_time_).InSeconds() <=
kPowerMonitorLogIntervalSeconds) {
return false;
}
*average_power_dbfs = AveragePower(*source);
*mic_volume_percent = static_cast<int>(100.0 * volume);
last_audio_level_log_time_ = now;
return true;
#else
return false;
#endif
}
void InputController::CheckMutedState() {
DCHECK(task_runner_->BelongsToCurrentThread());
DCHECK(stream_);
const bool new_state = stream_->IsMuted();
if (new_state != is_muted_) {
is_muted_ = new_state;
handler_->OnMuted(is_muted_);
}
}
// static
InputController::StreamType InputController::ParamsToStreamType(
const media::AudioParameters& params) {
switch (params.format()) {
case media::AudioParameters::Format::AUDIO_PCM_LINEAR:
return InputController::StreamType::HIGH_LATENCY;
case media::AudioParameters::Format::AUDIO_PCM_LOW_LATENCY:
return InputController::StreamType::LOW_LATENCY;
default:
// Currently, the remaining supported type is fake. Reconsider if other
// formats become supported.
return InputController::StreamType::FAKE;
}
}
} // namespace audio
// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef SERVICES_AUDIO_INPUT_CONTROLLER_H_
#define SERVICES_AUDIO_INPUT_CONTROLLER_H_
#include <stddef.h>
#include <stdint.h>
#include <memory>
#include <string>
#include "base/memory/weak_ptr.h"
#include "base/optional.h"
#include "base/strings/string_piece.h"
#include "base/timer/timer.h"
#include "build/build_config.h"
#include "media/base/audio_parameters.h"
// An InputController controls an media::AudioInputStream and records data
// from this input stream. The two main methods are Record() and Close() and
// they are both executed on the audio thread which is injected by the two
// alternative factory methods, Create() or CreateForStream().
//
// All public methods of InputController are synchronous if called from
// audio thread, or non-blocking if called from a different thread.
//
// Here is a state diagram for the InputController:
//
// .--> [ Closed / Error ] <--.
// | |
// | |
// [ Created ] ----------> [ Recording ]
// ^
// |
// *[ Empty ]
//
// * Initial state
//
// State sequences:
//
// [Creating Thread] [Audio Thread]
//
// User InputController EventHandler
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// Create() ==> DoCreate()
// media::AudioManager::Makemedia::AudioInputStream()
// media::AudioInputStream::Open()
// .- - - - - - - - - - - - -> OnError()
// .-------------------------> OnCreated()
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// Record() ==> DoRecord()
// media::AudioInputStream::Start()
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// Close() ==> DoClose()
// media::AudioInputStream::Stop()
// media::AudioInputStream::Close()
// SyncWriter::Close()
// Closure::Run() <-----------------.
// (closure-task)
//
// The audio thread itself is owned by the media::AudioManager that the
// InputController holds a reference to. When performing tasks on the
// audio thread, the controller must not add or release references to the
// media::AudioManager or itself (since it in turn holds a reference to the
// manager).
//
namespace base {
class SingleThreadTaskRunner;
}
namespace media {
class AudioBus;
class AudioInputStream;
class AudioManager;
class UserInputMonitor;
} // namespace media
namespace audio {
// Only do power monitoring for non-mobile platforms to save resources.
#if !defined(OS_ANDROID) && !defined(OS_IOS)
#define AUDIO_POWER_MONITORING
#endif
class InputController : public base::RefCountedThreadSafe<InputController> {
public:
// Error codes to make native logging more clear. These error codes are added
// to generic error strings to provide a higher degree of details.
// Changing these values can lead to problems when matching native debug
// logs with the actual cause of error.
enum ErrorCode {
// An unspecified error occured.
UNKNOWN_ERROR = 0,
// Failed to create an audio input stream.
STREAM_CREATE_ERROR, // = 1
// Failed to open an audio input stream.
STREAM_OPEN_ERROR, // = 2
// Native input stream reports an error. Exact reason differs between
// platforms.
STREAM_ERROR, // = 3
};
// An event handler that receives events from the InputController. The
// following methods are all called on the audio thread.
class EventHandler {
public:
// The initial "muted" state of the underlying stream is sent along with the
// OnCreated callback, to avoid the stream being treated as unmuted until an
// OnMuted callback has had time to be processed.
virtual void OnCreated(bool initially_muted) = 0;
virtual void OnError(ErrorCode error_code) = 0;
virtual void OnLog(base::StringPiece) = 0;
// Called whenever the muted state of the underlying stream changes.
virtual void OnMuted(bool is_muted) = 0;
protected:
virtual ~EventHandler() {}
};
// A synchronous writer interface used by InputController for
// synchronous writing.
class SyncWriter {
public:
virtual ~SyncWriter() {}
// Write certain amount of data from |data|.
virtual void Write(const media::AudioBus* data,
double volume,
bool key_pressed,
base::TimeTicks capture_time) = 0;
// Close this synchronous writer.
virtual void Close() = 0;
};
// enum used for determining what UMA stats to report.
enum StreamType {
VIRTUAL = 0,
HIGH_LATENCY = 1,
LOW_LATENCY = 2,
FAKE = 3,
};
media::AudioInputStream* stream_for_testing() { return stream_; }
// The audio device will be created on the audio thread, and when that is
// done, the event handler will receive an OnCreated() call from that same
// thread. |user_input_monitor| is used for typing detection and can be NULL.
static scoped_refptr<InputController> Create(
media::AudioManager* audio_manager,
EventHandler* event_handler,
SyncWriter* sync_writer,
media::UserInputMonitor* user_input_monitor,
const media::AudioParameters& params,
const std::string& device_id,
// External synchronous writer for audio controller.
bool agc_is_enabled);
// Factory method for creating an InputController with an existing
// |stream|. The stream will be opened on the audio thread, and when that is
// done, the event handler will receive an OnCreated() call from that same
// thread. |user_input_monitor| is used for typing detection and can be NULL.
static scoped_refptr<InputController> CreateForStream(
const scoped_refptr<base::SingleThreadTaskRunner>& task_runner,
EventHandler* event_handler,
media::AudioInputStream* stream,
// External synchronous writer for audio controller.
SyncWriter* sync_writer,
media::UserInputMonitor* user_input_monitor);
// Starts recording using the created audio input stream.
// This method is called on the creator thread.
virtual void Record();
// Closes the audio input stream. The state is changed and the resources
// are freed on the audio thread. |closed_task| is then executed on the thread
// that called Close().
// Callbacks (EventHandler and SyncWriter) must exist until |closed_task|
// is called.
// It is safe to call this method more than once. Calls after the first one
// will have no effect.
// This method trampolines to the audio thread.
virtual void Close(base::OnceClosure closed_task);
// Sets the capture volume of the input stream. The value 0.0 corresponds
// to muted and 1.0 to maximum volume.
virtual void SetVolume(double volume);
// Sets the output device which will be used to cancel audio from, if this
// input device supports echo cancellation.
virtual void SetOutputDeviceForAec(const std::string& output_device_id);
protected:
friend class base::RefCountedThreadSafe<InputController>;
// Used to log the result of capture startup.
// This was previously logged as a boolean with only the no callback and OK
// options. The enum order is kept to ensure backwards compatibility.
// Elements in this enum should not be deleted or rearranged; the only
// permitted operation is to add new elements before
// CAPTURE_STARTUP_RESULT_MAX and update CAPTURE_STARTUP_RESULT_MAX.
//
// The NO_DATA_CALLBACK enum has been replaced with NEVER_GOT_DATA,
// and there are also other histograms such as
// Media.Audio.InputStartupSuccessMac to cover issues similar
// to the ones the NO_DATA_CALLBACK was intended for.
enum CaptureStartupResult {
CAPTURE_STARTUP_OK = 0,
CAPTURE_STARTUP_CREATE_STREAM_FAILED = 1,
CAPTURE_STARTUP_OPEN_STREAM_FAILED = 2,
CAPTURE_STARTUP_NEVER_GOT_DATA = 3,
CAPTURE_STARTUP_STOPPED_EARLY = 4,
CAPTURE_STARTUP_RESULT_MAX = CAPTURE_STARTUP_STOPPED_EARLY,
};
#if defined(AUDIO_POWER_MONITORING)
// Used to log a silence report (see OnData).
// Elements in this enum should not be deleted or rearranged; the only
// permitted operation is to add new elements before SILENCE_STATE_MAX and
// update SILENCE_STATE_MAX.
// Possible silence state transitions:
// SILENCE_STATE_AUDIO_AND_SILENCE
// ^ ^
// SILENCE_STATE_ONLY_AUDIO SILENCE_STATE_ONLY_SILENCE
// ^ ^
// SILENCE_STATE_NO_MEASUREMENT
enum SilenceState {
SILENCE_STATE_NO_MEASUREMENT = 0,
SILENCE_STATE_ONLY_AUDIO = 1,
SILENCE_STATE_ONLY_SILENCE = 2,
SILENCE_STATE_AUDIO_AND_SILENCE = 3,
SILENCE_STATE_MAX = SILENCE_STATE_AUDIO_AND_SILENCE
};
#endif
InputController(scoped_refptr<base::SingleThreadTaskRunner> task_runner,
EventHandler* handler,
SyncWriter* sync_writer,
media::UserInputMonitor* user_input_monitor,
const media::AudioParameters& params,
StreamType type);
virtual ~InputController();
const scoped_refptr<base::SingleThreadTaskRunner>& GetTaskRunnerForTesting()
const {
return task_runner_;
}
EventHandler* GetHandlerForTesting() const { return handler_; }
private:
// Methods called on the audio thread (owned by the media::AudioManager).
void DoCreate(media::AudioManager* audio_manager,
const media::AudioParameters& params,
const std::string& device_id,
bool enable_agc);
void DoCreateForStream(media::AudioInputStream* stream_to_control,
bool enable_agc);
void DoRecord();
void DoClose();
void DoReportError();
void DoSetVolume(double volume);
void DoLogAudioLevels(float level_dbfs, int microphone_volume_percent);
void DoSetOutputDeviceForAec(const std::string& output_device_id);
#if defined(AUDIO_POWER_MONITORING)
// Updates the silence state, see enum SilenceState above for state
// transitions.
void UpdateSilenceState(bool silence);
// Logs the silence state as UMA stat.
void LogSilenceState(SilenceState value);
#endif
// Logs the result of creating an InputController.
void LogCaptureStartupResult(CaptureStartupResult result);
// Logs whether an error was encountered suring the stream.
void LogCallbackError();
// Called by the stream with log messages.
void LogMessage(const std::string& message);
// Called on the hw callback thread. Checks for keyboard input if
// user_input_monitor_ is set otherwise returns false.
bool CheckForKeyboardInput();
// Does power monitoring on supported platforms.
// Called on the hw callback thread.
// Returns true iff average power and mic volume was returned and should
// be posted to DoLogAudioLevels on the audio thread.
// Returns false if either power measurements are disabled or aren't needed
// right now (they're done periodically).
bool CheckAudioPower(const media::AudioBus* source,
double volume,
float* average_power_dbfs,
int* mic_volume_percent);
void CheckMutedState();
static StreamType ParamsToStreamType(const media::AudioParameters& params);
SEQUENCE_CHECKER(owning_sequence_);
// The task runner of audio-manager thread that this object runs on.
scoped_refptr<base::SingleThreadTaskRunner> const task_runner_;
// Contains the InputController::EventHandler which receives state
// notifications from this class.
EventHandler* const handler_;
// Pointer to the audio input stream object.
// Only used on the audio thread.
media::AudioInputStream* stream_ = nullptr;
// SyncWriter is used only in low-latency mode for synchronous writing.
SyncWriter* const sync_writer_;
StreamType type_;
double max_volume_ = 0.0;
media::UserInputMonitor* const user_input_monitor_;
#if defined(AUDIO_POWER_MONITORING)
// Whether the silence state and microphone levels should be checked and sent
// as UMA stats.
bool power_measurement_is_enabled_ = false;
// Updated each time a power measurement is performed.
base::TimeTicks last_audio_level_log_time_;
// The silence report sent as UMA stat at the end of a session.
SilenceState silence_state_ = SILENCE_STATE_NO_MEASUREMENT;
#endif
size_t prev_key_down_count_ = 0;
// Time when the stream started recording.
base::TimeTicks stream_create_time_;
bool is_muted_ = false;
base::RepeatingTimer check_muted_state_timer_;
class AudioCallback;
// Holds a pointer to the callback object that receives audio data from
// the lower audio layer. Valid only while 'recording' (between calls to
// stream_->Start() and stream_->Stop()).
// The value of this pointer is only set and read on the audio thread while
// the callbacks themselves occur on the hw callback thread. More details
// in the AudioCallback class in the cc file.
std::unique_ptr<AudioCallback> audio_callback_;
// A weak pointer factory that we use when posting tasks to the audio thread
// that we want to be automatically discarded after Close() has been called
// and that we do not want to keep the InputController instance alive
// beyond what is desired by the user of the instance. An example of where
// this is important is when we fire error notifications from the hw callback
// thread, post them to the audio thread. In that case, we do not want the
// error notification to keep the InputController alive for as long as
// the error notification is pending and then make a callback from an
// InputController that has already been closed.
// All outstanding weak pointers, are invalidated at the end of DoClose.
base::WeakPtrFactory<InputController> weak_ptr_factory_;
DISALLOW_COPY_AND_ASSIGN(InputController);
};
} // namespace audio
#endif // SERVICES_AUDIO_INPUT_CONTROLLER_H_
// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "services/audio/input_controller.h"
#include <memory>
#include "base/macros.h"
#include "base/message_loop/message_loop.h"
#include "base/run_loop.h"
#include "media/audio/audio_manager.h"
#include "media/audio/fake_audio_input_stream.h"
#include "media/audio/fake_audio_log_factory.h"
#include "media/audio/fake_audio_manager.h"
#include "media/audio/test_audio_thread.h"
#include "media/base/user_input_monitor.h"
#include "testing/gmock/include/gmock/gmock.h"
#include "testing/gtest/include/gtest/gtest.h"
using ::testing::_;
using ::testing::AtLeast;
using ::testing::Exactly;
using ::testing::InvokeWithoutArgs;
using ::testing::NotNull;
using base::WaitableEvent;
namespace audio {
namespace {
const int kSampleRate = media::AudioParameters::kAudioCDSampleRate;
const media::ChannelLayout kChannelLayout = media::CHANNEL_LAYOUT_STEREO;
const int kSamplesPerPacket = kSampleRate / 10;
const double kMaxVolume = 1.0;
// InputController will poll once every second, so wait at most a bit
// more than that for the callbacks.
constexpr base::TimeDelta kOnMuteWaitTimeout =
base::TimeDelta::FromMilliseconds(1500);
// Posts run_loop->QuitClosure() on specified
// message loop after a certain number of calls given by |limit|.
ACTION_P4(CheckCountAndPostQuitTask, count, limit, loop_or_proxy, run_loop) {
if (++*count >= limit) {
loop_or_proxy->PostTask(FROM_HERE, run_loop->QuitWhenIdleClosure());
}
}
void RunLoopWithTimeout(base::RunLoop* run_loop, base::TimeDelta timeout) {
base::OneShotTimer timeout_timer;
timeout_timer.Start(FROM_HERE, timeout, run_loop->QuitClosure());
run_loop->Run();
}
} // namespace
class MockInputControllerEventHandler : public InputController::EventHandler {
public:
MockInputControllerEventHandler() = default;
void OnLog(base::StringPiece) override {}
MOCK_METHOD1(OnCreated, void(bool initially_muted));
MOCK_METHOD1(OnError, void(InputController::ErrorCode error_code));
MOCK_METHOD1(OnMuted, void(bool is_muted));
private:
DISALLOW_COPY_AND_ASSIGN(MockInputControllerEventHandler);
};
class MockSyncWriter : public InputController::SyncWriter {
public:
MockSyncWriter() = default;
MOCK_METHOD4(Write,
void(const media::AudioBus* data,
double volume,
bool key_pressed,
base::TimeTicks capture_time));
MOCK_METHOD0(Close, void());
};
class MockUserInputMonitor : public media::UserInputMonitor {
public:
MockUserInputMonitor() = default;
uint32_t GetKeyPressCount() const override { return 0; }
MOCK_METHOD0(EnableKeyPressMonitoring, void());
MOCK_METHOD0(DisableKeyPressMonitoring, void());
};
class MockAudioInputStream : public media::AudioInputStream {
public:
MockAudioInputStream() {}
~MockAudioInputStream() override {}
void Start(AudioInputCallback*) override {}
void Stop() override {}
void Close() override {}
double GetMaxVolume() override { return kMaxVolume; }
double GetVolume() override { return 0; }
bool SetAutomaticGainControl(bool) override { return false; }
bool GetAutomaticGainControl() override { return false; }
bool IsMuted() override { return false; }
void SetOutputDeviceForAec(const std::string&) override {}
MOCK_METHOD0(Open, bool());
MOCK_METHOD1(SetVolume, void(double));
};
class InputControllerTest : public testing::TestWithParam<bool> {
public:
InputControllerTest()
: run_on_audio_thread_(GetParam()),
audio_manager_(std::make_unique<media::FakeAudioManager>(
std::make_unique<media::TestAudioThread>(!run_on_audio_thread_),
&log_factory_)),
params_(media::AudioParameters::AUDIO_FAKE,
kChannelLayout,
kSampleRate,
kSamplesPerPacket) {}
~InputControllerTest() override {
audio_manager_->Shutdown();
base::RunLoop().RunUntilIdle();
}
protected:
void CreateAudioController() {
controller_ = InputController::Create(
audio_manager_.get(), &event_handler_, &sync_writer_,
&user_input_monitor_, params_,
media::AudioDeviceDescription::kDefaultDeviceId, false);
}
void CloseAudioController() {
if (run_on_audio_thread_) {
controller_->Close(base::OnceClosure());
return;
}
base::RunLoop loop;
controller_->Close(loop.QuitWhenIdleClosure());
loop.Run();
}
base::MessageLoop message_loop_;
// Parameterize tests to run InputController either on audio thread
// (synchronously), or on a different thread (non-blocking).
bool run_on_audio_thread_;
scoped_refptr<InputController> controller_;
media::FakeAudioLogFactory log_factory_;
std::unique_ptr<media::AudioManager> audio_manager_;
MockInputControllerEventHandler event_handler_;
MockSyncWriter sync_writer_;
MockUserInputMonitor user_input_monitor_;
media::AudioParameters params_;
MockAudioInputStream stream_;
private:
DISALLOW_COPY_AND_ASSIGN(InputControllerTest);
};
TEST_P(InputControllerTest, CreateAndCloseWithoutRecording) {
EXPECT_CALL(event_handler_, OnCreated(_));
CreateAudioController();
base::RunLoop().RunUntilIdle();
ASSERT_TRUE(controller_.get());
EXPECT_CALL(sync_writer_, Close());
CloseAudioController();
}
// Test a normal call sequence of create, record and close.
TEST_P(InputControllerTest, CreateRecordAndClose) {
int count = 0;
EXPECT_CALL(event_handler_, OnCreated(_));
CreateAudioController();
ASSERT_TRUE(controller_.get());
base::RunLoop loop;
// Write() should be called ten times.
EXPECT_CALL(sync_writer_, Write(NotNull(), _, _, _))
.Times(AtLeast(10))
.WillRepeatedly(CheckCountAndPostQuitTask(
&count, 10, message_loop_.task_runner(), &loop));
EXPECT_CALL(user_input_monitor_, EnableKeyPressMonitoring());
controller_->Record();
// Record and wait until ten Write() callbacks are received.
loop.Run();
EXPECT_CALL(user_input_monitor_, DisableKeyPressMonitoring());
EXPECT_CALL(sync_writer_, Close());
CloseAudioController();
}
// Test that InputController rejects insanely large packet sizes.
TEST_P(InputControllerTest, SamplesPerPacketTooLarge) {
// Create an audio device with a very large packet size.
params_.set_frames_per_buffer(kSamplesPerPacket * 1000);
// OnCreated() shall not be called in this test.
EXPECT_CALL(event_handler_, OnCreated(_)).Times(Exactly(0));
CreateAudioController();
ASSERT_FALSE(controller_.get());
}
TEST_P(InputControllerTest, CloseTwice) {
EXPECT_CALL(event_handler_, OnCreated(_));
CreateAudioController();
ASSERT_TRUE(controller_.get());
EXPECT_CALL(user_input_monitor_, EnableKeyPressMonitoring());
controller_->Record();
EXPECT_CALL(user_input_monitor_, DisableKeyPressMonitoring());
EXPECT_CALL(sync_writer_, Close());
CloseAudioController();
CloseAudioController();
}
// Test that InputController sends OnMute callbacks properly.
TEST_P(InputControllerTest, TestOnmutedCallbackInitiallyUnmuted) {
const auto timeout = kOnMuteWaitTimeout;
WaitableEvent callback_event(WaitableEvent::ResetPolicy::AUTOMATIC,
WaitableEvent::InitialState::NOT_SIGNALED);
base::RunLoop unmute_run_loop;
base::RunLoop mute_run_loop;
base::RunLoop setup_run_loop;
EXPECT_CALL(event_handler_, OnCreated(false)).WillOnce(InvokeWithoutArgs([&] {
setup_run_loop.QuitWhenIdle();
}));
EXPECT_CALL(sync_writer_, Close());
EXPECT_CALL(event_handler_, OnMuted(true)).WillOnce(InvokeWithoutArgs([&] {
mute_run_loop.Quit();
}));
EXPECT_CALL(event_handler_, OnMuted(false)).WillOnce(InvokeWithoutArgs([&] {
unmute_run_loop.Quit();
}));
media::FakeAudioInputStream::SetGlobalMutedState(false);
CreateAudioController();
ASSERT_TRUE(controller_.get());
RunLoopWithTimeout(&setup_run_loop, timeout);
media::FakeAudioInputStream::SetGlobalMutedState(true);
RunLoopWithTimeout(&mute_run_loop, timeout);
media::FakeAudioInputStream::SetGlobalMutedState(false);
RunLoopWithTimeout(&unmute_run_loop, timeout);
CloseAudioController();
}
TEST_P(InputControllerTest, TestOnmutedCallbackInitiallyMuted) {
const auto timeout = kOnMuteWaitTimeout;
WaitableEvent callback_event(WaitableEvent::ResetPolicy::AUTOMATIC,
WaitableEvent::InitialState::NOT_SIGNALED);
base::RunLoop unmute_run_loop;
base::RunLoop setup_run_loop;
EXPECT_CALL(event_handler_, OnCreated(true)).WillOnce(InvokeWithoutArgs([&] {
setup_run_loop.QuitWhenIdle();
}));
EXPECT_CALL(sync_writer_, Close());
EXPECT_CALL(event_handler_, OnMuted(false)).WillOnce(InvokeWithoutArgs([&] {
unmute_run_loop.Quit();
}));
media::FakeAudioInputStream::SetGlobalMutedState(true);
CreateAudioController();
ASSERT_TRUE(controller_.get());
RunLoopWithTimeout(&setup_run_loop, timeout);
media::FakeAudioInputStream::SetGlobalMutedState(false);
RunLoopWithTimeout(&unmute_run_loop, timeout);
CloseAudioController();
}
INSTANTIATE_TEST_CASE_P(SyncAsync, InputControllerTest, testing::Bool());
} // namespace audio
...@@ -9,13 +9,13 @@ ...@@ -9,13 +9,13 @@
#include "base/bind_helpers.h" #include "base/bind_helpers.h"
#include "base/trace_event/trace_event.h" #include "base/trace_event/trace_event.h"
#include "media/audio/audio_input_sync_writer.h"
#include "media/audio/audio_manager.h" #include "media/audio/audio_manager.h"
#include "media/base/audio_parameters.h" #include "media/base/audio_parameters.h"
#include "media/base/user_input_monitor.h" #include "media/base/user_input_monitor.h"
#include "mojo/public/cpp/system/buffer.h" #include "mojo/public/cpp/system/buffer.h"
#include "mojo/public/cpp/system/handle.h" #include "mojo/public/cpp/system/handle.h"
#include "mojo/public/cpp/system/platform_handle.h" #include "mojo/public/cpp/system/platform_handle.h"
#include "services/audio/input_sync_writer.h"
#include "services/audio/user_input_monitor.h" #include "services/audio/user_input_monitor.h"
namespace audio { namespace audio {
...@@ -45,7 +45,7 @@ InputStream::InputStream(CreatedCallback created_callback, ...@@ -45,7 +45,7 @@ InputStream::InputStream(CreatedCallback created_callback,
created_callback_(std::move(created_callback)), created_callback_(std::move(created_callback)),
delete_callback_(std::move(delete_callback)), delete_callback_(std::move(delete_callback)),
foreign_socket_(), foreign_socket_(),
writer_(media::AudioInputSyncWriter::Create( writer_(InputSyncWriter::Create(
log_ ? base::BindRepeating(&media::mojom::AudioLog::OnLogMessage, log_ ? base::BindRepeating(&media::mojom::AudioLog::OnLogMessage,
base::Unretained(log_->get())) base::Unretained(log_->get()))
: base::DoNothing(), : base::DoNothing(),
...@@ -88,8 +88,8 @@ InputStream::InputStream(CreatedCallback created_callback, ...@@ -88,8 +88,8 @@ InputStream::InputStream(CreatedCallback created_callback,
return; return;
} }
controller_ = media::AudioInputController::Create( controller_ = InputController::Create(audio_manager, this, writer_.get(),
audio_manager, this, writer_.get(), user_input_monitor_.get(), params, user_input_monitor_.get(), params,
device_id, enable_agc); device_id, enable_agc);
} }
...@@ -116,7 +116,7 @@ InputStream::~InputStream() { ...@@ -116,7 +116,7 @@ InputStream::~InputStream() {
return; return;
} }
// TODO(https://crbug.com/803102): remove AudioInputController::Close() after // TODO(https://crbug.com/803102): remove InputController::Close() after
// content/ streams are removed, destructor should suffice. // content/ streams are removed, destructor should suffice.
controller_->Close(base::OnceClosure()); controller_->Close(base::OnceClosure());
...@@ -183,7 +183,7 @@ void InputStream::OnCreated(bool initially_muted) { ...@@ -183,7 +183,7 @@ void InputStream::OnCreated(bool initially_muted) {
initially_muted, id_); initially_muted, id_);
} }
void InputStream::OnError(media::AudioInputController::ErrorCode error_code) { void InputStream::OnError(InputController::ErrorCode error_code) {
DCHECK_CALLED_ON_VALID_SEQUENCE(owning_sequence_); DCHECK_CALLED_ON_VALID_SEQUENCE(owning_sequence_);
TRACE_EVENT_NESTABLE_ASYNC_INSTANT0("audio", "Error", this); TRACE_EVENT_NESTABLE_ASYNC_INSTANT0("audio", "Error", this);
......
...@@ -11,15 +11,14 @@ ...@@ -11,15 +11,14 @@
#include "base/memory/scoped_refptr.h" #include "base/memory/scoped_refptr.h"
#include "base/sync_socket.h" #include "base/sync_socket.h"
#include "base/unguessable_token.h" #include "base/unguessable_token.h"
#include "media/audio/audio_input_controller.h"
#include "media/mojo/interfaces/audio_data_pipe.mojom.h" #include "media/mojo/interfaces/audio_data_pipe.mojom.h"
#include "media/mojo/interfaces/audio_input_stream.mojom.h" #include "media/mojo/interfaces/audio_input_stream.mojom.h"
#include "media/mojo/interfaces/audio_logging.mojom.h" #include "media/mojo/interfaces/audio_logging.mojom.h"
#include "mojo/public/cpp/bindings/binding.h" #include "mojo/public/cpp/bindings/binding.h"
#include "services/audio/input_controller.h"
namespace media { namespace media {
class AudioInputSyncWriter;
class AudioManager; class AudioManager;
class AudioParameters; class AudioParameters;
...@@ -27,10 +26,11 @@ class AudioParameters; ...@@ -27,10 +26,11 @@ class AudioParameters;
namespace audio { namespace audio {
class InputSyncWriter;
class UserInputMonitor; class UserInputMonitor;
class InputStream final : public media::mojom::AudioInputStream, class InputStream final : public media::mojom::AudioInputStream,
public media::AudioInputController::EventHandler { public InputController::EventHandler {
public: public:
using CreatedCallback = using CreatedCallback =
base::OnceCallback<void(media::mojom::ReadOnlyAudioDataPipePtr, base::OnceCallback<void(media::mojom::ReadOnlyAudioDataPipePtr,
...@@ -59,9 +59,9 @@ class InputStream final : public media::mojom::AudioInputStream, ...@@ -59,9 +59,9 @@ class InputStream final : public media::mojom::AudioInputStream,
void Record() override; void Record() override;
void SetVolume(double volume) override; void SetVolume(double volume) override;
// media::AudioInputController::EventHandler implementation. // InputController::EventHandler implementation.
void OnCreated(bool initially_muted) override; void OnCreated(bool initially_muted) override;
void OnError(media::AudioInputController::ErrorCode error_code) override; void OnError(InputController::ErrorCode error_code) override;
void OnLog(base::StringPiece) override; void OnLog(base::StringPiece) override;
void OnMuted(bool is_muted) override; void OnMuted(bool is_muted) override;
...@@ -83,8 +83,8 @@ class InputStream final : public media::mojom::AudioInputStream, ...@@ -83,8 +83,8 @@ class InputStream final : public media::mojom::AudioInputStream,
DeleteCallback delete_callback_; DeleteCallback delete_callback_;
base::CancelableSyncSocket foreign_socket_; base::CancelableSyncSocket foreign_socket_;
const std::unique_ptr<media::AudioInputSyncWriter> writer_; const std::unique_ptr<InputSyncWriter> writer_;
scoped_refptr<media::AudioInputController> controller_; scoped_refptr<InputController> controller_;
const std::unique_ptr<UserInputMonitor> user_input_monitor_; const std::unique_ptr<UserInputMonitor> user_input_monitor_;
SEQUENCE_CHECKER(owning_sequence_); SEQUENCE_CHECKER(owning_sequence_);
......
// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "services/audio/input_sync_writer.h"
#include <algorithm>
#include <utility>
#include "base/format_macros.h"
#include "base/metrics/histogram_macros.h"
#include "base/strings/stringprintf.h"
#include "base/trace_event/trace_event.h"
#include "build/build_config.h"
namespace audio {
namespace {
// Used to log if any audio glitches have been detected during an audio session.
// Elements in this enum should not be added, deleted or rearranged.
enum AudioGlitchResult {
AUDIO_CAPTURER_NO_AUDIO_GLITCHES = 0,
AUDIO_CAPTURER_AUDIO_GLITCHES = 1,
AUDIO_CAPTURER_AUDIO_GLITCHES_MAX = AUDIO_CAPTURER_AUDIO_GLITCHES
};
} // namespace
InputSyncWriter::OverflowData::OverflowData(
double volume,
bool key_pressed,
base::TimeTicks capture_time,
std::unique_ptr<media::AudioBus> audio_bus)
: volume_(volume),
key_pressed_(key_pressed),
capture_time_(capture_time),
audio_bus_(std::move(audio_bus)) {}
InputSyncWriter::OverflowData::~OverflowData() {}
InputSyncWriter::OverflowData::OverflowData(InputSyncWriter::OverflowData&&) =
default;
InputSyncWriter::OverflowData& InputSyncWriter::OverflowData::operator=(
InputSyncWriter::OverflowData&& other) = default;
InputSyncWriter::InputSyncWriter(
base::RepeatingCallback<void(const std::string&)> log_callback,
base::MappedReadOnlyRegion shared_memory,
std::unique_ptr<base::CancelableSyncSocket> socket,
uint32_t shared_memory_segment_count,
const media::AudioParameters& params)
: log_callback_(std::move(log_callback)),
socket_(std::move(socket)),
shared_memory_region_(std::move(shared_memory.region)),
shared_memory_mapping_(std::move(shared_memory.mapping)),
shared_memory_segment_size_(
(CHECK(shared_memory_segment_count > 0),
shared_memory_mapping_.size() / shared_memory_segment_count)),
creation_time_(base::TimeTicks::Now()),
audio_bus_memory_size_(media::AudioBus::CalculateMemorySize(params)) {
// We use CHECKs since this class is used for IPC.
DCHECK(log_callback_);
CHECK(socket_);
CHECK(shared_memory_region_.IsValid());
CHECK(shared_memory_mapping_.IsValid());
CHECK_EQ(shared_memory_segment_size_ * shared_memory_segment_count,
shared_memory_mapping_.size());
CHECK_EQ(shared_memory_segment_size_,
audio_bus_memory_size_ + sizeof(media::AudioInputBufferParameters));
DVLOG(1) << "shared memory size: " << shared_memory_mapping_.size();
DVLOG(1) << "shared memory segment count: " << shared_memory_segment_count;
DVLOG(1) << "audio bus memory size: " << audio_bus_memory_size_;
audio_buses_.resize(shared_memory_segment_count);
// Create vector of audio buses by wrapping existing blocks of memory.
uint8_t* ptr = static_cast<uint8_t*>(shared_memory_mapping_.memory());
CHECK(ptr);
for (auto& bus : audio_buses_) {
CHECK_EQ(0U, reinterpret_cast<uintptr_t>(ptr) &
(media::AudioBus::kChannelAlignment - 1));
media::AudioInputBuffer* buffer =
reinterpret_cast<media::AudioInputBuffer*>(ptr);
bus = media::AudioBus::WrapMemory(params, buffer->audio);
ptr += shared_memory_segment_size_;
}
}
InputSyncWriter::~InputSyncWriter() {
// We log the following:
// - Percentage of data written to fifo (and not to shared memory).
// - Percentage of data dropped (fifo reached max size or socket buffer full).
// - Glitch yes/no (at least 1 drop).
//
// Subtract 'trailing' counts that will happen if the renderer process was
// killed or e.g. the page refreshed while the input device was open etc.
// This trims off the end of both the error and write counts so that we
// preserve the proportion of counts before the teardown period. We pick
// the largest trailing count as the time we consider that the trailing errors
// begun, and subract that from the total write count.
DCHECK_LE(trailing_write_to_fifo_count_, write_to_fifo_count_);
DCHECK_LE(trailing_write_to_fifo_count_, write_count_);
DCHECK_LE(trailing_write_error_count_, write_error_count_);
DCHECK_LE(trailing_write_error_count_, write_count_);
write_to_fifo_count_ -= trailing_write_to_fifo_count_;
write_error_count_ -= trailing_write_error_count_;
write_count_ -=
std::max(trailing_write_to_fifo_count_, trailing_write_error_count_);
if (write_count_ == 0)
return;
UMA_HISTOGRAM_PERCENTAGE("Media.AudioCapturerMissedReadDeadline",
100.0 * write_to_fifo_count_ / write_count_);
UMA_HISTOGRAM_PERCENTAGE("Media.AudioCapturerDroppedData",
100.0 * write_error_count_ / write_count_);
UMA_HISTOGRAM_ENUMERATION("Media.AudioCapturerAudioGlitches",
write_error_count_ == 0
? AUDIO_CAPTURER_NO_AUDIO_GLITCHES
: AUDIO_CAPTURER_AUDIO_GLITCHES,
AUDIO_CAPTURER_AUDIO_GLITCHES_MAX + 1);
std::string log_string = base::StringPrintf(
"AISW: number of detected audio glitches: %" PRIuS " out of %" PRIuS,
write_error_count_, write_count_);
log_callback_.Run(log_string);
}
// static
std::unique_ptr<InputSyncWriter> InputSyncWriter::Create(
base::RepeatingCallback<void(const std::string&)> log_callback,
uint32_t shared_memory_segment_count,
const media::AudioParameters& params,
base::CancelableSyncSocket* foreign_socket) {
// Having no shared memory doesn't make sense, so fail creation in that case.
if (shared_memory_segment_count == 0)
return nullptr;
base::CheckedNumeric<uint32_t> requested_memory_size =
ComputeAudioInputBufferSizeChecked(params, shared_memory_segment_count);
if (!requested_memory_size.IsValid())
return nullptr;
// Make sure we can share the memory read-only with the client.
auto shared_memory = base::ReadOnlySharedMemoryRegion::Create(
requested_memory_size.ValueOrDie());
if (!shared_memory.IsValid())
return nullptr;
auto socket = std::make_unique<base::CancelableSyncSocket>();
if (!base::CancelableSyncSocket::CreatePair(socket.get(), foreign_socket)) {
return nullptr;
}
return std::make_unique<InputSyncWriter>(
std::move(log_callback), std::move(shared_memory), std::move(socket),
shared_memory_segment_count, params);
}
base::ReadOnlySharedMemoryRegion InputSyncWriter::TakeSharedMemoryRegion() {
DCHECK(shared_memory_region_.IsValid());
return std::move(shared_memory_region_);
}
void InputSyncWriter::Write(const media::AudioBus* data,
double volume,
bool key_pressed,
base::TimeTicks capture_time) {
TRACE_EVENT1("audio", "InputSyncWriter::Write", "capture time (ms)",
(capture_time - base::TimeTicks()).InMillisecondsF());
++write_count_;
CheckTimeSinceLastWrite();
// Check that the renderer side has read data so that we don't overwrite data
// that hasn't been read yet. The renderer side sends a signal over the socket
// each time it has read data. Here, we read those verifications before
// writing. We verify that each buffer index is in sequence.
size_t number_of_indices_available = socket_->Peek() / sizeof(uint32_t);
if (number_of_indices_available > 0) {
auto indices = std::make_unique<uint32_t[]>(number_of_indices_available);
size_t bytes_received = socket_->Receive(
&indices[0], number_of_indices_available * sizeof(indices[0]));
CHECK_EQ(number_of_indices_available * sizeof(indices[0]), bytes_received);
for (size_t i = 0; i < number_of_indices_available; ++i) {
++next_read_buffer_index_;
CHECK_EQ(indices[i], next_read_buffer_index_);
CHECK_GT(number_of_filled_segments_, 0u);
--number_of_filled_segments_;
}
}
bool write_error = !WriteDataFromFifoToSharedMemory();
// Write the current data to the shared memory if there is room, otherwise
// put it in the fifo.
if (number_of_filled_segments_ < audio_buses_.size()) {
WriteParametersToCurrentSegment(volume, key_pressed, capture_time);
// Copy data into shared memory using pre-allocated audio buses.
data->CopyTo(audio_buses_[current_segment_id_].get());
if (!SignalDataWrittenAndUpdateCounters())
write_error = true;
trailing_write_to_fifo_count_ = 0;
} else {
if (!PushDataToFifo(data, volume, key_pressed, capture_time))
write_error = true;
++write_to_fifo_count_;
++trailing_write_to_fifo_count_;
}
// Increase write error counts if error, or reset the trailing error counter
// if all write operations went well (no data dropped).
if (write_error) {
++write_error_count_;
++trailing_write_error_count_;
TRACE_EVENT_INSTANT0("audio", "InputSyncWriter write error",
TRACE_EVENT_SCOPE_THREAD);
} else {
trailing_write_error_count_ = 0;
}
}
void InputSyncWriter::Close() {
socket_->Close();
}
void InputSyncWriter::CheckTimeSinceLastWrite() {
#if !defined(OS_ANDROID)
static const base::TimeDelta kLogDelayThreadhold =
base::TimeDelta::FromMilliseconds(500);
base::TimeTicks new_write_time = base::TimeTicks::Now();
std::ostringstream oss;
if (last_write_time_.is_null()) {
// This is the first time Write is called.
base::TimeDelta interval = new_write_time - creation_time_;
oss << "AISW::Write: audio input data received for the first time: delay "
"= "
<< interval.InMilliseconds() << "ms";
} else {
base::TimeDelta interval = new_write_time - last_write_time_;
if (interval > kLogDelayThreadhold) {
oss << "AISW::Write: audio input data delay unexpectedly long: delay = "
<< interval.InMilliseconds() << "ms";
}
}
const std::string log_message = oss.str();
if (!log_message.empty()) {
log_callback_.Run(log_message);
}
last_write_time_ = new_write_time;
#endif
}
bool InputSyncWriter::PushDataToFifo(const media::AudioBus* data,
double volume,
bool key_pressed,
base::TimeTicks capture_time) {
TRACE_EVENT1("audio", "InputSyncWriter::PushDataToFifo", "capture time (ms)",
(capture_time - base::TimeTicks()).InMillisecondsF());
if (overflow_data_.size() == kMaxOverflowBusesSize) {
// We use |write_error_count_| for capping number of log messages.
// |write_error_count_| also includes socket Send() errors, but those should
// be rare.
TRACE_EVENT_INSTANT0("audio", "InputSyncWriter::PushDataToFifo - overflow",
TRACE_EVENT_SCOPE_THREAD);
if (write_error_count_ <= 50 && write_error_count_ % 10 == 0) {
static const char* error_message = "AISW: No room in fifo.";
LOG(WARNING) << error_message;
log_callback_.Run(error_message);
if (write_error_count_ == 50) {
static const char* cap_error_message =
"AISW: Log cap reached, suppressing further fifo overflow logs.";
LOG(WARNING) << cap_error_message;
log_callback_.Run(error_message);
}
}
return false;
}
if (overflow_data_.empty()) {
static const char* message = "AISW: Starting to use fifo.";
log_callback_.Run(message);
}
// Push data to fifo.
std::unique_ptr<media::AudioBus> audio_bus =
media::AudioBus::Create(data->channels(), data->frames());
data->CopyTo(audio_bus.get());
overflow_data_.emplace_back(volume, key_pressed, capture_time,
std::move(audio_bus));
DCHECK_LE(overflow_data_.size(), static_cast<size_t>(kMaxOverflowBusesSize));
return true;
}
bool InputSyncWriter::WriteDataFromFifoToSharedMemory() {
TRACE_EVENT0("audio", "InputSyncWriter::WriteDataFromFifoToSharedMemory");
if (overflow_data_.empty())
return true;
const size_t segment_count = audio_buses_.size();
bool write_error = false;
auto data_it = overflow_data_.begin();
while (data_it != overflow_data_.end() &&
number_of_filled_segments_ < segment_count) {
// Write parameters to shared memory.
WriteParametersToCurrentSegment(data_it->volume_, data_it->key_pressed_,
data_it->capture_time_);
// Copy data from the fifo into shared memory using pre-allocated audio
// buses.
data_it->audio_bus_->CopyTo(audio_buses_[current_segment_id_].get());
if (!SignalDataWrittenAndUpdateCounters())
write_error = true;
++data_it;
}
// Erase all copied data from fifo.
overflow_data_.erase(overflow_data_.begin(), data_it);
if (overflow_data_.empty()) {
static const char* message = "AISW: Fifo emptied.";
log_callback_.Run(message);
}
return !write_error;
}
void InputSyncWriter::WriteParametersToCurrentSegment(
double volume,
bool key_pressed,
base::TimeTicks capture_time) {
TRACE_EVENT1("audio", "WriteParametersToCurrentSegment", "capture time (ms)",
(capture_time - base::TimeTicks()).InMillisecondsF());
uint8_t* ptr = static_cast<uint8_t*>(shared_memory_mapping_.memory());
CHECK_LT(current_segment_id_, audio_buses_.size());
ptr += current_segment_id_ * shared_memory_segment_size_;
auto* buffer = reinterpret_cast<media::AudioInputBuffer*>(ptr);
buffer->params.volume = volume;
buffer->params.size = audio_bus_memory_size_;
buffer->params.key_pressed = key_pressed;
buffer->params.capture_time_us =
(capture_time - base::TimeTicks()).InMicroseconds();
buffer->params.id = next_buffer_id_;
}
bool InputSyncWriter::SignalDataWrittenAndUpdateCounters() {
if (socket_->Send(&current_segment_id_, sizeof(current_segment_id_)) !=
sizeof(current_segment_id_)) {
// Ensure we don't log consecutive errors as this can lead to a large
// amount of logs.
if (!had_socket_error_) {
had_socket_error_ = true;
static const char* error_message = "AISW: No room in socket buffer.";
PLOG(WARNING) << error_message;
log_callback_.Run(error_message);
TRACE_EVENT_INSTANT0("audio", "InputSyncWriter: No room in socket buffer",
TRACE_EVENT_SCOPE_THREAD);
}
return false;
} else {
had_socket_error_ = false;
}
if (++current_segment_id_ >= audio_buses_.size())
current_segment_id_ = 0;
++number_of_filled_segments_;
CHECK_LE(number_of_filled_segments_, audio_buses_.size());
++next_buffer_id_;
return true;
}
} // namespace audio
// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef SERVICES_AUDIO_INPUT_SYNC_WRITER_H_
#define SERVICES_AUDIO_INPUT_SYNC_WRITER_H_
#include <stddef.h>
#include <stdint.h>
#include <memory>
#include <string>
#include <vector>
#include "base/containers/circular_deque.h"
#include "base/gtest_prod_util.h"
#include "base/macros.h"
#include "base/memory/read_only_shared_memory_region.h"
#include "base/sync_socket.h"
#include "base/time/time.h"
#include "build/build_config.h"
#include "media/base/audio_bus.h"
#include "media/base/audio_parameters.h"
#include "services/audio/input_controller.h"
#if defined(OS_POSIX)
#include "base/file_descriptor_posix.h"
#endif
namespace audio {
// A InputController::SyncWriter implementation using SyncSocket. This
// is used by InputController to provide a low latency data source for
// transmitting audio packets between the browser process and the renderer
// process.
class InputSyncWriter : public InputController::SyncWriter {
public:
// Maximum fifo size (|overflow_buses_| and |overflow_params_|) in number of
// media::AudioBuses.
enum { kMaxOverflowBusesSize = 100 };
// Create() automatically initializes the InputSyncWriter correctly,
// and should be strongly preferred over calling the constructor directly!
InputSyncWriter(
base::RepeatingCallback<void(const std::string&)> log_callback,
base::MappedReadOnlyRegion shared_memory,
std::unique_ptr<base::CancelableSyncSocket> socket,
uint32_t shared_memory_segment_count,
const media::AudioParameters& params);
~InputSyncWriter() override;
static std::unique_ptr<InputSyncWriter> Create(
base::RepeatingCallback<void(const std::string&)> log_callback,
uint32_t shared_memory_segment_count,
const media::AudioParameters& params,
base::CancelableSyncSocket* foreign_socket);
// Transfers shared memory region ownership to a caller. It shouldn't be
// called more than once.
base::ReadOnlySharedMemoryRegion TakeSharedMemoryRegion();
size_t shared_memory_segment_count() const { return audio_buses_.size(); }
// InputController::SyncWriter implementation.
void Write(const media::AudioBus* data,
double volume,
bool key_pressed,
base::TimeTicks capture_time) override;
void Close() override;
private:
friend class InputSyncWriterTest;
// Called by Write(). Checks the time since last called and if larger than a
// threshold logs info about that.
void CheckTimeSinceLastWrite();
// Push |data| and metadata to |audio_buffer_fifo_|. Returns true if
// successful. Logs error and returns false if the fifo already reached the
// maximum size.
bool PushDataToFifo(const media::AudioBus* data,
double volume,
bool key_pressed,
base::TimeTicks capture_time);
// Writes as much data as possible from the fifo (|overflow_buses_|) to the
// shared memory ring buffer. Returns true if all operations were successful,
// otherwise false.
bool WriteDataFromFifoToSharedMemory();
// Write audio parameters to current segment in shared memory.
void WriteParametersToCurrentSegment(double volume,
bool key_pressed,
base::TimeTicks capture_time);
// Signals over the socket that data has been written to the current segment.
// Updates counters and returns true if successful. Logs error and returns
// false if failure.
bool SignalDataWrittenAndUpdateCounters();
const base::RepeatingCallback<void(const std::string&)> log_callback_;
// Socket used to signal that audio data is ready.
const std::unique_ptr<base::CancelableSyncSocket> socket_;
// Shared memory for audio data and associated metadata.
base::ReadOnlySharedMemoryRegion shared_memory_region_;
const base::WritableSharedMemoryMapping shared_memory_mapping_;
// The size in bytes of a single audio segment in the shared memory.
const uint32_t shared_memory_segment_size_;
// Index of next segment to write.
uint32_t current_segment_id_ = 0;
// The time of the creation of this object.
base::TimeTicks creation_time_;
// The time of the last Write call.
base::TimeTicks last_write_time_;
// Size in bytes of each audio bus.
const int audio_bus_memory_size_;
// Increasing ID used for checking audio buffers are in correct sequence at
// read side.
uint32_t next_buffer_id_ = 0;
// Next expected audio buffer index to have been read at the other side. We
// will get the index read at the other side over the socket. Note that this
// index does not correspond to |next_buffer_id_|, it's two separate counters.
uint32_t next_read_buffer_index_ = 0;
// Keeps track of number of filled buffer segments in the ring buffer to
// ensure the we don't overwrite data that hasn't been read yet.
size_t number_of_filled_segments_ = 0;
// Counts the total number of calls to Write().
size_t write_count_ = 0;
// Counts the number of writes to the fifo instead of to the shared memory.
size_t write_to_fifo_count_ = 0;
// Counts the number of errors that causes data to be dropped, due to either
// the fifo or the socket buffer being full.
size_t write_error_count_ = 0;
// Denotes that the most recent socket error has been logged. Used to avoid
// log spam.
bool had_socket_error_ = false;
// Counts the fifo writes and errors we get during renderer process teardown
// so that we can account for that (subtract) when we calculate the overall
// counts.
size_t trailing_write_to_fifo_count_ = 0;
size_t trailing_write_error_count_ = 0;
// Vector of audio buses allocated during construction and deleted in the
// destructor.
std::vector<std::unique_ptr<media::AudioBus>> audio_buses_;
// Fifo for audio that is used in case there isn't room in the shared memory.
// This can for example happen under load when the consumer side is starved.
// It should ideally be rare, but we need to guarantee that the data arrives
// since audio processing such as echo cancelling requires that to perform
// properly.
struct OverflowData {
OverflowData(double volume,
bool key_pressed,
base::TimeTicks capture_time,
std::unique_ptr<media::AudioBus> audio_bus);
~OverflowData();
OverflowData(OverflowData&&);
OverflowData& operator=(OverflowData&& other);
double volume_;
bool key_pressed_;
base::TimeTicks capture_time_;
std::unique_ptr<media::AudioBus> audio_bus_;
private:
DISALLOW_COPY_AND_ASSIGN(OverflowData);
};
std::vector<OverflowData> overflow_data_;
DISALLOW_IMPLICIT_CONSTRUCTORS(InputSyncWriter);
};
} // namespace audio
#endif // SERVICES_AUDIO_INPUT_SYNC_WRITER_H_
// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "services/audio/input_sync_writer.h"
#include <stddef.h>
#include <stdint.h>
#include <memory>
#include <string>
#include <utility>
#include "base/compiler_specific.h"
#include "base/macros.h"
#include "base/memory/ptr_util.h"
#include "base/memory/read_only_shared_memory_region.h"
#include "base/sync_socket.h"
#include "base/test/mock_callback.h"
#include "base/test/scoped_task_environment.h"
#include "base/time/time.h"
#include "build/build_config.h"
#include "media/base/audio_bus.h"
#include "media/base/audio_parameters.h"
#include "media/base/channel_layout.h"
#include "testing/gmock/include/gmock/gmock.h"
#include "testing/gtest/include/gtest/gtest.h"
using ::testing::_;
namespace audio {
namespace {
// Number of audio buffers in the faked ring buffer.
const int kSegments = 10;
} // namespace
// Mocked out sockets used for Send/ReceiveWithTimeout. Counts the number of
// outstanding reads, i.e. the diff between send and receive calls.
class MockCancelableSyncSocket : public base::CancelableSyncSocket {
public:
explicit MockCancelableSyncSocket(int buffer_size)
: in_failure_mode_(false),
writes_(0),
reads_(0),
receives_(0),
buffer_size_(buffer_size),
read_buffer_index_(0) {}
size_t Send(const void* buffer, size_t length) override {
EXPECT_EQ(length, sizeof(uint32_t));
++writes_;
EXPECT_LE(NumberOfBuffersFilled(), buffer_size_);
return length;
}
size_t Receive(void* buffer, size_t length) override {
EXPECT_EQ(0u, length % sizeof(uint32_t));
if (in_failure_mode_)
return 0;
if (receives_ == reads_)
return 0;
uint32_t* ptr = static_cast<uint32_t*>(buffer);
size_t received = 0;
for (; received < length / sizeof(uint32_t) && receives_ < reads_;
++received, ++ptr) {
++receives_;
EXPECT_LE(receives_, reads_);
*ptr = ++read_buffer_index_;
}
return received * sizeof(uint32_t);
}
size_t Peek() override { return (reads_ - receives_) * sizeof(uint32_t); }
// Simluates reading |buffers| number of buffers from the ring buffer.
void Read(int buffers) {
reads_ += buffers;
EXPECT_LE(reads_, writes_);
}
// When |in_failure_mode_| == true, the socket fails to receive.
void SetFailureMode(bool in_failure_mode) {
in_failure_mode_ = in_failure_mode;
}
int NumberOfBuffersFilled() { return writes_ - reads_; }
private:
bool in_failure_mode_;
int writes_;
int reads_;
int receives_;
int buffer_size_;
uint32_t read_buffer_index_;
DISALLOW_COPY_AND_ASSIGN(MockCancelableSyncSocket);
};
class InputSyncWriterTest : public testing::Test {
public:
InputSyncWriterTest() {
const int sampling_frequency_hz = 16000;
const int frames = sampling_frequency_hz / 100; // 10 ms
const media::AudioParameters audio_params(
media::AudioParameters::AUDIO_FAKE, media::CHANNEL_LAYOUT_MONO,
sampling_frequency_hz, frames);
const uint32_t data_size =
ComputeAudioInputBufferSize(audio_params, kSegments);
auto shared_memory = base::ReadOnlySharedMemoryRegion::Create(data_size);
EXPECT_TRUE(shared_memory.IsValid());
auto socket = std::make_unique<MockCancelableSyncSocket>(kSegments);
socket_ = socket.get();
writer_ = std::make_unique<InputSyncWriter>(
mock_logger_.Get(), std::move(shared_memory), std::move(socket),
kSegments, audio_params);
audio_bus_ = media::AudioBus::Create(audio_params);
}
~InputSyncWriterTest() override {}
// Get total number of expected log calls. On non-Android we expect one log
// call at first Write() call, zero on Android. We also expect all call in the
// with a glitch summary from the destructor. Besides that only for errors
// and fifo info.
int GetTotalNumberOfExpectedLogCalls(int expected_calls_due_to_error) {
#if defined(OS_ANDROID)
return expected_calls_due_to_error + 1;
#else
return expected_calls_due_to_error + 2;
#endif
}
// Tests expected numbers which are given as arguments.
bool TestSocketAndFifoExpectations(int number_of_buffers_in_socket,
size_t number_of_verifications_in_socket,
size_t number_of_buffers_in_fifo) {
EXPECT_EQ(number_of_buffers_in_socket, socket_->NumberOfBuffersFilled());
EXPECT_EQ(number_of_verifications_in_socket, socket_->Peek());
EXPECT_EQ(number_of_buffers_in_fifo, writer_->overflow_data_.size());
return number_of_buffers_in_socket == socket_->NumberOfBuffersFilled() &&
number_of_verifications_in_socket == socket_->Peek() &&
number_of_buffers_in_fifo == writer_->overflow_data_.size();
}
protected:
using MockLogger =
base::MockCallback<base::RepeatingCallback<void(const std::string&)>>;
base::test::ScopedTaskEnvironment env_;
MockLogger mock_logger_;
std::unique_ptr<InputSyncWriter> writer_;
MockCancelableSyncSocket* socket_;
std::unique_ptr<media::AudioBus> audio_bus_;
private:
DISALLOW_COPY_AND_ASSIGN(InputSyncWriterTest);
};
TEST_F(InputSyncWriterTest, SingleWriteAndRead) {
EXPECT_CALL(mock_logger_, Run(_)).Times(GetTotalNumberOfExpectedLogCalls(0));
writer_->Write(audio_bus_.get(), 0, false, base::TimeTicks::Now());
EXPECT_TRUE(TestSocketAndFifoExpectations(1, 0, 0));
socket_->Read(1);
EXPECT_TRUE(TestSocketAndFifoExpectations(0, 1 * sizeof(uint32_t), 0));
}
TEST_F(InputSyncWriterTest, MultipleWritesAndReads) {
EXPECT_CALL(mock_logger_, Run(_)).Times(GetTotalNumberOfExpectedLogCalls(0));
for (int i = 1; i <= 2 * kSegments; ++i) {
writer_->Write(audio_bus_.get(), 0, false, base::TimeTicks::Now());
EXPECT_TRUE(TestSocketAndFifoExpectations(1, 0, 0));
socket_->Read(1);
EXPECT_TRUE(TestSocketAndFifoExpectations(0, 1 * sizeof(uint32_t), 0));
}
}
TEST_F(InputSyncWriterTest, MultipleWritesNoReads) {
EXPECT_CALL(mock_logger_, Run(_)).Times(GetTotalNumberOfExpectedLogCalls(1));
// Fill the ring buffer.
for (int i = 1; i <= kSegments; ++i) {
writer_->Write(audio_bus_.get(), 0, false, base::TimeTicks::Now());
EXPECT_TRUE(TestSocketAndFifoExpectations(i, 0, 0));
}
// Now the ring buffer is full, do more writes. We should start filling the
// fifo and should get one extra log call for that.
for (size_t i = 1; i <= kSegments; ++i) {
writer_->Write(audio_bus_.get(), 0, false, base::TimeTicks::Now());
EXPECT_TRUE(TestSocketAndFifoExpectations(kSegments, 0, i));
}
}
TEST_F(InputSyncWriterTest, FillAndEmptyRingBuffer) {
EXPECT_CALL(mock_logger_, Run(_)).Times(GetTotalNumberOfExpectedLogCalls(2));
// Fill the ring buffer.
for (int i = 1; i <= kSegments; ++i) {
writer_->Write(audio_bus_.get(), 0, false, base::TimeTicks::Now());
}
EXPECT_TRUE(TestSocketAndFifoExpectations(kSegments, 0, 0));
// Empty half of the ring buffer.
const int buffers_to_read = kSegments / 2;
socket_->Read(buffers_to_read);
EXPECT_TRUE(TestSocketAndFifoExpectations(
kSegments - buffers_to_read, buffers_to_read * sizeof(uint32_t), 0));
// Fill up again. The first write should do receive until that queue is
// empty.
for (int i = kSegments - buffers_to_read + 1; i <= kSegments; ++i) {
writer_->Write(audio_bus_.get(), 0, false, base::TimeTicks::Now());
EXPECT_TRUE(TestSocketAndFifoExpectations(i, 0, 0));
}
// Another write, should put the data in the fifo, and render an extra log
// call.
writer_->Write(audio_bus_.get(), 0, false, base::TimeTicks::Now());
EXPECT_TRUE(TestSocketAndFifoExpectations(kSegments, 0, 1));
// Empty the ring buffer.
socket_->Read(kSegments);
EXPECT_TRUE(
TestSocketAndFifoExpectations(0, kSegments * sizeof(uint32_t), 1));
// Another write, should do receive until that queue is empty and write both
// the data in the fifo and the new data, and render a log call.
writer_->Write(audio_bus_.get(), 0, false, base::TimeTicks::Now());
EXPECT_TRUE(TestSocketAndFifoExpectations(2, 0, 0));
// Read the two data blocks.
socket_->Read(2);
EXPECT_TRUE(TestSocketAndFifoExpectations(0, 2 * sizeof(uint32_t), 0));
}
TEST_F(InputSyncWriterTest, FillRingBufferAndFifo) {
EXPECT_CALL(mock_logger_, Run(_)).Times(GetTotalNumberOfExpectedLogCalls(2));
// Fill the ring buffer.
for (int i = 1; i <= kSegments; ++i) {
writer_->Write(audio_bus_.get(), 0, false, base::TimeTicks::Now());
}
EXPECT_TRUE(TestSocketAndFifoExpectations(kSegments, 0, 0));
// Fill the fifo. Should render one log call for starting filling it.
const size_t max_fifo_size = InputSyncWriter::kMaxOverflowBusesSize;
for (size_t i = 1; i <= max_fifo_size; ++i) {
writer_->Write(audio_bus_.get(), 0, false, base::TimeTicks::Now());
}
EXPECT_TRUE(TestSocketAndFifoExpectations(kSegments, 0, max_fifo_size));
// Another write, data should be dropped and render one log call.
writer_->Write(audio_bus_.get(), 0, false, base::TimeTicks::Now());
EXPECT_TRUE(TestSocketAndFifoExpectations(kSegments, 0, max_fifo_size));
}
TEST_F(InputSyncWriterTest, MultipleFillAndEmptyRingBufferAndPartOfFifo) {
EXPECT_CALL(mock_logger_, Run(_)).Times(GetTotalNumberOfExpectedLogCalls(4));
// Fill the ring buffer.
for (int i = 1; i <= kSegments; ++i) {
writer_->Write(audio_bus_.get(), 0, false, base::TimeTicks::Now());
}
EXPECT_TRUE(TestSocketAndFifoExpectations(kSegments, 0, 0));
// Write more data, should be put in the fifo and render one log call for
// starting filling it.
for (size_t i = 1; i <= 2 * kSegments; ++i) {
writer_->Write(audio_bus_.get(), 0, false, base::TimeTicks::Now());
}
EXPECT_TRUE(TestSocketAndFifoExpectations(kSegments, 0, 2 * kSegments));
// Empty the ring buffer.
socket_->Read(kSegments);
EXPECT_TRUE(TestSocketAndFifoExpectations(0, kSegments * sizeof(uint32_t),
2 * kSegments));
// Another write should fill up the ring buffer with data from the fifo and
// put this data into the fifo.
writer_->Write(audio_bus_.get(), 0, false, base::TimeTicks::Now());
EXPECT_TRUE(TestSocketAndFifoExpectations(kSegments, 0, kSegments + 1));
// Empty the ring buffer again.
socket_->Read(kSegments);
EXPECT_TRUE(TestSocketAndFifoExpectations(0, kSegments * sizeof(uint32_t),
kSegments + 1));
// Another write should fill up the ring buffer with data from the fifo and
// put this data into the fifo.
writer_->Write(audio_bus_.get(), 0, false, base::TimeTicks::Now());
EXPECT_TRUE(TestSocketAndFifoExpectations(kSegments, 0, 2));
// Empty the ring buffer again.
socket_->Read(kSegments);
EXPECT_TRUE(
TestSocketAndFifoExpectations(0, kSegments * sizeof(uint32_t), 2));
// Another write should put the remaining data in the fifo in the ring buffer
// together with this data. Should render a log call for emptying the fifo.
writer_->Write(audio_bus_.get(), 0, false, base::TimeTicks::Now());
EXPECT_TRUE(TestSocketAndFifoExpectations(3, 0, 0));
// Read the remaining data.
socket_->Read(3);
EXPECT_TRUE(TestSocketAndFifoExpectations(0, 3 * sizeof(uint32_t), 0));
// Fill the ring buffer and part of the fifo. Should render one log call for
// starting filling it.
for (int i = 1; i <= kSegments + 2; ++i) {
writer_->Write(audio_bus_.get(), 0, false, base::TimeTicks::Now());
}
EXPECT_TRUE(TestSocketAndFifoExpectations(kSegments, 0, 2));
// Empty both. Should render a log call for emptying the fifo.
socket_->Read(kSegments);
writer_->Write(audio_bus_.get(), 0, false, base::TimeTicks::Now());
socket_->Read(3);
EXPECT_TRUE(TestSocketAndFifoExpectations(0, 3 * sizeof(uint32_t), 0));
}
} // namespace audio
...@@ -68,8 +68,7 @@ LoopbackStream::LoopbackStream( ...@@ -68,8 +68,7 @@ LoopbackStream::LoopbackStream(
// the consumer. If successful, create the FlowNetwork too. // the consumer. If successful, create the FlowNetwork too.
if (base::TimeTicks::IsHighResolution()) { if (base::TimeTicks::IsHighResolution()) {
base::CancelableSyncSocket foreign_socket; base::CancelableSyncSocket foreign_socket;
std::unique_ptr<media::AudioInputSyncWriter> writer = std::unique_ptr<InputSyncWriter> writer = InputSyncWriter::Create(
media::AudioInputSyncWriter::Create(
base::BindRepeating( base::BindRepeating(
[](const std::string& message) { VLOG(1) << message; }), [](const std::string& message) { VLOG(1) << message; }),
shared_memory_count, params, &foreign_socket); shared_memory_count, params, &foreign_socket);
...@@ -232,7 +231,7 @@ void LoopbackStream::OnError() { ...@@ -232,7 +231,7 @@ void LoopbackStream::OnError() {
LoopbackStream::FlowNetwork::FlowNetwork( LoopbackStream::FlowNetwork::FlowNetwork(
scoped_refptr<base::SequencedTaskRunner> flow_task_runner, scoped_refptr<base::SequencedTaskRunner> flow_task_runner,
const media::AudioParameters& output_params, const media::AudioParameters& output_params,
std::unique_ptr<media::AudioInputSyncWriter> writer) std::unique_ptr<InputSyncWriter> writer)
: clock_(base::DefaultTickClock::GetInstance()), : clock_(base::DefaultTickClock::GetInstance()),
flow_task_runner_(flow_task_runner), flow_task_runner_(flow_task_runner),
output_params_(output_params), output_params_(output_params),
......
...@@ -21,14 +21,14 @@ ...@@ -21,14 +21,14 @@
#include "base/time/time.h" #include "base/time/time.h"
#include "base/timer/timer.h" #include "base/timer/timer.h"
#include "base/unguessable_token.h" #include "base/unguessable_token.h"
#include "media/audio/audio_input_controller.h"
#include "media/audio/audio_input_sync_writer.h"
#include "media/base/audio_parameters.h" #include "media/base/audio_parameters.h"
#include "media/mojo/interfaces/audio_data_pipe.mojom.h" #include "media/mojo/interfaces/audio_data_pipe.mojom.h"
#include "media/mojo/interfaces/audio_input_stream.mojom.h" #include "media/mojo/interfaces/audio_input_stream.mojom.h"
#include "mojo/public/cpp/bindings/binding.h" #include "mojo/public/cpp/bindings/binding.h"
#include "services/audio/group_coordinator.h" #include "services/audio/group_coordinator.h"
#include "services/audio/group_member.h" #include "services/audio/group_member.h"
#include "services/audio/input_controller.h"
#include "services/audio/input_sync_writer.h"
#include "services/audio/snooper_node.h" #include "services/audio/snooper_node.h"
namespace base { namespace base {
...@@ -89,7 +89,7 @@ class LoopbackStream : public media::mojom::AudioInputStream, ...@@ -89,7 +89,7 @@ class LoopbackStream : public media::mojom::AudioInputStream,
network_->set_clock_for_testing(clock); network_->set_clock_for_testing(clock);
} }
void set_sync_writer_for_testing( void set_sync_writer_for_testing(
std::unique_ptr<media::AudioInputController::SyncWriter> writer) { std::unique_ptr<InputController::SyncWriter> writer) {
network_->set_writer_for_testing(std::move(writer)); network_->set_writer_for_testing(std::move(writer));
} }
...@@ -114,12 +114,12 @@ class LoopbackStream : public media::mojom::AudioInputStream, ...@@ -114,12 +114,12 @@ class LoopbackStream : public media::mojom::AudioInputStream,
public: public:
FlowNetwork(scoped_refptr<base::SequencedTaskRunner> flow_task_runner, FlowNetwork(scoped_refptr<base::SequencedTaskRunner> flow_task_runner,
const media::AudioParameters& output_params, const media::AudioParameters& output_params,
std::unique_ptr<media::AudioInputSyncWriter> writer); std::unique_ptr<InputSyncWriter> writer);
// These must be called to override the Clock/SyncWriter before Start(). // These must be called to override the Clock/SyncWriter before Start().
void set_clock_for_testing(const base::TickClock* clock) { clock_ = clock; } void set_clock_for_testing(const base::TickClock* clock) { clock_ = clock; }
void set_writer_for_testing( void set_writer_for_testing(
std::unique_ptr<media::AudioInputController::SyncWriter> writer) { std::unique_ptr<InputController::SyncWriter> writer) {
writer_ = std::move(writer); writer_ = std::move(writer);
} }
...@@ -169,7 +169,7 @@ class LoopbackStream : public media::mojom::AudioInputStream, ...@@ -169,7 +169,7 @@ class LoopbackStream : public media::mojom::AudioInputStream,
const media::AudioParameters output_params_; const media::AudioParameters output_params_;
// Destination for the output of this FlowNetwork. // Destination for the output of this FlowNetwork.
std::unique_ptr<media::AudioInputController::SyncWriter> writer_; std::unique_ptr<InputController::SyncWriter> writer_;
// Ensures thread-safe access to changing the |inputs_| and |volume_| while // Ensures thread-safe access to changing the |inputs_| and |volume_| while
// running. // running.
......
...@@ -87,8 +87,7 @@ class MockClientAndObserver : public media::mojom::AudioInputStreamClient, ...@@ -87,8 +87,7 @@ class MockClientAndObserver : public media::mojom::AudioInputStreamClient,
// Subclass of FakeConsumer that adapts the SyncWriter interface to allow the // Subclass of FakeConsumer that adapts the SyncWriter interface to allow the
// tests to record and analyze the audio data from the LoopbackStream. // tests to record and analyze the audio data from the LoopbackStream.
class FakeSyncWriter : public FakeConsumer, class FakeSyncWriter : public FakeConsumer, public InputController::SyncWriter {
public media::AudioInputController::SyncWriter {
public: public:
FakeSyncWriter(int channels, int sample_rate) FakeSyncWriter(int channels, int sample_rate)
: FakeConsumer(channels, sample_rate) {} : FakeConsumer(channels, sample_rate) {}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment