Commit a0ab6cc8 authored by Sam Zackrisson's avatar Sam Zackrisson Committed by Commit Bot

Chrome+WebRTC audio processors: detect upmixed mono playout before analysis

This improves the WebRTC audio processing module adaptation on mono signals even when being played out on more than one speaker.

Tested: made appr.tc calls with all combinations of mono/stereo audio, multi-channel feature flag on/off, APM in audio service enabled/disabled: verified that (locally modified) log statement for entering multi-channel is printed, verified that aecdump recordings have the right content, verified that not too much processing happens before stereo is detected

Bug: chromium:1016708, chromium:1023337
Change-Id: Ia6827f26e01dba6779ab356687f7920ce9fa8703
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1906210Reviewed-by: default avatarOlga Sharonova <olka@chromium.org>
Reviewed-by: default avatarGuido Urdaneta <guidou@chromium.org>
Commit-Queue: Sam Zackrisson <saza@chromium.org>
Cr-Commit-Position: refs/heads/master@{#715252}
parent dee7c127
......@@ -102,13 +102,25 @@ void AudioProcessor::AnalyzePlayout(const AudioBus& audio,
DCHECK_GE(parameters.channels(), 1);
DCHECK_LE(parameters.channels(), audio.channels());
DCHECK_LE(parameters.channels(), media::limits::kMaxChannels);
webrtc::StreamConfig input_stream_config = CreateStreamConfig(parameters);
// If the input audio appears to contain upmixed mono audio, then APM is only
// given the left channel. This reduces computational complexity and improves
// convergence of audio processing algorithms.
// TODO(crbug.com/1023337): Ensure correct channel count in input audio bus.
assume_upmixed_mono_playout_ =
assume_upmixed_mono_playout_ && LeftAndRightChannelsAreSymmetric(audio);
if (assume_upmixed_mono_playout_) {
input_stream_config.set_num_channels(1);
}
std::array<const float*, media::limits::kMaxChannels> input_ptrs;
for (int i = 0; i < parameters.channels(); ++i) {
for (int i = 0; i < static_cast<int>(input_stream_config.num_channels());
++i) {
input_ptrs[i] = audio.channel(i);
}
const int apm_error = audio_processing_->AnalyzeReverseStream(
input_ptrs.data(), CreateStreamConfig(parameters));
input_ptrs.data(), input_stream_config);
DCHECK_EQ(apm_error, webrtc::AudioProcessing::kNoError);
}
......
......@@ -94,6 +94,10 @@ class COMPONENT_EXPORT(MEDIA_WEBRTC) AudioProcessor final
std::atomic<base::TimeDelta> render_delay_ = {base::TimeDelta()};
bool has_reverse_stream_ = false;
// Indicates whether the audio processor playout signal has ever had
// asymmetric left and right channel content.
bool assume_upmixed_mono_playout_ = true;
// The APM writes the processed data here.
std::unique_ptr<AudioBus> output_bus_;
std::vector<float*> output_ptrs_;
......
......@@ -19,4 +19,12 @@ webrtc::StreamConfig CreateStreamConfig(const AudioParameters& parameters) {
return webrtc::StreamConfig(rate, channels, has_keyboard);
}
bool LeftAndRightChannelsAreSymmetric(const AudioBus& audio) {
if (audio.channels() <= 1) {
return true;
}
return std::equal(audio.channel(0), audio.channel(0) + audio.frames(),
audio.channel(1));
}
} // namespace media
......@@ -6,6 +6,7 @@
#define MEDIA_WEBRTC_HELPERS_H_
#include "base/component_export.h"
#include "media/base/audio_bus.h"
#include "media/base/audio_parameters.h"
#include "third_party/webrtc/modules/audio_processing/include/audio_processing.h"
......@@ -14,6 +15,12 @@ namespace media {
COMPONENT_EXPORT(MEDIA_WEBRTC)
webrtc::StreamConfig CreateStreamConfig(const AudioParameters& parameters);
// Tests whether the audio bus data can be treated as upmixed mono audio:
// Returns true if there is at most one channel or if each sample is identical
// in the first two channels.
COMPONENT_EXPORT(MEDIA_WEBRTC)
bool LeftAndRightChannelsAreSymmetric(const AudioBus& audio);
} // namespace media
#endif // MEDIA_WEBRTC_WEBRTC_HELPERS_H_
......@@ -413,6 +413,7 @@ void MediaStreamAudioProcessor::OnPlayoutData(media::AudioBus* audio_bus,
int audio_delay_milliseconds) {
DCHECK_CALLED_ON_VALID_THREAD(render_thread_checker_);
DCHECK_GE(audio_bus->channels(), 1);
DCHECK_LE(audio_bus->channels(), media::limits::kMaxChannels);
int frames_per_10_ms = sample_rate / 100;
if (audio_bus->frames() != frames_per_10_ms) {
if (unsupported_buffer_size_log_count_ < 10) {
......@@ -429,16 +430,24 @@ void MediaStreamAudioProcessor::OnPlayoutData(media::AudioBus* audio_bus,
std::numeric_limits<base::subtle::Atomic32>::max());
base::subtle::Release_Store(&render_delay_ms_, audio_delay_milliseconds);
DCHECK_LE(audio_bus->channels(), media::limits::kMaxChannels);
webrtc::StreamConfig input_stream_config(sample_rate, audio_bus->channels());
// If the input audio appears to contain upmixed mono audio, then APM is only
// given the left channel. This reduces computational complexity and improves
// convergence of audio processing algorithms.
// TODO(crbug.com/1023337): Ensure correct channel count in input audio bus.
assume_upmixed_mono_playout_ = assume_upmixed_mono_playout_ &&
LeftAndRightChannelsAreSymmetric(*audio_bus);
if (assume_upmixed_mono_playout_) {
input_stream_config.set_num_channels(1);
}
std::array<const float*, media::limits::kMaxChannels> input_ptrs;
for (int i = 0; i < audio_bus->channels(); ++i)
for (int i = 0; i < static_cast<int>(input_stream_config.num_channels()); ++i)
input_ptrs[i] = audio_bus->channel(i);
// TODO(ajm): Should AnalyzeReverseStream() account for the
// |audio_delay_milliseconds|?
const int apm_error = audio_processing_->AnalyzeReverseStream(
input_ptrs.data(),
webrtc::StreamConfig(sample_rate, audio_bus->channels()));
input_ptrs.data(), input_stream_config);
if (apm_error != webrtc::AudioProcessing::kNoError &&
apm_playout_error_code_log_count_ < 10) {
LOG(ERROR) << "MSAP::OnPlayoutData: AnalyzeReverseStream error="
......
......@@ -169,6 +169,10 @@ class MODULES_EXPORT MediaStreamAudioProcessor
// Receives processing output.
std::unique_ptr<MediaStreamAudioBus> output_bus_;
// Indicates whether the audio processor playout signal has ever had
// asymmetric left and right channel content.
bool assume_upmixed_mono_playout_ = true;
// These are mutated on the main render thread in OnCaptureFormatChanged().
// The caller guarantees this does not run concurrently with accesses on the
// capture audio thread.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment