Chrome+WebRTC audio processors: detect upmixed mono playout before analysis

This improves the WebRTC audio processing module adaptation on mono signals even when being played out on more than one speaker.

Tested: made appr.tc calls with all combinations of mono/stereo audio, multi-channel feature flag on/off, APM in audio service enabled/disabled: verified that (locally modified) log statement for entering multi-channel is printed, verified that aecdump recordings have the right content, verified that not too much processing happens before stereo is detected

Bug: chromium:1016708, chromium:1023337
Change-Id: Ia6827f26e01dba6779ab356687f7920ce9fa8703
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1906210Reviewed-by: Olga Sharonova <olka@chromium.org>
Reviewed-by: Guido Urdaneta <guidou@chromium.org>
Commit-Queue: Sam Zackrisson <saza@chromium.org>
Cr-Commit-Position: refs/heads/master@{#715252}

media/webrtc/audio_processor.cc

@@ -102,13 +102,25 @@ void AudioProcessor::AnalyzePlayout(const AudioBus& audio,
   DCHECK_GE(parameters.channels(), 1);
   DCHECK_LE(parameters.channels(), audio.channels());
   DCHECK_LE(parameters.channels(), media::limits::kMaxChannels);
+  webrtc::StreamConfig input_stream_config = CreateStreamConfig(parameters);
+  // If the input audio appears to contain upmixed mono audio, then APM is only
+  // given the left channel. This reduces computational complexity and improves
+  // convergence of audio processing algorithms.
+  // TODO(crbug.com/1023337): Ensure correct channel count in input audio bus.
+  assume_upmixed_mono_playout_ =
+      assume_upmixed_mono_playout_ && LeftAndRightChannelsAreSymmetric(audio);
+  if (assume_upmixed_mono_playout_) {
+    input_stream_config.set_num_channels(1);
+  }
+
   std::array<const float*, media::limits::kMaxChannels> input_ptrs;
-  for (int i = 0; i < parameters.channels(); ++i) {
+  for (int i = 0; i < static_cast<int>(input_stream_config.num_channels());
+       ++i) {
     input_ptrs[i] = audio.channel(i);
   }
 
   const int apm_error = audio_processing_->AnalyzeReverseStream(
-      input_ptrs.data(), CreateStreamConfig(parameters));
+      input_ptrs.data(), input_stream_config);
 
   DCHECK_EQ(apm_error, webrtc::AudioProcessing::kNoError);
 }

media/webrtc/audio_processor.h

@@ -94,6 +94,10 @@ class COMPONENT_EXPORT(MEDIA_WEBRTC) AudioProcessor final
   std::atomic<base::TimeDelta> render_delay_ = {base::TimeDelta()};
   bool has_reverse_stream_ = false;
 
+  // Indicates whether the audio processor playout signal has ever had
+  // asymmetric left and right channel content.
+  bool assume_upmixed_mono_playout_ = true;
+
   // The APM writes the processed data here.
   std::unique_ptr<AudioBus> output_bus_;
   std::vector<float*> output_ptrs_;

media/webrtc/helpers.cc

@@ -19,4 +19,12 @@ webrtc::StreamConfig CreateStreamConfig(const AudioParameters& parameters) {
   return webrtc::StreamConfig(rate, channels, has_keyboard);
 }
 
+bool LeftAndRightChannelsAreSymmetric(const AudioBus& audio) {
+  if (audio.channels() <= 1) {
+    return true;
+  }
+  return std::equal(audio.channel(0), audio.channel(0) + audio.frames(),
+                    audio.channel(1));
+}
+
 }  // namespace media

media/webrtc/helpers.h

@@ -6,6 +6,7 @@
 #define MEDIA_WEBRTC_HELPERS_H_
 
 #include "base/component_export.h"
+#include "media/base/audio_bus.h"
 #include "media/base/audio_parameters.h"
 #include "third_party/webrtc/modules/audio_processing/include/audio_processing.h"
 
@@ -14,6 +15,12 @@ namespace media {
 COMPONENT_EXPORT(MEDIA_WEBRTC)
 webrtc::StreamConfig CreateStreamConfig(const AudioParameters& parameters);
 
+// Tests whether the audio bus data can be treated as upmixed mono audio:
+// Returns true if there is at most one channel or if each sample is identical
+// in the first two channels.
+COMPONENT_EXPORT(MEDIA_WEBRTC)
+bool LeftAndRightChannelsAreSymmetric(const AudioBus& audio);
+
 }  // namespace media
 
 #endif  // MEDIA_WEBRTC_WEBRTC_HELPERS_H_

third_party/blink/renderer/modules/mediastream/media_stream_audio_processor.cc

@@ -413,6 +413,7 @@ void MediaStreamAudioProcessor::OnPlayoutData(media::AudioBus* audio_bus,
                                               int audio_delay_milliseconds) {
   DCHECK_CALLED_ON_VALID_THREAD(render_thread_checker_);
   DCHECK_GE(audio_bus->channels(), 1);
+  DCHECK_LE(audio_bus->channels(), media::limits::kMaxChannels);
   int frames_per_10_ms = sample_rate / 100;
   if (audio_bus->frames() != frames_per_10_ms) {
     if (unsupported_buffer_size_log_count_ < 10) {
@@ -429,16 +430,24 @@ void MediaStreamAudioProcessor::OnPlayoutData(media::AudioBus* audio_bus,
             std::numeric_limits<base::subtle::Atomic32>::max());
   base::subtle::Release_Store(&render_delay_ms_, audio_delay_milliseconds);
 
-  DCHECK_LE(audio_bus->channels(), media::limits::kMaxChannels);
+  webrtc::StreamConfig input_stream_config(sample_rate, audio_bus->channels());
+  // If the input audio appears to contain upmixed mono audio, then APM is only
+  // given the left channel. This reduces computational complexity and improves
+  // convergence of audio processing algorithms.
+  // TODO(crbug.com/1023337): Ensure correct channel count in input audio bus.
+  assume_upmixed_mono_playout_ = assume_upmixed_mono_playout_ &&
+                                 LeftAndRightChannelsAreSymmetric(*audio_bus);
+  if (assume_upmixed_mono_playout_) {
+    input_stream_config.set_num_channels(1);
+  }
   std::array<const float*, media::limits::kMaxChannels> input_ptrs;
-  for (int i = 0; i < audio_bus->channels(); ++i)
+  for (int i = 0; i < static_cast<int>(input_stream_config.num_channels()); ++i)
     input_ptrs[i] = audio_bus->channel(i);
 
   // TODO(ajm): Should AnalyzeReverseStream() account for the
   // |audio_delay_milliseconds|?
   const int apm_error = audio_processing_->AnalyzeReverseStream(
-      input_ptrs.data(),
-      webrtc::StreamConfig(sample_rate, audio_bus->channels()));
+      input_ptrs.data(), input_stream_config);
   if (apm_error != webrtc::AudioProcessing::kNoError &&
       apm_playout_error_code_log_count_ < 10) {
     LOG(ERROR) << "MSAP::OnPlayoutData: AnalyzeReverseStream error="

third_party/blink/renderer/modules/mediastream/media_stream_audio_processor.h

@@ -169,6 +169,10 @@ class MODULES_EXPORT MediaStreamAudioProcessor
   // Receives processing output.
   std::unique_ptr<MediaStreamAudioBus> output_bus_;
 
+  // Indicates whether the audio processor playout signal has ever had
+  // asymmetric left and right channel content.
+  bool assume_upmixed_mono_playout_ = true;
+
   // These are mutated on the main render thread in OnCaptureFormatChanged().
   // The caller guarantees this does not run concurrently with accesses on the
   // capture audio thread.