Fix resampling crash in MediaElementSourceNode

Remove blink::MultiChannelResampler and replace it with the
blink::MediaMultiChannelResampler.

Bug: 1033408
Change-Id: Ia90022fbf186cf85d1fe5f589819774e63c87746
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1975713Reviewed-by: Hongchan Choi <hongchan@chromium.org>
Reviewed-by: Raymond Toy <rtoy@chromium.org>
Commit-Queue: Andrew MacPherson <andrew.macpherson@soundtrap.com>
Cr-Commit-Position: refs/heads/master@{#745388}

third_party/blink/renderer/modules/webaudio/media_element_audio_source_node.cc

@@ -138,8 +138,12 @@ void MediaElementAudioSourceHandler::SetFormat(uint32_t number_of_channels,
 
     if (source_sample_rate != Context()->sampleRate()) {
       double scale_factor = source_sample_rate / Context()->sampleRate();
-      multi_channel_resampler_ = std::make_unique<MultiChannelResampler>(
-          scale_factor, number_of_channels);
+      multi_channel_resampler_.reset(new MediaMultiChannelResampler(
+          number_of_channels, scale_factor,
+          audio_utilities::kRenderQuantumFrames,
+          CrossThreadBindRepeating(
+              &MediaElementAudioSourceHandler::ProvideResamplerInput,
+              CrossThreadUnretained(this))));
     } else {
       // Bypass resampling.
       multi_channel_resampler_.reset();
@@ -172,6 +176,15 @@ void MediaElementAudioSourceHandler::PrintCorsMessage(const String& message) {
   }
 }
 
+void MediaElementAudioSourceHandler::ProvideResamplerInput(
+    int resampler_frame_delay,
+    AudioBus* dest) {
+  DCHECK(Context()->IsAudioThread());
+  DCHECK(MediaElement());
+  DCHECK(dest);
+  MediaElement()->GetAudioSourceProvider().ProvideInput(dest, dest->length());
+}
+
 void MediaElementAudioSourceHandler::Process(uint32_t number_of_frames) {
   AudioBus* output_bus = Output(0).Bus();
 
@@ -199,8 +212,7 @@ void MediaElementAudioSourceHandler::Process(uint32_t number_of_frames) {
     // progress, even if we're going to output silence anyway.
     if (multi_channel_resampler_.get()) {
       DCHECK_NE(source_sample_rate_, Context()->sampleRate());
-      multi_channel_resampler_->Process(&provider, output_bus,
-                                        number_of_frames);
+      multi_channel_resampler_->Resample(number_of_frames, output_bus);
     } else {
       // Bypass the resampler completely if the source is at the context's
       // sample-rate.

third_party/blink/renderer/modules/webaudio/media_element_audio_source_node.h

@@ -32,7 +32,7 @@
 #include "base/thread_annotations.h"
 #include "third_party/blink/renderer/modules/webaudio/audio_node.h"
 #include "third_party/blink/renderer/platform/audio/audio_source_provider_client.h"
-#include "third_party/blink/renderer/platform/audio/multi_channel_resampler.h"
+#include "third_party/blink/renderer/platform/audio/media_multi_channel_resampler.h"
 #include "third_party/blink/renderer/platform/heap/persistent.h"
 #include "third_party/blink/renderer/platform/wtf/threading_primitives.h"
 
@@ -86,6 +86,9 @@ class MediaElementAudioSourceHandler final : public AudioHandler {
   // zeroes.
   void PrintCorsMessage(const String& message);
 
+  // Provide input to the resampler (if used).
+  void ProvideResamplerInput(int resampler_frame_delay, AudioBus* dest);
+
   // The HTMLMediaElement is held alive by MediaElementAudioSourceNode which is
   // an AudioNode. AudioNode uses pre-finalizers to dispose the handler, so
   // holding a weak reference is ok here and will not interfer with garbage
@@ -99,7 +102,7 @@ class MediaElementAudioSourceHandler final : public AudioHandler {
   unsigned source_number_of_channels_;
   double source_sample_rate_;
 
-  std::unique_ptr<MultiChannelResampler> multi_channel_resampler_;
+  std::unique_ptr<MediaMultiChannelResampler> multi_channel_resampler_;
 
   scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
 

third_party/blink/renderer/platform/BUILD.gn

@@ -359,8 +359,6 @@ jumbo_component("platform") {
     "audio/mac/vector_math_mac.h",
     "audio/media_multi_channel_resampler.cc",
     "audio/media_multi_channel_resampler.h",
-    "audio/multi_channel_resampler.cc",
-    "audio/multi_channel_resampler.h",
     "audio/panner.cc",
     "audio/panner.h",
     "audio/pffft/fft_frame_pffft.cc",

third_party/blink/renderer/platform/audio/audio_destination.cc

@@ -244,7 +244,7 @@ void AudioDestination::RequestRender(size_t frames_requested,
       output_position_.position = 0.0;
 
     if (resampler_) {
-      resampler_->Resample(audio_utilities::kRenderQuantumFrames,
+      resampler_->ResampleInternal(audio_utilities::kRenderQuantumFrames,
                                    resampler_bus_.get());
     } else {
       // Process WebAudio graph and push the rendered output to FIFO.

third_party/blink/renderer/platform/audio/media_multi_channel_resampler.cc

@@ -24,8 +24,21 @@ MediaMultiChannelResampler::MediaMultiChannelResampler(
 }
 
 void MediaMultiChannelResampler::Resample(int frames,
+                                          blink::AudioBus* audio_bus) {
+  // Create a media::AudioBus wrapper around the memory provided by the
+  // blink::AudioBus.
+  std::unique_ptr<media::AudioBus> resampler_bus_ =
+      media::AudioBus::CreateWrapper(audio_bus->NumberOfChannels());
+  for (unsigned int i = 0; i < audio_bus->NumberOfChannels(); ++i) {
+    resampler_bus_->SetChannelData(i, audio_bus->Channel(i)->MutableData());
+  }
+  resampler_bus_->set_frames(audio_bus->length());
+  ResampleInternal(frames, resampler_bus_.get());
+}
+
+void MediaMultiChannelResampler::ResampleInternal(int frames,
                                                   media::AudioBus* audio_bus) {
-  resampler_->Resample(audio_bus->frames(), audio_bus);
+  resampler_->Resample(frames, audio_bus);
 }
 
 void MediaMultiChannelResampler::ProvideResamplerInput(

third_party/blink/renderer/platform/audio/media_multi_channel_resampler.h

@@ -31,7 +31,7 @@ class PLATFORM_EXPORT MediaMultiChannelResampler {
   // frames are available to satisfy the request.  |frame_delay| is the number
   // of output frames already processed and can be used to estimate delay.
   typedef WTF::CrossThreadRepeatingFunction<void(int frame_delay,
-                                                 AudioBus* audio_bus)>
+                                                 blink::AudioBus* audio_bus)>
       ReadCB;
 
  public:
@@ -44,8 +44,16 @@ class PLATFORM_EXPORT MediaMultiChannelResampler {
                              size_t request_frames,
                              ReadCB read_cb);
 
-  // Resamples |frames| of data from |read_cb_| into AudioBus.
-  void Resample(int frames, media::AudioBus* audio_bus);
+  // Resamples |frames| of data from |read_cb_| into a blink::AudioBus, this
+  // requires creating a wrapper for the media::AudioBus on each call and so
+  // resampling directly into a media::AudioBus using ResampleInternal() is
+  // preferred if possible.
+  void Resample(int frames, blink::AudioBus* audio_bus);
+
+  // Resamples |frames| of data from |read_cb_| into a media::AudioBus by
+  // directly calling Resample() on the underlying
+  // media::MultiChannelResampler.
+  void ResampleInternal(int frames, media::AudioBus* audio_bus);
 
  private:
   // Wrapper method used to provide input to the media::MultiChannelResampler

third_party/blink/renderer/platform/audio/multi_channel_resampler.cc

-/*
- * Copyright (C) 2011 Google Inc. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1.  Redistributions of source code must retain the above copyright
- *     notice, this list of conditions and the following disclaimer.
- * 2.  Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- * 3.  Neither the name of Apple Computer, Inc. ("Apple") nor the names of
- *     its contributors may be used to endorse or promote products derived
- *     from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "third_party/blink/renderer/platform/audio/multi_channel_resampler.h"
-
-#include <memory>
-#include "third_party/blink/renderer/platform/audio/audio_bus.h"
-
-namespace blink {
-
-namespace {
-
-// ChannelProvider provides a single channel of audio data (one channel at a
-// time) for each channel of data provided to us in a multi-channel provider.
-
-class ChannelProvider final : public AudioSourceProvider {
- public:
-  ChannelProvider(AudioSourceProvider* multi_channel_provider,
-                  unsigned number_of_channels)
-      : multi_channel_provider_(multi_channel_provider),
-        number_of_channels_(number_of_channels),
-        current_channel_(0),
-        frames_to_process_(0) {}
-
-  // provideInput() will be called once for each channel, starting with the
-  // first channel.  Each time it's called, it will provide the next channel of
-  // data.
-  void ProvideInput(AudioBus* bus, uint32_t frames_to_process) override {
-    DCHECK(bus);
-    DCHECK_EQ(bus->NumberOfChannels(), 1u);
-
-    // Get the data from the multi-channel provider when the first channel asks
-    // for it.  For subsequent channels, we can just dish out the channel data
-    // from that (stored in m_multiChannelBus).
-    if (!current_channel_) {
-      frames_to_process_ = frames_to_process;
-      multi_channel_bus_ =
-          AudioBus::Create(number_of_channels_, frames_to_process);
-      multi_channel_provider_->ProvideInput(multi_channel_bus_.get(),
-                                            frames_to_process);
-    }
-
-    DCHECK(multi_channel_bus_.get());
-    DCHECK_EQ(frames_to_process, frames_to_process_);
-
-    // Copy the channel data from what we received from m_multiChannelProvider.
-    DCHECK_LT(current_channel_, number_of_channels_);
-    memcpy(bus->Channel(0)->MutableData(),
-           multi_channel_bus_->Channel(current_channel_)->Data(),
-           sizeof(float) * frames_to_process);
-    ++current_channel_;
-  }
-
- private:
-  AudioSourceProvider* multi_channel_provider_;
-  scoped_refptr<AudioBus> multi_channel_bus_;
-  unsigned number_of_channels_;
-  unsigned current_channel_;
-  // Used to verify that all channels ask for the same amount.
-  uint32_t frames_to_process_;
-};
-
-}  // namespace
-
-MultiChannelResampler::MultiChannelResampler(double scale_factor,
-                                             unsigned number_of_channels)
-    : number_of_channels_(number_of_channels) {
-  // Create each channel's resampler.
-  for (unsigned channel_index = 0; channel_index < number_of_channels;
-       ++channel_index)
-    kernels_.push_back(std::make_unique<SincResampler>(scale_factor));
-}
-
-void MultiChannelResampler::Process(AudioSourceProvider* provider,
-                                    AudioBus* destination,
-                                    uint32_t frames_to_process) {
-  // The provider can provide us with multi-channel audio data. But each of our
-  // single-channel resamplers (kernels) below requires a provider which
-  // provides a single unique channel of data.  channelProvider wraps the
-  // original multi-channel provider and dishes out one channel at a time.
-  ChannelProvider channel_provider(provider, number_of_channels_);
-
-  for (unsigned channel_index = 0; channel_index < number_of_channels_;
-       ++channel_index) {
-    // Depending on the sample-rate scale factor, and the internal buffering
-    // used in a SincResampler kernel, this call to process() will only
-    // sometimes call provideInput() on the channelProvider.  However, if it
-    // calls provideInput() for the first channel, then it will call it for the
-    // remaining channels, since they all buffer in the same way and are
-    // processing the same number of frames.
-    kernels_[channel_index]->Process(
-        &channel_provider, destination->Channel(channel_index)->MutableData(),
-        frames_to_process);
-  }
-}
-
-}  // namespace blink

third_party/blink/renderer/platform/audio/multi_channel_resampler.h

-/*
- * Copyright (C) 2011 Google Inc. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1.  Redistributions of source code must retain the above copyright
- *     notice, this list of conditions and the following disclaimer.
- * 2.  Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- * 3.  Neither the name of Apple Computer, Inc. ("Apple") nor the names of
- *     its contributors may be used to endorse or promote products derived
- *     from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef THIRD_PARTY_BLINK_RENDERER_PLATFORM_AUDIO_MULTI_CHANNEL_RESAMPLER_H_
-#define THIRD_PARTY_BLINK_RENDERER_PLATFORM_AUDIO_MULTI_CHANNEL_RESAMPLER_H_
-
-#include <memory>
-
-#include "base/macros.h"
-#include "third_party/blink/renderer/platform/audio/sinc_resampler.h"
-#include "third_party/blink/renderer/platform/wtf/allocator/allocator.h"
-#include "third_party/blink/renderer/platform/wtf/vector.h"
-
-namespace blink {
-
-class AudioBus;
-
-class PLATFORM_EXPORT MultiChannelResampler {
-  USING_FAST_MALLOC(MultiChannelResampler);
-
- public:
-  MultiChannelResampler(double scale_factor, unsigned number_of_channels);
-
-  // Process given AudioSourceProvider for streaming applications.
-  void Process(AudioSourceProvider*,
-               AudioBus* destination,
-               uint32_t frames_to_process);
-
- private:
-  // FIXME: the mac port can have a more highly optimized implementation based
-  // on CoreAudio instead of SincResampler. For now the default implementation
-  // will be used on all ports.
-  // https://bugs.webkit.org/show_bug.cgi?id=75118
-
-  // Each channel will be resampled using a high-quality SincResampler.
-  Vector<std::unique_ptr<SincResampler>> kernels_;
-
-  unsigned number_of_channels_;
-
-  DISALLOW_COPY_AND_ASSIGN(MultiChannelResampler);
-};
-
-}  // namespace blink
-
-#endif  // THIRD_PARTY_BLINK_RENDERER_PLATFORM_AUDIO_MULTI_CHANNEL_RESAMPLER_H_

third_party/blink/web_tests/webaudio/MediaElementAudioSource/resampling-crash.html

+<!DOCTYPE html>
+<html>
+  <head>
+    <title>
+      resampling-crash.html
+    </title>
+    <script src="../../resources/testharness.js"></script>
+    <script src="../../resources/testharnessreport.js"></script>
+    <script src="../resources/audit-util.js"></script>
+    <script src="../resources/audit.js"></script>
+  </head>
+  <body>
+    <script>
+      let audit = Audit.createTaskRunner();
+
+      audit.define(
+          {
+            label: 'resampling-crash',
+            description: 'Verify no crash when resampling from 192k to 44.1k'
+          },
+          (task, should) => {
+            let context = new AudioContext({sampleRate: 44100});
+            let audioElement = new Audio('../resources/media/192khz-10ms.ogg');
+            let mediaSource = context.createMediaElementSource(audioElement);
+            mediaSource.connect(context.destination);
+
+            // If connecting doesn't crash then the test passed.
+            // See https://crbug.com/1033408.
+            task.done();
+          });
+
+      audit.run();
+    </script>
+  </body>
+</html>