Add PushableMediaStreamAudioSource

This CL adds PushableMediaStreamAudioSource, which mirrors the video
equivalent PushableMediaStreamVideoSource. This class allows one to
easily push audio data into a MediaStreamTrack.

Bug: 1157608
Change-Id: I6fe3adc26f257eb6f9a6e7d8ff8dad8929ab2a38
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2625829Reviewed-by: Guido Urdaneta <guidou@chromium.org>
Reviewed-by: Kinuko Yasuda <kinuko@chromium.org>
Auto-Submit: Thomas Guilbert <tguilbert@chromium.org>
Commit-Queue: Guido Urdaneta <guidou@chromium.org>
Cr-Commit-Position: refs/heads/master@{#844758}

third_party/blink/renderer/modules/BUILD.gn

@@ -371,6 +371,7 @@ source_set("unit_tests") {
     "mediastream/mock_mojo_media_stream_dispatcher_host.cc",
     "mediastream/mock_mojo_media_stream_dispatcher_host.h",
     "mediastream/processed_local_audio_source_test.cc",
+    "mediastream/pushable_media_stream_audio_source_test.cc",
     "mediastream/pushable_media_stream_video_source_test.cc",
     "mediastream/user_media_client_test.cc",
     "mediastream/video_track_adapter_unittest.cc",

third_party/blink/renderer/modules/mediastream/BUILD.gn

@@ -78,6 +78,8 @@ blink_modules_sources("mediastream") {
     "overconstrained_error.h",
     "processed_local_audio_source.cc",
     "processed_local_audio_source.h",
+    "pushable_media_stream_audio_source.cc",
+    "pushable_media_stream_audio_source.h",
     "pushable_media_stream_video_source.cc",
     "pushable_media_stream_video_source.h",
     "remote_media_stream_track_adapter.cc",

third_party/blink/renderer/modules/mediastream/DEPS

@@ -45,6 +45,7 @@ include_rules = [
     "+third_party/blink/renderer/modules/mediastream",
     "+third_party/blink/renderer/modules/modules_export.h",
     "+third_party/blink/renderer/modules/peerconnection",
+    "+third_party/blink/renderer/modules/webcodecs/audio_frame.h",
     "+third_party/blink/renderer/modules/webcodecs/video_frame.h",
     "+third_party/blink/renderer/modules/webrtc",
     "+ui/gfx/geometry/size.h",

third_party/blink/renderer/modules/mediastream/pushable_media_stream_audio_source.cc

+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "third_party/blink/renderer/modules/mediastream/pushable_media_stream_audio_source.h"
+
+#include "third_party/blink/public/mojom/mediastream/media_stream.mojom-blink.h"
+#include "third_party/blink/renderer/platform/scheduler/public/post_cross_thread_task.h"
+#include "third_party/blink/renderer/platform/wtf/cross_thread_functional.h"
+#include "third_party/blink/renderer/platform/wtf/functional.h"
+
+namespace blink {
+PushableMediaStreamAudioSource::LivenessBroker::LivenessBroker(
+    PushableMediaStreamAudioSource* source)
+    : source_(source) {}
+
+void PushableMediaStreamAudioSource::LivenessBroker::
+    OnSourceDestroyedOrStopped() {
+  WTF::MutexLocker locker(mutex_);
+  source_ = nullptr;
+}
+
+void PushableMediaStreamAudioSource::LivenessBroker::PushAudioData(
+    std::unique_ptr<PushableAudioData> data,
+    base::TimeTicks reference_time) {
+  WTF::MutexLocker locker(mutex_);
+  if (!source_)
+    return;
+
+  source_->DeliverData(std::move(data), reference_time);
+}
+
+PushableMediaStreamAudioSource::PushableMediaStreamAudioSource(
+    scoped_refptr<base::SingleThreadTaskRunner> main_task_runner,
+    scoped_refptr<base::SequencedTaskRunner> audio_task_runner)
+    : MediaStreamAudioSource(std::move(main_task_runner), /* is_local */ true),
+      audio_task_runner_(std::move(audio_task_runner)),
+      liveness_broker_(
+          base::MakeRefCounted<PushableMediaStreamAudioSource::LivenessBroker>(
+              this)) {}
+
+PushableMediaStreamAudioSource::~PushableMediaStreamAudioSource() {
+  liveness_broker_->OnSourceDestroyedOrStopped();
+}
+
+void PushableMediaStreamAudioSource::PushAudioData(
+    std::unique_ptr<PushableAudioData> data,
+    base::TimeTicks reference_time) {
+  if (audio_task_runner_->RunsTasksInCurrentSequence()) {
+    DeliverData(std::move(data), reference_time);
+    return;
+  }
+
+  PostCrossThreadTask(
+      *audio_task_runner_, FROM_HERE,
+      CrossThreadBindOnce(
+          &PushableMediaStreamAudioSource::LivenessBroker::PushAudioData,
+          liveness_broker_, std::move(data), reference_time));
+}
+
+void PushableMediaStreamAudioSource::DeliverData(
+    std::unique_ptr<PushableAudioData> data,
+    base::TimeTicks reference_time) {
+  DCHECK(audio_task_runner_->RunsTasksInCurrentSequence());
+
+  const media::AudioBus& audio_bus = *data->data();
+  int sample_rate = data->sampleRate();
+
+  media::AudioParameters params = GetAudioParameters();
+  if (!params.IsValid() ||
+      params.format() != media::AudioParameters::AUDIO_PCM_LOW_LATENCY ||
+      last_channels_ != audio_bus.channels() ||
+      last_sample_rate_ != sample_rate || last_frames_ != audio_bus.frames()) {
+    SetFormat(
+        media::AudioParameters(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
+                               media::GuessChannelLayout(audio_bus.channels()),
+                               sample_rate, audio_bus.frames()));
+    last_channels_ = audio_bus.channels();
+    last_sample_rate_ = sample_rate;
+    last_frames_ = audio_bus.frames();
+  }
+
+  DeliverDataToTracks(audio_bus, reference_time);
+}
+
+bool PushableMediaStreamAudioSource::EnsureSourceIsStarted() {
+  DCHECK(GetTaskRunner()->BelongsToCurrentThread());
+  is_running_ = true;
+  return true;
+}
+
+void PushableMediaStreamAudioSource::EnsureSourceIsStopped() {
+  DCHECK(GetTaskRunner()->BelongsToCurrentThread());
+  liveness_broker_->OnSourceDestroyedOrStopped();
+  is_running_ = false;
+}
+
+}  // namespace blink

third_party/blink/renderer/modules/mediastream/pushable_media_stream_audio_source.h

+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIASTREAM_PUSHABLE_MEDIA_STREAM_AUDIO_SOURCE_H_
+#define THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIASTREAM_PUSHABLE_MEDIA_STREAM_AUDIO_SOURCE_H_
+
+#include "base/memory/scoped_refptr.h"
+#include "third_party/blink/renderer/modules/modules_export.h"
+#include "third_party/blink/renderer/platform/mediastream/media_stream_audio_source.h"
+#include "third_party/blink/renderer/platform/wtf/thread_safe_ref_counted.h"
+#include "third_party/blink/renderer/platform/wtf/threading_primitives.h"
+
+namespace blink {
+
+// Wrapper that abstracts how audio data is actually backed, to simplify
+// lifetime guarantees when jumping threads.
+class PushableAudioData {
+ public:
+  virtual ~PushableAudioData() = default;
+  virtual media::AudioBus* data() = 0;
+  virtual int sampleRate() = 0;
+};
+
+// Simplifies the creation of audio tracks.
+class MODULES_EXPORT PushableMediaStreamAudioSource
+    : public MediaStreamAudioSource {
+ public:
+  PushableMediaStreamAudioSource(
+      scoped_refptr<base::SingleThreadTaskRunner> main_task_runner,
+      scoped_refptr<base::SequencedTaskRunner> audio_task_runner);
+
+  ~PushableMediaStreamAudioSource() override;
+
+  // This can be called from any thread, and will push the data on
+  // |audio_task_runner_|
+  void PushAudioData(std::unique_ptr<PushableAudioData> data, base::TimeTicks);
+
+  bool running() const {
+    DCHECK(GetTaskRunner()->BelongsToCurrentThread());
+    return is_running_;
+  }
+
+ private:
+  // Helper class that facilitates the jump to |audio_task_runner_|, by
+  // outliving the given |source| if there is a pending task.
+  class LivenessBroker : public WTF::ThreadSafeRefCounted<LivenessBroker> {
+   public:
+    explicit LivenessBroker(PushableMediaStreamAudioSource* source);
+
+    void OnSourceDestroyedOrStopped();
+    void PushAudioData(std::unique_ptr<PushableAudioData> data,
+                       base::TimeTicks reference_time);
+
+   private:
+    WTF::Mutex mutex_;
+    PushableMediaStreamAudioSource* source_ GUARDED_BY(mutex_);
+  };
+
+  // Actually push data to the audio tracks. Only called on
+  // |audio_task_runner_|.
+  void DeliverData(std::unique_ptr<PushableAudioData> data,
+                   base::TimeTicks reference_time);
+
+  // MediaStreamAudioSource implementation.
+  bool EnsureSourceIsStarted() final;
+  void EnsureSourceIsStopped() final;
+
+  bool is_running_ = false;
+
+  int last_channels_ = 0;
+  int last_frames_ = 0;
+  int last_sample_rate_ = 0;
+
+  scoped_refptr<base::SequencedTaskRunner> audio_task_runner_;
+  scoped_refptr<LivenessBroker> liveness_broker_;
+};
+
+}  // namespace blink
+
+#endif  // THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIASTREAM_PUSHABLE_MEDIA_STREAM_AUDIO_SOURCE_H_

third_party/blink/renderer/modules/mediastream/pushable_media_stream_audio_source_test.cc

+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "third_party/blink/renderer/modules/mediastream/pushable_media_stream_audio_source.h"
+
+#include "base/run_loop.h"
+#include "media/base/bind_to_current_loop.h"
+#include "testing/gmock/include/gmock/gmock.h"
+#include "testing/gtest/include/gtest/gtest.h"
+#include "third_party/blink/public/mojom/mediastream/media_stream.mojom-blink.h"
+#include "third_party/blink/public/platform/modules/mediastream/web_media_stream_audio_sink.h"
+#include "third_party/blink/public/web/web_heap.h"
+#include "third_party/blink/renderer/platform/mediastream/media_stream_audio_track.h"
+#include "third_party/blink/renderer/platform/mediastream/media_stream_component.h"
+#include "third_party/blink/renderer/platform/mediastream/media_stream_source.h"
+#include "third_party/blink/renderer/platform/testing/io_task_runner_testing_platform_support.h"
+#include "third_party/blink/renderer/platform/wtf/cross_thread_functional.h"
+
+using testing::_;
+using testing::WithArg;
+
+namespace blink {
+
+namespace {
+
+class MockMediaStreamAudioSink : public WebMediaStreamAudioSink {
+ public:
+  MockMediaStreamAudioSink() = default;
+
+  MOCK_METHOD2(OnData, void(const media::AudioBus&, base::TimeTicks));
+  MOCK_METHOD1(OnSetFormat, void(const media::AudioParameters&));
+};
+
+class FakeAudioData : public PushableAudioData {
+ public:
+  FakeAudioData(int channels, int frames, int sample_rate)
+      : sample_rate_(sample_rate),
+        audio_bus_(media::AudioBus::Create(channels, frames)) {}
+
+  // PushableMediaStreamAudioSource::AudioData implementation.
+  media::AudioBus* data() override { return audio_bus_.get(); }
+  int sampleRate() override { return sample_rate_; }
+
+ private:
+  int sample_rate_;
+  std::unique_ptr<media::AudioBus> audio_bus_;
+};
+
+}  // namespace
+
+class PushableMediaStreamAudioSourceTest : public testing::Test {
+ public:
+  PushableMediaStreamAudioSourceTest() {
+    // Use the IO thread for testing purposes. This is stricter than an audio
+    // sequenced task runner needs to be.
+    audio_thread_ = Platform::Current()->GetIOTaskRunner();
+    main_thread_ = Thread::MainThread()->GetTaskRunner();
+
+    pushable_audio_source_ =
+        new PushableMediaStreamAudioSource(main_thread_, audio_thread_);
+    stream_source_ = MakeGarbageCollected<MediaStreamSource>(
+        "dummy_source_id", MediaStreamSource::kTypeAudio, "dummy_source_name",
+        false /* remote */);
+    stream_source_->SetPlatformSource(base::WrapUnique(pushable_audio_source_));
+    stream_component_ = MakeGarbageCollected<MediaStreamComponent>(
+        stream_source_->Id(), stream_source_);
+  }
+
+  void TearDown() override {
+    stream_source_ = nullptr;
+    stream_component_ = nullptr;
+    WebHeap::CollectAllGarbageForTesting();
+  }
+
+  bool ConnectSourceToTrack() {
+    return pushable_audio_source_->ConnectToTrack(stream_component_);
+  }
+
+  void SendAndVerifyAudioData(MockMediaStreamAudioSink* mock_sink,
+                              int channels,
+                              int frames,
+                              int sample_rate,
+                              bool expect_format_change) {
+    base::TimeTicks reference_time = base::TimeTicks::Now();
+
+    base::RunLoop run_loop;
+    if (expect_format_change) {
+      EXPECT_CALL(*mock_sink, OnSetFormat(_))
+          .WillOnce([&](const media::AudioParameters& params) {
+            EXPECT_EQ(params.sample_rate(), sample_rate);
+            EXPECT_EQ(params.channels(), channels);
+            EXPECT_EQ(params.frames_per_buffer(), frames);
+          });
+    } else {
+      EXPECT_CALL(*mock_sink, OnSetFormat(_)).Times(0);
+    }
+
+    EXPECT_CALL(*mock_sink, OnData(_, reference_time))
+        .WillOnce(WithArg<0>([&](const media::AudioBus& data) {
+          // Make sure we are on the right thread, and that the threads are
+          // distinct.
+          DCHECK(!main_thread_->BelongsToCurrentThread());
+          DCHECK(audio_thread_->BelongsToCurrentThread());
+
+          EXPECT_EQ(data.channels(), channels);
+          EXPECT_EQ(data.frames(), frames);
+          run_loop.Quit();
+        }));
+
+    pushable_audio_source_->PushAudioData(
+        std::make_unique<FakeAudioData>(channels, frames, sample_rate),
+        reference_time);
+    run_loop.Run();
+  }
+
+ protected:
+  ScopedTestingPlatformSupport<IOTaskRunnerTestingPlatformSupport> platform_;
+
+  Persistent<MediaStreamSource> stream_source_;
+  Persistent<MediaStreamComponent> stream_component_;
+
+  scoped_refptr<base::SingleThreadTaskRunner> main_thread_;
+  scoped_refptr<base::SingleThreadTaskRunner> audio_thread_;
+
+  PushableMediaStreamAudioSource* pushable_audio_source_;
+};
+
+TEST_F(PushableMediaStreamAudioSourceTest, ConnectAndStop) {
+  EXPECT_EQ(MediaStreamSource::kReadyStateLive,
+            stream_source_->GetReadyState());
+  EXPECT_FALSE(pushable_audio_source_->running());
+
+  EXPECT_TRUE(ConnectSourceToTrack());
+  EXPECT_EQ(MediaStreamSource::kReadyStateLive,
+            stream_source_->GetReadyState());
+  EXPECT_TRUE(pushable_audio_source_->running());
+
+  // If the pushable source stops, the MediaStreamSource should stop.
+  pushable_audio_source_->StopSource();
+  EXPECT_EQ(MediaStreamSource::kReadyStateEnded,
+            stream_source_->GetReadyState());
+  EXPECT_FALSE(pushable_audio_source_->running());
+}
+
+TEST_F(PushableMediaStreamAudioSourceTest, FramesPropagateToSink) {
+  EXPECT_TRUE(ConnectSourceToTrack());
+  auto mock_sink =
+      std::make_unique<::testing::StrictMock<MockMediaStreamAudioSink>>();
+
+  WebMediaStreamAudioSink::AddToAudioTrack(
+      mock_sink.get(), WebMediaStreamTrack(stream_component_.Get()));
+
+  constexpr int kChannels = 1;
+  constexpr int kFrames = 256;
+  constexpr int kSampleRate = 8000;
+
+  // The first audio data pushed should trigger a call to OnSetFormat().
+  SendAndVerifyAudioData(mock_sink.get(), kChannels, kFrames, kSampleRate,
+                         /*expect_format_change=*/true);
+
+  // Using the same audio parameters should not trigger OnSetFormat().
+  SendAndVerifyAudioData(mock_sink.get(), kChannels, kFrames, kSampleRate,
+                         /*expect_format_change=*/false);
+
+  // Format changes should trigger OnSetFormat().
+  SendAndVerifyAudioData(mock_sink.get(), kChannels * 2, kFrames * 4,
+                         /*sample_rate=*/44100, /*expect_format_change=*/true);
+
+  WebMediaStreamAudioSink::RemoveFromAudioTrack(
+      mock_sink.get(), WebMediaStreamTrack(stream_component_.Get()));
+}
+
+}  // namespace blink