Add audio support to MediaStreamTrackGenerator

This CL allows MediaStreamGenerator to accept AudioFrames as input, and
to connect to sinks as an audio track.

Bug: 1157608
Change-Id: I0319b77f3b4f537bb62c83bd26ab3e6adafd8cad
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2625279
Commit-Queue: Thomas Guilbert <tguilbert@chromium.org>
Reviewed-by: Guido Urdaneta <guidou@chromium.org>
Cr-Commit-Position: refs/heads/master@{#845088}

third_party/blink/renderer/modules/mediacapturefromelement/html_audio_element_capturer_source_unittest.cc

@@ -17,6 +17,7 @@
 #include "third_party/blink/public/platform/web_string.h"
 #include "third_party/blink/public/platform/webaudiosourceprovider_impl.h"
 #include "third_party/blink/public/web/web_heap.h"
+#include "third_party/blink/renderer/modules/mediastream/mock_media_stream_audio_sink.h"
 #include "third_party/blink/renderer/platform/mediastream/media_stream_audio_track.h"
 #include "third_party/blink/renderer/platform/mediastream/media_stream_component.h"
 #include "third_party/blink/renderer/platform/mediastream/media_stream_source.h"
@@ -31,19 +32,6 @@ static const int kAudioTrackSamplesPerBuffer =
     kAudioTrackSampleRate * kBufferDurationMs /
     base::Time::kMillisecondsPerSecond;
 
-class MockMediaStreamAudioSink final : public blink::WebMediaStreamAudioSink {
- public:
-  MockMediaStreamAudioSink() : blink::WebMediaStreamAudioSink() {}
-  ~MockMediaStreamAudioSink() override = default;
-
-  MOCK_METHOD1(OnSetFormat, void(const media::AudioParameters& params));
-  MOCK_METHOD2(OnData,
-               void(const media::AudioBus& audio_bus,
-                    base::TimeTicks estimated_capture_time));
-
-  DISALLOW_COPY_AND_ASSIGN(MockMediaStreamAudioSink);
-};
-
 // This test needs to bundle together plenty of objects, namely:
 // - a WebAudioSourceProviderImpl, which in turn needs an Audio Sink, in this
 //  case a NullAudioSink. This is needed to plug HTMLAudioElementCapturerSource

third_party/blink/renderer/modules/mediastream/BUILD.gn

@@ -132,6 +132,8 @@ source_set("test_support") {
     "mock_constraint_factory.cc",
     "mock_constraint_factory.h",
     "mock_encoded_video_frame.h",
+    "mock_media_stream_audio_sink.cc",
+    "mock_media_stream_audio_sink.h",
     "mock_media_stream_registry.cc",
     "mock_media_stream_registry.h",
     "mock_media_stream_video_sink.cc",

third_party/blink/renderer/modules/mediastream/media_stream_audio_track_underlying_sink_test.cc

@@ -15,6 +15,7 @@
 #include "third_party/blink/renderer/bindings/modules/v8/v8_audio_frame.h"
 #include "third_party/blink/renderer/core/streams/writable_stream.h"
 #include "third_party/blink/renderer/core/streams/writable_stream_default_writer.h"
+#include "third_party/blink/renderer/modules/mediastream/mock_media_stream_audio_sink.h"
 #include "third_party/blink/renderer/modules/mediastream/pushable_media_stream_audio_source.h"
 #include "third_party/blink/renderer/platform/bindings/exception_state.h"
 #include "third_party/blink/renderer/platform/bindings/to_v8.h"
@@ -26,18 +27,6 @@ using testing::_;
 
 namespace blink {
 
-namespace {
-
-class MockMediaStreamAudioSink : public WebMediaStreamAudioSink {
- public:
-  MockMediaStreamAudioSink() = default;
-
-  MOCK_METHOD2(OnData, void(const media::AudioBus&, base::TimeTicks));
-  MOCK_METHOD1(OnSetFormat, void(const media::AudioParameters&));
-};
-
-}  // namespace
-
 class MediaStreamAudioTrackUnderlyingSinkTest : public testing::Test {
  public:
   MediaStreamAudioTrackUnderlyingSinkTest() {

third_party/blink/renderer/modules/mediastream/media_stream_track_generator.cc

@@ -5,17 +5,21 @@
 #include "third_party/blink/renderer/modules/mediastream/media_stream_track_generator.h"
 
 #include "third_party/blink/public/mojom/web_feature/web_feature.mojom-blink.h"
+#include "third_party/blink/public/platform/platform.h"
 #include "third_party/blink/renderer/core/execution_context/execution_context.h"
 #include "third_party/blink/renderer/core/streams/readable_stream.h"
 #include "third_party/blink/renderer/core/streams/writable_stream.h"
+#include "third_party/blink/renderer/modules/mediastream/media_stream_audio_track_underlying_sink.h"
 #include "third_party/blink/renderer/modules/mediastream/media_stream_utils.h"
 #include "third_party/blink/renderer/modules/mediastream/media_stream_video_track.h"
 #include "third_party/blink/renderer/modules/mediastream/media_stream_video_track_underlying_sink.h"
-#include "third_party/blink/renderer/modules/mediastream/media_stream_video_track_underlying_source.h"
+#include "third_party/blink/renderer/modules/mediastream/pushable_media_stream_audio_source.h"
 #include "third_party/blink/renderer/modules/mediastream/pushable_media_stream_video_source.h"
 #include "third_party/blink/renderer/platform/bindings/exception_state.h"
 #include "third_party/blink/renderer/platform/bindings/script_state.h"
 #include "third_party/blink/renderer/platform/instrumentation/use_counter.h"
+#include "third_party/blink/renderer/platform/mediastream/media_stream_audio_track.h"
+#include "third_party/blink/renderer/platform/scheduler/main_thread/main_thread.h"
 #include "third_party/blink/renderer/platform/wtf/uuid.h"
 
 namespace blink {
@@ -31,19 +35,29 @@ MediaStreamTrackGenerator::MediaStreamTrackGenerator(
                                                       type,
                                                       track_id,
                                                       /*remote=*/false))) {
-  CreateOutputPlatformTrack();
+  if (type == MediaStreamSource::kTypeVideo) {
+    CreateVideoOutputPlatformTrack();
+  } else {
+    DCHECK_EQ(type, MediaStreamSource::kTypeAudio);
+    CreateAudioOutputPlatformTrack();
+  }
   UseCounter::Count(ExecutionContext::From(script_state),
                     WebFeature::kMediaStreamTrackGenerator);
 }
 
 WritableStream* MediaStreamTrackGenerator::writable(ScriptState* script_state) {
-  DCHECK_EQ(kind(), "video");
-  if (!writable_)
+  if (writable_)
+    return writable_;
+
+  if (kind() == "video")
     CreateVideoStream(script_state);
+  else if (kind() == "audio")
+    CreateAudioStream(script_state);
+
   return writable_;
 }
 
-void MediaStreamTrackGenerator::CreateOutputPlatformTrack() {
+void MediaStreamTrackGenerator::CreateVideoOutputPlatformTrack() {
   std::unique_ptr<PushableMediaStreamVideoSource> platform_source =
       std::make_unique<PushableMediaStreamVideoSource>();
   PushableMediaStreamVideoSource* platform_source_ptr = platform_source.get();
@@ -56,6 +70,20 @@ void MediaStreamTrackGenerator::CreateOutputPlatformTrack() {
   Component()->SetPlatformTrack(std::move(platform_track));
 }
 
+void MediaStreamTrackGenerator::CreateAudioOutputPlatformTrack() {
+  // TODO(https:/crbug.com/1168281): use a different thread than the IO thread
+  // to deliver Audio.
+  std::unique_ptr<PushableMediaStreamAudioSource> platform_source =
+      std::make_unique<PushableMediaStreamAudioSource>(
+          GetExecutionContext()->GetTaskRunner(
+              TaskType::kInternalMediaRealTime),
+          Platform::Current()->GetIOTaskRunner());
+
+  platform_source->ConnectToTrack(Component());
+
+  Component()->Source()->SetPlatformSource(std::move(platform_source));
+}
+
 void MediaStreamTrackGenerator::CreateVideoStream(ScriptState* script_state) {
   DCHECK(!writable_);
   PushableMediaStreamVideoSource* source =
@@ -67,30 +95,46 @@ void MediaStreamTrackGenerator::CreateVideoStream(ScriptState* script_state) {
       script_state, video_underlying_sink_, /*high_water_mark=*/1);
 }
 
+void MediaStreamTrackGenerator::CreateAudioStream(ScriptState* script_state) {
+  DCHECK(!writable_);
+  PushableMediaStreamAudioSource* source =
+      static_cast<PushableMediaStreamAudioSource*>(
+          Component()->Source()->GetPlatformSource());
+  audio_underlying_sink_ =
+      MakeGarbageCollected<MediaStreamAudioTrackUnderlyingSink>(source);
+  writable_ = WritableStream::CreateWithCountQueueingStrategy(
+      script_state, audio_underlying_sink_, /*high_water_mark=*/1);
+}
+
 MediaStreamTrackGenerator* MediaStreamTrackGenerator::Create(
     ScriptState* script_state,
     const String& kind,
     ExceptionState& exception_state) {
-  if (kind != "video") {
+  if (!script_state->ContextIsValid()) {
     exception_state.ThrowDOMException(DOMExceptionCode::kNotSupportedError,
-                                      "Only video tracks are supported");
+                                      "Invalid context");
     return nullptr;
   }
 
-  if (!script_state->ContextIsValid()) {
-    exception_state.ThrowDOMException(DOMExceptionCode::kNotSupportedError,
-                                      "Invalid context");
+  MediaStreamSource::StreamType type;
 
+  if (kind == "video") {
+    type = MediaStreamSource::kTypeVideo;
+  } else if (kind == "audio") {
+    type = MediaStreamSource::kTypeAudio;
+  } else {
+    exception_state.ThrowTypeError("Invalid track generator kind");
     return nullptr;
   }
 
   return MakeGarbageCollected<MediaStreamTrackGenerator>(
-      script_state, MediaStreamSource::kTypeVideo,
+      script_state, type,
       /*track_id=*/WTF::CreateCanonicalUUIDString());
 }
 
 void MediaStreamTrackGenerator::Trace(Visitor* visitor) const {
   visitor->Trace(video_underlying_sink_);
+  visitor->Trace(audio_underlying_sink_);
   visitor->Trace(writable_);
   MediaStreamTrack::Trace(visitor);
 }

third_party/blink/renderer/modules/mediastream/media_stream_track_generator.h

@@ -11,6 +11,7 @@
 
 namespace blink {
 
+class MediaStreamAudioTrackUnderlyingSink;
 class MediaStreamVideoTrackUnderlyingSink;
 class ScriptState;
 class WritableStream;
@@ -34,9 +35,13 @@ class MODULES_EXPORT MediaStreamTrackGenerator : public MediaStreamTrack {
   void Trace(Visitor* visitor) const override;
 
  private:
-  void CreateOutputPlatformTrack();
+  void CreateAudioOutputPlatformTrack();
+  void CreateAudioStream(ScriptState* script_state);
+
+  void CreateVideoOutputPlatformTrack();
   void CreateVideoStream(ScriptState* script_state);
 
+  Member<MediaStreamAudioTrackUnderlyingSink> audio_underlying_sink_;
   Member<MediaStreamVideoTrackUnderlyingSink> video_underlying_sink_;
   Member<WritableStream> writable_;
 };

third_party/blink/renderer/modules/mediastream/media_stream_track_generator_test.cc

@@ -17,8 +17,10 @@
 #include "third_party/blink/renderer/core/streams/writable_stream.h"
 #include "third_party/blink/renderer/core/streams/writable_stream_default_writer.h"
 #include "third_party/blink/renderer/modules/mediastream/media_stream_video_track.h"
+#include "third_party/blink/renderer/modules/mediastream/mock_media_stream_audio_sink.h"
 #include "third_party/blink/renderer/modules/mediastream/mock_media_stream_video_sink.h"
 #include "third_party/blink/renderer/modules/mediastream/pushable_media_stream_video_source.h"
+#include "third_party/blink/renderer/modules/webcodecs/audio_frame.h"
 #include "third_party/blink/renderer/modules/webcodecs/video_frame.h"
 #include "third_party/blink/renderer/platform/bindings/exception_code.h"
 #include "third_party/blink/renderer/platform/testing/io_task_runner_testing_platform_support.h"
@@ -39,6 +41,18 @@ ScriptValue CreateVideoFrameChunk(ScriptState* script_state) {
                           script_state->GetIsolate()));
 }
 
+ScriptValue CreateAudioFrameChunk(ScriptState* script_state) {
+  AudioFrame* audio_frame =
+      MakeGarbageCollected<AudioFrame>(media::AudioBuffer::CreateEmptyBuffer(
+          media::ChannelLayout::CHANNEL_LAYOUT_STEREO,
+          /*channel_count=*/2,
+          /*sample_rate=*/44100,
+          /*frame_count=*/500, base::TimeDelta()));
+  return ScriptValue(script_state->GetIsolate(),
+                     ToV8(audio_frame, script_state->GetContext()->Global(),
+                          script_state->GetIsolate()));
+}
+
 }  // namespace
 
 class MediaStreamTrackGeneratorTest : public testing::Test {
@@ -92,6 +106,49 @@ TEST_F(MediaStreamTrackGeneratorTest, VideoFramesAreWritten) {
   EXPECT_TRUE(generator->Ended());
 }
 
+TEST_F(MediaStreamTrackGeneratorTest, AudioFramesAreWritten) {
+  V8TestingScope v8_scope;
+  ScriptState* script_state = v8_scope.GetScriptState();
+  MediaStreamTrackGenerator* generator = MediaStreamTrackGenerator::Create(
+      script_state, "audio", v8_scope.GetExceptionState());
+
+  MockMediaStreamAudioSink media_stream_audio_sink;
+  WebMediaStreamAudioSink::AddToAudioTrack(
+      &media_stream_audio_sink, WebMediaStreamTrack(generator->Component()));
+
+  base::RunLoop sink_loop;
+  EXPECT_CALL(media_stream_audio_sink, OnData(_, _))
+      .WillOnce(testing::WithArg<0>([&](const media::AudioBus& data) {
+        EXPECT_NE(data.frames(), 0);
+        sink_loop.Quit();
+      }));
+  ExceptionState& exception_state = v8_scope.GetExceptionState();
+  auto* writer = generator->writable(script_state)
+                     ->getWriter(script_state, exception_state);
+  ScriptPromiseTester write_tester(
+      script_state,
+      writer->write(script_state, CreateAudioFrameChunk(script_state),
+                    exception_state));
+  EXPECT_FALSE(write_tester.IsFulfilled());
+  write_tester.WaitUntilSettled();
+  sink_loop.Run();
+  EXPECT_TRUE(write_tester.IsFulfilled());
+  EXPECT_FALSE(exception_state.HadException());
+
+  // Closing the writable stream should stop the track.
+  writer->releaseLock(script_state);
+  EXPECT_FALSE(generator->Ended());
+  ScriptPromiseTester close_tester(
+      script_state,
+      generator->writable(script_state)->close(script_state, exception_state));
+  close_tester.WaitUntilSettled();
+  EXPECT_FALSE(exception_state.HadException());
+  EXPECT_TRUE(generator->Ended());
+
+  WebMediaStreamAudioSink::RemoveFromAudioTrack(
+      &media_stream_audio_sink, WebMediaStreamTrack(generator->Component()));
+}
+
 TEST_F(MediaStreamTrackGeneratorTest, FramesDoNotFlowOnStoppedGenerator) {
   V8TestingScope v8_scope;
   ScriptState* script_state = v8_scope.GetScriptState();
@@ -216,16 +273,4 @@ TEST_F(MediaStreamTrackGeneratorTest, CloneStopSource) {
   EXPECT_TRUE(clone->Ended());
 }
 
-// TODO(crbug.com/1142955): Add support for audio.
-TEST_F(MediaStreamTrackGeneratorTest, Audio) {
-  V8TestingScope v8_scope;
-  ScriptState* script_state = v8_scope.GetScriptState();
-  MediaStreamTrackGenerator* generator = MediaStreamTrackGenerator::Create(
-      script_state, "audio", v8_scope.GetExceptionState());
-  EXPECT_EQ(generator, nullptr);
-  EXPECT_TRUE(v8_scope.GetExceptionState().HadException());
-  EXPECT_EQ(static_cast<DOMExceptionCode>(v8_scope.GetExceptionState().Code()),
-            DOMExceptionCode::kNotSupportedError);
-}
-
 }  // namespace blink

third_party/blink/renderer/modules/mediastream/mock_media_stream_audio_sink.cc

+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "third_party/blink/renderer/modules/mediastream/mock_media_stream_audio_sink.h"
+
+namespace blink {
+MockMediaStreamAudioSink::MockMediaStreamAudioSink() = default;
+MockMediaStreamAudioSink::~MockMediaStreamAudioSink() = default;
+}  // namespace blink

third_party/blink/renderer/modules/mediastream/mock_media_stream_audio_sink.h

+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIASTREAM_MOCK_MEDIA_STREAM_AUDIO_SINK_H_
+#define THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIASTREAM_MOCK_MEDIA_STREAM_AUDIO_SINK_H_
+
+#include "media/base/audio_bus.h"
+#include "media/base/audio_parameters.h"
+#include "testing/gmock/include/gmock/gmock.h"
+#include "third_party/blink/public/platform/modules/mediastream/web_media_stream_audio_sink.h"
+
+namespace blink {
+
+class MockMediaStreamAudioSink : public WebMediaStreamAudioSink {
+ public:
+  MockMediaStreamAudioSink();
+  ~MockMediaStreamAudioSink() override;
+
+  MOCK_METHOD2(OnData, void(const media::AudioBus&, base::TimeTicks));
+  MOCK_METHOD1(OnSetFormat, void(const media::AudioParameters&));
+};
+
+}  // namespace blink
+
+#endif  // THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIASTREAM_MOCK_MEDIA_STREAM_AUDIO_SINK_H_

third_party/blink/renderer/modules/mediastream/processed_local_audio_source_test.cc

@@ -50,10 +50,10 @@ constexpr int kExpectedSourceBufferSize = kRequestedBufferSize;
 // output end of its FIFO.
 constexpr int kExpectedOutputBufferSize = kSampleRate / 100;
 
-class MockMediaStreamAudioSink : public WebMediaStreamAudioSink {
+class FormatCheckingMockAudioSink : public WebMediaStreamAudioSink {
  public:
-  MockMediaStreamAudioSink() {}
-  ~MockMediaStreamAudioSink() override {}
+  FormatCheckingMockAudioSink() = default;
+  ~FormatCheckingMockAudioSink() override = default;
 
   void OnData(const media::AudioBus& audio_bus,
               base::TimeTicks estimated_capture_time) override {
@@ -173,8 +173,7 @@ TEST_F(ProcessedLocalAudioSourceTest, VerifyAudioFlowWithoutAudioProcessing) {
   CheckOutputFormatMatches(audio_source()->GetAudioParameters());
 
   // Connect a sink to the track.
-  std::unique_ptr<MockMediaStreamAudioSink> sink(
-      new MockMediaStreamAudioSink());
+  auto sink = std::make_unique<FormatCheckingMockAudioSink>();
   EXPECT_CALL(*sink, FormatIsSet(_))
       .WillOnce(Invoke(this, &ThisTest::CheckOutputFormatMatches));
   MediaStreamAudioTrack::From(audio_track())->AddSink(sink.get());

third_party/blink/renderer/modules/mediastream/pushable_media_stream_audio_source_test.cc

@@ -11,6 +11,7 @@
 #include "third_party/blink/public/mojom/mediastream/media_stream.mojom-blink.h"
 #include "third_party/blink/public/platform/modules/mediastream/web_media_stream_audio_sink.h"
 #include "third_party/blink/public/web/web_heap.h"
+#include "third_party/blink/renderer/modules/mediastream/mock_media_stream_audio_sink.h"
 #include "third_party/blink/renderer/platform/mediastream/media_stream_audio_track.h"
 #include "third_party/blink/renderer/platform/mediastream/media_stream_component.h"
 #include "third_party/blink/renderer/platform/mediastream/media_stream_source.h"
@@ -24,14 +25,6 @@ namespace blink {
 
 namespace {
 
-class MockMediaStreamAudioSink : public WebMediaStreamAudioSink {
- public:
-  MockMediaStreamAudioSink() = default;
-
-  MOCK_METHOD2(OnData, void(const media::AudioBus&, base::TimeTicks));
-  MOCK_METHOD1(OnSetFormat, void(const media::AudioParameters&));
-};
-
 class FakeAudioData : public PushableAudioData {
  public:
   FakeAudioData(int channels, int frames, int sample_rate)