Add Audio support to MediaStreamTrackProcessor

This CL allows a MediaStreamTrackProcessor to be created from an audio
track. This will allow AudioFrames to be read from an audio track, via
a ReadableStream interface.

Bug: 1157608
Change-Id: Id0406309a42dd76896c8481c83a85d5a376cad42
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2602598
Commit-Queue: Thomas Guilbert <tguilbert@chromium.org>
Reviewed-by: Guido Urdaneta <guidou@chromium.org>
Cr-Commit-Position: refs/heads/master@{#846403}

third_party/blink/renderer/modules/mediastream/media_stream_track_generator.cc

@@ -111,7 +111,7 @@ MediaStreamTrackGenerator* MediaStreamTrackGenerator::Create(
     const String& kind,
     ExceptionState& exception_state) {
   if (!script_state->ContextIsValid()) {
-    exception_state.ThrowDOMException(DOMExceptionCode::kNotSupportedError,
+    exception_state.ThrowDOMException(DOMExceptionCode::kInvalidStateError,
                                       "Invalid context");
     return nullptr;
   }

third_party/blink/renderer/modules/mediastream/media_stream_track_processor.cc

@@ -7,6 +7,7 @@
 #include "third_party/blink/public/mojom/web_feature/web_feature.mojom-blink.h"
 #include "third_party/blink/renderer/core/execution_context/execution_context.h"
 #include "third_party/blink/renderer/core/streams/readable_stream.h"
+#include "third_party/blink/renderer/modules/mediastream/media_stream_audio_track_underlying_source.h"
 #include "third_party/blink/renderer/modules/mediastream/media_stream_utils.h"
 #include "third_party/blink/renderer/modules/mediastream/media_stream_video_track.h"
 #include "third_party/blink/renderer/modules/mediastream/media_stream_video_track_underlying_source.h"
@@ -28,9 +29,14 @@ MediaStreamTrackProcessor::MediaStreamTrackProcessor(
 }
 
 ReadableStream* MediaStreamTrackProcessor::readable(ScriptState* script_state) {
-  DCHECK_EQ(input_track_->Source()->GetType(), MediaStreamSource::kTypeVideo);
-  if (!source_stream_)
+  if (source_stream_)
+    return source_stream_;
+
+  if (input_track_->Source()->GetType() == MediaStreamSource::kTypeVideo)
     CreateVideoSourceStream(script_state);
+  else
+    CreateAudioSourceStream(script_state);
+
   return source_stream_;
 }
 
@@ -44,26 +50,28 @@ void MediaStreamTrackProcessor::CreateVideoSourceStream(
       script_state, video_underlying_source_, /*high_water_mark=*/0);
 }
 
+void MediaStreamTrackProcessor::CreateAudioSourceStream(
+    ScriptState* script_state) {
+  DCHECK(!source_stream_);
+  audio_underlying_source_ =
+      MakeGarbageCollected<MediaStreamAudioTrackUnderlyingSource>(
+          script_state, input_track_, buffer_size_);
+  source_stream_ = ReadableStream::CreateWithCountQueueingStrategy(
+      script_state, audio_underlying_source_, /*high_water_mark=*/0);
+}
+
 MediaStreamTrackProcessor* MediaStreamTrackProcessor::Create(
     ScriptState* script_state,
     MediaStreamTrack* track,
     uint16_t buffer_size,
     ExceptionState& exception_state) {
   if (!track) {
-    exception_state.ThrowDOMException(DOMExceptionCode::kOperationError,
-                                      "Input track cannot be null");
+    exception_state.ThrowTypeError("Input track cannot be null");
     return nullptr;
   }
 
   if (track->readyState() == "ended") {
-    exception_state.ThrowDOMException(DOMExceptionCode::kInvalidStateError,
-                                      "Input track cannot be ended");
-    return nullptr;
-  }
-
-  if (track->kind() != "video") {
-    exception_state.ThrowDOMException(DOMExceptionCode::kNotSupportedError,
-                                      "Only video tracks are supported");
+    exception_state.ThrowTypeError("Input track cannot be ended");
     return nullptr;
   }
 
@@ -83,8 +91,7 @@ MediaStreamTrackProcessor* MediaStreamTrackProcessor::Create(
     MediaStreamTrack* track,
     ExceptionState& exception_state) {
   if (!track) {
-    exception_state.ThrowDOMException(DOMExceptionCode::kOperationError,
-                                      "Input track cannot be null");
+    exception_state.ThrowTypeError("Input track cannot be null");
     return nullptr;
   }
   // Using 1 as default buffer size for video since by default we do not want
@@ -98,6 +105,7 @@ MediaStreamTrackProcessor* MediaStreamTrackProcessor::Create(
 
 void MediaStreamTrackProcessor::Trace(Visitor* visitor) const {
   visitor->Trace(input_track_);
+  visitor->Trace(audio_underlying_source_);
   visitor->Trace(video_underlying_source_);
   visitor->Trace(source_stream_);
   ScriptWrappable::Trace(visitor);

third_party/blink/renderer/modules/mediastream/media_stream_track_processor.h

@@ -14,6 +14,7 @@ namespace blink {
 
 class MediaStreamComponent;
 class MediaStreamVideoTrackUnderlyingSource;
+class MediaStreamAudioTrackUnderlyingSource;
 class ScriptState;
 class ReadableStream;
 
@@ -44,9 +45,11 @@ class MODULES_EXPORT MediaStreamTrackProcessor : public ScriptWrappable {
 
  private:
   void CreateVideoSourceStream(ScriptState* script_state);
+  void CreateAudioSourceStream(ScriptState* script_state);
 
   Member<MediaStreamComponent> input_track_;
   Member<MediaStreamVideoTrackUnderlyingSource> video_underlying_source_;
+  Member<MediaStreamAudioTrackUnderlyingSource> audio_underlying_source_;
   Member<ReadableStream> source_stream_;
   uint16_t buffer_size_;
 };

third_party/blink/renderer/modules/mediastream/media_stream_track_processor_test.cc

@@ -18,9 +18,12 @@
 #include "third_party/blink/renderer/modules/mediastream/media_stream_track.h"
 #include "third_party/blink/renderer/modules/mediastream/media_stream_track_generator.h"
 #include "third_party/blink/renderer/modules/mediastream/media_stream_video_track.h"
+#include "third_party/blink/renderer/modules/mediastream/mock_media_stream_audio_sink.h"
 #include "third_party/blink/renderer/modules/mediastream/mock_media_stream_video_sink.h"
 #include "third_party/blink/renderer/modules/mediastream/mock_media_stream_video_source.h"
+#include "third_party/blink/renderer/modules/mediastream/pushable_media_stream_audio_source.h"
 #include "third_party/blink/renderer/modules/mediastream/pushable_media_stream_video_source.h"
+#include "third_party/blink/renderer/modules/webcodecs/audio_frame_serialization_data.h"
 #include "third_party/blink/renderer/platform/bindings/exception_state.h"
 #include "third_party/blink/renderer/platform/mediastream/media_stream_audio_source.h"
 #include "third_party/blink/renderer/platform/mediastream/media_stream_audio_track.h"
@@ -33,6 +36,13 @@ namespace blink {
 
 namespace {
 
+std::unique_ptr<PushableMediaStreamAudioSource> CreatePushableAudioSource() {
+  // Use the IO thread for testing purposes.
+  return std::make_unique<PushableMediaStreamAudioSource>(
+      Thread::MainThread()->GetTaskRunner(),
+      Platform::Current()->GetIOTaskRunner());
+}
+
 PushableMediaStreamVideoSource* CreatePushableVideoSource() {
   PushableMediaStreamVideoSource* pushable_video_source =
       new PushableMediaStreamVideoSource();
@@ -53,21 +63,22 @@ MediaStreamTrack* CreateVideoMediaStreamTrack(ExecutionContext* context,
                    /*enabled=*/true));
 }
 
-MediaStreamTrack* CreateAudioMediaStreamTrack(ExecutionContext* context) {
-  std::unique_ptr<MediaStreamAudioSource> audio_source =
-      std::make_unique<MediaStreamAudioSource>(
-          blink::scheduler::GetSingleThreadTaskRunnerForTesting(),
-          /*is_local_source=*/false);
+MediaStreamTrack* CreateAudioMediaStreamTrack(
+    ExecutionContext* context,
+    std::unique_ptr<PushableMediaStreamAudioSource> source) {
+  auto* source_ptr = source.get();
+
   MediaStreamSource* media_stream_source =
       MakeGarbageCollected<MediaStreamSource>(
           "source_id", MediaStreamSource::kTypeAudio, "source_name",
-          /*is_remote=*/false);
-  media_stream_source->SetPlatformSource(std::move(audio_source));
-  std::unique_ptr<MediaStreamAudioTrack> audio_track =
-      std::make_unique<MediaStreamAudioTrack>(/*is_local_track=*/false);
+          /*remote=*/false);
+  media_stream_source->SetPlatformSource(std::move(source));
+
   MediaStreamComponent* component =
       MakeGarbageCollected<MediaStreamComponent>(media_stream_source);
-  component->SetPlatformTrack(std::move(audio_track));
+
+  source_ptr->ConnectToTrack(component);
+
   return MakeGarbageCollected<MediaStreamTrack>(context, component);
 }
 
@@ -131,6 +142,53 @@ TEST_F(MediaStreamTrackProcessorTest, VideoFramesAreExposed) {
   EXPECT_EQ(mock_video_sink.last_frame(), frame);
 }
 
+TEST_F(MediaStreamTrackProcessorTest, AudioFramesAreExposed) {
+  V8TestingScope v8_scope;
+  ScriptState* script_state = v8_scope.GetScriptState();
+  ExceptionState& exception_state = v8_scope.GetExceptionState();
+  std::unique_ptr<PushableMediaStreamAudioSource> pushable_audio_source =
+      CreatePushableAudioSource();
+  auto* pushable_source_ptr = pushable_audio_source.get();
+  MediaStreamTrackProcessor* track_processor =
+      MediaStreamTrackProcessor::Create(
+          script_state,
+          CreateAudioMediaStreamTrack(v8_scope.GetExecutionContext(),
+                                      std::move(pushable_audio_source)),
+          exception_state);
+  EXPECT_FALSE(exception_state.HadException());
+  EXPECT_EQ(track_processor->input_track()->Source()->GetPlatformSource(),
+            pushable_source_ptr);
+
+  MockMediaStreamAudioSink mock_audio_sink;
+  WebMediaStreamAudioSink::AddToAudioTrack(
+      &mock_audio_sink, WebMediaStreamTrack(track_processor->input_track()));
+
+  auto* reader =
+      track_processor->readable(script_state)
+          ->GetDefaultReaderForTesting(script_state, exception_state);
+  EXPECT_FALSE(exception_state.HadException());
+
+  // Deliver a frame.
+  base::RunLoop sink_loop;
+  EXPECT_CALL(mock_audio_sink, OnData(_, _))
+      .WillOnce(base::test::RunOnceClosure(sink_loop.QuitClosure()));
+  pushable_source_ptr->PushAudioData(AudioFrameSerializationData::Wrap(
+      media::AudioBus::Create(/*channels=*/2, /*frames=*/100),
+      /*sample_rate=*/8000, base::TimeDelta::FromSeconds(1)));
+
+  ScriptPromiseTester read_tester(script_state,
+                                  reader->read(script_state, exception_state));
+  EXPECT_FALSE(read_tester.IsFulfilled());
+  read_tester.WaitUntilSettled();
+  EXPECT_FALSE(exception_state.HadException());
+  EXPECT_TRUE(read_tester.IsFulfilled());
+  EXPECT_TRUE(read_tester.Value().IsObject());
+  sink_loop.Run();
+
+  WebMediaStreamAudioSink::RemoveFromAudioTrack(
+      &mock_audio_sink, WebMediaStreamTrack(track_processor->input_track()));
+}
+
 TEST_F(MediaStreamTrackProcessorTest, CanceledReadableDisconnects) {
   V8TestingScope v8_scope;
   ScriptState* script_state = v8_scope.GetScriptState();
@@ -233,8 +291,8 @@ TEST_F(MediaStreamTrackProcessorTest, NullInputTrack) {
 
   EXPECT_EQ(track_processor, nullptr);
   EXPECT_TRUE(exception_state.HadException());
-  EXPECT_EQ(static_cast<DOMExceptionCode>(v8_scope.GetExceptionState().Code()),
-            DOMExceptionCode::kOperationError);
+  EXPECT_EQ(static_cast<ESErrorType>(v8_scope.GetExceptionState().Code()),
+            ESErrorType::kTypeError);
 }
 
 TEST_F(MediaStreamTrackProcessorTest, EndedTrack) {
@@ -251,25 +309,8 @@ TEST_F(MediaStreamTrackProcessorTest, EndedTrack) {
 
   EXPECT_EQ(track_processor, nullptr);
   EXPECT_TRUE(exception_state.HadException());
-  EXPECT_EQ(static_cast<DOMExceptionCode>(v8_scope.GetExceptionState().Code()),
-            DOMExceptionCode::kInvalidStateError);
-}
-
-// TODO(crbug.com/1142955): Add support for audio.
-TEST_F(MediaStreamTrackProcessorTest, Audio) {
-  V8TestingScope v8_scope;
-  ScriptState* script_state = v8_scope.GetScriptState();
-  ExceptionState& exception_state = v8_scope.GetExceptionState();
-  MediaStreamTrack* media_stream_track =
-      CreateAudioMediaStreamTrack(v8_scope.GetExecutionContext());
-  MediaStreamTrackProcessor* track_processor =
-      MediaStreamTrackProcessor::Create(script_state, media_stream_track,
-                                        exception_state);
-
-  EXPECT_EQ(track_processor, nullptr);
-  EXPECT_TRUE(exception_state.HadException());
-  EXPECT_EQ(static_cast<DOMExceptionCode>(v8_scope.GetExceptionState().Code()),
-            DOMExceptionCode::kNotSupportedError);
+  EXPECT_EQ(static_cast<ESErrorType>(v8_scope.GetExceptionState().Code()),
+            ESErrorType::kTypeError);
 }
 
 }  // namespace blink