Commit 5f1112ab authored by Guido Urdaneta's avatar Guido Urdaneta Committed by Commit Bot

[BreakoutBox] Add MediaStreamTrackProcessor

This CL introduces MediaStreamTrackProcessor, an object that allows
exposing a MediaStreamTrack's media frames as a readable stream,
enabling applications to perform custom processing.
This basically allows the creation of a custom sink, in terms of the
model proposed by the Media Capture and Streams spec [1].
A MediaStreamTrackProcessor can be combined with a TransformStream and a
MediaStreamTrackGenerator to effectively insert a processing step inside
an existing track.

[1] https://w3c.github.io/mediacapture-main/#the-model-sources-sinks-constraints-and-settings

Bug: 1142955
Change-Id: I2a119f0c2ea9ce50432509286bd0f43758d8a272
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2524322
Commit-Queue: Guido Urdaneta <guidou@chromium.org>
Reviewed-by: default avatarHarald Alvestrand <hta@chromium.org>
Reviewed-by: default avatarKinuko Yasuda <kinuko@chromium.org>
Cr-Commit-Position: refs/heads/master@{#826153}
parent 145656d7
......@@ -3055,6 +3055,7 @@ enum WebFeature {
kV8HTMLDialogElement_ShowModal_Method = 3726,
kAdFrameDetected = 3727,
kMediaStreamTrackGenerator = 3728,
kMediaStreamTrackProcessor = 3729,
// Add new features immediately above this line. Don't change assigned
// numbers of any item, and don't reuse removed slots.
......
......@@ -1679,6 +1679,8 @@ generated_interface_sources_in_modules = [
"$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_media_stream_track_event.h",
"$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_media_stream_track_generator.cc",
"$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_media_stream_track_generator.h",
"$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_media_stream_track_processor.cc",
"$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_media_stream_track_processor.h",
"$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_merchant_validation_event.cc",
"$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_merchant_validation_event.h",
"$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_metadata.cc",
......
......@@ -384,6 +384,7 @@ static_idl_files_in_modules = get_path_info(
"//third_party/blink/renderer/modules/mediastream/media_stream_track_event.idl",
"//third_party/blink/renderer/modules/mediastream/media_stream_track_event_init.idl",
"//third_party/blink/renderer/modules/mediastream/media_stream_track_generator.idl",
"//third_party/blink/renderer/modules/mediastream/media_stream_track_processor.idl",
"//third_party/blink/renderer/modules/mediastream/media_track_capabilities.idl",
"//third_party/blink/renderer/modules/mediastream/media_track_constraint_set.idl",
"//third_party/blink/renderer/modules/mediastream/media_track_constraints.idl",
......
......@@ -349,6 +349,7 @@ source_set("unit_tests") {
"mediastream/media_stream_constraints_util_video_device_test.cc",
"mediastream/media_stream_device_observer_test.cc",
"mediastream/media_stream_track_generator_test.cc",
"mediastream/media_stream_track_processor_test.cc",
"mediastream/media_stream_video_capturer_source_test.cc",
"mediastream/media_stream_video_renderer_sink_test.cc",
"mediastream/media_stream_video_source_test.cc",
......
......@@ -56,6 +56,8 @@ blink_modules_sources("mediastream") {
"media_stream_track_event.h",
"media_stream_track_generator.cc",
"media_stream_track_generator.h",
"media_stream_track_processor.cc",
"media_stream_track_processor.h",
"media_stream_utils.cc",
"media_stream_utils.h",
"media_stream_video_capturer_source.cc",
......
......@@ -11,6 +11,7 @@ modules_idl_files = [
"media_stream_track.idl",
"media_stream_track_event.idl",
"media_stream_track_generator.idl",
"media_stream_track_processor.idl",
"overconstrained_error.idl",
]
......
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "third_party/blink/renderer/modules/mediastream/media_stream_track_processor.h"
#include "third_party/blink/public/mojom/web_feature/web_feature.mojom-blink.h"
#include "third_party/blink/renderer/core/execution_context/execution_context.h"
#include "third_party/blink/renderer/core/streams/readable_stream.h"
#include "third_party/blink/renderer/modules/mediastream/media_stream_utils.h"
#include "third_party/blink/renderer/modules/mediastream/media_stream_video_track.h"
#include "third_party/blink/renderer/modules/mediastream/media_stream_video_track_underlying_source.h"
#include "third_party/blink/renderer/platform/bindings/exception_state.h"
#include "third_party/blink/renderer/platform/bindings/script_state.h"
#include "third_party/blink/renderer/platform/instrumentation/use_counter.h"
#include "third_party/blink/renderer/platform/wtf/uuid.h"
namespace blink {
MediaStreamTrackProcessor::MediaStreamTrackProcessor(
ScriptState* script_state,
MediaStreamComponent* input_track)
: input_track_(input_track) {
DCHECK(input_track_);
UseCounter::Count(ExecutionContext::From(script_state),
WebFeature::kMediaStreamTrackProcessor);
}
ReadableStream* MediaStreamTrackProcessor::readable(ScriptState* script_state) {
DCHECK_EQ(input_track_->Source()->GetType(), MediaStreamSource::kTypeVideo);
if (!source_stream_)
CreateVideoSourceStream(script_state);
return source_stream_;
}
void MediaStreamTrackProcessor::CreateVideoSourceStream(
ScriptState* script_state) {
DCHECK(!source_stream_);
video_underlying_source_ =
MakeGarbageCollected<MediaStreamVideoTrackUnderlyingSource>(script_state,
input_track_);
source_stream_ = ReadableStream::CreateWithCountQueueingStrategy(
script_state, video_underlying_source_, /*high_water_mark=*/0);
}
MediaStreamTrackProcessor* MediaStreamTrackProcessor::Create(
ScriptState* script_state,
MediaStreamTrack* track,
ExceptionState& exception_state) {
if (!track) {
exception_state.ThrowDOMException(DOMExceptionCode::kOperationError,
"Input track cannot be null");
return nullptr;
}
if (track->kind() != "video") {
exception_state.ThrowDOMException(DOMExceptionCode::kNotSupportedError,
"Only video tracks are supported");
return nullptr;
}
if (!script_state->ContextIsValid()) {
exception_state.ThrowDOMException(DOMExceptionCode::kNotSupportedError,
"The context has been destroyed");
return nullptr;
}
return MakeGarbageCollected<MediaStreamTrackProcessor>(script_state,
track->Component());
}
void MediaStreamTrackProcessor::Trace(Visitor* visitor) const {
visitor->Trace(input_track_);
visitor->Trace(video_underlying_source_);
visitor->Trace(source_stream_);
ScriptWrappable::Trace(visitor);
}
} // namespace blink
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIASTREAM_MEDIA_STREAM_TRACK_PROCESSOR_H_
#define THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIASTREAM_MEDIA_STREAM_TRACK_PROCESSOR_H_
#include "third_party/blink/renderer/modules/mediastream/media_stream_track.h"
#include "third_party/blink/renderer/modules/modules_export.h"
#include "third_party/blink/renderer/platform/bindings/script_wrappable.h"
#include "third_party/blink/renderer/platform/heap/impl/heap.h"
namespace blink {
class MediaStreamComponent;
class MediaStreamVideoTrackUnderlyingSource;
class ScriptState;
class ReadableStream;
class MODULES_EXPORT MediaStreamTrackProcessor : public ScriptWrappable {
DEFINE_WRAPPERTYPEINFO();
public:
static MediaStreamTrackProcessor* Create(ScriptState*,
MediaStreamTrack*,
ExceptionState&);
MediaStreamTrackProcessor(ScriptState*, MediaStreamComponent*);
MediaStreamTrackProcessor(const MediaStreamTrackProcessor&) = delete;
MediaStreamTrackProcessor& operator=(const MediaStreamTrackProcessor&) =
delete;
// MediaStreamTrackProcessor interface
ReadableStream* readable(ScriptState* script_state);
MediaStreamComponent* input_track() { return input_track_; }
void Trace(Visitor* visitor) const override;
private:
void CreateVideoSourceStream(ScriptState* script_state);
Member<MediaStreamComponent> input_track_;
Member<MediaStreamVideoTrackUnderlyingSource> video_underlying_source_;
Member<ReadableStream> source_stream_;
};
} // namespace blink
#endif // THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIASTREAM_MEDIA_STREAM_TRACK_PROCESSOR_H_
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// See spec in developement at
// https://w3c.github.io/mediacapture-insertable-streams/
[
Exposed=Window,
RuntimeEnabled=MediaStreamInsertableStreams
]
interface MediaStreamTrackProcessor {
[CallWith=ScriptState, RaisesException, MeasureAs=MediaStreamTrackProcessor]
constructor(MediaStreamTrack track);
// This stream returns VideoFrame objects.
// TODO(crbug.com/1142955): Add support for audio.
[CallWith=ScriptState] readonly attribute ReadableStream readable;
// TODO(crbug.com/1142955): Add |writable| attribute that takes control signals.
};
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "third_party/blink/renderer/modules/mediastream/media_stream_track_processor.h"
#include "base/run_loop.h"
#include "base/test/gmock_callback_support.h"
#include "media/base/video_frame.h"
#include "testing/gtest/include/gtest/gtest.h"
#include "third_party/blink/public/platform/scheduler/test/renderer_scheduler_test_support.h"
#include "third_party/blink/public/web/modules/mediastream/media_stream_video_source.h"
#include "third_party/blink/public/web/web_heap.h"
#include "third_party/blink/renderer/bindings/core/v8/script_promise_tester.h"
#include "third_party/blink/renderer/bindings/core/v8/v8_binding_for_testing.h"
#include "third_party/blink/renderer/core/streams/readable_stream.h"
#include "third_party/blink/renderer/core/streams/readable_stream_default_reader.h"
#include "third_party/blink/renderer/modules/mediastream/media_stream_track.h"
#include "third_party/blink/renderer/modules/mediastream/media_stream_track_generator.h"
#include "third_party/blink/renderer/modules/mediastream/media_stream_video_track.h"
#include "third_party/blink/renderer/modules/mediastream/mock_media_stream_video_sink.h"
#include "third_party/blink/renderer/modules/mediastream/mock_media_stream_video_source.h"
#include "third_party/blink/renderer/modules/mediastream/pushable_media_stream_video_source.h"
#include "third_party/blink/renderer/platform/bindings/exception_state.h"
#include "third_party/blink/renderer/platform/mediastream/media_stream_audio_source.h"
#include "third_party/blink/renderer/platform/mediastream/media_stream_audio_track.h"
#include "third_party/blink/renderer/platform/mediastream/media_stream_source.h"
#include "third_party/blink/renderer/platform/testing/io_task_runner_testing_platform_support.h"
using testing::_;
namespace blink {
namespace {
PushableMediaStreamVideoSource* CreatePushableVideoSource() {
PushableMediaStreamVideoSource* pushable_video_source =
new PushableMediaStreamVideoSource();
MediaStreamSource* media_stream_source =
MakeGarbageCollected<MediaStreamSource>(
"source_id", MediaStreamSource::kTypeVideo, "source_name",
/*remote=*/false);
media_stream_source->SetPlatformSource(
base::WrapUnique(pushable_video_source));
return pushable_video_source;
}
MediaStreamTrack* CreateVideoMediaStreamTrack(ExecutionContext* context,
MediaStreamVideoSource* source) {
return MakeGarbageCollected<MediaStreamTrack>(
context, MediaStreamVideoTrack::CreateVideoTrack(
source, MediaStreamVideoSource::ConstraintsOnceCallback(),
/*enabled=*/true));
}
MediaStreamTrack* CreateAudioMediaStreamTrack(ExecutionContext* context) {
std::unique_ptr<MediaStreamAudioSource> audio_source =
std::make_unique<MediaStreamAudioSource>(
blink::scheduler::GetSingleThreadTaskRunnerForTesting(),
/*is_local_source=*/false);
MediaStreamSource* media_stream_source =
MakeGarbageCollected<MediaStreamSource>(
"source_id", MediaStreamSource::kTypeAudio, "source_name",
/*is_remote=*/false);
media_stream_source->SetPlatformSource(std::move(audio_source));
std::unique_ptr<MediaStreamAudioTrack> audio_track =
std::make_unique<MediaStreamAudioTrack>(/*is_local_track=*/false);
MediaStreamComponent* component =
MakeGarbageCollected<MediaStreamComponent>(media_stream_source);
component->SetPlatformTrack(std::move(audio_track));
return MakeGarbageCollected<MediaStreamTrack>(context, component);
}
} // namespace
class MediaStreamTrackProcessorTest : public testing::Test {
public:
~MediaStreamTrackProcessorTest() override {
platform_->RunUntilIdle();
WebHeap::CollectAllGarbageForTesting();
}
protected:
ScopedTestingPlatformSupport<IOTaskRunnerTestingPlatformSupport> platform_;
};
TEST_F(MediaStreamTrackProcessorTest, VideoFramesAreExposed) {
V8TestingScope v8_scope;
ScriptState* script_state = v8_scope.GetScriptState();
ExceptionState& exception_state = v8_scope.GetExceptionState();
PushableMediaStreamVideoSource* pushable_video_source =
CreatePushableVideoSource();
MediaStreamTrackProcessor* track_processor =
MediaStreamTrackProcessor::Create(
script_state,
CreateVideoMediaStreamTrack(v8_scope.GetExecutionContext(),
pushable_video_source),
exception_state);
EXPECT_FALSE(exception_state.HadException());
EXPECT_EQ(track_processor->input_track()->Source()->GetPlatformSource(),
pushable_video_source);
MockMediaStreamVideoSink mock_video_sink;
mock_video_sink.ConnectToTrack(
WebMediaStreamTrack(track_processor->input_track()));
EXPECT_EQ(mock_video_sink.number_of_frames(), 0);
EXPECT_EQ(mock_video_sink.last_frame(), nullptr);
auto* reader = track_processor->readable(script_state)
->getReader(script_state, exception_state);
EXPECT_FALSE(exception_state.HadException());
// Deliver a frame
base::RunLoop sink_loop;
EXPECT_CALL(mock_video_sink, OnVideoFrame(_))
.WillOnce(base::test::RunOnceClosure(sink_loop.QuitClosure()));
scoped_refptr<media::VideoFrame> frame =
media::VideoFrame::CreateBlackFrame(gfx::Size(10, 5));
pushable_video_source->PushFrame(frame, base::TimeTicks());
ScriptPromiseTester read_tester(script_state,
reader->read(script_state, exception_state));
EXPECT_FALSE(read_tester.IsFulfilled());
read_tester.WaitUntilSettled();
EXPECT_FALSE(exception_state.HadException());
EXPECT_TRUE(read_tester.IsFulfilled());
EXPECT_TRUE(read_tester.Value().IsObject());
sink_loop.Run();
EXPECT_EQ(mock_video_sink.number_of_frames(), 1);
EXPECT_EQ(mock_video_sink.last_frame(), frame);
}
TEST_F(MediaStreamTrackProcessorTest, CanceledReadableDisconnects) {
V8TestingScope v8_scope;
ScriptState* script_state = v8_scope.GetScriptState();
PushableMediaStreamVideoSource* pushable_video_source =
CreatePushableVideoSource();
ExceptionState& exception_state = v8_scope.GetExceptionState();
MediaStreamTrackProcessor* track_processor =
MediaStreamTrackProcessor::Create(
script_state,
CreateVideoMediaStreamTrack(v8_scope.GetExecutionContext(),
pushable_video_source),
exception_state);
// Initially the track has no sinks.
MediaStreamVideoTrack* video_track =
MediaStreamVideoTrack::From(track_processor->input_track());
EXPECT_EQ(video_track->CountSinks(), 0u);
MockMediaStreamVideoSink mock_video_sink;
mock_video_sink.ConnectToTrack(
WebMediaStreamTrack(track_processor->input_track()));
EXPECT_EQ(mock_video_sink.number_of_frames(), 0);
EXPECT_EQ(mock_video_sink.last_frame(), nullptr);
EXPECT_EQ(video_track->CountSinks(), 1u);
// Accessing the readable connects it to the track
auto* readable = track_processor->readable(script_state);
EXPECT_EQ(video_track->CountSinks(), 2u);
ScriptPromiseTester cancel_tester(
script_state, readable->cancel(script_state, exception_state));
cancel_tester.WaitUntilSettled();
EXPECT_FALSE(exception_state.HadException());
EXPECT_EQ(video_track->CountSinks(), 1u);
// Cancelling the readable does not stop the track.
// Push a frame and expect delivery to the mock sink.
base::RunLoop sink_loop;
EXPECT_CALL(mock_video_sink, OnVideoFrame(_))
.WillOnce(base::test::RunOnceClosure(sink_loop.QuitClosure()));
scoped_refptr<media::VideoFrame> frame =
media::VideoFrame::CreateBlackFrame(gfx::Size(10, 5));
pushable_video_source->PushFrame(frame, base::TimeTicks());
sink_loop.Run();
EXPECT_EQ(mock_video_sink.number_of_frames(), 1);
EXPECT_EQ(mock_video_sink.last_frame(), frame);
}
TEST_F(MediaStreamTrackProcessorTest, ProcessorConnectsToGenerator) {
V8TestingScope v8_scope;
ScriptState* script_state = v8_scope.GetScriptState();
// Create a processor connected to a pushable source.
PushableMediaStreamVideoSource* pushable_video_source =
CreatePushableVideoSource();
ExceptionState& exception_state = v8_scope.GetExceptionState();
MediaStreamTrackProcessor* track_processor =
MediaStreamTrackProcessor::Create(
script_state,
CreateVideoMediaStreamTrack(v8_scope.GetExecutionContext(),
pushable_video_source),
exception_state);
// Create generator and connect it to a mock sink.
MediaStreamTrackGenerator* track_generator =
MakeGarbageCollected<MediaStreamTrackGenerator>(
script_state, MediaStreamSource::kTypeVideo, "track_id");
MockMediaStreamVideoSink mock_video_sink;
mock_video_sink.ConnectToTrack(
WebMediaStreamTrack(track_generator->Component()));
EXPECT_EQ(mock_video_sink.number_of_frames(), 0);
EXPECT_EQ(mock_video_sink.last_frame(), nullptr);
// Connect the processor to the generator
track_processor->readable(script_state)
->pipeTo(script_state, track_generator->writable(script_state),
exception_state);
// Push a frame and verify that it makes it to the sink at the end of the
// chain.
base::RunLoop sink_loop;
EXPECT_CALL(mock_video_sink, OnVideoFrame(_))
.WillOnce(base::test::RunOnceClosure(sink_loop.QuitClosure()));
scoped_refptr<media::VideoFrame> frame =
media::VideoFrame::CreateBlackFrame(gfx::Size(10, 5));
pushable_video_source->PushFrame(frame, base::TimeTicks());
sink_loop.Run();
EXPECT_EQ(mock_video_sink.number_of_frames(), 1);
EXPECT_EQ(mock_video_sink.last_frame(), frame);
}
TEST_F(MediaStreamTrackProcessorTest, NullInputTrack) {
V8TestingScope v8_scope;
ScriptState* script_state = v8_scope.GetScriptState();
ExceptionState& exception_state = v8_scope.GetExceptionState();
MediaStreamTrackProcessor* track_processor =
MediaStreamTrackProcessor::Create(script_state, nullptr, exception_state);
EXPECT_EQ(track_processor, nullptr);
EXPECT_TRUE(exception_state.HadException());
EXPECT_EQ(static_cast<DOMExceptionCode>(v8_scope.GetExceptionState().Code()),
DOMExceptionCode::kOperationError);
}
// TODO(crbug.com/1142955): Add support for audio.
TEST_F(MediaStreamTrackProcessorTest, Audio) {
V8TestingScope v8_scope;
ScriptState* script_state = v8_scope.GetScriptState();
ExceptionState& exception_state = v8_scope.GetExceptionState();
MediaStreamTrack* media_stream_track =
CreateAudioMediaStreamTrack(v8_scope.GetExecutionContext());
MediaStreamTrackProcessor* track_processor =
MediaStreamTrackProcessor::Create(script_state, media_stream_track,
exception_state);
EXPECT_EQ(track_processor, nullptr);
EXPECT_TRUE(exception_state.HadException());
EXPECT_EQ(static_cast<DOMExceptionCode>(v8_scope.GetExceptionState().Code()),
DOMExceptionCode::kNotSupportedError);
}
} // namespace blink
......@@ -5182,6 +5182,10 @@ interface MediaStreamTrackGenerator : MediaStreamTrack
attribute @@toStringTag
getter writable
method constructor
interface MediaStreamTrackProcessor
attribute @@toStringTag
getter readable
method constructor
interface MerchantValidationEvent : Event
attribute @@toStringTag
getter methodName
......
......@@ -30231,6 +30231,7 @@ Called by update_use_counter_feature_enum.py.-->
<int value="3726" label="V8HTMLDialogElement_ShowModal_Method"/>
<int value="3727" label="AdFrameDetected"/>
<int value="3728" label="MediaStreamTrackGenerator"/>
<int value="3729" label="MediaStreamTrackProcessor"/>
</enum>
<enum name="FeaturePolicyAllowlistType">
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment