Skip to content

GitLab

T
tangled
Commit c0e73fce authored by Eugene Zemtsov's avatar Eugene Zemtsov Committed by Commit Bot
Browse files
Options

[webcodecs] Make sure callbacks aren't called after encoder.reset() 

Bug: 1146170
Change-Id: Idd807b7f6e2618c8b4e1f161a2c66d74b4b40374
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2531901Reviewed-by: default avatarDan Sanders <sandersd@chromium.org>
Commit-Queue: Eugene Zemtsov <eugene@chromium.org>
Cr-Commit-Position: refs/heads/master@{#826478}
parent 42e68028
Hide whitespace changes
Inline Side-by-side
Showing with 235 additions and 154 deletions
third_party/blink/renderer/modules/webcodecs/video_encoder.cc
View file @ c0e73fce
......@@ -345,6 +345,7 @@ void VideoEncoder::configure(const VideoEncoderConfig* config,
state_ = V8CodecState(V8CodecState::Enum::kConfigured);
Request* request = MakeGarbageCollected<Request>();
request->reset_count = reset_count_;
request->type = Request::Type::kConfigure;
active_config_ = std::move(parsed_config);
EnqueueRequest(request);
......@@ -390,11 +391,12 @@ void VideoEncoder::encode(VideoFrame* frame,
frame->destroy();
Request* request = MakeGarbageCollected<Request>();
request->reset_count = reset_count_;
request->type = Request::Type::kEncode;
request->frame = internal_frame;
request->encodeOpts = opts;
++requested_encodes_;
return EnqueueRequest(request);
EnqueueRequest(request);
}
void VideoEncoder::close(ExceptionState& exception_state) {
......@@ -405,7 +407,7 @@ void VideoEncoder::close(ExceptionState& exception_state) {
state_ = V8CodecState(V8CodecState::Enum::kClosed);
ClearRequests();
ResetInternal();
media_encoder_.reset();
output_callback_.Clear();
error_callback_.Clear();
......@@ -422,6 +424,7 @@ ScriptPromise VideoEncoder::flush(ExceptionState& exception_state) {
Request* request = MakeGarbageCollected<Request>();
request->resolver =
MakeGarbageCollected<ScriptPromiseResolver>(script_state_);
request->reset_count = reset_count_;
request->type = Request::Type::kFlush;
EnqueueRequest(request);
return request->resolver->Promise();
......@@ -429,17 +432,17 @@ ScriptPromise VideoEncoder::flush(ExceptionState& exception_state) {
void VideoEncoder::reset(ExceptionState& exception_state) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
// TODO: Not fully implemented yet
if (ThrowIfCodecStateClosed(state_, "reset", exception_state))
return;
ClearRequests();
ResetInternal();
state_ = V8CodecState(V8CodecState::Enum::kUnconfigured);
}
void VideoEncoder::ClearRequests() {
void VideoEncoder::ResetInternal() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
reset_count_++;
while (!requests_.empty()) {
Request* pending_req = requests_.TakeFirst();
DCHECK(pending_req);
......@@ -449,6 +452,7 @@ void VideoEncoder::ClearRequests() {
pending_req->resolver.Release()->Reject(ex);
}
}
stall_request_processing_ = false;
}
void VideoEncoder::HandleError(DOMException* ex) {
......@@ -457,7 +461,7 @@ void VideoEncoder::HandleError(DOMException* ex) {
state_ = V8CodecState(V8CodecState::Enum::kClosed);
ClearRequests();
ResetInternal();
// Errors are permanent. Shut everything down.
error_callback_.Clear();
......@@ -510,7 +514,7 @@ void VideoEncoder::ProcessEncode(Request* request) {
auto done_callback = [](VideoEncoder* self, Request* req,
media::Status status) {
if (!self)
if (!self || self->reset_count_ != req->reset_count)
return;
DCHECK_CALLED_ON_VALID_SEQUENCE(self->sequence_checker_);
if (!status.is_ok()) {
......@@ -556,11 +560,11 @@ void VideoEncoder::ProcessConfigure(Request* request) {
}
auto output_cb = WTF::BindRepeating(&VideoEncoder::CallOutputCallback,
WrapWeakPersistent(this));
WrapWeakPersistent(this), reset_count_);
auto done_callback = [](VideoEncoder* self, Request* req,
media::Status status) {
if (!self)
if (!self || self->reset_count_ != req->reset_count)
return;
DCHECK_CALLED_ON_VALID_SEQUENCE(self->sequence_checker_);
DCHECK(self->active_config_);
......@@ -589,9 +593,9 @@ void VideoEncoder::ProcessFlush(Request* request) {
auto done_callback = [](VideoEncoder* self, Request* req,
media::Status status) {
DCHECK(req->resolver);
if (!self)
if (!self || self->reset_count_ != req->reset_count)
return;
DCHECK(req->resolver);
DCHECK_CALLED_ON_VALID_SEQUENCE(self->sequence_checker_);
if (status.is_ok()) {
req->resolver.Release()->Resolve();
......@@ -612,10 +616,12 @@ void VideoEncoder::ProcessFlush(Request* request) {
}
void VideoEncoder::CallOutputCallback(
uint32_t reset_count,
media::VideoEncoderOutput output,
base::Optional<media::VideoEncoder::CodecDescription> codec_desc) {
if (!script_state_->ContextIsValid() || !output_callback_ ||
state_.AsEnum() != V8CodecState::Enum::kConfigured)
state_.AsEnum() != V8CodecState::Enum::kConfigured ||
reset_count != reset_count_)
return;
EncodedVideoMetadata metadata;
......
third_party/blink/renderer/modules/webcodecs/video_encoder.h
View file @ c0e73fce
......@@ -90,12 +90,15 @@ class MODULES_EXPORT VideoEncoder final : public ScriptWrappable {
void Trace(Visitor*) const;
Type type;
// Current value of VideoEncoder.reset_count_ when request was created.
uint32_t reset_count = 0;
Member<VideoFrame> frame; // used by kEncode
Member<const VideoEncoderEncodeOptions> encodeOpts; // used by kEncode
Member<ScriptPromiseResolver> resolver; // used by kFlush
};
void CallOutputCallback(
uint32_t reset_count,
media::VideoEncoderOutput output,
base::Optional<media::VideoEncoder::CodecDescription> codec_desc);
void HandleError(DOMException* ex);
......@@ -106,7 +109,7 @@ class MODULES_EXPORT VideoEncoder final : public ScriptWrappable {
void ProcessConfigure(Request* request);
void ProcessFlush(Request* request);
void ClearRequests();
void ResetInternal();
std::unique_ptr<ParsedConfig> ParseConfig(const VideoEncoderConfig*,
ExceptionState&);
......@@ -124,6 +127,10 @@ class MODULES_EXPORT VideoEncoder final : public ScriptWrappable {
Member<V8WebCodecsErrorCallback> error_callback_;
HeapDeque<Member<Request>> requests_;
int32_t requested_encodes_ = 0;
// How many times reset() was called on the encoder. It's used to decide
// when a callback needs to be dismissed because reset() was called between
// an operation and its callback.
uint32_t reset_count_ = 0;
// Some kConfigure and kFlush requests can't be executed in parallel with
// kEncode. This flag stops processing of new requests in the requests_ queue
......
third_party/blink/web_tests/external/wpt/webcodecs/video-encoder.html
View file @ c0e73fce
<!DOCTYPE html>
<html>
<title>Test the VideoTrackReader API.</title>
<head>
<title>Test the VideoTrackReader API.</title>
</head>
<body>
<img id='frame_image' src="pattern.png">
</body>
......@@ -10,204 +12,270 @@
<script src="/webcodecs/utils.js"></script>
<script>
const defaultConfig = {
codec: 'vp8',
framerate: 25,
width: 640,
height: 480
};
const defaultConfig = {
codec: 'vp8',
framerate: 25,
width: 640,
height: 480
};
async function waitTillLoaded(img) {
if (img.complete)
return Promise.resolve();
return new Promise((resolve, reject) => {
img.onload = () => resolve()
});
}
async function generateBitmap(width, height) {
return createImageBitmap(document.getElementById('frame_image'),
{ resizeWidth: width,
resizeHeight: height });
}
async function generateBitmap(width, height) {
let img = document.getElementById('frame_image');
await waitTillLoaded(img);
return createImageBitmap(img,
{
resizeWidth: width,
resizeHeight: height
});
}
async function createVideoFrame(width, height, timestamp) {
let bitmap = await generateBitmap(width, height);
return new VideoFrame(bitmap, { timestamp: timestamp });
}
async function createVideoFrame(width, height, timestamp) {
let bitmap = await generateBitmap(width, height);
return new VideoFrame(bitmap, { timestamp: timestamp });
}
promise_test(t => {
// VideoEncoderInit lacks required fields.
assert_throws_js(TypeError, () => { new VideoEncoder({}); });
promise_test(t => {
// VideoEncoderInit lacks required fields.
assert_throws_js(TypeError, () => { new VideoEncoder({}); });
// VideoEncoderInit has required fields.
let encoder = new VideoEncoder(getDefaultCodecInit(t));
// VideoEncoderInit has required fields.
let encoder = new VideoEncoder(getDefaultCodecInit(t));
assert_equals(encoder.state, "unconfigured");
assert_equals(encoder.state, "unconfigured");
encoder.close();
encoder.close();
return endAfterEventLoopTurn();
}, 'Test VideoEncoder construction');
return endAfterEventLoopTurn();
}, 'Test VideoEncoder construction');
promise_test(t => {
let encoder = new VideoEncoder(getDefaultCodecInit(t));
promise_test(t => {
let encoder = new VideoEncoder(getDefaultCodecInit(t));
let badCodecsList = [
'', // Empty codec
'bogus', // Non exsitent codec
'vorbis', // Audio codec
'vp9', // Ambiguous codec
'video/webm; codecs="vp9"' // Codec with mime type
]
let badCodecsList = [
'', // Empty codec
'bogus', // Non exsitent codec
'vorbis', // Audio codec
'vp9', // Ambiguous codec
'video/webm; codecs="vp9"' // Codec with mime type
]
testConfigurations(encoder, defaultConfig, badCodecsList);
testConfigurations(encoder, defaultConfig, badCodecsList);
return endAfterEventLoopTurn();
}, 'Test VideoEncoder.configure()');
return endAfterEventLoopTurn();
}, 'Test VideoEncoder.configure()');
promise_test(async t => {
let output_chunks = [];
let codecInit = getDefaultCodecInit(t);
codecInit.output = chunk => output_chunks.push(chunk);
promise_test(async t => {
let output_chunks = [];
let codecInit = getDefaultCodecInit(t);
codecInit.output = chunk => output_chunks.push(chunk);
let encoder = new VideoEncoder(codecInit);
let encoder = new VideoEncoder(codecInit);
// No encodes yet.
assert_equals(encoder.encodeQueueSize, 0);
// No encodes yet.
assert_equals(encoder.encodeQueueSize, 0);
encoder.configure(defaultConfig);
encoder.configure(defaultConfig);
// Still no encodes.
assert_equals(encoder.encodeQueueSize, 0);
// Still no encodes.
assert_equals(encoder.encodeQueueSize, 0);
let frame1 = await createVideoFrame(640, 480, 0);
let frame2 = await createVideoFrame(640, 480, 33333);
let frame1 = await createVideoFrame(640, 480, 0);
let frame2 = await createVideoFrame(640, 480, 33333);
encoder.encode(frame1.clone());
encoder.encode(frame2.clone());
encoder.encode(frame1.clone());
encoder.encode(frame2.clone());
// Could be 0, 1, or 2. We can't guarantee this check runs before the UA has
// processed the encodes.
assert_true(encoder.encodeQueueSize >= 0 && encoder.encodeQueueSize <= 2)
// Could be 0, 1, or 2. We can't guarantee this check runs before the UA has
// processed the encodes.
assert_true(encoder.encodeQueueSize >= 0 && encoder.encodeQueueSize <= 2)
await encoder.flush();
await encoder.flush();
// We can guarantee that all encodes are processed after a flush.
assert_equals(encoder.encodeQueueSize, 0);
// We can guarantee that all encodes are processed after a flush.
assert_equals(encoder.encodeQueueSize, 0);
assert_equals(output_chunks.length, 2);
assert_equals(output_chunks[0].timestamp, frame1.timestamp);
assert_equals(output_chunks[1].timestamp, frame2.timestamp);
}, 'Test successful configure(), encode(), and flush()');
assert_equals(output_chunks.length, 2);
assert_equals(output_chunks[0].timestamp, frame1.timestamp);
assert_equals(output_chunks[1].timestamp, frame2.timestamp);
}, 'Test successful configure(), encode(), and flush()');
promise_test(async t => {
let callbacks_before_reset = 0;
let callbacks_after_reset = 0;
let codecInit = getDefaultCodecInit(t);
codecInit.output = chunk => {
if (chunk.timestamp % 2 == 0) {
// pre-reset frames have even timestamp
callbacks_before_reset++;
} else {
// after-reset frames have odd timestamp
callbacks_after_reset++;
}
}
let encoder = new VideoEncoder(codecInit);
encoder.configure(defaultConfig);
await encoder.flush();
let frames = [];
for (let i = 0; i < 200; i++) {
let frame = await createVideoFrame(640, 480, i * 40_000);
frames.push(frame);
}
promise_test(async t => {
let output_chunks = [];
let codecInit = getDefaultCodecInit(t);
codecInit.output = chunk => output_chunks.push(chunk);
for (frame of frames)
encoder.encode(frame);
let encoder = new VideoEncoder(codecInit);
// Wait for the first frame to be encoded
await t.step_wait(() => callbacks_before_reset > 0,
"Encoded outputs started coming", 10000, 1);
// No encodes yet.
assert_equals(encoder.encodeQueueSize, 0);
let saved_callbacks_before_reset = callbacks_before_reset;
assert_greater_than(callbacks_before_reset, 0);
assert_less_than_equal(callbacks_before_reset, frames.length);
let config = defaultConfig;
encoder.reset();
assert_equals(encoder.encodeQueueSize, 0);
encoder.configure(config);
let newConfig = { ...defaultConfig };
newConfig.width = 800;
newConfig.height = 600;
encoder.configure(newConfig);
let frame1 = await createVideoFrame(640, 480, 0);
let frame2 = await createVideoFrame(640, 480, 33333);
for (let i = frames.length; i < frames.length + 5; i++) {
let frame = await createVideoFrame(800, 600, i * 40_000 + 1);
encoder.encode(frame);
}
await encoder.flush();
assert_equals(callbacks_after_reset, 5);
assert_equals(saved_callbacks_before_reset, callbacks_before_reset);
assert_equals(encoder.encodeQueueSize, 0);
}, 'Test successful reset() and re-confiugre()');
encoder.encode(frame1.clone());
encoder.configure(config);
promise_test(async t => {
let output_chunks = [];
let codecInit = getDefaultCodecInit(t);
codecInit.output = chunk => output_chunks.push(chunk);
encoder.encode(frame2.clone());
let encoder = new VideoEncoder(codecInit);
await encoder.flush();
// No encodes yet.
assert_equals(encoder.encodeQueueSize, 0);
// We can guarantee that all encodes are processed after a flush.
assert_equals(encoder.encodeQueueSize, 0);
let config = defaultConfig;
// The first frame may have been dropped when reconfiguring.
// This shouldn't happen, and should be fixed/called out in the spec, but
// this is preptively added to prevent flakiness.
// TODO: Remove these checks when implementations handle this correctly.
assert_true(output_chunks.length == 1 || output_chunks.length == 2);
encoder.configure(config);
if (output_chunks.length == 1) {
// If we only have one chunk frame, make sure we droped the frame that was
// in flight when we reconfigured.
assert_equals(output_chunks[0].timestamp, frame2.timestamp);
} else {
assert_equals(output_chunks[0].timestamp, frame1.timestamp);
assert_equals(output_chunks[1].timestamp, frame2.timestamp);
}
let frame1 = await createVideoFrame(640, 480, 0);
let frame2 = await createVideoFrame(640, 480, 33333);
output_chunks = [];
encoder.encode(frame1.clone());
encoder.configure(config);
let frame3 = await createVideoFrame(640, 480, 66666);
let frame4 = await createVideoFrame(640, 480, 100000);
encoder.encode(frame2.clone());
encoder.encode(frame3.clone());
await encoder.flush();
// Verify that a failed call to configure does not change the encoder's state.
let badConfig = {...defaultConfig};
badConfig.codec = 'bogus';
assert_throws_js(TypeError, () => encoder.configure(badConfig));
// We can guarantee that all encodes are processed after a flush.
assert_equals(encoder.encodeQueueSize, 0);
encoder.encode(frame4.clone());
// The first frame may have been dropped when reconfiguring.
// This shouldn't happen, and should be fixed/called out in the spec, but
// this is preptively added to prevent flakiness.
// TODO: Remove these checks when implementations handle this correctly.
assert_true(output_chunks.length == 1 || output_chunks.length == 2);
await encoder.flush();
if (output_chunks.length == 1) {
// If we only have one chunk frame, make sure we droped the frame that was
// in flight when we reconfigured.
assert_equals(output_chunks[0].timestamp, frame2.timestamp);
} else {
assert_equals(output_chunks[0].timestamp, frame1.timestamp);
assert_equals(output_chunks[1].timestamp, frame2.timestamp);
}
assert_equals(output_chunks[0].timestamp, frame3.timestamp);
assert_equals(output_chunks[1].timestamp, frame4.timestamp);
}, 'Test successful encode() after re-configure().');
output_chunks = [];
promise_test(async t => {
let output_chunks = [];
let codecInit = getDefaultCodecInit(t);
codecInit.output = chunk => output_chunks.push(chunk);
let frame3 = await createVideoFrame(640, 480, 66666);
let frame4 = await createVideoFrame(640, 480, 100000);
let encoder = new VideoEncoder(codecInit);
encoder.encode(frame3.clone());
let timestamp = 33333;
let frame = await createVideoFrame(640, 480, timestamp);
// Verify that a failed call to configure does not change the encoder's state.
let badConfig = { ...defaultConfig };
badConfig.codec = 'bogus';
assert_throws_js(TypeError, () => encoder.configure(badConfig));
encoder.configure(defaultConfig);
assert_equals(encoder.state, "configured");
encoder.encode(frame4.clone());
encoder.encode(frame);
await encoder.flush();
// |frame| is not longer valid since it has been destroyed.
assert_not_equals(frame.timestamp, timestamp);
assert_throws_dom("InvalidStateError", () => frame.clone());
assert_equals(output_chunks[0].timestamp, frame3.timestamp);
assert_equals(output_chunks[1].timestamp, frame4.timestamp);
}, 'Test successful encode() after re-configure().');
encoder.close();
promise_test(async t => {
let output_chunks = [];
let codecInit = getDefaultCodecInit(t);
codecInit.output = chunk => output_chunks.push(chunk);
return endAfterEventLoopTurn();
}, 'Test encoder consumes (destroys) frames.');
let encoder = new VideoEncoder(codecInit);
promise_test(async t => {
let encoder = new VideoEncoder(getDefaultCodecInit(t));
let timestamp = 33333;
let frame = await createVideoFrame(640, 480, timestamp);
let frame = await createVideoFrame(640, 480, 0);
encoder.configure(defaultConfig);
assert_equals(encoder.state, "configured");
return testClosedCodec(t, encoder, defaultConfig, frame);
}, 'Verify closed VideoEncoder operations');
encoder.encode(frame);
promise_test(async t => {
let encoder = new VideoEncoder(getDefaultCodecInit(t));
// |frame| is not longer valid since it has been destroyed.
assert_not_equals(frame.timestamp, timestamp);
assert_throws_dom("InvalidStateError", () => frame.clone());
let frame = await createVideoFrame(640, 480, 0);
encoder.close();
return testUnconfiguredCodec(t, encoder, frame);
}, 'Verify unconfigured VideoEncoder operations');
return endAfterEventLoopTurn();
}, 'Test encoder consumes (destroys) frames.');
promise_test(async t => {
let encoder = new VideoEncoder(getDefaultCodecInit(t));
promise_test(async t => {
let encoder = new VideoEncoder(getDefaultCodecInit(t));
let frame = await createVideoFrame(640, 480, 0);
frame.destroy();
let frame = await createVideoFrame(640, 480, 0);
encoder.configure(defaultConfig);
return testClosedCodec(t, encoder, defaultConfig, frame);
}, 'Verify closed VideoEncoder operations');
frame.destroy();
assert_throws_dom("OperationError", () => {
encoder.encode(frame)
});
}, 'Verify encoding destroyed frames throws.');
promise_test(async t => {
let encoder = new VideoEncoder(getDefaultCodecInit(t));
let frame = await createVideoFrame(640, 480, 0);
return testUnconfiguredCodec(t, encoder, frame);
}, 'Verify unconfigured VideoEncoder operations');
promise_test(async t => {
let encoder = new VideoEncoder(getDefaultCodecInit(t));
let frame = await createVideoFrame(640, 480, 0);
frame.destroy();
encoder.configure(defaultConfig);
frame.destroy();
assert_throws_dom("OperationError", () => {
encoder.encode(frame)
});
}, 'Verify encoding destroyed frames throws.');
</script>
</html>
</html>
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment