Commit 19caef57 authored by Dan Sanders's avatar Dan Sanders Committed by Commit Bot

[webcodecs] Implement planar access to VideoFrames.

This initial version supports only I420. We'll want to implement
explicit conversion and probably color space metadata before adding
many more formats.

Bug: 1096722
Change-Id: I42949e2a7e9e2135fab260e73da2f2aa420eb424
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2355390
Commit-Queue: Dan Sanders <sandersd@chromium.org>
Reviewed-by: default avatarChrome Cunningham <chcunningham@chromium.org>
Cr-Commit-Position: refs/heads/master@{#799773}
parent f6e6cae5
......@@ -37,6 +37,8 @@ blink_modules_sources("webcodecs") {
"video_frame.h",
"video_frame_attachment.cc",
"video_frame_attachment.h",
"video_frame_handle.cc",
"video_frame_handle.h",
"video_track_reader.cc",
"video_track_reader.h",
"video_track_writer.cc",
......
......@@ -4,34 +4,93 @@
#include "third_party/blink/renderer/modules/webcodecs/plane.h"
#include <string.h>
#include "third_party/blink/renderer/platform/bindings/exception_code.h"
#include "third_party/blink/renderer/platform/bindings/exception_state.h"
namespace blink {
Plane::Plane(VideoFrame* frame, uint32_t plane)
: frame_(frame), plane_(plane) {}
Plane::Plane(scoped_refptr<VideoFrameHandle> handle, size_t plane)
: handle_(std::move(handle)), plane_(plane) {
#if DCHECK_IS_ON()
// Validate the plane index, but only if the handle is valid.
auto local_frame = handle_->frame();
if (local_frame) {
DCHECK(local_frame->IsMappable());
DCHECK_LT(plane, local_frame->layout().num_planes());
}
#endif // DCHECK_IS_ON()
}
uint32_t Plane::stride() const {
// Use |plane_|, to satisfy the compiler.
static_cast<void>(plane_);
// TODO(sandersd): Look up via |frame_|.
return 0;
auto local_frame = handle_->frame();
if (!local_frame)
return 0;
// TODO(sandersd): Consider returning row_bytes() instead. This would imply
// removing padding bytes in copyInto().
return local_frame->stride(plane_);
}
uint32_t Plane::rows() const {
// TODO(sandersd): Look up via |frame_|.
return 0;
auto local_frame = handle_->frame();
if (!local_frame)
return 0;
return local_frame->rows(plane_);
}
uint32_t Plane::length() const {
// TODO(sandersd): Look up via |frame_|.
return 0;
auto local_frame = handle_->frame();
if (!local_frame)
return 0;
// Note: this could be slightly larger than the actual data size. readInto()
// will pad with zeros.
return local_frame->rows(plane_) * local_frame->stride(plane_);
}
void Plane::readInto(MaybeShared<DOMArrayBufferView> dst, ExceptionState&) {}
void Plane::readInto(MaybeShared<DOMArrayBufferView> dst,
ExceptionState& exception_state) {
auto local_frame = handle_->frame();
if (!local_frame) {
exception_state.ThrowDOMException(DOMExceptionCode::kInvalidStateError,
"Cannot read from destroyed VideoFrame.");
return;
}
// Note: these methods all return int.
size_t rows = local_frame->rows(plane_);
size_t row_bytes = local_frame->row_bytes(plane_);
size_t stride = local_frame->stride(plane_);
DCHECK_GT(rows, 0u); // should fail VideoFrame::IsValidConfig()
DCHECK_GT(row_bytes, 0u); // should fail VideoFrame::IsValidConfig()
DCHECK_GE(stride, row_bytes);
size_t total_size = rows * stride;
size_t trailing_zeros_size = stride - row_bytes;
size_t copy_size = total_size - trailing_zeros_size;
// Note: byteLength is zero if the buffer is detached.
DOMArrayBufferView* view = dst.View();
uint8_t* base = static_cast<uint8_t*>(view->BaseAddressMaybeShared());
if (!base) {
exception_state.ThrowDOMException(DOMExceptionCode::kInvalidStateError,
"Destination buffer is not valid.");
return;
}
if (total_size > view->byteLengthAsSizeT()) {
exception_state.ThrowDOMException(
DOMExceptionCode::kInvalidStateError,
"Destination buffer is not large enough.");
return;
}
// Copy plane bytes.
memcpy(base, local_frame->data(plane_), copy_size);
void Plane::Trace(Visitor* visitor) const {
visitor->Trace(frame_);
ScriptWrappable::Trace(visitor);
// Zero trailing padding bytes.
memset(base + copy_size, 0, trailing_zeros_size);
}
} // namespace blink
......@@ -7,12 +7,12 @@
#include <stdint.h>
#include "base/memory/scoped_refptr.h"
#include "third_party/blink/renderer/core/typed_arrays/array_buffer_view_helpers.h"
#include "third_party/blink/renderer/core/typed_arrays/dom_typed_array.h"
#include "third_party/blink/renderer/modules/modules_export.h"
#include "third_party/blink/renderer/modules/webcodecs/video_frame.h"
#include "third_party/blink/renderer/modules/webcodecs/video_frame_handle.h"
#include "third_party/blink/renderer/platform/bindings/script_wrappable.h"
#include "third_party/blink/renderer/platform/heap/member.h"
namespace blink {
......@@ -22,21 +22,26 @@ class MODULES_EXPORT Plane final : public ScriptWrappable {
DEFINE_WRAPPERTYPEINFO();
public:
// Construct a Plane given a |handle| and a plane index.
// It is legal for |handle| to be invalid, the resulting Plane will also be
// invalid.
Plane(scoped_refptr<VideoFrameHandle> handle, size_t plane);
~Plane() override = default;
// TODO(sandersd): Review return types. There is some implicit casting going
// on here.
// TODO(sandersd): Consider throwing if |handle_| is invalidated. Currently
// everything returns 0, which is not a bad API either.
uint32_t stride() const;
uint32_t rows() const;
uint32_t length() const;
// Throws InvalidStateError if |handle_| has been invalidated.
void readInto(MaybeShared<DOMArrayBufferView> dst, ExceptionState&);
void Trace(Visitor*) const override;
private:
Plane(VideoFrame* frame, uint32_t plane);
Member<VideoFrame> frame_;
uint32_t plane_;
scoped_refptr<VideoFrameHandle> handle_;
size_t plane_;
};
} // namespace blink
......
......@@ -63,27 +63,12 @@ bool IsValidSkColorType(SkColorType sk_color_type) {
} // namespace
VideoFrame::Handle::Handle(scoped_refptr<media::VideoFrame> frame)
: frame_(std::move(frame)) {
DCHECK(frame_);
}
scoped_refptr<media::VideoFrame> VideoFrame::Handle::frame() {
WTF::MutexLocker locker(mutex_);
return frame_;
}
void VideoFrame::Handle::Invalidate() {
WTF::MutexLocker locker(mutex_);
frame_.reset();
}
VideoFrame::VideoFrame(scoped_refptr<media::VideoFrame> frame)
: handle_(base::MakeRefCounted<Handle>(std::move(frame))) {
: handle_(base::MakeRefCounted<VideoFrameHandle>(std::move(frame))) {
DCHECK(handle_->frame());
}
VideoFrame::VideoFrame(scoped_refptr<Handle> handle)
VideoFrame::VideoFrame(scoped_refptr<VideoFrameHandle> handle)
: handle_(std::move(handle)) {
DCHECK(handle_);
}
......@@ -184,14 +169,45 @@ VideoFrame* VideoFrame::Create(ImageBitmap* source,
return result;
}
// static
bool VideoFrame::IsSupportedPlanarFormat(media::VideoFrame* frame) {
// For now only I420 in CPU memory is supported.
return frame && frame->IsMappable() &&
frame->format() == media::PIXEL_FORMAT_I420 &&
frame->layout().num_planes() == 3;
}
String VideoFrame::format() const {
// TODO(sandersd): Look up on |handle_->frame()|.
return String();
auto local_frame = handle_->frame();
if (!local_frame || !IsSupportedPlanarFormat(local_frame.get()))
return String();
switch (local_frame->format()) {
case media::PIXEL_FORMAT_I420:
return "I420";
default:
NOTREACHED();
return String();
}
}
HeapVector<Member<Plane>> VideoFrame::planes() const {
// TODO(sandersd): Should probably be extrated and cached.
return HeapVector<Member<Plane>>();
base::Optional<HeapVector<Member<Plane>>> VideoFrame::planes() {
// Verify that |this| has not been invalidated, and that the format is
// supported.
auto local_frame = handle_->frame();
if (!local_frame || !IsSupportedPlanarFormat(local_frame.get()))
return base::nullopt;
// Create a Plane for each VideoFrame plane, but only the first time.
if (planes_.IsEmpty()) {
for (size_t i = 0; i < local_frame->layout().num_planes(); i++) {
// Note: |handle_| may have been invalidated since |local_frame| was read.
planes_.push_back(MakeGarbageCollected<Plane>(handle_, i));
}
}
return planes_;
}
uint32_t VideoFrame::codedWidth() const {
......@@ -282,7 +298,7 @@ VideoFrame* VideoFrame::clone(ExceptionState& exception_state) {
return MakeGarbageCollected<VideoFrame>(std::move(frame));
}
scoped_refptr<VideoFrame::Handle> VideoFrame::handle() {
scoped_refptr<VideoFrameHandle> VideoFrame::handle() {
return handle_;
}
......@@ -417,4 +433,9 @@ ScriptPromise VideoFrame::CreateImageBitmap(ScriptState* script_state,
return ScriptPromise();
}
void VideoFrame::Trace(Visitor* visitor) const {
visitor->Trace(planes_);
ScriptWrappable::Trace(visitor);
}
} // namespace blink
......@@ -11,18 +11,17 @@
#include "media/base/video_frame.h"
#include "third_party/blink/renderer/core/imagebitmap/image_bitmap_source.h"
#include "third_party/blink/renderer/modules/modules_export.h"
#include "third_party/blink/renderer/modules/webcodecs/plane.h"
#include "third_party/blink/renderer/modules/webcodecs/video_frame_handle.h"
#include "third_party/blink/renderer/platform/bindings/script_wrappable.h"
#include "third_party/blink/renderer/platform/heap/heap_allocator.h"
#include "third_party/blink/renderer/platform/heap/member.h"
#include "third_party/blink/renderer/platform/wtf/text/wtf_string.h"
#include "third_party/blink/renderer/platform/wtf/thread_safe_ref_counted.h"
#include "third_party/blink/renderer/platform/wtf/threading_primitives.h"
namespace blink {
class ImageBitmap;
class ExceptionState;
class Plane;
class ScriptPromise;
class ScriptState;
class VideoFrameInit;
......@@ -32,41 +31,19 @@ class MODULES_EXPORT VideoFrame final : public ScriptWrappable,
DEFINE_WRAPPERTYPEINFO();
public:
// Wrapper class that allows sharing a single |frame_| reference across
// multiple VideoFrames, which can be invalidated for all frames at once.
class MODULES_EXPORT Handle : public WTF::ThreadSafeRefCounted<Handle> {
public:
explicit Handle(scoped_refptr<media::VideoFrame>);
// Returns a copy of |frame_|, which should be re-used throughout the scope
// of a function call, instead of calling frame() multiple times.
scoped_refptr<media::VideoFrame> frame();
// Releases the underlying media::VideoFrame reference, affecting all
// blink::VideoFrames that hold a reference to |this|.
void Invalidate();
private:
friend class WTF::ThreadSafeRefCounted<Handle>;
~Handle() = default;
WTF::Mutex mutex_;
scoped_refptr<media::VideoFrame> frame_;
};
// Creates a VideoFrame with a new Handle wrapping |frame|.
// Creates a VideoFrame with a new VideoFrameHandle wrapping |frame|.
explicit VideoFrame(scoped_refptr<media::VideoFrame> frame);
// Creates a VideoFrame from an existing handle.
// All frames sharing |handle| will have their |handle_| invalidated if any of
// the frames receives a call to destroy().
explicit VideoFrame(scoped_refptr<Handle> handle);
explicit VideoFrame(scoped_refptr<VideoFrameHandle> handle);
// video_frame.idl implementation.
static VideoFrame* Create(ImageBitmap*, VideoFrameInit*, ExceptionState&);
String format() const;
HeapVector<Member<Plane>> planes() const;
base::Optional<HeapVector<Member<Plane>>> planes();
uint32_t codedWidth() const;
uint32_t codedHeight() const;
......@@ -95,13 +72,18 @@ class MODULES_EXPORT VideoFrame final : public ScriptWrappable,
const ImageBitmapOptions*,
ExceptionState&);
scoped_refptr<VideoFrame::Handle> handle();
scoped_refptr<VideoFrameHandle> handle();
// Convenience functions
scoped_refptr<media::VideoFrame> frame();
scoped_refptr<const media::VideoFrame> frame() const;
// GarbageCollected override
void Trace(Visitor*) const override;
private:
static bool IsSupportedPlanarFormat(media::VideoFrame*);
// ImageBitmapSource implementation
static constexpr uint64_t kCpuEfficientFrameSize = 320u * 240u;
IntSize BitmapSourceSize() const override;
......@@ -111,7 +93,8 @@ class MODULES_EXPORT VideoFrame final : public ScriptWrappable,
const ImageBitmapOptions*,
ExceptionState&) override;
scoped_refptr<VideoFrame::Handle> handle_;
scoped_refptr<VideoFrameHandle> handle_;
HeapVector<Member<Plane>> planes_;
};
} // namespace blink
......
......@@ -8,7 +8,7 @@
#include "base/optional.h"
#include "third_party/blink/renderer/bindings/core/v8/serialization/serialized_script_value.h"
#include "third_party/blink/renderer/modules/modules_export.h"
#include "third_party/blink/renderer/modules/webcodecs/video_frame.h"
#include "third_party/blink/renderer/modules/webcodecs/video_frame_handle.h"
namespace blink {
......@@ -26,16 +26,14 @@ class MODULES_EXPORT VideoFrameAttachment
size_t size() const { return frame_handles_.size(); }
Vector<scoped_refptr<VideoFrame::Handle>>& Handles() {
return frame_handles_;
}
Vector<scoped_refptr<VideoFrameHandle>>& Handles() { return frame_handles_; }
const Vector<scoped_refptr<VideoFrame::Handle>>& Handles() const {
const Vector<scoped_refptr<VideoFrameHandle>>& Handles() const {
return frame_handles_;
}
private:
Vector<scoped_refptr<VideoFrame::Handle>> frame_handles_;
Vector<scoped_refptr<VideoFrameHandle>> frame_handles_;
};
} // namespace blink
......
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "third_party/blink/renderer/modules/webcodecs/video_frame_handle.h"
namespace blink {
VideoFrameHandle::VideoFrameHandle(scoped_refptr<media::VideoFrame> frame)
: frame_(std::move(frame)) {
DCHECK(frame_);
}
scoped_refptr<media::VideoFrame> VideoFrameHandle::frame() {
WTF::MutexLocker locker(mutex_);
return frame_;
}
void VideoFrameHandle::Invalidate() {
WTF::MutexLocker locker(mutex_);
frame_.reset();
}
} // namespace blink
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_VIDEO_FRAME_HANDLE_H_
#define THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_VIDEO_FRAME_HANDLE_H_
#include "base/memory/scoped_refptr.h"
#include "media/base/video_frame.h"
#include "third_party/blink/renderer/modules/modules_export.h"
#include "third_party/blink/renderer/platform/wtf/thread_safe_ref_counted.h"
#include "third_party/blink/renderer/platform/wtf/threading_primitives.h"
namespace blink {
// Wrapper class that allows sharing a single |frame_| reference across
// multiple VideoFrames, which can be invalidated for all frames at once.
class MODULES_EXPORT VideoFrameHandle
: public WTF::ThreadSafeRefCounted<VideoFrameHandle> {
public:
explicit VideoFrameHandle(scoped_refptr<media::VideoFrame>);
// Returns a copy of |frame_|, which should be re-used throughout the scope
// of a function call, instead of calling frame() multiple times. Otherwise
// the frame could be destroyed between calls.
scoped_refptr<media::VideoFrame> frame();
// Releases the underlying media::VideoFrame reference, affecting all
// blink::VideoFrames and blink::Planes that hold a reference to |this|.
void Invalidate();
private:
friend class WTF::ThreadSafeRefCounted<VideoFrameHandle>;
~VideoFrameHandle() = default;
WTF::Mutex mutex_;
scoped_refptr<media::VideoFrame> frame_;
};
} // namespace blink
#endif // THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_VIDEO_FRAME_HANDLE_H_
......@@ -6,6 +6,7 @@
#include "testing/gtest/include/gtest/gtest.h"
#include "third_party/blink/renderer/bindings/core/v8/v8_binding_for_testing.h"
#include "third_party/blink/renderer/modules/webcodecs/video_frame.h"
#include "third_party/blink/renderer/modules/webcodecs/video_frame_handle.h"
#include "ui/gfx/geometry/rect.h"
#include "ui/gfx/geometry/size.h"
......@@ -20,7 +21,7 @@ class VideoFrameTest : public testing::Test {
return MakeGarbageCollected<VideoFrame>(std::move(media_frame));
}
VideoFrame* CreateBlinkVideoFrameFromHandle(
scoped_refptr<VideoFrame::Handle> handle) {
scoped_refptr<VideoFrameHandle> handle) {
return MakeGarbageCollected<VideoFrame>(std::move(handle));
}
scoped_refptr<media::VideoFrame> CreateDefaultBlackMediaVideoFrame() {
......@@ -92,7 +93,7 @@ TEST_F(VideoFrameTest, FramesNotSharingHandleDestruction) {
VideoFrame* blink_frame = CreateBlinkVideoFrame(media_frame);
auto new_handle =
base::MakeRefCounted<VideoFrame::Handle>(blink_frame->frame());
base::MakeRefCounted<VideoFrameHandle>(blink_frame->frame());
VideoFrame* frame_with_new_handle =
CreateBlinkVideoFrameFromHandle(std::move(new_handle));
......
<!DOCTYPE html>
<html>
<title>Test the VideoFrame API.</title>
<body></body>
<script src="/resources/testharness.js"></script>
<script src="/resources/testharnessreport.js"></script>
<script>
function makeImageBitmap(width, height) {
let canvas = new OffscreenCanvas(width, height);
let ctx = canvas.getContext('2d');
ctx.fillStyle = 'rgba(50, 100, 150, 255)';
ctx.fillRect(0, 0, width, height);
return canvas.transferToImageBitmap();
}
test(t => {
let image = makeImageBitmap(32, 16);
let frame = new VideoFrame(image, {timestamp: 10});
assert_equals(frame.timestamp, 10, "timestamp");
assert_equals(frame.duration, null, "duration");
assert_equals(frame.cropWidth, 32, "cropWidth");
assert_equals(frame.cropHeight, 16, "cropHeight");
assert_equals(frame.cropWidth, 32, "displayWidth");
assert_equals(frame.cropHeight, 16, "displayHeight");
frame.destroy();
}, 'Test we can construct a VideoFrame.');
test(t => {
let image = makeImageBitmap(1, 1);
let frame = new VideoFrame(image, {timestamp: 10});
assert_equals(frame.cropWidth, 1, "cropWidth");
assert_equals(frame.cropHeight, 1, "cropHeight");
assert_equals(frame.cropWidth, 1, "displayWidth");
assert_equals(frame.cropHeight, 1, "displayHeight");
frame.destroy();
}, 'Test we can construct an odd-sized VideoFrame.');
test(t => {
let image = makeImageBitmap(32, 16);
let frame = new VideoFrame(image, {timestamp: 0});
// TODO(sandersd): This would be more clear as RGBA, but conversion has
// not be specified (or implemented) yet.
if (frame.format !== "I420") {
return;
}
assert_equals(frame.planes.length, 3, "number of planes");
// Validate Y plane metadata.
let yPlane = frame.planes[0];
let yStride = yPlane.stride;
let yRows = yPlane.rows;
let yLength = yPlane.length;
// Required minimums to contain the visible data.
assert_greater_than_equal(yRows, 16, "Y plane rows");
assert_greater_than_equal(yStride, 32, "Y plane stride");
assert_greater_than_equal(yLength, 32 * 16, "Y plane length");
// Not required by spec, but sets limit at 50% padding per dimension.
assert_less_than_equal(yRows, 32, "Y plane rows");
assert_less_than_equal(yStride, 64, "Y plane stride");
assert_less_than_equal(yLength, 32 * 64, "Y plane length");
// Validate Y plane data.
let buffer = new ArrayBuffer(yLength);
let view = new Uint8Array(buffer);
frame.planes[0].readInto(view);
// TODO(sandersd): This probably needs to be fuzzy unless we can make
// guarantees about the color space.
assert_equals(view[0], 94, "Y value at (0, 0)");
frame.destroy();
}, 'Test we can read planar data from a VideoFrame.');
test(t => {
let image = makeImageBitmap(32, 16);
let frame = new VideoFrame(image, {timestamp: 0});
// TODO(sandersd): This would be more clear as RGBA, but conversion has
// not be specified (or implemented) yet.
if (frame.format !== "I420") {
return;
}
assert_equals(frame.planes.length, 3, "number of planes");
// Attempt to read Y plane data, but destroy the frame first.
let yPlane = frame.planes[0];
let yLength = yPlane.length;
frame.destroy();
let buffer = new ArrayBuffer(yLength);
let view = new Uint8Array(buffer);
assert_throws_dom("InvalidStateError", () => yPlane.readInto(view));
}, 'Test we cannot read planar data from a destroyed VideoFrame.');
</script>
</html>
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment