Commit 2f029071 authored by Eugene Zemtsov's avatar Eugene Zemtsov Committed by Commit Bot

[webcodecs] VideoEncoder now emits decoder configuration if needed

1. VideoEncodeAcceleratorAdapter calls H264AnnexBToAvcBitstreamConverter
   to convert H264 bitstreams
2. webcodecs::VideoEncoder and media::VideoEncoder have callbacks
   to report decoder config

Bug: 1122955
Change-Id: I897974a04b60cdb2a5c1ce98a9812b6ca9648d80
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2415221Reviewed-by: default avatarDan Sanders <sandersd@chromium.org>
Commit-Queue: Eugene Zemtsov <eugene@chromium.org>
Cr-Commit-Position: refs/heads/master@{#809188}
parent b94a6751
......@@ -47,9 +47,16 @@ class MEDIA_EXPORT VideoEncoder {
base::Optional<int> keyframe_interval = 10000;
};
// A sequence of codec specific bytes, commonly known as extradata.
// If available, it should be given to the decoder as part of the
// decoder config.
using CodecDescription = std::vector<uint8_t>;
// Callback for VideoEncoder to report an encoded video frame whenever it
// becomes available.
using OutputCB = base::RepeatingCallback<void(VideoEncoderOutput output)>;
using OutputCB =
base::RepeatingCallback<void(VideoEncoderOutput output,
base::Optional<CodecDescription>)>;
// Callback to report success and errors in encoder calls.
using StatusCB = base::OnceCallback<void(Status error)>;
......
......@@ -101,8 +101,15 @@ source_set("formats") {
"mpeg/adts_stream_parser.cc",
"mpeg/adts_stream_parser.h",
]
}
# We need this to make h264_annex_b_to_avc_bitstream_converter.h accessible
# from video_encode_accelerator_adapter.cc, unfortunately we can't move
# h264_annex_b_to_avc_bitstream_converter.h since it depends on
# box_definitions.h
# TODO(eugene): consider moving media/video/h264_parser.(h|cc) to
# media/formats. It might help to break this dependency cycle.
allow_circular_includes_from = [ "//media/video" ]
}
if (proprietary_codecs && enable_platform_hevc) {
sources += [
"mp4/hevc.cc",
......
......@@ -14,6 +14,9 @@
#include "base/time/time.h"
#include "media/base/bind_to_current_loop.h"
#include "media/base/video_frame.h"
#if BUILDFLAG(USE_PROPRIETARY_CODECS)
#include "media/formats/mp4/h264_annex_b_to_avc_bitstream_converter.h"
#endif // BUILDFLAG(USE_PROPRIETARY_CODECS)
#include "media/video/gpu_video_accelerator_factories.h"
#include "third_party/libyuv/include/libyuv.h"
......@@ -179,6 +182,11 @@ void VideoEncodeAcceleratorAdapter::InitializeOnAcceleratorThread(
return;
}
#if BUILDFLAG(USE_PROPRIETARY_CODECS)
if (profile >= H264PROFILE_MIN && profile <= H264PROFILE_MAX)
h264_converter_ = std::make_unique<H264AnnexBToAvcBitstreamConverter>();
#endif // BUILDFLAG(USE_PROPRIETARY_CODECS)
output_cb_ = std::move(output_cb);
state_ = State::kInitializing;
pending_init_ = std::make_unique<PendingOp>();
......@@ -336,16 +344,51 @@ void VideoEncodeAcceleratorAdapter::RequireBitstreamBuffers(
void VideoEncodeAcceleratorAdapter::BitstreamBufferReady(
int32_t buffer_id,
const BitstreamBufferMetadata& metadata) {
base::Optional<CodecDescription> desc;
VideoEncoderOutput result;
result.key_frame = metadata.key_frame;
result.timestamp = metadata.timestamp;
result.size = metadata.payload_size_bytes;
result.data.reset(new uint8_t[result.size]);
base::WritableSharedMemoryMapping* mapping =
output_pool_->GetMapping(buffer_id);
DCHECK_LE(result.size, mapping->size());
memcpy(result.data.get(), mapping->memory(), result.size);
#if BUILDFLAG(USE_PROPRIETARY_CODECS)
if (h264_converter_) {
uint8_t* src = static_cast<uint8_t*>(mapping->memory());
size_t dst_size = result.size;
size_t actual_output_size = 0;
bool config_changed = false;
std::unique_ptr<uint8_t[]> dst(new uint8_t[dst_size]);
auto status =
h264_converter_->ConvertChunk(base::span<uint8_t>(src, result.size),
base::span<uint8_t>(dst.get(), dst_size),
&config_changed, &actual_output_size);
if (!status.is_ok()) {
LOG(ERROR) << status.message();
NotifyError(VideoEncodeAccelerator::kPlatformFailureError);
return;
}
result.size = actual_output_size;
result.data = std::move(dst);
if (config_changed) {
const auto& config = h264_converter_->GetCurrentConfig();
desc = CodecDescription();
if (!config.Serialize(desc.value())) {
NotifyError(VideoEncodeAccelerator::kPlatformFailureError);
return;
}
}
} else {
#endif // BUILDFLAG(USE_PROPRIETARY_CODECS)
result.data.reset(new uint8_t[result.size]);
memcpy(result.data.get(), mapping->memory(), result.size);
#if BUILDFLAG(USE_PROPRIETARY_CODECS)
}
#endif // BUILDFLAG(USE_PROPRIETARY_CODECS)
// Give the buffer back to |accelerator_|
base::UnsafeSharedMemoryRegion* region = output_pool_->GetRegion(buffer_id);
......@@ -359,7 +402,7 @@ void VideoEncodeAcceleratorAdapter::BitstreamBufferReady(
break;
}
}
output_cb_.Run(std::move(result));
output_cb_.Run(std::move(result), std::move(desc));
if (pending_encodes_.empty() && !accelerator_->IsFlushSupported()) {
// Manually call FlushCompleted(), since |accelerator_| won't do it for us.
FlushCompleted(true);
......@@ -388,6 +431,7 @@ void VideoEncodeAcceleratorAdapter::NotifyError(
std::move(encode->done_callback).Run(Status());
}
pending_encodes_.clear();
state_ = State::kNotInitialized;
}
void VideoEncodeAcceleratorAdapter::NotifyEncoderInfoChange(
......
......@@ -19,6 +19,7 @@
namespace media {
class GpuVideoAcceleratorFactories;
class H264AnnexBToAvcBitstreamConverter;
// This class is a somewhat complex adapter from VideoEncodeAccelerator
// to VideoEncoder, it takes cares of such things as
......@@ -95,6 +96,10 @@ class MEDIA_EXPORT VideoEncodeAcceleratorAdapter
std::unique_ptr<VideoEncodeAccelerator> accelerator_;
media::GpuVideoAcceleratorFactories* gpu_factories_;
#if BUILDFLAG(USE_PROPRIETARY_CODECS)
std::unique_ptr<H264AnnexBToAvcBitstreamConverter> h264_converter_;
#endif // BUILDFLAG(USE_PROPRIETARY_CODECS)
base::circular_deque<std::unique_ptr<PendingOp>> pending_encodes_;
std::unique_ptr<PendingOp> pending_flush_;
std::unique_ptr<PendingOp> pending_init_;
......
......@@ -371,7 +371,7 @@ void VpxVideoEncoder::DrainOutputs() {
result.size = pkt->data.frame.sz;
result.data.reset(new uint8_t[result.size]);
memcpy(result.data.get(), pkt->data.frame.buf, result.size);
output_cb_.Run(std::move(result));
output_cb_.Run(std::move(result), {});
}
}
}
......
......@@ -26,6 +26,7 @@
#include "third_party/blink/renderer/bindings/core/v8/script_function.h"
#include "third_party/blink/renderer/bindings/core/v8/script_promise.h"
#include "third_party/blink/renderer/bindings/core/v8/v8_dom_exception.h"
#include "third_party/blink/renderer/bindings/modules/v8/v8_video_decoder_config.h"
#include "third_party/blink/renderer/bindings/modules/v8/v8_video_encoder_config.h"
#include "third_party/blink/renderer/bindings/modules/v8/v8_video_encoder_encode_options.h"
#include "third_party/blink/renderer/bindings/modules/v8/v8_video_encoder_init.h"
......@@ -170,6 +171,7 @@ std::unique_ptr<VideoEncoder::ParsedConfig> VideoEncoder::ParseConfig(
parsed->profile = media::VIDEO_CODEC_PROFILE_UNKNOWN;
parsed->color_space = media::VideoColorSpace::REC709();
parsed->level = 0;
parsed->codec_string = config->codec();
bool parse_succeeded = media::ParseVideoCodecString(
"", config->codec().Utf8(), &is_codec_ambiguous, &parsed->codec,
......@@ -260,7 +262,7 @@ void VideoEncoder::configure(const VideoEncoderConfig* config,
Request* request = MakeGarbageCollected<Request>();
request->type = Request::Type::kConfigure;
request->config = std::move(parsed_config);
active_config_ = std::move(parsed_config);
EnqueueRequest(request);
}
......@@ -286,8 +288,10 @@ void VideoEncoder::encode(VideoFrame* frame,
return;
}
if (internal_frame->cropWidth() != uint32_t{frame_size_.width()} ||
internal_frame->cropHeight() != uint32_t{frame_size_.height()}) {
DCHECK(active_config_);
if (internal_frame->cropWidth() != uint32_t{active_config_->options.width} ||
internal_frame->cropHeight() !=
uint32_t{active_config_->options.height}) {
exception_state.ThrowDOMException(
DOMExceptionCode::kOperationError,
"Frame size doesn't match initial encoder parameters.");
......@@ -363,14 +367,6 @@ void VideoEncoder::ClearRequests() {
}
}
void VideoEncoder::CallOutputCallback(EncodedVideoChunk* chunk) {
if (!script_state_->ContextIsValid() || !output_callback_ ||
state_.AsEnum() != V8CodecState::Enum::kConfigured)
return;
ScriptState::Scope scope(script_state_);
output_callback_->InvokeAndReportException(nullptr, chunk);
}
void VideoEncoder::HandleError(DOMException* ex) {
// Save a temp before we clear the callback.
V8WebCodecsErrorCallback* error_callback = error_callback_.Get();
......@@ -462,13 +458,11 @@ void VideoEncoder::ProcessEncode(Request* request) {
void VideoEncoder::ProcessConfigure(Request* request) {
DCHECK_NE(state_.AsEnum(), V8CodecState::Enum::kClosed);
DCHECK(request->config);
DCHECK_EQ(request->type, Request::Type::kConfigure);
DCHECK(active_config_);
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
auto config = std::move(request->config);
switch (config->codec) {
switch (active_config_->codec) {
case media::kCodecVP8:
case media::kCodecVP9:
media_encoder_ = CreateVpxVideoEncoder();
......@@ -491,9 +485,7 @@ void VideoEncoder::ProcessConfigure(Request* request) {
return;
}
frame_size_ = gfx::Size(config->options.width, config->options.height);
auto output_cb = WTF::BindRepeating(&VideoEncoder::MediaEncoderOutputCallback,
auto output_cb = WTF::BindRepeating(&VideoEncoder::CallOutputCallback,
WrapWeakPersistent(this));
auto done_callback = [](VideoEncoder* self, Request* req,
......@@ -501,16 +493,20 @@ void VideoEncoder::ProcessConfigure(Request* request) {
if (!self)
return;
DCHECK_CALLED_ON_VALID_SEQUENCE(self->sequence_checker_);
DCHECK(self->active_config_);
if (!status.is_ok()) {
std::string msg = "Encoder initialization error: " + status.message();
self->HandleError(DOMExceptionCode::kOperationError, msg.c_str());
}
self->stall_request_processing_ = false;
self->ProcessRequests();
};
stall_request_processing_ = true;
media_encoder_->Initialize(config->profile, config->options, output_cb,
media_encoder_->Initialize(active_config_->profile, active_config_->options,
std::move(output_cb),
WTF::Bind(done_callback, WrapWeakPersistent(this),
WrapPersistent(request)));
}
......@@ -545,8 +541,13 @@ void VideoEncoder::ProcessFlush(Request* request) {
WrapPersistentIfNeeded(request)));
}
void VideoEncoder::MediaEncoderOutputCallback(
media::VideoEncoderOutput output) {
void VideoEncoder::CallOutputCallback(
media::VideoEncoderOutput output,
base::Optional<media::VideoEncoder::CodecDescription> codec_desc) {
if (!script_state_->ContextIsValid() || !output_callback_ ||
state_.AsEnum() != V8CodecState::Enum::kConfigured)
return;
EncodedVideoMetadata metadata;
metadata.timestamp = output.timestamp;
metadata.key_frame = output.key_frame;
......@@ -556,7 +557,21 @@ void VideoEncoder::MediaEncoderOutputCallback(
ArrayBufferContents data(output.data.release(), output.size, deleter);
auto* dom_array = MakeGarbageCollected<DOMArrayBuffer>(std::move(data));
auto* chunk = MakeGarbageCollected<EncodedVideoChunk>(metadata, dom_array);
CallOutputCallback(chunk);
DCHECK(active_config_);
VideoDecoderConfig* decoder_config =
MakeGarbageCollected<VideoDecoderConfig>();
decoder_config->setCodec(active_config_->codec_string);
decoder_config->setCodedHeight(active_config_->options.height);
decoder_config->setCodedWidth(active_config_->options.width);
if (codec_desc.has_value()) {
auto* desc_array_buf = DOMArrayBuffer::Create(codec_desc.value().data(),
codec_desc.value().size());
decoder_config->setDescription(
ArrayBufferOrArrayBufferView::FromArrayBuffer(desc_array_buf));
}
ScriptState::Scope scope(script_state_);
output_callback_->InvokeAndReportException(nullptr, chunk, decoder_config);
}
void VideoEncoder::Trace(Visitor* visitor) const {
......
......@@ -69,8 +69,6 @@ class MODULES_EXPORT VideoEncoder final : public ScriptWrappable {
// TODO(ezemtsov): Replace this with a {Audio|Video}EncoderConfig.
struct ParsedConfig final {
void Trace(Visitor*) const;
media::VideoCodec codec;
media::VideoCodecProfile profile;
uint8_t level;
......@@ -79,6 +77,7 @@ class MODULES_EXPORT VideoEncoder final : public ScriptWrappable {
AccelerationPreference acc_pref;
media::VideoEncoder::Options options;
String codec_string;
};
struct Request final : public GarbageCollected<Request> {
......@@ -91,13 +90,14 @@ class MODULES_EXPORT VideoEncoder final : public ScriptWrappable {
void Trace(Visitor*) const;
Type type;
std::unique_ptr<ParsedConfig> config; // used by kConfigure
Member<VideoFrame> frame; // used by kEncode
Member<const VideoEncoderEncodeOptions> encodeOpts; // used by kEncode
Member<ScriptPromiseResolver> resolver; // used by kFlush
};
void CallOutputCallback(EncodedVideoChunk* chunk);
void CallOutputCallback(
media::VideoEncoderOutput output,
base::Optional<media::VideoEncoder::CodecDescription> codec_desc);
void HandleError(DOMException* ex);
void HandleError(DOMExceptionCode code, const String& message);
void EnqueueRequest(Request* request);
......@@ -108,13 +108,11 @@ class MODULES_EXPORT VideoEncoder final : public ScriptWrappable {
void ClearRequests();
void MediaEncoderOutputCallback(media::VideoEncoderOutput output);
std::unique_ptr<ParsedConfig> ParseConfig(const VideoEncoderConfig*,
ExceptionState&);
bool VerifyCodecSupport(ParsedConfig*, ExceptionState&);
gfx::Size frame_size_;
std::unique_ptr<ParsedConfig> active_config_;
std::unique_ptr<media::VideoEncoder> media_encoder_;
V8CodecState state_;
......
......@@ -5,4 +5,5 @@
// https://github.com/WICG/web-codecs
// Handles a new encoded video chunk on the consumer side of the video encoder.
callback VideoEncoderOutputCallback = void (EncodedVideoChunk chunk);
\ No newline at end of file
callback VideoEncoderOutputCallback =
void (EncodedVideoChunk chunk, VideoDecoderConfig decoder_config);
\ No newline at end of file
<!DOCTYPE html>
<html>
<head>
<script src="../resources/testharness.js"></script>
<script src="../resources/testharnessreport.js"></script>
<script src="../resources/testharness.js"></script>
<script src="../resources/testharnessreport.js"></script>
</head>
<body>
<img id='frame_image' style='display: none;' src="pattern.png">
<script>
'use strict';
async function generateBitmap(width, height, text) {
let img = document.getElementById('frame_image');
let cnv = document.createElement("canvas");
cnv.height = height;
cnv.width = width;
var ctx = cnv.getContext('2d');
ctx.drawImage(img, 0, 0, width, height);
ctx.font = '30px fantasy';
ctx.fillText(text, 5, 40);
return createImageBitmap(cnv);
}
async function createFrame(width, height, ts) {
let imageBitmap = await generateBitmap(width, height, ts.toString());
return new VideoFrame(imageBitmap, { timestamp: ts });
}
async function encode_test(codec, acc) {
let w = 640;
let h = 480;
let next_ts = 0
let frames_to_encode = 20;
let frames_processed = 0;
let process_video_chunk = function(chunk) {
assert_greater_than_equal(chunk.timestamp, next_ts++);
let type = (chunk.timestamp % 5 == 0) ? "key" : "delta";
assert_equals(chunk.type, type);
var data = new Uint8Array(chunk.data);
assert_greater_than_equal(data.length, 0);
console.log("data len: " + data.length);
frames_processed++;
};
const init = {
output : process_video_chunk,
error: (e) => { console.log(e.message); }
};
const params = {
codec : codec,
acceleration: acc,
width : w,
height : h,
bitrate: 10e6,
framerate: 30,
};
let encoder = new VideoEncoder(init);
encoder.configure(params);
for (let i = 0; i < frames_to_encode; i++) {
var frame = await createFrame(w, h, i);
let keyframe = (i % 5 == 0);
encoder.encode(frame, { keyFrame : keyframe});
}
await encoder.flush();
encoder.close();
assert_equals(frames_processed, frames_to_encode);
}
promise_test(encode_test.bind(null, "vp09.00.10.08", "allow"),
"encoding vp9 profile0");
promise_test(encode_test.bind(null, "vp09.02.10.10", "allow"),
"encoding vp9 profile2");
promise_test(encode_test.bind(null, "vp8", "allow"),
"encoding vp8");
/* Uncomment this for manual testing, before we have GPU tests for that
promise_test(encode_test.
bind(null, "avc1.42001E", "avc1.42001E", "require"),
"encoding avc1.42001E");
*/
</script>
<img id='frame_image' style='display: none;' src="pattern.png">
<script>
'use strict';
async function generateBitmap(width, height, text) {
let img = document.getElementById('frame_image');
let cnv = document.createElement("canvas");
cnv.height = height;
cnv.width = width;
var ctx = cnv.getContext('2d');
ctx.drawImage(img, 0, 0, width, height);
ctx.font = '30px fantasy';
ctx.fillText(text, 5, 40);
return createImageBitmap(cnv);
}
async function createFrame(width, height, ts) {
let imageBitmap = await generateBitmap(width, height, ts.toString());
return new VideoFrame(imageBitmap, { timestamp: ts });
}
function delay(time_ms) {
return new Promise((resolve, reject) => {
setTimeout(resolve, time_ms);
});
};
async function encode_decode_test(codec) {
const w = 120;
const h = 60;
let next_ts = 0
let frames_to_encode = 16;
let frames_encoded = 0;
let frames_decoded = 0;
let decoder = new VideoDecoder({
output(frame) {
assert_equals(frame.cropWidth, w, "cropWidth");
assert_equals(frame.cropHeight, h, "cropHeight");
assert_equals(frame.timestamp, next_ts++, "timestamp");
frames_decoded++;
frame.destroy();
},
error(e) { console.log(e.message); }
});
const encoder_init = {
output(chunk, config) {
frames_encoded++;
if (decoder.state != "configured")
decoder.configure(config);
decoder.decode(chunk);
},
error(e) { console.log(e.message); }
};
const encoder_config = {
codec: codec,
acceleration: "allow",
width: w,
height: h,
bitrate: 10e6,
framerate: 30,
};
let encoder = new VideoEncoder(encoder_init);
encoder.configure(encoder_config);
for (let i = 0; i < frames_to_encode; i++) {
var frame = await createFrame(w, h, i);
let keyframe = (i % 5 == 0);
encoder.encode(frame, { keyFrame: keyframe });
await delay(1);
}
await encoder.flush();
await decoder.flush();
encoder.close();
decoder.close();
assert_equals(frames_encoded, frames_to_encode);
assert_equals(frames_decoded, frames_to_encode);
}
async function encode_test(codec, acc) {
let w = 120;
let h = 60;
let next_ts = 0
let frames_to_encode = 16;
let frames_processed = 0;
let process_video_chunk = function (chunk, config) {
assert_greater_than_equal(chunk.timestamp, next_ts++);
let type = (chunk.timestamp % 5 == 0) ? "key" : "delta";
assert_equals(chunk.type, type);
var data = new Uint8Array(chunk.data);
assert_greater_than_equal(data.length, 0);
if (config) {
assert_equals(config.codec, codec);
assert_equals(config.codedWidth, w);
assert_equals(config.codedHeight, h);
let data = new Uint8Array(config.description);
console.log("extra data len: " + data.length);
}
frames_processed++;
};
const init = {
output: process_video_chunk,
error: (e) => { console.log(e.message); },
};
const params = {
codec: codec,
acceleration: acc,
width: w,
height: h,
bitrate: 10e6,
framerate: 30,
};
let encoder = new VideoEncoder(init);
encoder.configure(params);
for (let i = 0; i < frames_to_encode; i++) {
var frame = await createFrame(w, h, i);
let keyframe = (i % 5 == 0);
encoder.encode(frame, { keyFrame: keyframe });
await delay(1);
}
await encoder.flush();
encoder.close();
assert_equals(frames_processed, frames_to_encode);
}
promise_test(encode_test.bind(null, "vp09.00.10.08", "allow"),
"encoding vp9 profile0");
promise_test(encode_test.bind(null, "vp09.02.10.10", "allow"),
"encoding vp9 profile2");
promise_test(encode_test.bind(null, "vp8", "allow"),
"encoding vp8");
promise_test(encode_decode_test.bind(null, "vp8"),
"encoding and decoding vp8");
promise_test(encode_decode_test.bind(null, "vp09.02.10.10"),
"encoding and decoding vp09.02.10.10");
/* Uncomment this for manual testing, before we have GPU tests for that */
// promise_test(encode_test.bind(null, "avc1.42001E", "require"),
// "encoding avc1.42001E");
// promise_test(encode_decode_test.bind(null, "avc1.42001E"),
// "encoding and decoding avc1.42001E");
</script>
</body>
</html>
</html>
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment