Commit fb915a42 authored by Ren-Pei Zeng's avatar Ren-Pei Zeng Committed by Commit Bot

Enable MjpegDecodeAccelerator to input DMA buffer

This CL adds a new Decode() interface in MjpegDecodeAccelerator to use
DMA buffer FD, size and offset as input, and enables it in VAAPI and
V4L2 implementations.

Bug: b:120057531
Test: jpeg_decode_accelerator_unittest
Test: media_unittests --gtest_filter=MojoMjpeg*
Change-Id: I5b53e599cb887d9e8fa8f85cd7f5d83ddfc3aa94
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1730891
Commit-Queue: Ren-Pei Zeng <kamesan@chromium.org>
Reviewed-by: default avatarDaniele Castagna <dcastagna@chromium.org>
Reviewed-by: default avatarRicky Liang <jcliang@chromium.org>
Reviewed-by: default avatarAndres Calderon Jaramillo <andrescj@chromium.org>
Cr-Commit-Position: refs/heads/master@{#693014}
parent 0f39ef65
......@@ -5,9 +5,12 @@
#include "components/chromeos_camera/fake_mjpeg_decode_accelerator.h"
#include "base/bind.h"
#include "base/logging.h"
#include "base/single_thread_task_runner.h"
#include "base/threading/thread_task_runner_handle.h"
#include "media/base/unaligned_shared_memory.h"
#include "media/base/video_frame.h"
#include "media/base/video_types.h"
namespace chromeos_camera {
......@@ -58,8 +61,17 @@ void FakeMjpegDecodeAccelerator::Decode(
std::move(video_frame), base::Passed(&src_shm)));
}
void FakeMjpegDecodeAccelerator::Decode(
int32_t task_id,
base::ScopedFD src_dmabuf_fd,
size_t src_size,
off_t src_offset,
scoped_refptr<media::VideoFrame> dst_frame) {
NOTIMPLEMENTED();
}
void FakeMjpegDecodeAccelerator::DecodeOnDecoderThread(
int32_t bitstream_buffer_id,
int32_t task_id,
scoped_refptr<media::VideoFrame> video_frame,
std::unique_ptr<media::UnalignedSharedMemory> src_shm) {
DCHECK(decoder_task_runner_->BelongsToCurrentThread());
......@@ -73,32 +85,29 @@ void FakeMjpegDecodeAccelerator::DecodeOnDecoderThread(
client_task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&FakeMjpegDecodeAccelerator::OnDecodeDoneOnClientThread,
weak_factory_.GetWeakPtr(), bitstream_buffer_id));
weak_factory_.GetWeakPtr(), task_id));
}
bool FakeMjpegDecodeAccelerator::IsSupported() {
return true;
}
void FakeMjpegDecodeAccelerator::NotifyError(int32_t bitstream_buffer_id,
Error error) {
void FakeMjpegDecodeAccelerator::NotifyError(int32_t task_id, Error error) {
client_task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&FakeMjpegDecodeAccelerator::NotifyErrorOnClientThread,
weak_factory_.GetWeakPtr(), bitstream_buffer_id, error));
weak_factory_.GetWeakPtr(), task_id, error));
}
void FakeMjpegDecodeAccelerator::NotifyErrorOnClientThread(
int32_t bitstream_buffer_id,
Error error) {
void FakeMjpegDecodeAccelerator::NotifyErrorOnClientThread(int32_t task_id,
Error error) {
DCHECK(client_task_runner_->BelongsToCurrentThread());
client_->NotifyError(bitstream_buffer_id, error);
client_->NotifyError(task_id, error);
}
void FakeMjpegDecodeAccelerator::OnDecodeDoneOnClientThread(
int32_t input_buffer_id) {
void FakeMjpegDecodeAccelerator::OnDecodeDoneOnClientThread(int32_t task_id) {
DCHECK(client_task_runner_->BelongsToCurrentThread());
client_->VideoFrameReady(input_buffer_id);
client_->VideoFrameReady(task_id);
}
} // namespace chromeos_camera
......@@ -34,16 +34,21 @@ class FakeMjpegDecodeAccelerator : public MjpegDecodeAccelerator {
bool Initialize(MjpegDecodeAccelerator::Client* client) override;
void Decode(media::BitstreamBuffer bitstream_buffer,
scoped_refptr<media::VideoFrame> video_frame) override;
void Decode(int32_t task_id,
base::ScopedFD src_dmabuf_fd,
size_t src_size,
off_t src_offset,
scoped_refptr<media::VideoFrame> dst_frame) override;
bool IsSupported() override;
private:
void DecodeOnDecoderThread(
int32_t bitstream_buffer_id,
int32_t task_id,
scoped_refptr<media::VideoFrame> video_frame,
std::unique_ptr<media::UnalignedSharedMemory> src_shm);
void NotifyError(int32_t bitstream_buffer_id, Error error);
void NotifyErrorOnClientThread(int32_t bitstream_buffer_id, Error error);
void OnDecodeDoneOnClientThread(int32_t input_buffer_id);
void NotifyError(int32_t task_id, Error error);
void NotifyErrorOnClientThread(int32_t task_id, Error error);
void OnDecodeDoneOnClientThread(int32_t task_id);
// Task runner for calls to |client_|.
const scoped_refptr<base::SingleThreadTaskRunner> client_task_runner_;
......
......@@ -5,19 +5,26 @@
#ifndef COMPONENTS_CHROMEOS_CAMERA_MJPEG_DECODE_ACCELERATOR_H_
#define COMPONENTS_CHROMEOS_CAMERA_MJPEG_DECODE_ACCELERATOR_H_
#include <stddef.h>
#include <stdint.h>
#include <sys/types.h>
#include "base/callback.h"
#include "base/files/scoped_file.h"
#include "base/memory/scoped_refptr.h"
#include "media/base/bitstream_buffer.h"
#include "media/base/video_frame.h"
namespace media {
class VideoFrame;
}
namespace chromeos_camera {
// MJPEG decoder interface.
// The input are JPEG images including headers (Huffman tables may be omitted).
// The output color format is I420. The decoder will convert the color format
// to I420 if the color space or subsampling does not match that and if it is
// capable of doing so. The client is responsible for allocating buffers and
// keeps the ownership of them.
// The decoder will convert to the output color format if the input color format
// or subsampling does not match that and if it is capable of doing so. The
// client is responsible for allocating buffers and keeps the ownership of them.
// The intended use case of this interface is decoding MJPEG images coming
// from camera capture. It can also be used for normal still JPEG image
// decoding, but normal JPEG images may use more JPEG features that may not be
......@@ -27,7 +34,7 @@ class MjpegDecodeAccelerator {
// Callback for JPEG decoder initialization.
typedef base::Callback<void(bool success)> InitCB;
static const int32_t kInvalidBitstreamBufferId = -1;
static const int32_t kInvalidTaskId = -1;
// Enumeration of decode errors generated by NotifyError callback. These
// values are persisted to logs. Entries should not be renumbered and numeric
......@@ -58,9 +65,8 @@ class MjpegDecodeAccelerator {
public:
// Callback called after each successful Decode().
// Parameters:
// |bitstream_buffer_id| is the id of BitstreamBuffer corresponding to
// Decode() call.
virtual void VideoFrameReady(int32_t bitstream_buffer_id) = 0;
// |task_id| is the id passed to Decode() call.
virtual void VideoFrameReady(int32_t task_id) = 0;
// Callback to notify errors. Client is responsible for destroying JDA when
// receiving a fatal error, i.e. PLATFORM_FAILURE. For other errors, client
......@@ -68,11 +74,11 @@ class MjpegDecodeAccelerator {
// using the same instance of JDA.
// Parameters:
// |error| is the error code.
// |bitstream_buffer_id| is the bitstream buffer id that resulted in the
// recoverable error. For PLATFORM_FAILURE, |bitstream_buffer_id| may be
// kInvalidBitstreamBufferId if the error was not related to any
// particular buffer being processed.
virtual void NotifyError(int32_t bitstream_buffer_id, Error error) = 0;
// |task_id| is the id passed to Decode() call that resulted in the
// recoverable error. For PLATFORM_FAILURE, |task_id| may be
// |kInvalidTaskId| if the error was not related to any particular buffer
// being processed.
virtual void NotifyError(int32_t task_id, Error error) = 0;
protected:
virtual ~Client() {}
......@@ -100,27 +106,37 @@ class MjpegDecodeAccelerator {
// Decodes the given bitstream buffer that contains one JPEG frame. It
// supports at least baseline encoding defined in JPEG ISO/IEC 10918-1. The
// decoder will convert the output to |video_frame->format()| or return
// PLATFORM_FAILURE if it cannot convert. Client still owns this buffer, but
// should deallocate or access the buffer only after receiving a decode
// callback VideoFrameReady with the corresponding |bitstream_buffer_id|, or
// NotifyError.
// PLATFORM_FAILURE if it cannot convert.
// Parameters:
// |bitstream_buffer| contains encoded JPEG frame.
// |video_frame| contains an allocated video frame for the output, backed
// with an UnsafeSharedMemoryRegion.
// with an UnsafeSharedMemoryRegion or DMA buffer.
//
// Client is responsible for filling the |video_frame->coded_size()|,
// |video_frame->visible_rect()|, and allocating its backing buffer. For
// unsafe shared memory backed VideoFrames, only I420 format is supported.
// For DMA-buf backed VideoFrames, the supported formats depend on the
// underlying hardware implementation. After decode completes, the decoded
// JPEG frame will be filled into the |video_frame|. Ownership of the
// unsafe shared memory backed VideoFrames, I420 and NV12 formats are
// supported. For DMA-buf backed VideoFrames, the supported formats depend on
// the underlying hardware implementation. After decode completes, the
// decoded JPEG frame will be filled into the |video_frame|. Ownership of the
// |bitstream_buffer| and |video_frame| remains with the client. The client
// is not allowed to deallocate them before VideoFrameReady or NotifyError()
// is invoked for given id of |bitstream_buffer|, or destructor returns.
virtual void Decode(media::BitstreamBuffer bitstream_buffer,
scoped_refptr<media::VideoFrame> video_frame) = 0;
// The same as above but the JPEG image is stored in a DMA buffer.
// Parameters:
// |src_dmabuf_fd| contains encoded JPEG frame.
// |src_size| is the size of the JPEG frame.
// |src_offset| is the offset at which the JPEG data starts.
// |dst_frame| contains an allocated video frame for the output, backed with
// an UnsafeSharedMemoryRegion or DMA buffer.
virtual void Decode(int32_t task_id,
base::ScopedFD src_dmabuf_fd,
size_t src_size,
off_t src_offset,
scoped_refptr<media::VideoFrame> dst_frame) = 0;
// Returns true when the JPEG decoder is supported. This can be called before
// Initialize().
virtual bool IsSupported() = 0;
......
......@@ -76,6 +76,15 @@ void MojoMjpegDecodeAccelerator::Decode(
base::Unretained(this)));
}
void MojoMjpegDecodeAccelerator::Decode(
int32_t task_id,
base::ScopedFD src_dmabuf_fd,
size_t src_size,
off_t src_offset,
scoped_refptr<media::VideoFrame> dst_frame) {
NOTIMPLEMENTED();
}
bool MojoMjpegDecodeAccelerator::IsSupported() {
return true;
}
......@@ -116,7 +125,7 @@ void MojoMjpegDecodeAccelerator::OnDecodeAck(
void MojoMjpegDecodeAccelerator::OnLostConnectionToJpegDecoder() {
DCHECK(io_task_runner_->RunsTasksInCurrentSequence());
OnDecodeAck(
kInvalidBitstreamBufferId,
kInvalidTaskId,
::chromeos_camera::MjpegDecodeAccelerator::Error::PLATFORM_FAILURE);
}
......
......@@ -36,6 +36,11 @@ class MojoMjpegDecodeAccelerator : public MjpegDecodeAccelerator {
void InitializeAsync(Client* client, InitCB init_cb) override;
void Decode(media::BitstreamBuffer bitstream_buffer,
scoped_refptr<media::VideoFrame> video_frame) override;
void Decode(int32_t task_id,
base::ScopedFD src_dmabuf_fd,
size_t src_size,
off_t src_offset,
scoped_refptr<media::VideoFrame> dst_frame) override;
bool IsSupported() override;
private:
......
......@@ -22,9 +22,8 @@ VideoCaptureJpegDecoderImpl::VideoCaptureJpegDecoderImpl(
decode_done_cb_(std::move(decode_done_cb)),
send_log_message_cb_(std::move(send_log_message_cb)),
has_received_decoded_frame_(false),
next_bitstream_buffer_id_(0),
in_buffer_id_(
chromeos_camera::MjpegDecodeAccelerator::kInvalidBitstreamBufferId),
next_task_id_(0),
task_id_(chromeos_camera::MjpegDecodeAccelerator::kInvalidTaskId),
decoder_status_(INIT_PENDING) {}
VideoCaptureJpegDecoderImpl::~VideoCaptureJpegDecoderImpl() {
......@@ -59,7 +58,7 @@ void VideoCaptureJpegDecoderImpl::DecodeCapturedData(
DCHECK(decoder_);
TRACE_EVENT_ASYNC_BEGIN0("jpeg", "VideoCaptureJpegDecoderImpl decoding",
next_bitstream_buffer_id_);
next_task_id_);
TRACE_EVENT0("jpeg", "VideoCaptureJpegDecoderImpl::DecodeCapturedData");
// TODO(kcwu): enqueue decode requests in case decoding is not fast enough
......@@ -96,12 +95,12 @@ void VideoCaptureJpegDecoderImpl::DecodeCapturedData(
}
memcpy(in_shared_mapping_.memory(), data, in_buffer_size);
// No need to lock for |in_buffer_id_| since IsDecoding_Locked() is false.
in_buffer_id_ = next_bitstream_buffer_id_;
media::BitstreamBuffer in_buffer(in_buffer_id_, in_shared_region_.Duplicate(),
// No need to lock for |task_id_| since IsDecoding_Locked() is false.
task_id_ = next_task_id_;
media::BitstreamBuffer in_buffer(task_id_, in_shared_region_.Duplicate(),
in_buffer_size);
// Mask against 30 bits, to avoid (undefined) wraparound on signed integer.
next_bitstream_buffer_id_ = (next_bitstream_buffer_id_ + 1) & 0x3FFFFFFF;
next_task_id_ = (next_task_id_ + 1) & 0x3FFFFFFF;
// The API of |decoder_| requires us to wrap the |out_buffer| in a VideoFrame.
const gfx::Size dimensions = frame_format.frame_size;
......@@ -155,12 +154,16 @@ void VideoCaptureJpegDecoderImpl::DecodeCapturedData(
// |decoder_task_runner_|.
decoder_task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&chromeos_camera::MjpegDecodeAccelerator::Decode,
base::Unretained(decoder_.get()), std::move(in_buffer),
std::move(out_frame)));
base::BindOnce(
[](chromeos_camera::MjpegDecodeAccelerator* decoder,
BitstreamBuffer in_buffer, scoped_refptr<VideoFrame> out_frame) {
decoder->Decode(std::move(in_buffer), std::move(out_frame));
},
base::Unretained(decoder_.get()), std::move(in_buffer),
std::move(out_frame)));
}
void VideoCaptureJpegDecoderImpl::VideoFrameReady(int32_t bitstream_buffer_id) {
void VideoCaptureJpegDecoderImpl::VideoFrameReady(int32_t task_id) {
DCHECK(decoder_task_runner_->RunsTasksInCurrentSequence());
TRACE_EVENT0("jpeg", "VideoCaptureJpegDecoderImpl::VideoFrameReady");
if (!has_received_decoded_frame_) {
......@@ -174,26 +177,23 @@ void VideoCaptureJpegDecoderImpl::VideoFrameReady(int32_t bitstream_buffer_id) {
return;
}
if (bitstream_buffer_id != in_buffer_id_) {
LOG(ERROR) << "Unexpected bitstream_buffer_id " << bitstream_buffer_id
<< ", expected " << in_buffer_id_;
if (task_id != task_id_) {
LOG(ERROR) << "Unexpected task_id " << task_id << ", expected " << task_id_;
return;
}
in_buffer_id_ =
chromeos_camera::MjpegDecodeAccelerator::kInvalidBitstreamBufferId;
task_id_ = chromeos_camera::MjpegDecodeAccelerator::kInvalidTaskId;
std::move(decode_done_closure_).Run();
TRACE_EVENT_ASYNC_END0("jpeg", "VideoCaptureJpegDecoderImpl decoding",
bitstream_buffer_id);
task_id);
}
void VideoCaptureJpegDecoderImpl::NotifyError(
int32_t bitstream_buffer_id,
int32_t task_id,
chromeos_camera::MjpegDecodeAccelerator::Error error) {
DCHECK(decoder_task_runner_->RunsTasksInCurrentSequence());
LOG(ERROR) << "Decode error, bitstream_buffer_id=" << bitstream_buffer_id
<< ", error=" << error;
LOG(ERROR) << "Decode error, task_id=" << task_id << ", error=" << error;
send_log_message_cb_.Run("Gpu Jpeg decoder failed");
base::AutoLock lock(lock_);
decode_done_closure_.Reset();
......@@ -241,11 +241,4 @@ void VideoCaptureJpegDecoderImpl::RecordInitDecodeUMA_Locked() {
decoder_status_ == INIT_PASSED);
}
void VideoCaptureJpegDecoderImpl::DestroyDecoderOnIOThread(
base::WaitableEvent* event) {
DCHECK(decoder_task_runner_->RunsTasksInCurrentSequence());
decoder_.reset();
event->Signal();
}
} // namespace media
......@@ -94,10 +94,10 @@ class CAPTURE_EXPORT VideoCaptureJpegDecoderImpl
base::OnceClosure decode_done_closure_;
// Next id for input BitstreamBuffer.
int32_t next_bitstream_buffer_id_;
int32_t next_task_id_;
// The id for current input BitstreamBuffer being decoded.
int32_t in_buffer_id_;
int32_t task_id_;
// Shared memory to store JPEG stream buffer. The input BitstreamBuffer is
// backed by this.
......
......@@ -18,8 +18,6 @@
#include "base/single_thread_task_runner.h"
#include "base/threading/thread.h"
#include "components/chromeos_camera/mjpeg_decode_accelerator.h"
#include "media/base/bitstream_buffer.h"
#include "media/base/unaligned_shared_memory.h"
#include "media/gpu/media_gpu_export.h"
#include "media/gpu/v4l2/v4l2_device.h"
......@@ -30,6 +28,14 @@ class VideoFrame;
class MEDIA_GPU_EXPORT V4L2MjpegDecodeAccelerator
: public chromeos_camera::MjpegDecodeAccelerator {
public:
// Job record. Jobs are processed in a FIFO order. This is separate from
// BufferRecord of input, because a BufferRecord of input may be returned
// before we dequeue the corresponding output buffer. It can't always be
// associated with a BufferRecord of output immediately either, because at
// the time of submission we may not have one available (and don't need one
// to submit input to the device).
class JobRecord;
V4L2MjpegDecodeAccelerator(
const scoped_refptr<V4L2Device>& device,
const scoped_refptr<base::SingleThreadTaskRunner>& io_task_runner);
......@@ -40,6 +46,11 @@ class MEDIA_GPU_EXPORT V4L2MjpegDecodeAccelerator
chromeos_camera::MjpegDecodeAccelerator::Client* client) override;
void Decode(BitstreamBuffer bitstream_buffer,
scoped_refptr<VideoFrame> video_frame) override;
void Decode(int32_t task_id,
base::ScopedFD src_dmabuf_fd,
size_t src_size,
off_t src_offset,
scoped_refptr<media::VideoFrame> dst_frame) override;
bool IsSupported() override;
private:
......@@ -54,27 +65,6 @@ class MEDIA_GPU_EXPORT V4L2MjpegDecodeAccelerator
bool at_device;
};
// Job record. Jobs are processed in a FIFO order. This is separate from
// BufferRecord of input, because a BufferRecord of input may be returned
// before we dequeue the corresponding output buffer. It can't always be
// associated with a BufferRecord of output immediately either, because at
// the time of submission we may not have one available (and don't need one
// to submit input to the device).
struct JobRecord {
JobRecord(BitstreamBuffer bitstream_buffer,
scoped_refptr<VideoFrame> video_frame);
~JobRecord();
// Input image buffer ID.
int32_t bitstream_buffer_id;
// Memory mapped from |bitstream_buffer|.
UnalignedSharedMemory shm;
// Offset used for shm.
off_t offset;
// Output frame buffer.
scoped_refptr<VideoFrame> out_frame;
};
void EnqueueInput();
void EnqueueOutput();
void Dequeue();
......@@ -103,9 +93,9 @@ class MEDIA_GPU_EXPORT V4L2MjpegDecodeAccelerator
// Destroy and create output buffers. Return false on error.
bool RecreateOutputBuffers();
void VideoFrameReady(int32_t bitstream_buffer_id);
void NotifyError(int32_t bitstream_buffer_id, Error error);
void PostNotifyError(int32_t bitstream_buffer_id, Error error);
void VideoFrameReady(int32_t task_id);
void NotifyError(int32_t task_id, Error error);
void PostNotifyError(int32_t task_id, Error error);
// Run on |decoder_thread_| to enqueue the coming frame.
void DecodeTask(std::unique_ptr<JobRecord> job_record);
......
......@@ -9,6 +9,7 @@
#include <memory>
#include "base/containers/span.h"
#include "base/macros.h"
#include "base/memory/scoped_refptr.h"
#include "base/memory/weak_ptr.h"
......@@ -53,22 +54,38 @@ class MEDIA_GPU_EXPORT VaapiMjpegDecodeAccelerator
chromeos_camera::MjpegDecodeAccelerator::Client* client) override;
void Decode(BitstreamBuffer bitstream_buffer,
scoped_refptr<VideoFrame> video_frame) override;
void Decode(int32_t task_id,
base::ScopedFD src_dmabuf_fd,
size_t src_size,
off_t src_offset,
scoped_refptr<VideoFrame> dst_frame) override;
bool IsSupported() override;
private:
// Notifies the client that an error has occurred and decoding cannot
// continue. The client is notified on the |task_runner_|, i.e., the thread in
// which |*this| was created.
void NotifyError(int32_t bitstream_buffer_id, Error error);
void NotifyError(int32_t task_id, Error error);
// Notifies the client that a decode is ready. The client is notified on the
// |task_runner_|, i.e., the thread in which |*this| was created.
void VideoFrameReady(int32_t bitstream_buffer_id);
void VideoFrameReady(int32_t task_id);
// Processes one decode request.
void DecodeTask(int32_t bitstream_buffer_id,
std::unique_ptr<UnalignedSharedMemory> shm,
scoped_refptr<VideoFrame> video_frame);
void DecodeFromShmTask(int32_t task_id,
std::unique_ptr<UnalignedSharedMemory> shm,
scoped_refptr<VideoFrame> dst_frame);
void DecodeFromDmaBufTask(int32_t task_id,
base::ScopedFD src_dmabuf_fd,
size_t src_size,
off_t src_offset,
scoped_refptr<VideoFrame> dst_frame);
// Decodes the JPEG in |src_image| into |dst_frame| and notifies the client
// when finished or when an error occurs.
void DecodeImpl(int32_t task_id,
base::span<const uint8_t> src_image,
scoped_refptr<VideoFrame> dst_frame);
// Puts contents of |surface| into given |video_frame| using VA-API Video
// Processing Pipeline (VPP), and passes the |input_buffer_id| of the
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment