media/gpu/v4l2: add GetVideoFrame() method to buffer references

Creating a VideoFrame instance from a V4L2 buffer reference is a common
operation. The best place to do this is in the buffer reference itself,
since it can export the required DMABUFs itself, has a reference to its
own device, knows about its own layout, and can cache and reuse the
VideoFrame instance instead of creating a new one for each request.

This CL adds (and uses) such a method. Although it only works for MMAP
buffers, it will probably be extended in the future to other kinds of
buffers when they support queueing a VideoFrame directly.

Bug: 792790
Test: VDA unittest passed in regular and import mode on Hana.
Test: VEA unittest passed on Peach_pit.

Change-Id: If387d8f11f8a5c72d7fc26907f5c9bea4835d8b6
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1517503
Commit-Queue: Alexandre Courbot <acourbot@chromium.org>
Reviewed-by: Hirokazu Honda <hiroh@chromium.org>
Cr-Commit-Position: refs/heads/master@{#642386}

media/gpu/v4l2/v4l2_device.cc

@@ -4,6 +4,7 @@
 
 #include "media/gpu/v4l2/v4l2_device.h"
 
+#include <algorithm>
 #include <set>
 
 #include <libdrm/drm_fourcc.h>
@@ -37,21 +38,23 @@ class V4L2Buffer {
   static std::unique_ptr<V4L2Buffer> Create(scoped_refptr<V4L2Device> device,
                                             enum v4l2_buf_type type,
                                             enum v4l2_memory memory,
-                                            size_t planes_count,
+                                            const struct v4l2_format& format,
                                             size_t buffer_id);
   ~V4L2Buffer();
 
   void* GetPlaneMapping(const size_t plane);
   size_t GetMemoryUsage() const;
   const struct v4l2_buffer* v4l2_buffer() const { return &v4l2_buffer_; }
+  const scoped_refptr<VideoFrame>& GetVideoFrame();
 
  private:
   V4L2Buffer(scoped_refptr<V4L2Device> device,
              enum v4l2_buf_type type,
              enum v4l2_memory memory,
-             size_t planes_count,
+             const struct v4l2_format& format,
              size_t buffer_id);
   bool Query();
+  scoped_refptr<VideoFrame> CreateVideoFrame();
 
   scoped_refptr<V4L2Device> device_;
   std::vector<void*> plane_mappings_;
@@ -63,17 +66,20 @@ class V4L2Buffer {
   // the number of allocated planes, resulting in memory corruption.
   struct v4l2_plane v4l2_planes_[VIDEO_MAX_PLANES] = {{}};
 
+  struct v4l2_format format_;
+  scoped_refptr<VideoFrame> video_frame_;
+
   DISALLOW_COPY_AND_ASSIGN(V4L2Buffer);
 };
 
 std::unique_ptr<V4L2Buffer> V4L2Buffer::Create(scoped_refptr<V4L2Device> device,
                                                enum v4l2_buf_type type,
                                                enum v4l2_memory memory,
-                                               size_t planes_count,
+                                               const struct v4l2_format& format,
                                                size_t buffer_id) {
   // Not using std::make_unique because constructor is private.
   std::unique_ptr<V4L2Buffer> buffer(
-      new V4L2Buffer(device, type, memory, planes_count, buffer_id));
+      new V4L2Buffer(device, type, memory, format, buffer_id));
 
   if (!buffer->Query())
     return nullptr;
@@ -84,14 +90,16 @@ std::unique_ptr<V4L2Buffer> V4L2Buffer::Create(scoped_refptr<V4L2Device> device,
 V4L2Buffer::V4L2Buffer(scoped_refptr<V4L2Device> device,
                        enum v4l2_buf_type type,
                        enum v4l2_memory memory,
-                       size_t planes_count,
+                       const struct v4l2_format& format,
                        size_t buffer_id)
-    : device_(device) {
+    : device_(device), format_(format) {
   DCHECK(V4L2_TYPE_IS_MULTIPLANAR(type));
-  DCHECK_LE(planes_count, base::size(v4l2_planes_));
+  DCHECK_LE(format.fmt.pix_mp.num_planes, base::size(v4l2_planes_));
   v4l2_buffer_.m.planes = v4l2_planes_;
   // Just in case we got more planes than we want.
-  v4l2_buffer_.length = std::min(planes_count, base::size(v4l2_planes_));
+  v4l2_buffer_.length =
+      std::min(static_cast<size_t>(format.fmt.pix_mp.num_planes),
+               base::size(v4l2_planes_));
   v4l2_buffer_.index = buffer_id;
   v4l2_buffer_.type = type;
   v4l2_buffer_.memory = memory;
@@ -156,6 +164,43 @@ size_t V4L2Buffer::GetMemoryUsage() const {
   return usage;
 }
 
+scoped_refptr<VideoFrame> V4L2Buffer::CreateVideoFrame() {
+  auto layout = V4L2Device::V4L2FormatToVideoFrameLayout(format_);
+  if (!layout) {
+    DVLOG(1) << "Cannot create frame layout for V4L2 buffers";
+    return nullptr;
+  }
+
+  std::vector<base::ScopedFD> dmabuf_fds = device_->GetDmabufsForV4L2Buffer(
+      v4l2_buffer_.index, v4l2_buffer_.length,
+      static_cast<enum v4l2_buf_type>(v4l2_buffer_.type));
+  if (dmabuf_fds.empty()) {
+    VLOGF(1) << "Failed to get DMABUFs of V4L2 buffer";
+    return nullptr;
+  }
+
+  gfx::Size size(format_.fmt.pix_mp.width, format_.fmt.pix_mp.height);
+
+  return VideoFrame::WrapExternalDmabufs(
+      *layout, gfx::Rect(size), size, std::move(dmabuf_fds), base::TimeDelta());
+}
+
+const scoped_refptr<VideoFrame>& V4L2Buffer::GetVideoFrame() {
+  // We can create the VideoFrame only when using MMAP buffers.
+  if (v4l2_buffer_.memory != V4L2_MEMORY_MMAP) {
+    DVLOGF(1) << "Cannot create video frame from non-MMAP buffer";
+    // video_frame_ should be null since that's its default value.
+    DCHECK_EQ(video_frame_, nullptr);
+    return video_frame_;
+  }
+
+  // Create the video frame instance if requiring it for the first time.
+  if (!video_frame_)
+    video_frame_ = CreateVideoFrame();
+
+  return video_frame_;
+}
+
 // A thread-safe pool of buffer indexes, allowing buffers to be obtained and
 // returned from different threads. All the methods of this class are
 // thread-safe. Users should keep a scoped_refptr to instances of this class
@@ -217,6 +262,8 @@ class V4L2BufferRefBase {
   bool QueueBuffer();
   void* GetPlaneMapping(const size_t plane);
 
+  const scoped_refptr<VideoFrame>& GetVideoFrame();
+
   // Data from the buffer, that users can query and/or write.
   struct v4l2_buffer v4l2_buffer_;
   // WARNING: do not change this to a vector or something smaller than
@@ -283,6 +330,20 @@ void* V4L2BufferRefBase::GetPlaneMapping(const size_t plane) {
   return queue_->buffers_[BufferId()]->GetPlaneMapping(plane);
 }
 
+const scoped_refptr<VideoFrame>& V4L2BufferRefBase::GetVideoFrame() {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+
+  // Used so we can return a const scoped_refptr& in all cases.
+  static const scoped_refptr<VideoFrame> null_videoframe;
+
+  if (!queue_)
+    return null_videoframe;
+
+  DCHECK_LE(BufferId(), queue_->buffers_.size());
+
+  return queue_->buffers_[BufferId()]->GetVideoFrame();
+}
+
 V4L2WritableBufferRef::V4L2WritableBufferRef() {
   // Invalid buffers can be created from any thread.
   DETACH_FROM_SEQUENCE(sequence_checker_);
@@ -322,6 +383,12 @@ V4L2WritableBufferRef& V4L2WritableBufferRef::operator=(
   return *this;
 }
 
+const scoped_refptr<VideoFrame>& V4L2WritableBufferRef::GetVideoFrame() {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+
+  return buffer_data_->GetVideoFrame();
+}
+
 bool V4L2WritableBufferRef::IsValid() const {
   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 
@@ -497,6 +564,12 @@ V4L2ReadableBuffer::V4L2ReadableBuffer(const struct v4l2_buffer* v4l2_buffer,
   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 }
 
+const scoped_refptr<VideoFrame>& V4L2ReadableBuffer::GetVideoFrame() {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+
+  return buffer_data_->GetVideoFrame();
+}
+
 V4L2ReadableBuffer::~V4L2ReadableBuffer() {
   // This method is thread-safe. Since we are the destructor, we are guaranteed
   // to be called from the only remaining reference to us. Also, we are just
@@ -642,7 +715,7 @@ size_t V4L2Queue::AllocateBuffers(size_t count, enum v4l2_memory memory) {
 
   // Now query all buffer information.
   for (size_t i = 0; i < reqbufs.count; i++) {
-    auto buffer = V4L2Buffer::Create(device_, type_, memory_, planes_count_, i);
+    auto buffer = V4L2Buffer::Create(device_, type_, memory_, format, i);
 
     if (!buffer) {
       DeallocateBuffers();

media/gpu/v4l2/v4l2_device.h

@@ -105,6 +105,15 @@ class MEDIA_GPU_EXPORT V4L2WritableBufferRef {
   // Returns the previously-set number of bytes used for |plane|.
   size_t GetPlaneBytesUsed(const size_t plane) const;
 
+  // Return the VideoFrame underlying this buffer. The VideoFrame's layout
+  // will match that of the V4L2 format. This method will *always* return the
+  // same VideoFrame instance for a given V4L2 buffer. Moreover, the VideoFrame
+  // instance will also be the same across V4L2WritableBufferRef and
+  // V4L2ReadableBufferRef if both references point to the same V4L2 buffer.
+  // Note: at the moment, this method is valid for MMAP buffers only. It will
+  // return nullptr for any other buffer type.
+  const scoped_refptr<VideoFrame>& GetVideoFrame() WARN_UNUSED_RESULT;
+
   // Return the V4L2 buffer ID of the underlying buffer.
   // TODO(acourbot) This is used for legacy clients but should be ultimately
   // removed. See crbug/879971
@@ -155,6 +164,15 @@ class MEDIA_GPU_EXPORT V4L2ReadableBuffer
   // removed. See crbug/879971
   size_t BufferId() const;
 
+  // Return the VideoFrame underlying this buffer. The VideoFrame's layout
+  // will match that of the V4L2 format. This method will *always* return the
+  // same VideoFrame instance for a given V4L2 buffer. Moreover, the VideoFrame
+  // instance will also be the same across V4L2WritableBufferRef and
+  // V4L2ReadableBufferRef if both references point to the same V4L2 buffer.
+  // Note: at the moment, this method is valid for MMAP buffers only. It will
+  // return nullptr for any other buffer type.
+  const scoped_refptr<VideoFrame>& GetVideoFrame() WARN_UNUSED_RESULT;
+
  private:
   friend class V4L2BufferRefFactory;
   friend class base::RefCountedThreadSafe<V4L2ReadableBuffer>;

media/gpu/v4l2/v4l2_video_decode_accelerator.cc

@@ -434,24 +434,12 @@ void V4L2VideoDecodeAccelerator::AssignPictureBuffersTask(
     DCHECK_EQ(output_record.egl_image, EGL_NO_IMAGE_KHR);
     DCHECK_EQ(output_record.picture_id, -1);
     DCHECK(!output_record.cleared);
-    DCHECK(output_record.processor_input_fds.empty());
 
     output_record.picture_id = buffers[i].id();
     output_record.texture_id = buffers[i].service_texture_ids().empty()
                                    ? 0
                                    : buffers[i].service_texture_ids()[0];
 
-    if (image_processor_device_) {
-      std::vector<base::ScopedFD> dmabuf_fds = device_->GetDmabufsForV4L2Buffer(
-          i, output_planes_count_, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
-      if (dmabuf_fds.empty()) {
-        VLOGF(1) << "Failed to get DMABUFs of decoder.";
-        NOTIFY_ERROR(PLATFORM_FAILURE);
-        return;
-      }
-      output_record.processor_input_fds = std::move(dmabuf_fds);
-    }
-
     if (output_mode_ == Config::OutputMode::ALLOCATE) {
       std::vector<base::ScopedFD> dmabuf_fds;
       dmabuf_fds = egl_image_device_->GetDmabufsForV4L2Buffer(
@@ -2497,17 +2485,9 @@ bool V4L2VideoDecodeAccelerator::ProcessFrame(int32_t bitstream_buffer_id,
   DVLOGF(4);
   DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread());
 
-  auto layout = VideoFrameLayout::Create(
-      V4L2Device::V4L2PixFmtToVideoPixelFormat(output_format_fourcc_),
-      coded_size_);
-  if (!layout) {
-    return false;
-  }
   OutputRecord& output_record = output_buffer_map_[buf->BufferId()];
-  scoped_refptr<VideoFrame> input_frame = VideoFrame::WrapExternalDmabufs(
-      *layout, gfx::Rect(visible_size_), visible_size_,
-      DuplicateFDs(output_record.processor_input_fds), base::TimeDelta());
 
+  scoped_refptr<VideoFrame> input_frame = buf->GetVideoFrame();
   if (!input_frame) {
     VLOGF(1) << "Failed wrapping input frame!";
     return false;

media/gpu/v4l2/v4l2_video_decode_accelerator.h

@@ -193,8 +193,6 @@ class MEDIA_GPU_EXPORT V4L2VideoDecodeAccelerator
     GLuint texture_id;
     bool cleared;           // Whether the texture is cleared and safe to render
                             // from. See TextureManager for details.
-    // Input fds of the processor. Exported from the decoder.
-    std::vector<base::ScopedFD> processor_input_fds;
     // Output fds. Used only when OutputMode is IMPORT.
     std::vector<base::ScopedFD> output_fds;
   };