media/gpu: Use StorageType instead of v4l2_memory to create image processor.

V4L2ImageProcessor::Create() has input_memory_type and output_memory_type
parameters of type v4l2_memory. In order to have a generic ImageProcessor
factory method, we use StorageType to replace v4l2_memory.

Also, since V4L2ImageProcessor supports both allocate and import output mode,
and for a storage type (e.g. STORAGE_OWNED_MEMORY), its v4l2_memory mapping
is V4L2_MEMORY_MMAP for allocate mode and V4L2_MEMORY_USERPTR for import mode.
We have to pass output_mode to v4l2 IP factory till all its users move to
use import mode.

BUG=b:73752373, chromium:907767
TEST=TEST=Run VEA VDA unittest on devices (peach_pit and elm)
VEA:
video_encode_accelerator_unittest \
--test_stream_data=bear-320x180.yuv:320:180:1:bear.mp4:100000:30 \
--disable_flush --single-process-tests -v=0

VDA:
video_decode_accelerator_unittest \
--test_stream_data=/usr/local/video/test-25fps.h264:320:240:250:258:\
35:150:1 -v=0 --disable_flush --single-process-tests --ozone-platform=gbm

Change-Id: I60f8eeea937dc6566cf844a8a74069711c29f5a0
Reviewed-on: https://chromium-review.googlesource.com/c/1341739
Commit-Queue: Shuo-Peng Liao <deanliao@google.com>
Reviewed-by: Hirokazu Honda <hiroh@chromium.org>
Cr-Commit-Position: refs/heads/master@{#610764}

media/gpu/image_processor.h

@@ -10,12 +10,11 @@
 #include "base/callback_forward.h"
 #include "base/files/scoped_file.h"
 #include "base/memory/ref_counted.h"
+#include "media/base/video_frame.h"
 #include "ui/gfx/geometry/size.h"
 
 namespace media {
 
-class VideoFrame;
-
 // An image processor is used to convert from one image format to another (e.g.
 // I420 to NV12) while optionally scaling. It is useful in situations where
 // a given video hardware (e.g. decoder or encoder) accepts or produces data
@@ -24,6 +23,14 @@ class VideoFrame;
 // This class exposes the interface that an image processor should implement.
 class ImageProcessor {
  public:
+  // OutputMode is used as intermediate stage. The ultimate goal is to make
+  // ImageProcessor's clients all use IMPORT output mode.
+  // TODO(907767): Remove this once ImageProcessor always works as IMPORT mode
+  // for output.
+  enum class OutputMode {
+    ALLOCATE,
+    IMPORT
+  };
 
   // Returns input allocated size required by the processor to be fed with.
   virtual gfx::Size input_allocated_size() const = 0;
@@ -31,6 +38,15 @@ class ImageProcessor {
   // Returns output allocated size required by the processor.
   virtual gfx::Size output_allocated_size() const = 0;
 
+  // Returns input storage type.
+  virtual VideoFrame::StorageType input_storage_type() const = 0;
+
+  // Returns output storage type.
+  virtual VideoFrame::StorageType output_storage_type() const = 0;
+
+  // Returns output mode.
+  virtual OutputMode output_mode() const = 0;
+
   // Callback to be used to return the index of a processed image to the
   // client. After the client is done with the frame, call Process with the
   // index to return the output buffer to the image processor.

media/gpu/v4l2/v4l2_image_processor.cc

@@ -65,9 +65,13 @@ V4L2ImageProcessor::JobRecord::JobRecord() : output_buffer_index(-1) {}
 
 V4L2ImageProcessor::JobRecord::~JobRecord() {}
 
-V4L2ImageProcessor::V4L2ImageProcessor(scoped_refptr<V4L2Device> device,
+V4L2ImageProcessor::V4L2ImageProcessor(
+    scoped_refptr<V4L2Device> device,
+    VideoFrame::StorageType input_storage_type,
+    VideoFrame::StorageType output_storage_type,
     v4l2_memory input_memory_type,
     v4l2_memory output_memory_type,
+    OutputMode output_mode,
     const VideoFrameLayout& input_layout,
     const VideoFrameLayout& output_layout,
     gfx::Size input_visible_size,
@@ -77,9 +81,12 @@ V4L2ImageProcessor::V4L2ImageProcessor(scoped_refptr<V4L2Device> device,
     : input_layout_(input_layout),
       input_visible_size_(input_visible_size),
       input_memory_type_(input_memory_type),
+      input_storage_type_(input_storage_type),
       output_layout_(output_layout),
       output_visible_size_(output_visible_size),
       output_memory_type_(output_memory_type),
+      output_storage_type_(output_storage_type),
+      output_mode_(output_mode),
       child_task_runner_(base::ThreadTaskRunnerHandle::Get()),
       device_(device),
       device_thread_("V4L2ImageProcessorThread"),
@@ -120,11 +127,30 @@ void V4L2ImageProcessor::NotifyErrorOnChildThread(
   error_cb_.Run();
 }
 
+namespace {
+
+v4l2_memory InputStorageTypeToV4L2Memory(VideoFrame::StorageType storage_type) {
+  switch (storage_type) {
+    case VideoFrame::STORAGE_OWNED_MEMORY:
+    case VideoFrame::STORAGE_UNOWNED_MEMORY:
+    case VideoFrame::STORAGE_SHMEM:
+    case VideoFrame::STORAGE_MOJO_SHARED_BUFFER:
+      return V4L2_MEMORY_USERPTR;
+    case VideoFrame::STORAGE_DMABUFS:
+      return V4L2_MEMORY_DMABUF;
+    default:
+      return static_cast<v4l2_memory>(0);
+  }
+}
+
+}  // namespace
+
 // static
 std::unique_ptr<V4L2ImageProcessor> V4L2ImageProcessor::Create(
       scoped_refptr<V4L2Device> device,
-      v4l2_memory input_memory_type,
-      v4l2_memory output_memory_type,
+      VideoFrame::StorageType input_storage_type,
+      VideoFrame::StorageType output_storage_type,
+      OutputMode output_mode,
       const VideoFrameLayout& input_layout,
       const VideoFrameLayout& output_layout,
       gfx::Size input_visible_size,
@@ -137,10 +163,23 @@ std::unique_ptr<V4L2ImageProcessor> V4L2ImageProcessor::Create(
     VLOGF(1) << "Failed creating V4L2Device";
     return nullptr;
   }
-  DCHECK(input_memory_type == V4L2_MEMORY_USERPTR ||
-         input_memory_type == V4L2_MEMORY_DMABUF);
-  DCHECK(output_memory_type == V4L2_MEMORY_MMAP ||
-         output_memory_type == V4L2_MEMORY_DMABUF);
+
+  const v4l2_memory input_memory_type = InputStorageTypeToV4L2Memory(
+      input_storage_type);
+  if (input_memory_type == 0) {
+    VLOGF(1) << "Unsupported input storage type: " << input_storage_type;
+    return nullptr;
+  }
+
+  // Note that for v4l2 IP, output storage type must be STORAGE_DMABUFS.
+  // And output_memory_type depends on its output mode.
+  if (output_storage_type != VideoFrame::STORAGE_DMABUFS) {
+    VLOGF(1) << "Unsupported output storage type: " << output_storage_type;
+    return nullptr;
+  }
+  const v4l2_memory output_memory_type =
+      output_mode == ImageProcessor::OutputMode::ALLOCATE ? V4L2_MEMORY_MMAP
+                                                          : V4L2_MEMORY_DMABUF;
 
   if (!device->IsImageProcessingSupported()) {
     VLOGF(1) << "V4L2ImageProcessor not supported in this platform";
@@ -214,7 +253,8 @@ std::unique_ptr<V4L2ImageProcessor> V4L2ImageProcessor::Create(
   }
 
   auto processor = base::WrapUnique(new V4L2ImageProcessor(
-      std::move(device), input_memory_type, output_memory_type,
+      std::move(device), input_storage_type, output_storage_type,
+      input_memory_type, output_memory_type, output_mode,
       *negotiated_input_layout, *negotiated_output_layout, input_visible_size,
       output_visible_size, num_buffers, std::move(error_cb)));
   if (!processor->Initialize()) {
@@ -322,6 +362,18 @@ gfx::Size V4L2ImageProcessor::output_allocated_size() const {
   return output_layout_.coded_size();
 }
 
+VideoFrame::StorageType V4L2ImageProcessor::input_storage_type() const {
+  return input_storage_type_;
+}
+
+VideoFrame::StorageType V4L2ImageProcessor::output_storage_type() const {
+  return output_storage_type_;
+}
+
+ImageProcessor::OutputMode V4L2ImageProcessor::output_mode() const {
+  return output_mode_;
+}
+
 bool V4L2ImageProcessor::Process(scoped_refptr<VideoFrame> frame,
                                  int output_buffer_index,
                                  std::vector<base::ScopedFD> output_dmabuf_fds,

media/gpu/v4l2/v4l2_image_processor.h

@@ -18,6 +18,7 @@
 #include "base/memory/ref_counted.h"
 #include "base/memory/weak_ptr.h"
 #include "base/threading/thread.h"
+#include "media/base/video_frame.h"
 #include "media/base/video_frame_layout.h"
 #include "media/gpu/image_processor.h"
 #include "media/gpu/media_gpu_export.h"
@@ -34,6 +35,9 @@ class MEDIA_GPU_EXPORT V4L2ImageProcessor : public ImageProcessor {
   ~V4L2ImageProcessor() override;
   gfx::Size input_allocated_size() const override;
   gfx::Size output_allocated_size() const override;
+  VideoFrame::StorageType input_storage_type() const override;
+  VideoFrame::StorageType output_storage_type() const override;
+  OutputMode output_mode() const override;
   bool Process(scoped_refptr<VideoFrame> frame,
                int output_buffer_index,
                std::vector<base::ScopedFD> output_dmabuf_fds,
@@ -60,14 +64,17 @@ class MEDIA_GPU_EXPORT V4L2ImageProcessor : public ImageProcessor {
                               size_t* num_planes);
 
   // Factory method to create V4L2ImageProcessor to convert from
-  // input_format to output_format. The number of input buffers and output
+  // input_format to output_format. Caller shall provide input and output
+  // storage type as well as output mode. The number of input buffers and output
   // buffers will be |num_buffers|. Provided |error_cb| will be posted to the
   // child thread if an error occurs after initialization. Returns nullptr if
   // V4L2ImageProcessor fails to create.
+  // Note: output_mode will be removed once all its clients use import mode.
   static std::unique_ptr<V4L2ImageProcessor> Create(
       scoped_refptr<V4L2Device> device,
-      v4l2_memory input_memory_type,
-      v4l2_memory output_memory_type,
+      VideoFrame::StorageType input_storage_type,
+      VideoFrame::StorageType output_storage_type,
+      OutputMode output_mode,
       const VideoFrameLayout& input_layout,
       const VideoFrameLayout& output_layout,
       gfx::Size input_visible_size,
@@ -113,8 +120,11 @@ class MEDIA_GPU_EXPORT V4L2ImageProcessor : public ImageProcessor {
   };
 
   V4L2ImageProcessor(scoped_refptr<V4L2Device> device,
+                     VideoFrame::StorageType input_storage_type,
+                     VideoFrame::StorageType output_storage_type,
                      v4l2_memory input_memory_type,
                      v4l2_memory output_memory_type,
+                     OutputMode output_mode,
                      const VideoFrameLayout& input_layout,
                      const VideoFrameLayout& output_layout,
                      gfx::Size input_visible_size,
@@ -157,11 +167,14 @@ class MEDIA_GPU_EXPORT V4L2ImageProcessor : public ImageProcessor {
   const VideoFrameLayout input_layout_;
   const gfx::Size input_visible_size_;
   const v4l2_memory input_memory_type_;
+  const VideoFrame::StorageType input_storage_type_;
 
   // Stores input frame's format, coded_size, buffer and plane layout.
   const VideoFrameLayout output_layout_;
   const gfx::Size output_visible_size_;
   const v4l2_memory output_memory_type_;
+  const VideoFrame::StorageType output_storage_type_;
+  const OutputMode output_mode_;
 
   // Our original calling task runner for the child thread.
   const scoped_refptr<base::SingleThreadTaskRunner> child_task_runner_;

media/gpu/v4l2/v4l2_video_decode_accelerator.cc

@@ -2353,10 +2353,10 @@ bool V4L2VideoDecodeAccelerator::ResetImageProcessor() {
 bool V4L2VideoDecodeAccelerator::CreateImageProcessor() {
   VLOGF(2);
   DCHECK(!image_processor_);
-  const v4l2_memory input_memory_type = V4L2_MEMORY_DMABUF;
-  const v4l2_memory output_memory_type =
-      (output_mode_ == Config::OutputMode::ALLOCATE ? V4L2_MEMORY_MMAP
-                                                    : V4L2_MEMORY_DMABUF);
+  const ImageProcessor::OutputMode image_processor_output_mode =
+      (output_mode_ == Config::OutputMode::ALLOCATE
+           ? ImageProcessor::OutputMode::ALLOCATE
+           : ImageProcessor::OutputMode::IMPORT);
 
   auto input_layout = VideoFrameLayout::Create(
       V4L2Device::V4L2PixFmtToVideoPixelFormat(output_format_fourcc_),
@@ -2376,9 +2376,9 @@ bool V4L2VideoDecodeAccelerator::CreateImageProcessor() {
   // Unretained is safe because |this| owns image processor and there will be
   // no callbacks after processor destroys.
   image_processor_ = V4L2ImageProcessor::Create(
-      image_processor_device_, input_memory_type, output_memory_type,
-      *input_layout, *output_layout, visible_size_, visible_size_,
-      output_buffer_map_.size(),
+      image_processor_device_, VideoFrame::STORAGE_DMABUFS,
+      VideoFrame::STORAGE_DMABUFS, image_processor_output_mode, *input_layout,
+      *output_layout, visible_size_, visible_size_, output_buffer_map_.size(),
       base::Bind(&V4L2VideoDecodeAccelerator::ImageProcessorError,
                  base::Unretained(this)));
 

media/gpu/v4l2/v4l2_video_encode_accelerator.cc

@@ -227,8 +227,13 @@ bool V4L2VideoEncodeAccelerator::Initialize(const Config& config,
     // size at |visible_size_| and requiring the output buffers to be of at
     // least |input_allocated_size_|. Unretained is safe because |this| owns
     // image processor and there will be no callbacks after processor destroys.
+    // |input_storage_type| can be STORAGE_SHMEM and STORAGE_MOJO_SHARED_BUFFER.
+    // However, it doesn't matter VideoFrame::STORAGE_OWNED_MEMORY is specified
+    // for |input_storage_type| here, as long as VideoFrame on Process()'s data
+    // can be accessed by VideoFrame::data().
     image_processor_ = V4L2ImageProcessor::Create(
-        V4L2Device::Create(), V4L2_MEMORY_USERPTR, V4L2_MEMORY_MMAP,
+        V4L2Device::Create(), VideoFrame::STORAGE_OWNED_MEMORY,
+        VideoFrame::STORAGE_DMABUFS, ImageProcessor::OutputMode::ALLOCATE,
         *input_layout, *output_layout, visible_size_, visible_size_,
         kImageProcBufferCount,
         base::Bind(&V4L2VideoEncodeAccelerator::ImageProcessorError,