media/gpu/VaapiVEA: Set a size of a reconstructed surface to coded size

There are two modes in VEA, native input mode and non native input mode. In
native input mode, a input buffer given by VEA client is a graphic native input
buffer. The buffer alignment is decided in minigbm implementation. It is 64 and
4 for width and height, respectively. In non native input mode, the buffer is a
shared memory based buffer. The width and height is specified by VEA.

crrev.com/c/1810453 changes a dimension alignment for an input buffer in HW
video encoder using VA-API. The width and height are aligned by 16 before the
CL. The width is aligned by 64 and height aligned by 4 since the CL.
For example, if a configured encode stream size is 176x144.
             | A size of buffer | VA Surface size     | VA surface size for
             | given by client  | for an input buffer | a reconstructed surface
N-I     mode | 192x144          | 192x144             | 192x144
non N-I mode | 192x144          | 192x144             | 192x144

This change causes a lot of failures in chrome encoder test (crbug.com/1009297)
and CTS (b:142210669). This CL is expected to mitigate the failures. A
reconstructed surface size is the same as coded size, whose width and height
both are aligned by 16. The size of a buffer is the same as coded size in non
native input mode. This is fine because an input buffer is copied in a client
size in non native input mode. The sizes in the above example are below.
             | A size of buffer | VA Surface size     | VA surface size for
             | given by client  | for an input buffer | a reconstructed surface
N-I     mode | 192x144          | 192x144             | 176x144
non N-I mode | 176x144          | 176x144             | 176x144

Bug: 1009297, 1005205
Bug: b:142210669, b:142217112, b:142091557
Test: EncodeDecodeTest#testEncodeDecodeVideoFromBufferToBufferQCIF on caroline
Test: VEA unittest on eve
Change-Id: Ibca0f951ace886fcbeb8031e886c196913f69cca
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1847432Reviewed-by: Pawel Osciak <posciak@chromium.org>
Commit-Queue: Hirokazu Honda <hiroh@chromium.org>
Cr-Commit-Position: refs/heads/master@{#705017}

media/gpu/vaapi/vaapi_video_encode_accelerator.cc

@@ -133,7 +133,7 @@ class VaapiEncodeJob : public AcceleratedVideoEncoder::EncodeJob {
  private:
   ~VaapiEncodeJob() override = default;
 
-  // Input surface for video frame data.
+  // Input surface for video frame data or scaled data.
   const scoped_refptr<VASurface> input_surface_;
 
   // Surface for the reconstructed picture, used for reference
@@ -383,39 +383,48 @@ void VaapiVideoEncodeAccelerator::InitializeTask(const Config& config) {
     return;
   }
 
-  aligned_input_size_ =
-      GetInputFrameSize(config.input_format, config.input_visible_size);
-  if (aligned_input_size_.IsEmpty()) {
-    NOTIFY_ERROR(kPlatformFailureError, "Failed to get frame size");
-    return;
-  }
-
   output_buffer_byte_size_ = encoder_->GetBitstreamBufferSize();
-  const size_t max_ref_frames = encoder_->GetMaxNumOfRefFrames();
-
-  // Use at least kMinNumFramesInFlight if encoder requested less for
-  // pipeline depth.
-  const size_t num_frames_in_flight =
-      std::max(kMinNumFramesInFlight, max_ref_frames);
-  DVLOGF(1) << "Frames in flight: " << num_frames_in_flight;
 
   va_surface_release_cb_ = BindToCurrentLoop(
       base::BindRepeating(&VaapiVideoEncodeAccelerator::RecycleVASurfaceID,
                           base::Unretained(this)));
+  vpp_va_surface_release_cb_ = BindToCurrentLoop(
+      base::BindRepeating(&VaapiVideoEncodeAccelerator::RecycleVPPVASurfaceID,
+                          base::Unretained(this)));
 
-  // In native input mode, an input surface is needed only if scaling
-  // is not required. Since we cannot find the necessity of the scaling here,
-  // we allocate input surfaces always, which is redundant.
-  //
-  // TODO(hiroh): Think about moving this surface creation in the first
-  // Encode().
-  va_surfaces_per_video_frame_ =
-      kNumSurfacesForOutputPicture + kNumSurfacesPerInputVideoFrame;
+  // The surface size for a reconstructed surface is a coded size.
+  gfx::Size reconstructed_surface_size = encoder_->GetCodedSize();
+  if (native_input_mode_) {
+    // In native input mode, we do not need surfaces for input frames.
+    va_surfaces_per_video_frame_ = kNumSurfacesForOutputPicture;
+    // The aligned input size must be the same as a size of a native graphic
+    // buffer.
+    aligned_input_size_ =
+        GetInputFrameSize(config.input_format, config.input_visible_size);
+    if (aligned_input_size_.IsEmpty()) {
+      NOTIFY_ERROR(kPlatformFailureError, "Failed to get frame size");
+      return;
+    }
+  } else {
+    // In non-native mode, we need to create additional surfaces for input
+    // frames.
+    va_surfaces_per_video_frame_ =
+        kNumSurfacesForOutputPicture + kNumSurfacesPerInputVideoFrame;
+    // There is no way to know aligned size that a client provided, so we
+    // request coded size.
+    aligned_input_size_ = encoder_->GetCodedSize();
+  }
+
+  // The number of required buffers is the number of required reference frames
+  // + 1 for the current frame to be encoded.
+  const size_t max_ref_frames = encoder_->GetMaxNumOfRefFrames();
+  num_frames_in_flight_ = std::max(kMinNumFramesInFlight, max_ref_frames);
+  DVLOGF(1) << "Frames in flight: " << num_frames_in_flight_;
 
   if (!vaapi_wrapper_->CreateContextAndSurfaces(
-          kVaSurfaceFormat, aligned_input_size_,
+          kVaSurfaceFormat, reconstructed_surface_size,
           VaapiWrapper::SurfaceUsageHint::kVideoEncoder,
-          (num_frames_in_flight + 1) * va_surfaces_per_video_frame_,
+          (num_frames_in_flight_ + 1) * va_surfaces_per_video_frame_,
           &available_va_surface_ids_)) {
     NOTIFY_ERROR(kPlatformFailureError, "Failed creating VASurfaces");
     return;
@@ -423,7 +432,7 @@ void VaapiVideoEncodeAccelerator::InitializeTask(const Config& config) {
 
   child_task_runner_->PostTask(
       FROM_HERE, base::BindOnce(&Client::RequireBitstreamBuffers, client_,
-                                num_frames_in_flight, aligned_input_size_,
+                                num_frames_in_flight_, aligned_input_size_,
                                 output_buffer_byte_size_));
 
   SetState(kEncoding);
@@ -438,6 +447,15 @@ void VaapiVideoEncodeAccelerator::RecycleVASurfaceID(
   EncodePendingInputs();
 }
 
+void VaapiVideoEncodeAccelerator::RecycleVPPVASurfaceID(
+    VASurfaceID va_surface_id) {
+  DVLOGF(4) << "va_surface_id: " << va_surface_id;
+  DCHECK(encoder_thread_task_runner_->BelongsToCurrentThread());
+
+  available_vpp_va_surface_ids_.push_back(va_surface_id);
+  EncodePendingInputs();
+}
+
 void VaapiVideoEncodeAccelerator::ExecuteEncode(VASurfaceID va_surface_id) {
   DCHECK(encoder_thread_task_runner_->BelongsToCurrentThread());
   if (!vaapi_wrapper_->ExecuteAndDestroyPendingBuffers(va_surface_id))
@@ -566,7 +584,8 @@ scoped_refptr<VaapiEncodeJob> VaapiVideoEncodeAccelerator::CreateEncodeJob(
     return nullptr;
   }
 
-  if (available_va_surface_ids_.size() < va_surfaces_per_video_frame_) {
+  if (available_va_surface_ids_.size() < va_surfaces_per_video_frame_ ||
+      (vpp_vaapi_wrapper_ && available_vpp_va_surface_ids_.empty())) {
     DVLOGF(4) << "Not enough surfaces available";
     return nullptr;
   }
@@ -618,15 +637,27 @@ scoped_refptr<VaapiEncodeJob> VaapiVideoEncodeAccelerator::CreateEncodeJob(
                      "Failed to initialize VppVaapiWrapper");
         return nullptr;
       }
+
+      // Allocate the same number of surfaces as reconstructed surfaces.
+      if (vpp_vaapi_wrapper_->CreateContextAndSurfaces(
+              kVaSurfaceFormat, aligned_input_size_,
+              VaapiWrapper::SurfaceUsageHint::kVideoProcessWrite,
+              num_frames_in_flight_ + 1, &available_vpp_va_surface_ids_)) {
+        NOTIFY_ERROR(kPlatformFailureError,
+                     "Failed creating VASurfaces for scaling");
+        vpp_vaapi_wrapper_ = nullptr;
+        return nullptr;
+      };
     }
-    scoped_refptr<VASurface> scaled_surface =
-        new VASurface(available_va_surface_ids_.back(), aligned_input_size_,
-                      kVaSurfaceFormat, base::BindOnce(va_surface_release_cb_));
-    available_va_surface_ids_.pop_back();
+
+    scoped_refptr<VASurface> scaled_surface = new VASurface(
+        available_vpp_va_surface_ids_.back(), aligned_input_size_,
+        kVaSurfaceFormat, base::BindOnce(vpp_va_surface_release_cb_));
+    available_vpp_va_surface_ids_.pop_back();
     // Scale frame->coded_size() -> |aligned_input_size_| here.
     vpp_vaapi_wrapper_->BlitSurface(input_surface, scaled_surface);
-    // We can destroy the original |input_surface| because the buffer is alive
-    // as long as |frame| is alive.
+    // We can destroy the original |input_surface| because the buffer is already
+    // copied to scaled_surface.
     input_surface = std::move(scaled_surface);
   }
 
@@ -815,6 +846,11 @@ void VaapiVideoEncodeAccelerator::DestroyTask() {
   // Clean up members that are to be accessed on the encoder thread only.
   if (vaapi_wrapper_)
     vaapi_wrapper_->DestroyContextAndSurfaces(available_va_surface_ids_);
+  if (vpp_vaapi_wrapper_) {
+    vpp_vaapi_wrapper_->DestroyContextAndSurfaces(
+        available_vpp_va_surface_ids_);
+  }
+
   available_va_buffer_ids_.clear();
 
   while (!available_bitstream_buffers_.empty())

media/gpu/vaapi/vaapi_video_encode_accelerator.h

@@ -109,9 +109,13 @@ class MEDIA_GPU_EXPORT VaapiVideoEncodeAccelerator
   void ExecuteEncode(VASurfaceID va_surface_id);
 
   // Callback that returns a no longer used VASurfaceID to
-  // available_va_surface_ids_ for reuse.
+  // |available_va_surface_ids_| for reuse.
   void RecycleVASurfaceID(VASurfaceID va_surface_id);
 
+  // Callback that returns a no longer used VASurfaceID to
+  // |available_vpp_va_surface_ids_| for reuse.
+  void RecycleVPPVASurfaceID(VASurfaceID va_surface_id);
+
   // Returns a bitstream buffer to the client if both a previously executed job
   // awaits to be completed and we have bitstream buffers available to download
   // the encoded data into.
@@ -163,6 +167,9 @@ class MEDIA_GPU_EXPORT VaapiVideoEncodeAccelerator
   // and two otherwise.
   size_t va_surfaces_per_video_frame_;
 
+  // The number of frames that needs to be held on encoding.
+  size_t num_frames_in_flight_;
+
   // All of the members below must be accessed on the encoder_thread_,
   // while it is running.
 
@@ -172,8 +179,10 @@ class MEDIA_GPU_EXPORT VaapiVideoEncodeAccelerator
   // Encoder instance managing video codec state and preparing encode jobs.
   std::unique_ptr<AcceleratedVideoEncoder> encoder_;
 
-  // VA surfaces available for reuse.
+  // VA surfaces available for encoding.
   std::vector<VASurfaceID> available_va_surface_ids_;
+  // VA surfaces available for scaling.
+  std::vector<VASurfaceID> available_vpp_va_surface_ids_;
 
   // VASurfaceIDs internal format.
   static constexpr unsigned int kVaSurfaceFormat = VA_RT_FORMAT_YUV420;
@@ -183,6 +192,7 @@ class MEDIA_GPU_EXPORT VaapiVideoEncodeAccelerator
 
   // Callback via which finished VA surfaces are returned to us.
   base::RepeatingCallback<void(VASurfaceID)> va_surface_release_cb_;
+  base::RepeatingCallback<void(VASurfaceID)> vpp_va_surface_release_cb_;
 
   // Queue of input frames to be encoded.
   base::queue<std::unique_ptr<InputFrameRef>> input_queue_;

media/gpu/vaapi/vaapi_wrapper.cc

@@ -2114,6 +2114,9 @@ bool VaapiWrapper::CreateSurfaces(unsigned int va_format,
     case SurfaceUsageHint::kVideoEncoder:
       attribute.value.value.i = VA_SURFACE_ATTRIB_USAGE_HINT_ENCODER;
       break;
+    case SurfaceUsageHint::kVideoProcessWrite:
+      attribute.value.value.i = VA_SURFACE_ATTRIB_USAGE_HINT_VPP_WRITE;
+      break;
     case SurfaceUsageHint::kGeneric:
       attribute.value.value.i = VA_SURFACE_ATTRIB_USAGE_HINT_GENERIC;
       break;

media/gpu/vaapi/vaapi_wrapper.h

@@ -98,6 +98,7 @@ class MEDIA_GPU_EXPORT VaapiWrapper
   enum class SurfaceUsageHint : uint8_t {
     kVideoDecoder,
     kVideoEncoder,
+    kVideoProcessWrite,
     kGeneric,
   };