Video capture with GpuMemoryBuffer - Handle buffer drops

When the system gets busy, it's possible for the video capture buffer
pool to run out of buffers temporarily.  Instead of raising an exception
we should handle the case gracefully, dropping a camera frame in the
worst case.

Design doc: go/cros-camera:dd:zero-copy

Bug: 982201
Change-Id: I23d2e45b8427bd36ea663410817e4361252823c3
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1712844Reviewed-by: Shik Chen <shik@chromium.org>
Commit-Queue: Ricky Liang <jcliang@chromium.org>
Cr-Commit-Position: refs/heads/master@{#682158}

media/capture/video/chromeos/camera_device_delegate_unittest.cc

@@ -221,6 +221,17 @@ class CameraDeviceDelegateTest : public ::testing::Test {
     entry->data.assign(as_int8, as_int8 + entry->count * sizeof(int32_t));
     static_metadata->entries->push_back(std::move(entry));
 
+    entry = cros::mojom::CameraMetadataEntry::New();
+    entry->index = 3;
+    entry->tag =
+        cros::mojom::CameraMetadataTag::ANDROID_REQUEST_PIPELINE_MAX_DEPTH;
+    entry->type = cros::mojom::EntryType::TYPE_BYTE;
+    entry->count = 1;
+    uint8_t pipeline_max_depth = 1;
+    entry->data.assign(&pipeline_max_depth,
+                       &pipeline_max_depth + entry->count * sizeof(uint8_t));
+    static_metadata->entries->push_back(std::move(entry));
+
     switch (camera_id) {
       case 0:
         camera_info->facing = cros::mojom::CameraFacing::CAMERA_FACING_FRONT;

media/capture/video/chromeos/request_manager.cc

@@ -89,6 +89,13 @@ void RequestManager::SetUpStreamsAndBuffers(
         *reinterpret_cast<int32_t*>((*partial_count)->data.data());
   }
 
+  auto pipeline_depth = GetMetadataEntryAsSpan<uint8_t>(
+      static_metadata,
+      cros::mojom::CameraMetadataTag::ANDROID_REQUEST_PIPELINE_MAX_DEPTH);
+  CHECK_EQ(pipeline_depth.size(), 1u);
+  pipeline_depth_ = pipeline_depth[0];
+  preview_buffers_queued_ = 0;
+
   // Set the last received frame number for each stream types to be undefined.
   for (const auto& stream : streams) {
     StreamType stream_type = StreamIdToStreamType(stream->id);
@@ -96,7 +103,7 @@ void RequestManager::SetUpStreamsAndBuffers(
   }
 
   stream_buffer_manager_->SetUpStreamsAndBuffers(
-      capture_format, std::move(static_metadata), std::move(streams));
+      capture_format, static_metadata, std::move(streams));
 }
 
 cros::mojom::Camera3StreamPtr RequestManager::GetStreamConfiguration(
@@ -294,11 +301,18 @@ void RequestManager::PrepareCaptureRequest() {
   }
 
   if (!is_reprocess_request && !is_oneshot_request && !is_preview_request) {
+    // We have to keep the pipeline full.
+    if (preview_buffers_queued_ < pipeline_depth_) {
+      ipc_task_runner_->PostTask(
+          FROM_HERE,
+          base::BindOnce(&RequestManager::PrepareCaptureRequest, GetWeakPtr()));
+    }
     return;
   }
 
   auto capture_request = request_builder_->BuildRequest(
       std::move(stream_types), std::move(settings), input_buffer_id);
+  CHECK_GT(capture_request->output_buffers.size(), 0u);
 
   CaptureResult& pending_result =
       pending_results_[capture_request->frame_number];
@@ -318,6 +332,10 @@ void RequestManager::PrepareCaptureRequest() {
     pending_reprocess_tasks_queue_.pop();
   }
 
+  if (is_preview_request) {
+    ++preview_buffers_queued_;
+  }
+
   UpdateCaptureSettings(&capture_request->settings);
   capture_interface_->ProcessCaptureRequest(
       std::move(capture_request),
@@ -375,9 +393,19 @@ bool RequestManager::TryPrepareReprocessRequest(
 bool RequestManager::TryPreparePreviewRequest(
     std::set<StreamType>* stream_types,
     cros::mojom::CameraMetadataPtr* settings) {
-  if (!stream_buffer_manager_->HasFreeBuffers({StreamType::kPreviewOutput})) {
+  if (preview_buffers_queued_ == pipeline_depth_) {
     return false;
   }
+  if (!stream_buffer_manager_->HasFreeBuffers({StreamType::kPreviewOutput})) {
+    // Try our best to reserve an usable buffer.  If the reservation still
+    // fails, then we'd have to drop the camera frame.
+    DLOG(WARNING) << "Late request for reserving preview buffer";
+    stream_buffer_manager_->ReserveBuffer(StreamType::kPreviewOutput);
+    if (!stream_buffer_manager_->HasFreeBuffers({StreamType::kPreviewOutput})) {
+      DLOG(WARNING) << "No free buffer for preview stream";
+      return false;
+    }
+  }
 
   stream_types->insert({StreamType::kPreviewOutput});
   *settings = repeating_request_settings_.Clone();
@@ -767,6 +795,11 @@ void RequestManager::SubmitCaptureResult(
     stream_buffer_manager_->ReleaseBufferFromCaptureResult(stream_type,
                                                            buffer_ipc_id);
   }
+
+  if (stream_type == StreamType::kPreviewOutput) {
+    --preview_buffers_queued_;
+  }
+
   pending_result.unsubmitted_buffer_count--;
 
   if (pending_result.unsubmitted_buffer_count == 0) {

media/capture/video/chromeos/request_manager.h

@@ -302,6 +302,15 @@ class CAPTURE_EXPORT RequestManager final
   // shot.
   uint32_t partial_result_count_;
 
+  // The pipeline depth reported in the ANDROID_REQUEST_PIPELINE_MAX_DEPTH
+  // metadata.
+  size_t pipeline_depth_;
+
+  // The number of preview buffers queued to the camera service.  The request
+  // manager needs to try its best to queue |pipeline_depth_| preview buffers to
+  // avoid camera frame drops.
+  size_t preview_buffers_queued_;
+
   // The shutter time of the first frame.  We derive the |timestamp| of a
   // frame using the difference between the frame's shutter time and
   // |first_frame_shutter_time_|.

media/capture/video/chromeos/request_manager_unittest.cc

@@ -146,6 +146,17 @@ class RequestManagerTest : public ::testing::Test {
     entry->data.assign(as_int8, as_int8 + entry->count * sizeof(int32_t));
     static_metadata->entries->push_back(std::move(entry));
 
+    entry = cros::mojom::CameraMetadataEntry::New();
+    entry->index = 2;
+    entry->tag =
+        cros::mojom::CameraMetadataTag::ANDROID_REQUEST_PIPELINE_MAX_DEPTH;
+    entry->type = cros::mojom::EntryType::TYPE_BYTE;
+    entry->count = 1;
+    uint8_t pipeline_max_depth = 1;
+    entry->data.assign(&pipeline_max_depth,
+                       &pipeline_max_depth + entry->count * sizeof(uint8_t));
+    static_metadata->entries->push_back(std::move(entry));
+
     return static_metadata;
   }
 

media/capture/video/chromeos/stream_buffer_manager.cc

@@ -175,6 +175,8 @@ void StreamBufferManager::SetUpStreamsAndBuffers(
          ++j) {
       ReserveBuffer(stream_type);
     }
+    CHECK_EQ(stream_context_[stream_type]->free_buffers.size(),
+             stream_context_[stream_type]->stream->max_buffers);
     DVLOG(2) << "Allocated "
              << stream_context_[stream_type]->stream->max_buffers << " buffers";
   }
@@ -317,10 +319,7 @@ void StreamBufferManager::ReserveBufferFromPool(StreamType stream_type) {
   if (!device_context_->ReserveVideoCaptureBufferFromPool(
           stream_context->buffer_dimension,
           stream_context->capture_format.pixel_format, &vcd_buffer)) {
-    device_context_->SetErrorState(
-        media::VideoCaptureError::
-            kCrosHalV3BufferManagerFailedToCreateGpuMemoryBuffer,
-        FROM_HERE, "Failed to reserve video capture buffer");
+    DLOG(WARNING) << "Failed to reserve video capture buffer";
     return;
   }
   auto gmb = gmb_support_->CreateGpuMemoryBufferImplFromHandle(