Reland "Enable camera blob stream when needed"

This is reland of 10f4b936.
It addressed the issue in capture_unittests.

Original change's description:
> Since blob stream needs higher resolution, it causes higher cpu
loading
> to require higher resolution and resize to smaller resolution.
> In hangout app, we don't need blob stream. Enabling blob stream when
> needed can save a lot of cpu usage.
>
> BUG=b:114676133
> TEST=manually test in apprtc and CCA. make sure picture taking still
> works in CCA.
>
> Change-Id: I9144461bc76627903d0b3b359ce9cf962ff3628c
> Reviewed-on: https://chromium-review.googlesource.com/c/1261242
> Commit-Queue: Heng-ruey Hsu <henryhsu@chromium.org>
> Reviewed-by: Ricky Liang <jcliang@chromium.org>
> Reviewed-by: Xiaohan Wang <xhwang@chromium.org>
> Reviewed-by: Robert Sesek <rsesek@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#601492}

TBR=xhwang@chromium.org,rsesek@chromium.org,jcliang@chromium.org

BUG=b:114676133
TEST=run capture_unittests.

Change-Id: I977a4eade8d7f7ddc6188c4ce4a659d675e89d5f
Reviewed-on: https://chromium-review.googlesource.com/c/1295753Reviewed-by: Heng-ruey Hsu <henryhsu@chromium.org>
Reviewed-by: Ricky Liang <jcliang@chromium.org>
Commit-Queue: Heng-ruey Hsu <henryhsu@chromium.org>
Cr-Commit-Position: refs/heads/master@{#601854}

media/capture/mojom/video_capture_types.mojom

@@ -205,7 +205,8 @@ enum VideoCaptureError {
   kMacDeckLinkCouldNotStartCapturing,
   kMacDeckLinkUnsupportedPixelFormat,
   kMacAvFoundationReceivedAVCaptureSessionRuntimeErrorNotification,
-  kAndroidApi2ErrorConfiguringCamera
+  kAndroidApi2ErrorConfiguringCamera,
+  kCrosHalV3DeviceDelegateFailedToFlush
 };
 
 enum VideoCaptureFrameDropReason {

media/capture/mojom/video_capture_types_mojom_traits.cc

@@ -658,6 +658,9 @@ EnumTraits<media::mojom::VideoCaptureError, media::VideoCaptureError>::ToMojom(
     case media::VideoCaptureError::kAndroidApi2ErrorConfiguringCamera:
       return media::mojom::VideoCaptureError::
           kAndroidApi2ErrorConfiguringCamera;
+    case media::VideoCaptureError::kCrosHalV3DeviceDelegateFailedToFlush:
+      return media::mojom::VideoCaptureError::
+          kCrosHalV3DeviceDelegateFailedToFlush;
   }
   NOTREACHED();
   return media::mojom::VideoCaptureError::kNone;
@@ -1176,6 +1179,9 @@ bool EnumTraits<media::mojom::VideoCaptureError, media::VideoCaptureError>::
     case media::mojom::VideoCaptureError::kAndroidApi2ErrorConfiguringCamera:
       *output = media::VideoCaptureError::kAndroidApi2ErrorConfiguringCamera;
       return true;
+    case media::mojom::VideoCaptureError::kCrosHalV3DeviceDelegateFailedToFlush:
+      *output = media::VideoCaptureError::kCrosHalV3DeviceDelegateFailedToFlush;
+      return true;
   }
   NOTREACHED();
   return false;

media/capture/video/chromeos/camera_device_delegate.cc

@@ -133,6 +133,12 @@ class CameraDeviceDelegate::StreamCaptureInterfaceImpl final
     }
   }
 
+  void Flush(base::OnceCallback<void(int32_t)> callback) final {
+    if (camera_device_delegate_) {
+      camera_device_delegate_->Flush(std::move(callback));
+    }
+  }
+
  private:
   const base::WeakPtr<CameraDeviceDelegate> camera_device_delegate_;
 };
@@ -189,7 +195,7 @@ void CameraDeviceDelegate::StopAndDeAllocate(
     // The device delegate is in the process of opening the camera device.
     return;
   }
-  stream_buffer_manager_->StopPreview();
+  stream_buffer_manager_->StopPreview(base::NullCallback());
   device_ops_->Close(
       base::BindOnce(&CameraDeviceDelegate::OnClosed, GetWeakPtr()));
 }
@@ -226,18 +232,17 @@ void CameraDeviceDelegate::GetPhotoState(
     return;
   }
 
-  auto stream_config =
-      stream_buffer_manager_->GetStreamConfiguration(StreamType::kStillCapture);
-  if (stream_config) {
-    photo_state->width->current = stream_config->width;
-    photo_state->width->min = stream_config->width;
-    photo_state->width->max = stream_config->width;
-    photo_state->width->step = 0.0;
-    photo_state->height->current = stream_config->height;
-    photo_state->height->min = stream_config->height;
-    photo_state->height->max = stream_config->height;
-    photo_state->height->step = 0.0;
-  }
+  int32_t max_blob_width = 0, max_blob_height = 0;
+  GetMaxBlobStreamResolution(static_metadata_, &max_blob_width,
+                             &max_blob_height);
+  photo_state->width->current = max_blob_width;
+  photo_state->width->min = max_blob_width;
+  photo_state->width->max = max_blob_width;
+  photo_state->width->step = 0.0;
+  photo_state->height->current = max_blob_height;
+  photo_state->height->min = max_blob_height;
+  photo_state->height->max = max_blob_height;
+  photo_state->height->step = 0.0;
   std::move(callback).Run(std::move(photo_state));
 }
 
@@ -258,7 +263,14 @@ void CameraDeviceDelegate::SetPhotoOptions(
     return;
   }
 
-  std::move(callback).Run(true);
+  if (stream_buffer_manager_->GetStreamNumber() < kMaxConfiguredStreams) {
+    stream_buffer_manager_->StopPreview(
+        base::BindOnce(&CameraDeviceDelegate::OnFlushed, GetWeakPtr()));
+    set_photo_option_callback_ = std::move(callback);
+  } else {
+    set_photo_option_callback_.Reset();
+    std::move(callback).Run(true);
+  }
 }
 
 void CameraDeviceDelegate::SetRotation(int rotation) {
@@ -282,7 +294,7 @@ void CameraDeviceDelegate::OnMojoConnectionError() {
   } else {
     // The Mojo channel terminated unexpectedly.
     if (stream_buffer_manager_) {
-      stream_buffer_manager_->StopPreview();
+      stream_buffer_manager_->StopPreview(base::NullCallback());
     }
     device_context_->SetState(CameraDeviceContext::State::kStopped);
     device_context_->SetErrorState(
@@ -295,6 +307,19 @@ void CameraDeviceDelegate::OnMojoConnectionError() {
   }
 }
 
+void CameraDeviceDelegate::OnFlushed(int32_t result) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  if (result) {
+    device_context_->SetErrorState(
+        media::VideoCaptureError::kCrosHalV3DeviceDelegateFailedToFlush,
+        FROM_HERE,
+        std::string("Flush failed: ") + base::safe_strerror(-result));
+    return;
+  }
+  device_context_->SetState(CameraDeviceContext::State::kInitialized);
+  ConfigureStreams(true);
+}
+
 void CameraDeviceDelegate::OnClosed(int32_t result) {
   DCHECK(ipc_task_runner_->BelongsToCurrentThread());
   DCHECK_EQ(device_context_->GetState(), CameraDeviceContext::State::kStopping);
@@ -426,10 +451,10 @@ void CameraDeviceDelegate::OnInitialized(int32_t result) {
     return;
   }
   device_context_->SetState(CameraDeviceContext::State::kInitialized);
-  ConfigureStreams();
+  ConfigureStreams(false);
 }
 
-void CameraDeviceDelegate::ConfigureStreams() {
+void CameraDeviceDelegate::ConfigureStreams(bool require_photo) {
   DCHECK(ipc_task_runner_->BelongsToCurrentThread());
   DCHECK_EQ(device_context_->GetState(),
             CameraDeviceContext::State::kInitialized);
@@ -450,31 +475,34 @@ void CameraDeviceDelegate::ConfigureStreams() {
   preview_stream->rotation =
       cros::mojom::Camera3StreamRotation::CAMERA3_STREAM_ROTATION_0;
 
+  cros::mojom::Camera3StreamConfigurationPtr stream_config =
+      cros::mojom::Camera3StreamConfiguration::New();
+  stream_config->streams.push_back(std::move(preview_stream));
+
   // Set up context for still capture stream. We set still capture stream to the
   // JPEG stream configuration with maximum supported resolution.
   // TODO(jcliang): Once we support SetPhotoOptions() the still capture stream
   // should be configured dynamically per the photo options.
-  int32_t max_blob_width = 0, max_blob_height = 0;
-  GetMaxBlobStreamResolution(static_metadata_, &max_blob_width,
-                             &max_blob_height);
-
-  cros::mojom::Camera3StreamPtr still_capture_stream =
-      cros::mojom::Camera3Stream::New();
-  still_capture_stream->id = static_cast<uint64_t>(StreamType::kStillCapture);
-  still_capture_stream->stream_type =
-      cros::mojom::Camera3StreamType::CAMERA3_STREAM_OUTPUT;
-  still_capture_stream->width = max_blob_width;
-  still_capture_stream->height = max_blob_height;
-  still_capture_stream->format =
-      cros::mojom::HalPixelFormat::HAL_PIXEL_FORMAT_BLOB;
-  still_capture_stream->data_space = 0;
-  still_capture_stream->rotation =
-      cros::mojom::Camera3StreamRotation::CAMERA3_STREAM_ROTATION_0;
+  if (require_photo) {
+    int32_t max_blob_width = 0, max_blob_height = 0;
+    GetMaxBlobStreamResolution(static_metadata_, &max_blob_width,
+                               &max_blob_height);
+
+    cros::mojom::Camera3StreamPtr still_capture_stream =
+        cros::mojom::Camera3Stream::New();
+    still_capture_stream->id = static_cast<uint64_t>(StreamType::kStillCapture);
+    still_capture_stream->stream_type =
+        cros::mojom::Camera3StreamType::CAMERA3_STREAM_OUTPUT;
+    still_capture_stream->width = max_blob_width;
+    still_capture_stream->height = max_blob_height;
+    still_capture_stream->format =
+        cros::mojom::HalPixelFormat::HAL_PIXEL_FORMAT_BLOB;
+    still_capture_stream->data_space = 0;
+    still_capture_stream->rotation =
+        cros::mojom::Camera3StreamRotation::CAMERA3_STREAM_ROTATION_0;
+    stream_config->streams.push_back(std::move(still_capture_stream));
+  }
 
-  cros::mojom::Camera3StreamConfigurationPtr stream_config =
-      cros::mojom::Camera3StreamConfiguration::New();
-  stream_config->streams.push_back(std::move(preview_stream));
-  stream_config->streams.push_back(std::move(still_capture_stream));
   stream_config->operation_mode = cros::mojom::Camera3StreamConfigurationMode::
       CAMERA3_STREAM_CONFIGURATION_NORMAL_MODE;
   device_ops_->ConfigureStreams(
@@ -502,7 +530,8 @@ void CameraDeviceDelegate::OnConfiguredStreams(
     return;
   }
   if (!updated_config ||
-      updated_config->streams.size() != kMaxConfiguredStreams) {
+      updated_config->streams.size() > kMaxConfiguredStreams ||
+      updated_config->streams.size() < 1) {
     device_context_->SetErrorState(
         media::VideoCaptureError::
             kCrosHalV3DeviceDelegateWrongNumberOfStreamsConfigured,
@@ -569,6 +598,10 @@ void CameraDeviceDelegate::OnConstructedDefaultPreviewRequestSettings(
         base::BindOnce(&CameraDeviceDelegate::ConstructDefaultRequestSettings,
                        GetWeakPtr(), StreamType::kStillCapture));
   }
+
+  if (set_photo_option_callback_) {
+    std::move(set_photo_option_callback_).Run(true);
+  }
 }
 
 void CameraDeviceDelegate::OnConstructedDefaultStillCaptureRequestSettings(
@@ -633,6 +666,11 @@ void CameraDeviceDelegate::ProcessCaptureRequest(
   device_ops_->ProcessCaptureRequest(std::move(request), std::move(callback));
 }
 
+void CameraDeviceDelegate::Flush(base::OnceCallback<void(int32_t)> callback) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  device_ops_->Flush(std::move(callback));
+}
+
 bool CameraDeviceDelegate::SetPointsOfInterest(
     const std::vector<mojom::Point2DPtr>& points_of_interest) {
   if (points_of_interest.empty()) {

media/capture/video/chromeos/camera_device_delegate.h

@@ -60,6 +60,9 @@ class CAPTURE_EXPORT StreamCaptureInterface {
   virtual void ProcessCaptureRequest(
       cros::mojom::Camera3CaptureRequestPtr request,
       base::OnceCallback<void(int32_t)> callback) = 0;
+
+  // Send flush to cancel all pending requests to the camera HAL.
+  virtual void Flush(base::OnceCallback<void(int32_t)> callback) = 0;
 };
 
 // CameraDeviceDelegate is instantiated on the capture thread where
@@ -99,6 +102,9 @@ class CAPTURE_EXPORT CameraDeviceDelegate final {
   // Mojo connection error handler.
   void OnMojoConnectionError();
 
+  // Reconfigure streams for picture taking.
+  void OnFlushed(int32_t result);
+
   // Callback method for the Close Mojo IPC call.  This method resets the Mojo
   // connection and closes the camera device.
   void OnClosed(int32_t result);
@@ -124,7 +130,7 @@ class CAPTURE_EXPORT CameraDeviceDelegate final {
   // indicates.  If there's no error OnConfiguredStreams notifies
   // |client_| the capture has started by calling OnStarted, and proceeds to
   // ConstructDefaultRequestSettings.
-  void ConfigureStreams();
+  void ConfigureStreams(bool require_photo);
   void OnConfiguredStreams(
       int32_t result,
       cros::mojom::Camera3StreamConfigurationPtr updated_config);
@@ -155,6 +161,7 @@ class CAPTURE_EXPORT CameraDeviceDelegate final {
                       base::OnceCallback<void(int32_t)> callback);
   void ProcessCaptureRequest(cros::mojom::Camera3CaptureRequestPtr request,
                              base::OnceCallback<void(int32_t)> callback);
+  void Flush(base::OnceCallback<void(int32_t)> callback);
 
   bool SetPointsOfInterest(
       const std::vector<mojom::Point2DPtr>& points_of_interest);
@@ -187,6 +194,8 @@ class CAPTURE_EXPORT CameraDeviceDelegate final {
 
   base::OnceClosure device_close_callback_;
 
+  VideoCaptureDevice::SetPhotoOptionsCallback set_photo_option_callback_;
+
   base::WeakPtrFactory<CameraDeviceDelegate> weak_ptr_factory_;
 
   DISALLOW_IMPLICIT_CONSTRUCTORS(CameraDeviceDelegate);

media/capture/video/chromeos/camera_device_delegate_unittest.cc

@@ -149,6 +149,7 @@ class CameraDeviceDelegateTest : public ::testing::Test {
     camera_device_delegate_ = std::make_unique<CameraDeviceDelegate>(
         descriptor, camera_hal_delegate_,
         device_delegate_thread_.task_runner());
+    num_streams_ = 0;
   }
 
   void GetFakeCameraInfo(uint32_t camera_id,
@@ -242,11 +243,13 @@ class CameraDeviceDelegateTest : public ::testing::Test {
       base::OnceCallback<void(int32_t,
                               cros::mojom::Camera3StreamConfigurationPtr)>&
           callback) {
-    ASSERT_EQ(2u, config->streams.size());
+    ASSERT_GE(2u, config->streams.size());
+    ASSERT_LT(0u, config->streams.size());
     for (size_t i = 0; i < config->streams.size(); ++i) {
       config->streams[i]->usage = 0;
       config->streams[i]->max_buffers = 1;
     }
+    num_streams_ = config->streams.size();
     std::move(callback).Run(0, std::move(config));
   }
 
@@ -332,14 +335,16 @@ class CameraDeviceDelegateTest : public ::testing::Test {
         .Times(1)
         .WillOnce(Invoke(&unittest_internal::MockGpuMemoryBufferManager::
                              CreateFakeGpuMemoryBuffer));
-    EXPECT_CALL(
-        mock_gpu_memory_buffer_manager_,
-        CreateGpuMemoryBuffer(_, gfx::BufferFormat::R_8,
-                              gfx::BufferUsage::CAMERA_AND_CPU_READ_WRITE,
-                              gpu::kNullSurfaceHandle))
-        .Times(1)
-        .WillOnce(Invoke(&unittest_internal::MockGpuMemoryBufferManager::
-                             CreateFakeGpuMemoryBuffer));
+    if (num_streams_ == 2) {
+      EXPECT_CALL(
+          mock_gpu_memory_buffer_manager_,
+          CreateGpuMemoryBuffer(_, gfx::BufferFormat::R_8,
+                                gfx::BufferUsage::CAMERA_AND_CPU_READ_WRITE,
+                                gpu::kNullSurfaceHandle))
+          .Times(1)
+          .WillOnce(Invoke(&unittest_internal::MockGpuMemoryBufferManager::
+                               CreateFakeGpuMemoryBuffer));
+    }
     EXPECT_CALL(
         mock_gpu_memory_buffer_manager_,
         CreateGpuMemoryBuffer(gfx::Size(kDefaultWidth, kDefaultHeight),
@@ -349,14 +354,16 @@ class CameraDeviceDelegateTest : public ::testing::Test {
         .Times(1)
         .WillOnce(Invoke(&unittest_internal::MockGpuMemoryBufferManager::
                              CreateFakeGpuMemoryBuffer));
-    EXPECT_CALL(mock_gpu_memory_buffer_manager_,
-                CreateGpuMemoryBuffer(
-                    gfx::Size(kJpegMaxBufferSize, 1), gfx::BufferFormat::R_8,
-                    gfx::BufferUsage::CAMERA_AND_CPU_READ_WRITE,
-                    gpu::kNullSurfaceHandle))
-        .Times(1)
-        .WillOnce(Invoke(&unittest_internal::MockGpuMemoryBufferManager::
-                             CreateFakeGpuMemoryBuffer));
+    if (num_streams_ == 2) {
+      EXPECT_CALL(mock_gpu_memory_buffer_manager_,
+                  CreateGpuMemoryBuffer(
+                      gfx::Size(kJpegMaxBufferSize, 1), gfx::BufferFormat::R_8,
+                      gfx::BufferUsage::CAMERA_AND_CPU_READ_WRITE,
+                      gpu::kNullSurfaceHandle))
+          .Times(1)
+          .WillOnce(Invoke(&unittest_internal::MockGpuMemoryBufferManager::
+                               CreateFakeGpuMemoryBuffer));
+    }
   }
 
   void SetUpExpectationUntilCapturing(
@@ -422,6 +429,7 @@ class CameraDeviceDelegateTest : public ::testing::Test {
     ASSERT_TRUE(camera_device_delegate_);
     device_delegate_thread_.Stop();
     camera_device_delegate_.reset();
+    num_streams_ = 0;
   }
 
   void DoLoop() {
@@ -461,6 +469,8 @@ class CameraDeviceDelegateTest : public ::testing::Test {
 
   std::unique_ptr<CameraDeviceContext> device_context_;
 
+  size_t num_streams_;
+
  private:
   base::test::ScopedTaskEnvironment scoped_task_environment_;
   base::Thread hal_delegate_thread_;
@@ -603,7 +613,7 @@ TEST_F(CameraDeviceDelegateTest, FailToOpenDevice) {
   };
   EXPECT_CALL(*mock_client, OnError(_, _, _))
       .Times(AtLeast(1))
-      .WillOnce(InvokeWithoutArgs(stop_on_error));
+      .WillRepeatedly(InvokeWithoutArgs(stop_on_error));
 
   EXPECT_CALL(mock_camera_module_, DoGetCameraInfo(0, _))
       .Times(1)

media/capture/video/chromeos/stream_buffer_manager.cc

@@ -187,9 +187,13 @@ void StreamBufferManager::StartPreview(
   RegisterBuffer(StreamType::kPreview);
 }
 
-void StreamBufferManager::StopPreview() {
+void StreamBufferManager::StopPreview(
+    base::OnceCallback<void(int32_t)> callback) {
   DCHECK(ipc_task_runner_->BelongsToCurrentThread());
   capturing_ = false;
+  if (callback) {
+    capture_interface_->Flush(std::move(callback));
+  }
 }
 
 cros::mojom::Camera3StreamPtr StreamBufferManager::GetStreamConfiguration(
@@ -223,6 +227,10 @@ void StreamBufferManager::TakePhoto(
   RegisterBuffer(StreamType::kStillCapture);
 }
 
+size_t StreamBufferManager::GetStreamNumber() {
+  return stream_context_.size();
+}
+
 void StreamBufferManager::AddResultMetadataObserver(
     ResultMetadataObserver* observer) {
   DCHECK(ipc_task_runner_->BelongsToCurrentThread());
@@ -416,7 +424,6 @@ void StreamBufferManager::OnRegisteredBuffer(StreamType stream_type,
 void StreamBufferManager::ProcessCaptureRequest() {
   DCHECK(ipc_task_runner_->BelongsToCurrentThread());
   DCHECK(stream_context_[StreamType::kPreview]);
-  DCHECK(stream_context_[StreamType::kStillCapture]);
 
   cros::mojom::Camera3CaptureRequestPtr request =
       cros::mojom::Camera3CaptureRequest::New();
@@ -439,7 +446,8 @@ void StreamBufferManager::ProcessCaptureRequest() {
     request->output_buffers.push_back(std::move(buffer));
   }
 
-  if (!stream_context_[StreamType::kStillCapture]->registered_buffers.empty()) {
+  if (stream_context_.count(StreamType::kStillCapture) &&
+      !stream_context_[StreamType::kStillCapture]->registered_buffers.empty()) {
     DCHECK(!still_capture_callbacks_currently_processing_.empty());
     cros::mojom::Camera3StreamBufferPtr buffer =
         cros::mojom::Camera3StreamBuffer::New();

media/capture/video/chromeos/stream_buffer_manager.h

@@ -116,13 +116,15 @@ class CAPTURE_EXPORT StreamBufferManager final
 
   // Stops the capture loop.  After StopPreview is called |callback_ops_| is
   // unbound, so no new capture request or result will be processed.
-  void StopPreview();
+  void StopPreview(base::OnceCallback<void(int32_t)> callback);
 
   cros::mojom::Camera3StreamPtr GetStreamConfiguration(StreamType stream_type);
 
   void TakePhoto(cros::mojom::CameraMetadataPtr settings,
                  VideoCaptureDevice::TakePhotoCallback callback);
 
+  size_t GetStreamNumber();
+
   // CaptureMetadataDispatcher implementations.
   void AddResultMetadataObserver(ResultMetadataObserver* observer) override;
   void RemoveResultMetadataObserver(ResultMetadataObserver* observer) override;

media/capture/video/chromeos/stream_buffer_manager_unittest.cc

@@ -63,6 +63,9 @@ class MockStreamCaptureInterface : public StreamCaptureInterface {
   MOCK_METHOD2(DoProcessCaptureRequest,
                void(cros::mojom::Camera3CaptureRequestPtr& request,
                     base::OnceCallback<void(int32_t)>& callback));
+
+  void Flush(base::OnceCallback<void(int32_t)> callback) { DoFlush(callback); }
+  MOCK_METHOD1(DoFlush, void(base::OnceCallback<void(int32_t)>& callback));
 };
 
 const VideoCaptureFormat kDefaultCaptureFormat(gfx::Size(1280, 720),

media/capture/video_capture_types.h

@@ -178,7 +178,8 @@ enum class VideoCaptureError {
   kMacDeckLinkUnsupportedPixelFormat = 112,
   kMacAvFoundationReceivedAVCaptureSessionRuntimeErrorNotification = 113,
   kAndroidApi2ErrorConfiguringCamera = 114,
-  kMaxValue = 114
+  kCrosHalV3DeviceDelegateFailedToFlush = 115,
+  kMaxValue = 115
 };
 
 // WARNING: Do not change the values assigned to the entries. They are used for