[CrOS] video_capture: pre-rotate frames for non-zero sensor orientation

There are some devices with camera sensors of non-zero physical
orientation.  For these devices, we still have to rotate the camera
frames to compensate for the sensor orientation if we disable
pre-rotation in the video capture stack.  Otherwise, the video
recorded in the camera app will have wrong orientation because we
don't rotate the frames at all on the video stack.

Bug: 990682
Test: Manually on Nocturne
Change-Id: Iba471e318f6d15770491b5677769e5d6c5d1c241
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2335849
Commit-Queue: Ricky Liang <jcliang@chromium.org>
Reviewed-by: Wei Lee <wtlee@chromium.org>
Cr-Commit-Position: refs/heads/master@{#795406}

media/capture/video/chromeos/camera_device_context.cc

@@ -60,8 +60,8 @@ void CameraDeviceContext::SubmitCapturedGpuMemoryBuffer(
     base::TimeTicks reference_time,
     base::TimeDelta timestamp) {
   client_->OnIncomingCapturedGfxBuffer(buffer, frame_format,
-                                       GetCameraFrameOrientation(),
-                                       reference_time, timestamp);
+                                       GetCameraFrameRotation(), reference_time,
+                                       timestamp);
 }
 
 void CameraDeviceContext::SetSensorOrientation(int sensor_orientation) {
@@ -78,10 +78,19 @@ void CameraDeviceContext::SetScreenRotation(int screen_rotation) {
   screen_rotation_ = screen_rotation;
 }
 
-int CameraDeviceContext::GetCameraFrameOrientation() {
+int CameraDeviceContext::GetCameraFrameRotation() {
   return (sensor_orientation_ + screen_rotation_) % 360;
 }
 
+int CameraDeviceContext::GetRotationForDisplay() {
+  return screen_rotation_;
+}
+
+int CameraDeviceContext::GetRotationFromSensorOrientation() {
+  // We need to rotate the frame back to 0 degree.
+  return (360 - sensor_orientation_);
+}
+
 bool CameraDeviceContext::ReserveVideoCaptureBufferFromPool(
     gfx::Size size,
     VideoPixelFormat format,

media/capture/video/chromeos/camera_device_context.h

@@ -134,7 +134,17 @@ class CAPTURE_EXPORT CameraDeviceContext {
 
   void SetScreenRotation(int screen_rotation);
 
-  int GetCameraFrameOrientation();
+  // Gets the accumulated rotation that the camera frame needs to be rotated
+  // to match the display orientation.  This includes the sensor orientation and
+  // the screen rotation.
+  int GetCameraFrameRotation();
+
+  // Gets the rotation needed to match the camera frame to the display, assuming
+  // the camera frame has 0 degree orientation.
+  int GetRotationForDisplay();
+
+  // Gets the rotation needed to get the camera frame to 0 degree orientation.
+  int GetRotationFromSensorOrientation();
 
   // Reserves a video capture buffer from the buffer pool provided by the video
   // |client_|.  Returns true if the operation succeeds; false otherwise.

media/capture/video/chromeos/camera_device_delegate.cc

@@ -1070,7 +1070,7 @@ bool CameraDeviceDelegate::SetPointsOfInterest(
 
   // Handle rotation, still in normalized square space.
   std::tie(x, y) = [&]() -> std::pair<double, double> {
-    switch (device_context_->GetCameraFrameOrientation()) {
+    switch (device_context_->GetCameraFrameRotation()) {
       case 0:
         return {x, y};
       case 90:

media/capture/video/chromeos/request_manager.cc

@@ -319,7 +319,7 @@ void RequestManager::PrepareCaptureRequest() {
   pending_result.input_buffer_id = input_buffer_id;
   pending_result.reprocess_effect = reprocess_effect;
   pending_result.still_capture_callback = std::move(callback);
-  pending_result.orientation = device_context_->GetCameraFrameOrientation();
+  pending_result.orientation = device_context_->GetCameraFrameRotation();
 
   // For reprocess supported devices, bind the ReprocessTaskQueue with this
   // frame number. Once the shot result is returned, we will rebind the
@@ -441,7 +441,7 @@ bool RequestManager::TryPrepareOneShotRequest(
     take_photo_callback_queue_.pop();
 
     *settings = std::move(take_photo_settings_queue_.front());
-    SetJpegOrientation(settings, device_context_->GetCameraFrameOrientation());
+    SetJpegOrientation(settings, device_context_->GetCameraFrameRotation());
   }
   SetZeroShutterLag(settings, true);
   take_photo_settings_queue_.pop();
@@ -832,8 +832,7 @@ void RequestManager::SubmitCapturedPreviewBuffer(uint32_t frame_number,
     VideoCaptureFormat format;
     base::Optional<VideoCaptureDevice::Client::Buffer> buffer =
         stream_buffer_manager_->AcquireBufferForClientById(
-            StreamType::kPreviewOutput, buffer_ipc_id,
-            device_context_->GetCameraFrameOrientation(), &format);
+            StreamType::kPreviewOutput, buffer_ipc_id, &format);
     CHECK(buffer);
 
     // TODO: Figure out the right color space for the camera frame.  We may need
@@ -859,7 +858,7 @@ void RequestManager::SubmitCapturedPreviewBuffer(uint32_t frame_number,
         return VideoRotation::VIDEO_ROTATION_0;
       };
       metadata.rotation =
-          translate_rotation(device_context_->GetCameraFrameOrientation());
+          translate_rotation(device_context_->GetRotationForDisplay());
     } else {
       // All frames are pre-rotated to the display orientation.
       metadata.rotation = VideoRotation::VIDEO_ROTATION_0;

media/capture/video/chromeos/stream_buffer_manager.cc

@@ -67,7 +67,6 @@ gfx::GpuMemoryBuffer* StreamBufferManager::GetGpuMemoryBufferById(
 base::Optional<StreamBufferManager::Buffer>
 StreamBufferManager::AcquireBufferForClientById(StreamType stream_type,
                                                 uint64_t buffer_ipc_id,
-                                                int rotation,
                                                 VideoCaptureFormat* format) {
   DCHECK(stream_context_.count(stream_type));
   auto& stream_context = stream_context_[stream_type];
@@ -83,9 +82,14 @@ StreamBufferManager::AcquireBufferForClientById(StreamType stream_type,
   // We only support NV12 at the moment.
   DCHECK_EQ(format->pixel_format, PIXEL_FORMAT_NV12);
 
+  int rotation = device_context_->GetCameraFrameRotation();
   if (base::FeatureList::IsEnabled(
           features::kDisableCameraFrameRotationAtSource)) {
-    return std::move(buffer_pair.vcd_buffer);
+    // For a device that don't have the camera sensor installed to match the
+    // device's natural orientation, we have to fix the sensor orientation here.
+    // Otherwise the recorded video in Chrome camera app would have wrong
+    // orientation because we no longer rotate the frames for the video encoder.
+    rotation = device_context_->GetRotationFromSensorOrientation();
   }
 
   if (rotation == 0) {

media/capture/video/chromeos/stream_buffer_manager.h

@@ -69,7 +69,6 @@ class CAPTURE_EXPORT StreamBufferManager final {
   // camera frame rotation for good.
   base::Optional<Buffer> AcquireBufferForClientById(StreamType stream_type,
                                                     uint64_t buffer_ipc_id,
-                                                    int rotation,
                                                     VideoCaptureFormat* format);
 
   VideoCaptureFormat GetStreamCaptureFormat(StreamType stream_type);