[Fuchsia] rotate frame in video capturer based on the camera orientation.

fuchsia.camera3 API provides camera application to clients, which are
expected to rotate the image when presenting it on the screen. Updated
VideoCaptureDeviceFuchsia to watch camera orientation and apply
corresponding transformation to the video frames.

Bug: 866669
Change-Id: I05ea0a3d4a59975ae0ed111c949b434fb6482b95
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2189619
Commit-Queue: Sergey Ulanov <sergeyu@chromium.org>
Reviewed-by: Kevin Marshall <kmarshall@chromium.org>
Cr-Commit-Position: refs/heads/master@{#768552}

media/capture/video/fuchsia/video_capture_device_fuchsia.cc

@@ -11,6 +11,7 @@
 #include "base/time/time.h"
 #include "media/base/video_types.h"
 #include "third_party/libyuv/include/libyuv/convert.h"
+#include "third_party/libyuv/include/libyuv/video_common.h"
 #include "ui/gfx/buffer_format_util.h"
 
 namespace media {
@@ -21,71 +22,68 @@ size_t RoundUp(size_t value, size_t alignment) {
   return ((value + alignment - 1) / alignment) * alignment;
 }
 
-void CopyAndConvertFrame(
-    base::span<const uint8_t> src_span,
-    fuchsia::sysmem::PixelFormatType src_pixel_format,
-    size_t src_stride_y,
-    size_t src_coded_height,
-    std::unique_ptr<VideoCaptureBufferHandle> output_handle,
-    gfx::Size output_size) {
-  const uint8_t* src_y = src_span.data();
-  size_t src_y_plane_size = src_stride_y * src_coded_height;
-
-  // Calculate offsets and strides for the output buffer.
-  uint8_t* dst_y = output_handle->data();
-  int dst_stride_y = output_size.width();
-  size_t dst_y_plane_size = output_size.width() * output_size.height();
-  uint8_t* dst_u = dst_y + dst_y_plane_size;
-  int dst_stride_u = output_size.width() / 2;
-  uint8_t* dst_v = dst_u + dst_y_plane_size / 4;
-  int dst_stride_v = output_size.width() / 2;
-
-  // Check that the output fits in the buffer.
-  const uint8_t* dst_end = dst_v + dst_y_plane_size / 4;
-  CHECK_LE(dst_end, output_handle->data() + output_handle->mapped_size());
-
+libyuv::FourCC GetFourccForPixelFormat(
+    fuchsia::sysmem::PixelFormatType src_pixel_format) {
   switch (src_pixel_format) {
+    case fuchsia::sysmem::PixelFormatType::I420:
+      return libyuv::FourCC::FOURCC_I420;
     case fuchsia::sysmem::PixelFormatType::YV12:
-    case fuchsia::sysmem::PixelFormatType::I420: {
-      const uint8_t* src_u = src_y + src_y_plane_size;
-      int src_stride_u = src_stride_y / 2;
-      size_t src_u_plane_size = src_stride_u * src_coded_height / 2;
-      const uint8_t* src_v = src_u + src_u_plane_size;
-      int src_stride_v = src_stride_y / 2;
-
-      if (src_pixel_format == fuchsia::sysmem::PixelFormatType::YV12) {
-        // Swap U and V planes to account for different plane order in YV12.
-        std::swap(src_u, src_v);
-      }
-
-      size_t src_v_plane_size = src_stride_v * src_coded_height / 2;
-      const uint8_t* src_end = src_v + src_v_plane_size;
-      CHECK_LE(src_end, src_span.data() + src_span.size());
-
-      libyuv::I420Copy(src_y, src_stride_y, src_u, src_stride_u, src_v,
-                       src_stride_v, dst_y, dst_stride_y, dst_u, dst_stride_u,
-                       dst_v, dst_stride_v, output_size.width(),
-                       output_size.height());
-      break;
-    }
-
-    case fuchsia::sysmem::PixelFormatType::NV12: {
-      const uint8_t* src_uv = src_y + src_stride_y * src_coded_height;
-      int src_stride_uv = src_stride_y;
-
-      int src_uv_plane_size = src_stride_uv * src_coded_height / 2;
-      const uint8_t* src_end = src_uv + src_uv_plane_size;
-      CHECK_LE(src_end, src_span.data() + src_span.size());
-
-      libyuv::NV12ToI420(src_y, src_stride_y, src_uv, src_stride_uv, dst_y,
-                         dst_stride_y, dst_u, dst_stride_u, dst_v, dst_stride_v,
-                         output_size.width(), output_size.height());
-
-      break;
-    }
-
+      return libyuv::FourCC::FOURCC_YV12;
+    case fuchsia::sysmem::PixelFormatType::NV12:
+      return libyuv::FourCC::FOURCC_NV12;
     default:
       NOTREACHED();
+      return libyuv::FourCC::FOURCC_I420;
+  }
+}
+
+libyuv::RotationMode CameraOrientationToLibyuvRotation(
+    fuchsia::camera3::Orientation orientation,
+    bool* flip_y) {
+  switch (orientation) {
+    case fuchsia::camera3::Orientation::UP:
+      *flip_y = false;
+      return libyuv::RotationMode::kRotate0;
+
+    case fuchsia::camera3::Orientation::DOWN:
+      *flip_y = false;
+      return libyuv::RotationMode::kRotate180;
+
+    case fuchsia::camera3::Orientation::LEFT:
+      *flip_y = false;
+      return libyuv::RotationMode::kRotate270;
+
+    case fuchsia::camera3::Orientation::RIGHT:
+      *flip_y = false;
+      return libyuv::RotationMode::kRotate90;
+
+    case fuchsia::camera3::Orientation::UP_FLIPPED:
+      *flip_y = true;
+      return libyuv::RotationMode::kRotate180;
+
+    case fuchsia::camera3::Orientation::DOWN_FLIPPED:
+      *flip_y = true;
+      return libyuv::RotationMode::kRotate0;
+
+    case fuchsia::camera3::Orientation::LEFT_FLIPPED:
+      *flip_y = true;
+      return libyuv::RotationMode::kRotate90;
+
+    case fuchsia::camera3::Orientation::RIGHT_FLIPPED:
+      *flip_y = true;
+      return libyuv::RotationMode::kRotate270;
+  }
+}
+
+gfx::Size RotateSize(gfx::Size size, libyuv::RotationMode rotation) {
+  switch (rotation) {
+    case libyuv::RotationMode::kRotate0:
+    case libyuv::RotationMode::kRotate180:
+      return size;
+
+    case libyuv::RotationMode::kRotate90:
+    case libyuv::RotationMode::kRotate270:
+      return gfx::Size(size.height(), size.width());
   }
 }
 
@@ -150,6 +148,7 @@ void VideoCaptureDeviceFuchsia::AllocateAndStart(
       fit::bind_member(this, &VideoCaptureDeviceFuchsia::OnStreamError));
 
   WatchResolution();
+  WatchOrientation();
 
   // Call SetBufferCollection() with a new buffer collection token to indicate
   // that we are interested in buffer collection negotiation. The collection
@@ -213,6 +212,21 @@ void VideoCaptureDeviceFuchsia::OnWatchResolutionResult(
   WatchResolution();
 }
 
+void VideoCaptureDeviceFuchsia::WatchOrientation() {
+  DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+
+  stream_->WatchOrientation(fit::bind_member(
+      this, &VideoCaptureDeviceFuchsia::OnWatchOrientationResult));
+}
+
+void VideoCaptureDeviceFuchsia::OnWatchOrientationResult(
+    fuchsia::camera3::Orientation orientation) {
+  DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+
+  orientation_ = orientation;
+  WatchOrientation();
+}
+
 void VideoCaptureDeviceFuchsia::WatchBufferCollection() {
   DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
 
@@ -333,8 +347,15 @@ void VideoCaptureDeviceFuchsia::ProcessNewFrame(
       sysmem_buffer_format.bytes_per_row_divisor);
   gfx::Size visible_size =
       frame_size_.value_or(gfx::Size(src_coded_width, src_coded_height));
-  gfx::Size output_size((visible_size.width() + 1) & ~1,
-                        (visible_size.height() + 1) & ~1);
+  gfx::Size nonrotated_output_size((visible_size.width() + 1) & ~1,
+                                   (visible_size.height() + 1) & ~1);
+
+  bool flip_y;
+  libyuv::RotationMode rotation =
+      CameraOrientationToLibyuvRotation(orientation_, &flip_y);
+
+  gfx::Size output_size = RotateSize(nonrotated_output_size, rotation);
+  visible_size = RotateSize(visible_size, rotation);
 
   base::TimeTicks reference_time =
       base::TimeTicks::FromZxTime(frame_info.timestamp);
@@ -375,11 +396,36 @@ void VideoCaptureDeviceFuchsia::ProcessNewFrame(
     return;
   }
 
-  auto src_pixel_format = buffer_reader_->buffer_settings()
-                              .image_format_constraints.pixel_format.type;
-  CopyAndConvertFrame(src_span, src_pixel_format, src_stride, src_coded_height,
-                      buffer.handle_provider->GetHandleForInProcessAccess(),
-                      output_size);
+  std::unique_ptr<VideoCaptureBufferHandle> output_handle =
+      buffer.handle_provider->GetHandleForInProcessAccess();
+
+  // Calculate offsets and strides for the output buffer.
+  uint8_t* dst_y = output_handle->data();
+  int dst_stride_y = output_size.width();
+  size_t dst_y_plane_size = output_size.width() * output_size.height();
+  uint8_t* dst_u = dst_y + dst_y_plane_size;
+  int dst_stride_u = output_size.width() / 2;
+  uint8_t* dst_v = dst_u + dst_y_plane_size / 4;
+  int dst_stride_v = output_size.width() / 2;
+
+  // Check that the output fits in the buffer.
+  const uint8_t* dst_end = dst_v + dst_y_plane_size / 4;
+  CHECK_LE(dst_end, output_handle->data() + output_handle->mapped_size());
+
+  // Vertical flip is indicated to ConvertToI420() by negating src_height.
+  int flipped_src_height = static_cast<int>(src_coded_height);
+  if (flip_y)
+    flipped_src_height = -flipped_src_height;
+
+  auto four_cc =
+      GetFourccForPixelFormat(buffer_reader_->buffer_settings()
+                                  .image_format_constraints.pixel_format.type);
+
+  libyuv::ConvertToI420(src_span.data(), src_span.size(), dst_y, dst_stride_y,
+                        dst_u, dst_stride_u, dst_v, dst_stride_v,
+                        /*crop_x=*/0, /*crop_y=*/0, src_stride,
+                        flipped_src_height, nonrotated_output_size.width(),
+                        nonrotated_output_size.height(), rotation, four_cc);
 
   client_->OnIncomingCapturedBufferExt(
       std::move(buffer), capture_format, gfx::ColorSpace(), reference_time,

media/capture/video/fuchsia/video_capture_device_fuchsia.h

@@ -60,6 +60,12 @@ class CAPTURE_EXPORT VideoCaptureDeviceFuchsia : public VideoCaptureDevice {
   // Callback for WatchResolution().
   void OnWatchResolutionResult(fuchsia::math::Size frame_size);
 
+  // Watches for orientation updates and updates |orientation_| accordingly.
+  void WatchOrientation();
+
+  // Callback for WatchOrientation().
+  void OnWatchOrientationResult(fuchsia::camera3::Orientation orientation);
+
   // Watches for sysmem buffer collection updates from the camera.
   void WatchBufferCollection();
 
@@ -95,6 +101,8 @@ class CAPTURE_EXPORT VideoCaptureDeviceFuchsia : public VideoCaptureDevice {
   std::unique_ptr<SysmemBufferReader> buffer_reader_;
 
   base::Optional<gfx::Size> frame_size_;
+  fuchsia::camera3::Orientation orientation_ =
+      fuchsia::camera3::Orientation::UP;
 
   base::TimeTicks start_time_;
 

media/capture/video/fuchsia/video_capture_device_fuchsia_test.cc

@@ -256,6 +256,37 @@ TEST_F(VideoCaptureDeviceFuchsiaTest, MultipleFrames) {
   }
 }
 
+TEST_F(VideoCaptureDeviceFuchsiaTest, FrameRotation) {
+  const gfx::Size kResolution(4, 2);
+  fake_stream_.SetFakeResolution(kResolution);
+
+  StartCapturer();
+
+  EXPECT_TRUE(fake_stream_.WaitBuffersAllocated());
+
+  for (int i = static_cast<int>(fuchsia::camera3::Orientation::UP);
+       i <= static_cast<int>(fuchsia::camera3::Orientation::RIGHT_FLIPPED);
+       ++i) {
+    SCOPED_TRACE(testing::Message() << "Orientation " << i);
+
+    auto orientation = static_cast<fuchsia::camera3::Orientation>(i);
+
+    ASSERT_TRUE(fake_stream_.WaitFreeBuffer());
+    fake_stream_.SetFakeOrientation(orientation);
+    fake_stream_.ProduceFrame(base::TimeTicks::Now(), i);
+    client_->WaitFrame();
+
+    gfx::Size expected_size = kResolution;
+    if (orientation == fuchsia::camera3::Orientation::LEFT ||
+        orientation == fuchsia::camera3::Orientation::LEFT_FLIPPED ||
+        orientation == fuchsia::camera3::Orientation::RIGHT ||
+        orientation == fuchsia::camera3::Orientation::RIGHT_FLIPPED) {
+      expected_size = gfx::Size(expected_size.height(), expected_size.width());
+    }
+    ValidateReceivedFrame(client_->received_frames().back(), expected_size, i);
+  }
+}
+
 TEST_F(VideoCaptureDeviceFuchsiaTest, FrameDimensionsNotDivisibleBy2) {
   const gfx::Size kOddResolution(21, 7);
   fake_stream_.SetFakeResolution(kOddResolution);

media/fuchsia/camera/fake_fuchsia_camera.cc

@@ -25,11 +25,62 @@ uint8_t GetTestFrameValue(gfx::Size size, int x, int y, uint8_t salt) {
   return static_cast<uint8_t>(y + x * size.height() + salt);
 }
 
+// Fills one plane of a test frame. |data| points at the location of the pixel
+// (0, 0). |orientation| specifies frame orientation transformation that will be
+// applied on the receiving end, so this function applies _reverse_ of the
+// |orientation| transformation.
 void FillPlane(uint8_t* data,
                gfx::Size size,
-               size_t x_step,
-               size_t y_step,
+               int x_step,
+               int y_step,
+               fuchsia::camera3::Orientation orientation,
                uint8_t salt) {
+  // First flip X axis for flipped orientation.
+  if (orientation == fuchsia::camera3::Orientation::UP_FLIPPED ||
+      orientation == fuchsia::camera3::Orientation::DOWN_FLIPPED ||
+      orientation == fuchsia::camera3::Orientation::RIGHT_FLIPPED ||
+      orientation == fuchsia::camera3::Orientation::LEFT_FLIPPED) {
+    // Move the origin to the top right corner and flip the X axis.
+    data += (size.width() - 1) * x_step;
+    x_step = -x_step;
+  }
+
+  switch (orientation) {
+    case fuchsia::camera3::Orientation::UP:
+    case fuchsia::camera3::Orientation::UP_FLIPPED:
+      break;
+
+    case fuchsia::camera3::Orientation::DOWN:
+    case fuchsia::camera3::Orientation::DOWN_FLIPPED:
+      // Move |data| to point to the bottom right corner and reverse direction
+      // of both axes.
+      data += (size.width() - 1) * x_step + (size.height() - 1) * y_step;
+      x_step = -x_step;
+      y_step = -y_step;
+      break;
+
+    case fuchsia::camera3::Orientation::LEFT:
+    case fuchsia::camera3::Orientation::LEFT_FLIPPED:
+      // Rotate 90 degrees clockwise by moving |data| to point to the right top
+      // corner, swapping the axes and reversing direction of the Y axis.
+      data += (size.width() - 1) * x_step;
+      size = gfx::Size(size.height(), size.width());
+      std::swap(x_step, y_step);
+      y_step = -y_step;
+      break;
+
+    case fuchsia::camera3::Orientation::RIGHT:
+    case fuchsia::camera3::Orientation::RIGHT_FLIPPED:
+      // Rotate 90 degrees counter-clockwise by moving |data| to point to the
+      // bottom left corner, swapping the axes and reversing direction of the X
+      // axis.
+      data += (size.height() - 1) * y_step;
+      size = gfx::Size(size.height(), size.width());
+      std::swap(x_step, y_step);
+      x_step = -x_step;
+      break;
+  }
+
   for (int y = 0; y < size.height(); ++y) {
     for (int x = 0; x < size.width(); ++x) {
       data[x * x_step + y * y_step] = GetTestFrameValue(size, x, y, salt);
@@ -111,6 +162,13 @@ void FakeCameraStream::SetFakeResolution(gfx::Size resolution) {
   SendResolution();
 }
 
+void FakeCameraStream::SetFakeOrientation(
+    fuchsia::camera3::Orientation orientation) {
+  orientation_ = orientation;
+  orientation_update_ = orientation;
+  SendOrientation();
+}
+
 bool FakeCameraStream::WaitBuffersAllocated() {
   EXPECT_FALSE(wait_buffers_allocated_run_loop_);
 
@@ -158,16 +216,16 @@ void FakeCameraStream::ProduceFrame(base::TimeTicks timestamp, uint8_t salt) {
   // Fill Y plane.
   uint8_t* y_plane = reinterpret_cast<uint8_t*>(buffer->mapping.memory());
   size_t stride = kMaxFrameSize.width();
-  FillPlane(y_plane, coded_size, /*x_step=*/1, /*y_step=*/stride,
+  FillPlane(y_plane, coded_size, /*x_step=*/1, /*y_step=*/stride, orientation_,
             salt + kYPlaneSalt);
 
   // Fill UV plane.
   gfx::Size uv_size(coded_size.width() / 2, coded_size.height() / 2);
   uint8_t* uv_plane = y_plane + kMaxFrameSize.width() * kMaxFrameSize.height();
-  FillPlane(uv_plane, uv_size, /*x_step=*/2, /*y_step=*/stride,
+  FillPlane(uv_plane, uv_size, /*x_step=*/2, /*y_step=*/stride, orientation_,
             salt + kUPlaneSalt);
   FillPlane(uv_plane + 1, uv_size, /*x_step=*/2, /*y_step=*/stride,
-            salt + kVPlaneSalt);
+            orientation_, salt + kVPlaneSalt);
 
   // Create FrameInfo.
   fuchsia::camera3::FrameInfo frame;
@@ -194,6 +252,12 @@ void FakeCameraStream::WatchResolution(WatchResolutionCallback callback) {
   SendResolution();
 }
 
+void FakeCameraStream::WatchOrientation(WatchOrientationCallback callback) {
+  EXPECT_FALSE(watch_orientation_callback_);
+  watch_orientation_callback_ = std::move(callback);
+  SendOrientation();
+}
+
 void FakeCameraStream::SetBufferCollection(
     fidl::InterfaceHandle<fuchsia::sysmem::BufferCollectionToken>
         token_handle) {
@@ -327,6 +391,14 @@ void FakeCameraStream::SendResolution() {
   resolution_update_.reset();
 }
 
+void FakeCameraStream::SendOrientation() {
+  if (!watch_orientation_callback_ || !orientation_update_)
+    return;
+  watch_orientation_callback_(orientation_update_.value());
+  watch_orientation_callback_ = {};
+  orientation_update_.reset();
+}
+
 void FakeCameraStream::SendBufferCollection() {
   if (!watch_buffer_collection_callback_ || !new_buffer_collection_token_)
     return;

media/fuchsia/camera/fake_fuchsia_camera.h

@@ -39,6 +39,7 @@ class FakeCameraStream : public fuchsia::camera3::testing::Stream_TestBase,
   void Bind(fidl::InterfaceRequest<fuchsia::camera3::Stream> request);
 
   void SetFakeResolution(gfx::Size resolution);
+  void SetFakeOrientation(fuchsia::camera3::Orientation orientation);
 
   // Waits for the buffer collection to be allocated. Returns true if the buffer
   // collection was allocated successfully.
@@ -55,6 +56,7 @@ class FakeCameraStream : public fuchsia::camera3::testing::Stream_TestBase,
 
   // fuchsia::camera3::Stream implementation.
   void WatchResolution(WatchResolutionCallback callback) final;
+  void WatchOrientation(WatchOrientationCallback callback) final;
   void SetBufferCollection(
       fidl::InterfaceHandle<fuchsia::sysmem::BufferCollectionToken>
           token_handle) final;
@@ -74,6 +76,10 @@ class FakeCameraStream : public fuchsia::camera3::testing::Stream_TestBase,
   // resolution has been updated.
   void SendResolution();
 
+  // Calls callback for the pending WatchOrientation() if the call is pending
+  // and orientation has been updated.
+  void SendOrientation();
+
   // Calls callback for the pending WatchBufferCollection() if we have a new
   // token and the call is pending.
   void SendBufferCollection();
@@ -89,11 +95,17 @@ class FakeCameraStream : public fuchsia::camera3::testing::Stream_TestBase,
   fidl::Binding<fuchsia::camera3::Stream> binding_;
 
   gfx::Size resolution_ = kDefaultFrameSize;
+  fuchsia::camera3::Orientation orientation_ =
+      fuchsia::camera3::Orientation::UP;
 
   base::Optional<fuchsia::math::Size> resolution_update_ = fuchsia::math::Size{
       kDefaultFrameSize.width(), kDefaultFrameSize.height()};
   WatchResolutionCallback watch_resolution_callback_;
 
+  base::Optional<fuchsia::camera3::Orientation> orientation_update_ =
+      fuchsia::camera3::Orientation::UP;
+  WatchOrientationCallback watch_orientation_callback_;
+
   base::Optional<fidl::InterfaceHandle<fuchsia::sysmem::BufferCollectionToken>>
       new_buffer_collection_token_;
   WatchBufferCollectionCallback watch_buffer_collection_callback_;