Implement decoding to YUV for AVIF image format

The ReadYUV() function and the StaticAVIFTests.YUV test in
avif_image_decoder_test.cc are adapted from the ReadYUV() function and
the JPEGImageDecoderTest.yuv test in jpeg_image_decoder_test.cc.

Declare ImageDecoder::CanDecodeToYUV() as a const method.

Test:
1. blink_platform_unittests --gtest_filter=*AVIF*
2. Run chrome with --enable-features=AVIF and visit
   https://resources.link-u.co.jp/avif/images.html.

Bug: 1087056
Change-Id: I8cd6375cc0824e71ec80622328f288a19c67bc90
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2229840
Commit-Queue: Wan-Teh Chang <wtc@google.com>
Reviewed-by: Peter Kasting <pkasting@chromium.org>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Reviewed-by: Andres Calderon Jaramillo <andrescj@chromium.org>
Cr-Commit-Position: refs/heads/master@{#781330}

third_party/blink/renderer/platform/image-decoders/avif/avif_image_decoder.cc

@@ -4,13 +4,16 @@
 
 #include "third_party/blink/renderer/platform/image-decoders/avif/avif_image_decoder.h"
 
+#include <cstring>
 #include <memory>
 
+#include "base/bits.h"
 #include "base/containers/adapters.h"
 #include "base/feature_list.h"
 #include "base/logging.h"
 #include "base/memory/scoped_refptr.h"
 #include "base/numerics/ranges.h"
+#include "base/numerics/safe_conversions.h"
 #include "base/optional.h"
 #include "base/timer/elapsed_timer.h"
 #include "build/build_config.h"
@@ -166,6 +169,14 @@ media::VideoPixelFormat AvifToVideoPixelFormat(avifPixelFormat fmt, int depth) {
   }
 }
 
+// |y_size| is the width or height of the Y plane. Returns the width or height
+// of the U and V planes. |chroma_shift| represents the subsampling of the
+// chroma (U and V) planes in the x (for width) or y (for height) direction.
+int UVSize(int y_size, int chroma_shift) {
+  DCHECK(chroma_shift == 0 || chroma_shift == 1);
+  return (y_size + chroma_shift) >> chroma_shift;
+}
+
 inline void WritePixel(float max_channel,
                        const gfx::Point3F& pixel,
                        float alpha,
@@ -288,6 +299,116 @@ void AVIFImageDecoder::OnSetData(SegmentReader* data) {
     SetFailed();
 }
 
+IntSize AVIFImageDecoder::DecodedYUVSize(int component) const {
+  DCHECK_GE(component, 0);
+  // TODO(crbug.com/910276): Change after alpha support.
+  DCHECK_LE(component, 2);
+  DCHECK(IsDecodedSizeAvailable());
+  if (component == SkYUVAIndex::kU_Index ||
+      component == SkYUVAIndex::kV_Index) {
+    return IntSize(UVSize(Size().Width(), chroma_shift_x_),
+                   UVSize(Size().Height(), chroma_shift_y_));
+  }
+  return Size();
+}
+
+size_t AVIFImageDecoder::DecodedYUVWidthBytes(int component) const {
+  DCHECK_GE(component, 0);
+  // TODO(crbug.com/910276): Change after alpha support.
+  DCHECK_LE(component, 2);
+  DCHECK(IsDecodedSizeAvailable());
+  // Try to return the same width bytes as used by the dav1d library. This will
+  // allow DecodeToYUV() to copy each plane with a single memcpy() call.
+  //
+  // The comments for Dav1dPicAllocator in dav1d/picture.h require the pixel
+  // width be padded to a multiple of 128 pixels.
+  int aligned_width = base::bits::Align(Size().Width(), 128);
+  if (component == SkYUVAIndex::kU_Index ||
+      component == SkYUVAIndex::kV_Index) {
+    aligned_width >>= chroma_shift_x_;
+  }
+  // When the stride is a multiple of 1024, dav1d_default_picture_alloc()
+  // slightly pads the stride to avoid a reduction in cache hit rate in most
+  // L1/L2 cache implementations. Match that trick here. (Note that this padding
+  // is not documented in dav1d/picture.h.)
+  if ((aligned_width & 1023) == 0)
+    aligned_width += 64;
+  return aligned_width;
+}
+
+SkYUVColorSpace AVIFImageDecoder::GetYUVColorSpace() const {
+  DCHECK(CanDecodeToYUV());
+  DCHECK(yuv_color_space_);
+  return *yuv_color_space_;
+}
+
+void AVIFImageDecoder::DecodeToYUV() {
+  DCHECK(image_planes_);
+  DCHECK(CanDecodeToYUV());
+  DCHECK(IsAllDataReceived());
+
+  if (Failed())
+    return;
+
+  DCHECK(decoder_);
+  DCHECK_EQ(decoded_frame_count_, 1u);  // Not animation.
+
+  // libavif cannot decode to an external buffer. So we need to copy from
+  // libavif's internal buffer to |image_planes_|.
+  // TODO(wtc): Enhance libavif to decode to an external buffer.
+  if (!DecodeImage(0)) {
+    SetFailed();
+    return;
+  }
+
+  const auto* image = decoder_->image;
+  // All frames must be the same size.
+  if (Size() != IntSize(image->width, image->height)) {
+    DVLOG(1) << "All frames must be the same size";
+    SetFailed();
+    return;
+  }
+  DCHECK_EQ(image->depth, 8u);
+  DCHECK(!image->alphaPlane);
+  static_assert(SkYUVAIndex::kY_Index == static_cast<int>(AVIF_CHAN_Y), "");
+  static_assert(SkYUVAIndex::kU_Index == static_cast<int>(AVIF_CHAN_U), "");
+  static_assert(SkYUVAIndex::kV_Index == static_cast<int>(AVIF_CHAN_V), "");
+  // Initialize |width| and |height| to the width and height of the luma plane.
+  uint32_t width = image->width;
+  uint32_t height = image->height;
+  // |height| comes from the AV1 sequence header or frame header, which encodes
+  // max_frame_height_minus_1 and frame_height_minus_1, respectively, as n-bit
+  // unsigned integers for some n.
+  DCHECK_GT(height, 0u);
+  for (int plane = 0; plane < 3; ++plane) {
+    const uint8_t* src = image->yuvPlanes[plane];
+    size_t src_row_bytes = base::strict_cast<size_t>(image->yuvRowBytes[plane]);
+    uint8_t* dst = static_cast<uint8_t*>(image_planes_->Plane(plane));
+    size_t dst_row_bytes = image_planes_->RowBytes(plane);
+    DCHECK_LE(width, src_row_bytes);
+    DCHECK_LE(width, dst_row_bytes);
+    if (src_row_bytes == dst_row_bytes) {
+      // If |src| and |dst| have the same stride, we can copy the plane with a
+      // single memcpy() call. For the last row we copy only |width| bytes to
+      // avoid reading past the end of the last row. For all other rows we copy
+      // |src_row_bytes| bytes.
+      memcpy(dst, src, (height - 1) * src_row_bytes + width);
+    } else {
+      for (uint32_t j = 0; j < height; ++j) {
+        memcpy(dst, src, width);
+        src += src_row_bytes;
+        dst += dst_row_bytes;
+      }
+    }
+    if (plane == 0) {
+      // Having processed the luma plane, change |width| and |height| to the
+      // width and height of the chroma planes.
+      width = UVSize(width, chroma_shift_x_);
+      height = UVSize(height, chroma_shift_y_);
+    }
+  }
+}
+
 int AVIFImageDecoder::RepetitionCount() const {
   return decoded_frame_count_ > 1 ? kAnimationLoopInfinite : kAnimationNone;
 }
@@ -416,7 +537,7 @@ bool AVIFImageDecoder::MaybeCreateDemuxer() {
     return false;
 
   // TODO(dalecurtis): This may create a second copy of the media data in
-  // memory, which is not great. Upstream should provide a read() based API:
+  // memory, which is not great. libavif should provide a read() based API:
   // https://github.com/AOMediaCodec/libavif/issues/11
   image_data_ = data_->GetAsSkData();
   if (!image_data_)
@@ -458,6 +579,12 @@ bool AVIFImageDecoder::MaybeCreateDemuxer() {
       ImageIsHighBitDepth() &&
       high_bit_depth_decoding_option_ == kHighBitDepthToHalfFloat;
 
+  const avifPixelFormat yuv_format = container->yuvFormat;
+  avifPixelFormatInfo format_info;
+  avifGetPixelFormatInfo(yuv_format, &format_info);
+  chroma_shift_x_ = format_info.chromaShiftX;
+  chroma_shift_y_ = format_info.chromaShiftY;
+
   // SetEmbeddedColorProfile() must be called before IsSizeAvailable() becomes
   // true. So call SetEmbeddedColorProfile() before calling SetSize(). The color
   // profile is either an ICC profile or the CICP color description.
@@ -501,14 +628,26 @@ bool AVIFImageDecoder::MaybeCreateDemuxer() {
     }
   }
 
+  // Determine whether the image can be decoded to YUV.
+  // * Bit depths higher than 8 are not supported.
+  // * TODO(crbug.com/915972): Only YUV 4:2:0 subsampling format is supported.
+  // * Alpha channel is not supported.
+  // * Multi-frame images (animations) are not supported. (The DecodeToYUV()
+  //   method does not have an 'index' parameter.)
+  // * If ColorTransform() returns a non-null pointer, the decoder has to do a
+  //   color space conversion, so we don't decode to YUV.
+  allow_decode_to_yuv_ =
+      !ImageIsHighBitDepth() && yuv_format == AVIF_PIXEL_FORMAT_YUV420 &&
+      !decoder_->alphaPresent && decoded_frame_count_ == 1 &&
+      (yuv_color_space_ = GetSkYUVColorSpace(container)) && !ColorTransform();
+
   return SetSize(container->width, container->height);
 }
 
 bool AVIFImageDecoder::DecodeImage(size_t index) {
   const auto ret = avifDecoderNthImage(decoder_.get(), index);
-  // We shouldn't be called more times than specified in
-  // DecodeFrameCount(); possibly this should truncate if the initial
-  // count is wrong?
+  // |index| should be less than what DecodeFrameCount() returns, so we should
+  // not get the AVIF_RESULT_NO_IMAGES_REMAINING error.
   DCHECK_NE(ret, AVIF_RESULT_NO_IMAGES_REMAINING);
   return ret == AVIF_RESULT_OK;
 }
@@ -530,8 +669,6 @@ bool AVIFImageDecoder::RenderImage(const avifImage* image, ImageFrame* buffer) {
   const bool is_mono = image->yuvFormat == AVIF_PIXEL_FORMAT_YUV400;
   const bool premultiply_alpha = buffer->PremultiplyAlpha();
 
-  // TODO(dalecurtis): We should decode to YUV when possible. Currently the
-  // YUV path seems to only support still-image YUV8.
   if (decode_to_half_float_) {
     UpdateColorTransform(frame_cs, image->depth);
 

third_party/blink/renderer/platform/image-decoders/avif/avif_image_decoder.h

@@ -7,6 +7,7 @@
 
 #include <memory>
 
+#include "base/optional.h"
 #include "third_party/blink/renderer/platform/image-decoders/image_decoder.h"
 #include "third_party/skia/include/core/SkBitmap.h"
 #include "third_party/skia/include/core/SkData.h"
@@ -34,6 +35,10 @@ class PLATFORM_EXPORT AVIFImageDecoder final : public ImageDecoder {
   String FilenameExtension() const override { return "avif"; }
   bool ImageIsHighBitDepth() override;
   void OnSetData(SegmentReader* data) override;
+  IntSize DecodedYUVSize(int component) const override;
+  size_t DecodedYUVWidthBytes(int component) const override;
+  SkYUVColorSpace GetYUVColorSpace() const override;
+  void DecodeToYUV() override;
   int RepetitionCount() const override;
   base::TimeDelta FrameDurationAtIndex(size_t) const override;
 
@@ -69,7 +74,10 @@ class PLATFORM_EXPORT AVIFImageDecoder final : public ImageDecoder {
 
   uint8_t bit_depth_ = 0;
   bool decode_to_half_float_ = false;
+  uint8_t chroma_shift_x_ = 0;
+  uint8_t chroma_shift_y_ = 0;
   size_t decoded_frame_count_ = 0;
+  base::Optional<SkYUVColorSpace> yuv_color_space_;
   std::unique_ptr<avifDecoder, void (*)(avifDecoder*)> decoder_{nullptr,
                                                                 nullptr};
 

third_party/blink/renderer/platform/image-decoders/avif/avif_image_decoder_test.cc

@@ -467,6 +467,58 @@ void TestInvalidStaticImage(const char* avif_file, ErrorPhase error_phase) {
   }
 }
 
+void ReadYUV(int* output_y_width,
+             int* output_y_height,
+             int* output_uv_width,
+             int* output_uv_height,
+             const char* image_file_path) {
+  scoped_refptr<SharedBuffer> data = ReadFile(image_file_path);
+  ASSERT_TRUE(data);
+
+  auto decoder = std::make_unique<AVIFImageDecoder>(
+      ImageDecoder::kAlphaNotPremultiplied, ImageDecoder::kDefaultBitDepth,
+      ColorBehavior::Tag(), ImageDecoder::kNoDecodedImageByteLimit);
+  decoder->SetData(data.get(), true);
+  ASSERT_TRUE(decoder->CanDecodeToYUV());
+
+  ASSERT_TRUE(decoder->IsSizeAvailable());
+
+  IntSize size = decoder->DecodedSize();
+  IntSize y_size = decoder->DecodedYUVSize(0);
+  IntSize u_size = decoder->DecodedYUVSize(1);
+  IntSize v_size = decoder->DecodedYUVSize(2);
+
+  EXPECT_EQ(size.Width(), y_size.Width());
+  EXPECT_EQ(size.Height(), y_size.Height());
+  EXPECT_EQ(u_size.Width(), v_size.Width());
+  EXPECT_EQ(u_size.Height(), v_size.Height());
+
+  *output_y_width = y_size.Width();
+  *output_y_height = y_size.Height();
+  *output_uv_width = u_size.Width();
+  *output_uv_height = u_size.Height();
+
+  size_t row_bytes[3];
+  row_bytes[0] = decoder->DecodedYUVWidthBytes(0);
+  row_bytes[1] = decoder->DecodedYUVWidthBytes(1);
+  row_bytes[2] = decoder->DecodedYUVWidthBytes(2);
+
+  size_t planes_data_size = row_bytes[0] * y_size.Height() +
+                            row_bytes[1] * u_size.Height() +
+                            row_bytes[2] * v_size.Height();
+  auto planes_data = std::make_unique<char[]>(planes_data_size);
+
+  void* planes[3];
+  planes[0] = planes_data.get();
+  planes[1] = static_cast<char*>(planes[0]) + row_bytes[0] * y_size.Height();
+  planes[2] = static_cast<char*>(planes[1]) + row_bytes[1] * u_size.Height();
+
+  decoder->SetImagePlanes(std::make_unique<ImagePlanes>(planes, row_bytes));
+
+  decoder->DecodeToYUV();
+  EXPECT_FALSE(decoder->Failed());
+}
+
 }  // namespace
 
 TEST(AnimatedAVIFTests, ValidImages) {
@@ -525,6 +577,18 @@ TEST(StaticAVIFTests, ValidImages) {
       1, kAnimationNone);
 }
 
+TEST(StaticAVIFTests, YUV) {
+  const char* avif_file =
+      "/images/resources/avif/red-full-ranged-8bpc.avif";  // 3x3, YUV 4:2:0
+  int output_y_width, output_y_height, output_uv_width, output_uv_height;
+  ReadYUV(&output_y_width, &output_y_height, &output_uv_width,
+          &output_uv_height, avif_file);
+  EXPECT_EQ(3, output_y_width);
+  EXPECT_EQ(3, output_y_height);
+  EXPECT_EQ(2, output_uv_width);
+  EXPECT_EQ(2, output_uv_height);
+}
+
 using StaticAVIFColorTests = ::testing::TestWithParam<StaticColorCheckParam>;
 
 INSTANTIATE_TEST_CASE_P(Parameterized,

third_party/blink/renderer/platform/image-decoders/image_decoder.h

@@ -143,7 +143,7 @@ class PLATFORM_EXPORT ImageDecoder {
   };
 
   // Enforces YUV decoding to be disallowed in the image decoder. The default
-  // value defers to the YUV decoding decision to the decoder.
+  // value defers the YUV decoding decision to the decoder.
   enum class OverrideAllowDecodeToYuv {
     kDefault,
     kDeny,
@@ -376,7 +376,7 @@ class PLATFORM_EXPORT ImageDecoder {
     frame_buffer_cache_[0].SetMemoryAllocator(allocator);
   }
 
-  bool CanDecodeToYUV() { return allow_decode_to_yuv_; }
+  bool CanDecodeToYUV() const { return allow_decode_to_yuv_; }
   // Should only be called if CanDecodeToYuv() returns true, in which case
   // the subclass of ImageDecoder must override this method.
   virtual void DecodeToYUV() { NOTREACHED(); }