media/gpu/video_encode_accelerator_tests: Verify quality of encoded images

This CL adds VideoFrameValidators that check SSIM and PSNR,
respectively, to StreamValidator. It eenables
video_encode_accelerator_tests to check the quality of encoded
images produced by VideoEncodeAccelerator.

Bug: 1045825
Test: video_encode_accelerator_tests on eve
Change-Id: I5bca3b7f74e664488c38ab749d6d491883d7f356
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2190468Reviewed-by: David Staessens <dstaessens@chromium.org>
Commit-Queue: Hirokazu Honda <hiroh@chromium.org>
Cr-Commit-Position: refs/heads/master@{#773992}

media/gpu/BUILD.gn

@@ -559,7 +559,9 @@ if (use_v4l2_codec || use_vaapi) {
     data = [ "//media/test/data/" ]
     deps = [
       ":buildflags",
+      "test:frame_validator",
       "test:helpers",
+      "test:test_helpers",
       "test:video_encoder",
       "test:video_encoder_test_environment",
       "//media:test_support",

media/gpu/test/video.cc

@@ -127,6 +127,10 @@ const std::vector<uint8_t>& Video::Data() const {
   return data_;
 }
 
+std::vector<uint8_t>& Video::Data() {
+  return data_;
+}
+
 VideoCodec Video::Codec() const {
   return codec_;
 }
@@ -478,6 +482,5 @@ base::Optional<VideoPixelFormat> Video::ConvertStringtoPixelFormat(
     return base::nullopt;
   }
 }
-
 }  // namespace test
 }  // namespace media

media/gpu/test/video.h

@@ -42,6 +42,7 @@ class Video {
   const base::FilePath& FilePath() const;
   // Get the video data, will be empty if the video hasn't been loaded yet.
   const std::vector<uint8_t>& Data() const;
+  std::vector<uint8_t>& Data();
 
   // Decode the video, replacing the video stream data in |data_| with raw video
   // data. This is currently only supported for VP9 videos. Returns whether

media/gpu/test/video_encoder/video_encoder_test_environment.cc

@@ -107,7 +107,7 @@ VideoEncoderTestEnvironment::VideoEncoderTestEnvironment(
 
 VideoEncoderTestEnvironment::~VideoEncoderTestEnvironment() = default;
 
-const media::test::Video* VideoEncoderTestEnvironment::Video() const {
+media::test::Video* VideoEncoderTestEnvironment::Video() const {
   return video_.get();
 }
 

media/gpu/test/video_encoder/video_encoder_test_environment.h

@@ -34,7 +34,7 @@ class VideoEncoderTestEnvironment : public VideoTestEnvironment {
   ~VideoEncoderTestEnvironment() override;
 
   // Get the video the tests will be ran on.
-  const media::test::Video* Video() const;
+  media::test::Video* Video() const;
   // Get the output folder.
   const base::FilePath& OutputFolder() const;
   // Get the output codec profile.

media/gpu/test/video_frame_validator.cc

@@ -315,6 +315,7 @@ PSNRVideoFrameValidator::Validate(scoped_refptr<const VideoFrame> frame,
   CHECK(model_frame);
   double psnr = ComputePSNR(*frame, *model_frame);
   DVLOGF(4) << "frame_index: " << frame_index << ", psnr: " << psnr;
+  psnr_[frame_index] = psnr;
   if (psnr < tolerance_)
     return std::make_unique<PSNRMismatchedFrameInfo>(frame_index, psnr);
   return nullptr;
@@ -365,6 +366,7 @@ SSIMVideoFrameValidator::Validate(scoped_refptr<const VideoFrame> frame,
   CHECK(model_frame);
   double ssim = ComputeSSIM(*frame, *model_frame);
   DVLOGF(4) << "frame_index: " << frame_index << ", ssim: " << ssim;
+  ssim_[frame_index] = ssim;
   if (ssim < tolerance_)
     return std::make_unique<SSIMMismatchedFrameInfo>(frame_index, ssim);
   return nullptr;

media/gpu/test/video_frame_validator.h

@@ -185,6 +185,7 @@ class PSNRVideoFrameValidator : public VideoFrameValidator {
       const GetModelFrameCB& get_model_frame_cb,
       double tolerance = kDefaultTolerance,
       std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor = nullptr);
+  const std::map<size_t, double>& GetPSNRValues() { return psnr_; }
   ~PSNRVideoFrameValidator() override;
 
  private:
@@ -201,6 +202,7 @@ class PSNRVideoFrameValidator : public VideoFrameValidator {
 
   const GetModelFrameCB get_model_frame_cb_;
   const double tolerance_;
+  std::map<size_t, double> psnr_;
 };
 
 // Validate by computing SSIM from the frame to be validated and the model frame
@@ -214,6 +216,7 @@ class SSIMVideoFrameValidator : public VideoFrameValidator {
       const GetModelFrameCB& get_model_frame_cb,
       double tolerance = kDefaultTolerance,
       std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor = nullptr);
+  const std::map<size_t, double>& GetSSIMValues() { return ssim_; }
   ~SSIMVideoFrameValidator() override;
 
  private:
@@ -230,6 +233,7 @@ class SSIMVideoFrameValidator : public VideoFrameValidator {
 
   const GetModelFrameCB get_model_frame_cb_;
   const double tolerance_;
+  std::map<size_t, double> ssim_;
 };
 }  // namespace test
 }  // namespace media

media/gpu/test/video_test_helpers.cc

@@ -6,10 +6,12 @@
 
 #include <limits>
 
+#include "base/memory/ptr_util.h"
 #include "base/stl_util.h"
 #include "base/sys_byteorder.h"
 #include "media/base/video_decoder_config.h"
 #include "media/base/video_frame_layout.h"
+#include "media/gpu/test/video.h"
 #include "media/video/h264_parser.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
@@ -379,5 +381,83 @@ void AlignedDataHelper::CreateAlignedInputStream(
   }
 }
 
+// static
+std::unique_ptr<RawDataHelper> RawDataHelper::Create(Video* video) {
+  size_t frame_size = 0;
+  VideoPixelFormat pixel_format = video->PixelFormat();
+  const size_t num_planes = VideoFrame::NumPlanes(pixel_format);
+  size_t strides[VideoFrame::kMaxPlanes] = {};
+  size_t plane_sizes[VideoFrame::kMaxPlanes] = {};
+  const gfx::Size& resolution = video->Resolution();
+  // Calculate size of frames and their planes.
+  for (size_t i = 0; i < num_planes; ++i) {
+    const size_t bytes_per_line =
+        VideoFrame::RowBytes(i, pixel_format, resolution.width());
+    const size_t plane_size =
+        bytes_per_line * VideoFrame::Rows(i, pixel_format, resolution.height());
+    strides[i] = bytes_per_line;
+    plane_sizes[i] = plane_size;
+    frame_size += plane_size;
+  }
+
+  // Verify whether calculated frame size is valid.
+  const size_t data_size = video->Data().size();
+  if (frame_size == 0 || data_size % frame_size != 0 ||
+      data_size / frame_size != video->NumFrames()) {
+    LOG(ERROR) << "Invalid frame_size=" << frame_size
+               << ", file size=" << data_size;
+    return nullptr;
+  }
+
+  std::vector<ColorPlaneLayout> planes(num_planes);
+  size_t offset = 0;
+  for (size_t i = 0; i < num_planes; ++i) {
+    planes[i].stride =
+        VideoFrame::RowBytes(i, pixel_format, resolution.width());
+    planes[i].offset = offset;
+    planes[i].size = plane_sizes[i];
+    offset += plane_sizes[i];
+  }
+  auto layout = VideoFrameLayout::CreateWithPlanes(pixel_format, resolution,
+                                                   std::move(planes));
+  if (!layout) {
+    LOG(ERROR) << "Failed to create VideoFrameLayout";
+    return nullptr;
+  }
+
+  return base::WrapUnique(new RawDataHelper(video, frame_size, *layout));
+}
+
+RawDataHelper::RawDataHelper(Video* video,
+                             size_t frame_size,
+                             const VideoFrameLayout& layout)
+    : video_(video), frame_size_(frame_size), layout_(layout) {}
+
+RawDataHelper::~RawDataHelper() = default;
+
+scoped_refptr<const VideoFrame> RawDataHelper::GetFrame(size_t index) {
+  if (index >= video_->NumFrames()) {
+    LOG(ERROR) << "index is too big. index=" << index
+               << ", num_frames=" << video_->NumFrames();
+    return nullptr;
+  }
+
+  size_t offset = frame_size_ * index;
+  uint8_t* frame_data[VideoFrame::kMaxPlanes] = {};
+  const size_t num_planes = VideoFrame::NumPlanes(video_->PixelFormat());
+  for (size_t i = 0; i < num_planes; ++i) {
+    frame_data[i] = reinterpret_cast<uint8_t*>(video_->Data().data()) + offset;
+    offset += layout_->planes()[i].size;
+  }
+  // TODO(crbug.com/1045825): Investigate use of MOJO_SHARED_BUFFER, similar to
+  // changes made in crrev.com/c/2050895.
+  scoped_refptr<const VideoFrame> video_frame =
+      VideoFrame::WrapExternalYuvDataWithLayout(
+          *layout_, gfx::Rect(video_->Resolution()), video_->Resolution(),
+          frame_data[0], frame_data[1], frame_data[2],
+          base::TimeTicks::Now().since_origin());
+  return video_frame;
+}
+
 }  // namespace test
 }  // namespace media

media/gpu/test/video_test_helpers.h

@@ -5,6 +5,7 @@
 #ifndef MEDIA_GPU_TEST_VIDEO_TEST_HELPERS_H_
 #define MEDIA_GPU_TEST_VIDEO_TEST_HELPERS_H_
 
+#include <memory>
 #include <string>
 #include <vector>
 
@@ -19,12 +20,14 @@
 #include "media/base/decoder_buffer.h"
 #include "media/base/video_codecs.h"
 #include "media/base/video_frame.h"
+#include "media/base/video_frame_layout.h"
 #include "media/base/video_types.h"
 #include "ui/gfx/geometry/rect.h"
 #include "ui/gfx/geometry/size.h"
 
 namespace media {
 namespace test {
+class Video;
 
 // Helper class allowing one thread to wait on a notification from another.
 // If notifications come in faster than they are Wait()'d for, they are
@@ -206,6 +209,32 @@ class AlignedDataHelper {
   std::vector<size_t> aligned_plane_size_;
 };
 
+// Small helper class to extract video frames from raw data streams.
+// However, the data wrapped by VideoFrame is not guaranteed to be aligned.
+// This class doesn't change |video|, but cannot be mark it as constant because
+// GetFrame() returns non const |data_| wrapped by the returned VideoFrame.
+class RawDataHelper {
+ public:
+  static std::unique_ptr<RawDataHelper> Create(Video* video);
+  ~RawDataHelper();
+
+  // Returns i-th VideoFrame in |video|. The returned frame doesn't own the
+  // underlying video data.
+  scoped_refptr<const VideoFrame> GetFrame(size_t index);
+
+ private:
+  RawDataHelper(Video* video,
+                size_t frame_size,
+                const VideoFrameLayout& layout);
+  // |video| and its associated data must outlive this class and VideoFrames
+  // returned by GetFrame().
+  Video* const video_;
+
+  // The size of one video frame.
+  const size_t frame_size_;
+  // The layout of VideoFrames returned by GetFrame().
+  const base::Optional<VideoFrameLayout> layout_;
+};
 }  // namespace test
 }  // namespace media
 

media/gpu/video_encode_accelerator_tests.cc

@@ -15,6 +15,9 @@
 #include "media/gpu/test/video_encoder/video_encoder.h"
 #include "media/gpu/test/video_encoder/video_encoder_client.h"
 #include "media/gpu/test/video_encoder/video_encoder_test_environment.h"
+#include "media/gpu/test/video_frame_helpers.h"
+#include "media/gpu/test/video_frame_validator.h"
+#include "media/gpu/test/video_test_helpers.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 namespace media {
@@ -58,7 +61,7 @@ media::test::VideoEncoderTestEnvironment* g_env;
 class VideoEncoderTest : public ::testing::Test {
  public:
   std::unique_ptr<VideoEncoder> CreateVideoEncoder(
-      const Video* video,
+      Video* video,
       VideoEncoderClientConfig config = VideoEncoderClientConfig()) {
     LOG_ASSERT(video);
 
@@ -91,19 +94,41 @@ class VideoEncoderTest : public ::testing::Test {
         codec, config.output_profile, VideoDecoderConfig::AlphaMode::kIsOpaque,
         VideoColorSpace(), kNoTransformation, visible_rect.size(), visible_rect,
         visible_rect.size(), EmptyExtraData(), EncryptionScheme::kUnencrypted);
+    std::vector<std::unique_ptr<VideoFrameProcessor>> video_frame_processors;
+    // TODO(hiroh): Add corrupt frame processors.
+    VideoFrameValidator::GetModelFrameCB get_model_frame_cb =
+        base::BindRepeating(&VideoEncoderTest::GetModelFrame,
+                            base::Unretained(this));
+    auto psnr_validator = PSNRVideoFrameValidator::Create(get_model_frame_cb);
+    auto ssim_validator = SSIMVideoFrameValidator::Create(get_model_frame_cb);
+    video_frame_processors.push_back(std::move(psnr_validator));
+    video_frame_processors.push_back(std::move(ssim_validator));
     auto bitstream_validator =
-        BitstreamValidator::Create(decoder_config, video->NumFrames() - 1);
+        BitstreamValidator::Create(decoder_config, video->NumFrames() - 1,
+                                   std::move(video_frame_processors));
     LOG_ASSERT(bitstream_validator);
     bitstream_processors.emplace_back(std::move(bitstream_validator));
 
     auto video_encoder =
         VideoEncoder::Create(config, std::move(bitstream_processors));
-
     LOG_ASSERT(video_encoder);
     LOG_ASSERT(video_encoder->Initialize(video));
 
+    raw_data_helper_ = RawDataHelper::Create(video);
+    if (!raw_data_helper_) {
+      LOG(ERROR) << "Failed to create ";
+      return nullptr;
+    }
     return video_encoder;
   }
+
+ private:
+  scoped_refptr<const VideoFrame> GetModelFrame(size_t frame_index) {
+    LOG_ASSERT(raw_data_helper_);
+    return raw_data_helper_->GetFrame(frame_index);
+  }
+
+  std::unique_ptr<RawDataHelper> raw_data_helper_;
 };
 
 }  // namespace