media/gpu/test/VFValidator: Add validation by comparing each byte value

VideoFrameValidator validates frames by comparing with expected md5sum. The
validation is appropriate for deterministic image/video processing like video
decoding and format conversion among YUVs. There are various non-deterministic
image/video processing like frame scaling. So that VideoFrameValidator can
verify resulted frames in such cases, this enables VideoFrameValidator to
validate by comparing each byte of a given buffer with one of an expected
buffer.

Bug: 917951
Test: image_processor_test on atlas (pass) and eve (fail due to b/141724493)
Change-Id: I19dac5df329d9c838edc184dedf2582cd72beead
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1866114
Commit-Queue: Hirokazu Honda <hiroh@chromium.org>
Reviewed-by: David Staessens <dstaessens@chromium.org>
Cr-Commit-Position: refs/heads/master@{#710226}

media/gpu/chromeos/image_processor_test.cc

@@ -83,37 +83,51 @@ class ImageProcessorParamTest
   std::unique_ptr<test::ImageProcessorClient> CreateImageProcessorClient(
       const test::Image& input_image,
       const std::vector<VideoFrame::StorageType>& input_storage_types,
-      const test::Image& output_image,
+      test::Image* const output_image,
       const std::vector<VideoFrame::StorageType>& output_storage_types) {
     Fourcc input_fourcc =
         Fourcc::FromVideoPixelFormat(input_image.PixelFormat());
     Fourcc output_fourcc =
-        Fourcc::FromVideoPixelFormat(output_image.PixelFormat());
+        Fourcc::FromVideoPixelFormat(output_image->PixelFormat());
     auto input_layout = test::CreateVideoFrameLayout(input_image.PixelFormat(),
                                                      input_image.Size());
     auto output_layout = test::CreateVideoFrameLayout(
-        output_image.PixelFormat(), output_image.Size());
+        output_image->PixelFormat(), output_image->Size());
     LOG_ASSERT(input_layout && output_layout);
     ImageProcessor::PortConfig input_config(
         input_fourcc, input_image.Size(), input_layout->planes(),
         input_image.Size(), input_storage_types);
     ImageProcessor::PortConfig output_config(
-        output_fourcc, output_image.Size(), output_layout->planes(),
-        output_image.Size(), output_storage_types);
+        output_fourcc, output_image->Size(), output_layout->planes(),
+        output_image->Size(), output_storage_types);
     // TODO(crbug.com/917951): Select more appropriate number of buffers.
     constexpr size_t kNumBuffers = 1;
-    LOG_ASSERT(output_image.IsMetadataLoaded());
+    LOG_ASSERT(output_image->IsMetadataLoaded());
     std::vector<std::unique_ptr<test::VideoFrameProcessor>> frame_processors;
     // TODO(crbug.com/944823): Use VideoFrameValidator for RGB formats.
     // TODO(crbug.com/917951): We should validate a scaled image with SSIM.
     // Validating processed frames is currently not supported when a format is
     // not YUV or when scaling images.
     if (IsYuvPlanar(input_fourcc.ToVideoPixelFormat()) &&
-        IsYuvPlanar(output_fourcc.ToVideoPixelFormat()) &&
-        input_image.Size() == output_image.Size()) {
-      auto vf_validator = test::VideoFrameValidator::Create(
-          {output_image.Checksum()}, output_image.PixelFormat());
-      frame_processors.push_back(std::move(vf_validator));
+        IsYuvPlanar(output_fourcc.ToVideoPixelFormat())) {
+      if (input_image.Size() == output_image->Size()) {
+        auto vf_validator = test::VideoFrameValidator::Create(
+            {output_image->Checksum()}, output_image->PixelFormat());
+        frame_processors.push_back(std::move(vf_validator));
+      } else if (input_fourcc == output_fourcc) {
+        // Scaling case.
+        LOG_ASSERT(output_image->Load());
+        scoped_refptr<const VideoFrame> model_frame =
+            CreateVideoFrameFromImage(*output_image);
+        LOG_ASSERT(model_frame) << "Failed to create from image";
+        // Scaling is not deterministic process. There are various algorithms to
+        // scale images. We set a weaker tolerance value, 32, to avoid false
+        // negative.
+        constexpr uint32_t kImageProcessorTestTorelance = 32;
+        auto vf_validator = test::VideoFrameValidator::Create(
+            {model_frame}, kImageProcessorTestTorelance);
+        frame_processors.push_back(std::move(vf_validator));
+      }
     }
 
     if (g_save_images) {
@@ -151,7 +165,7 @@ TEST_P(ImageProcessorParamTest, ConvertOneTime_MemToMem) {
   }
 
   auto ip_client = CreateImageProcessorClient(
-      input_image, {VideoFrame::STORAGE_OWNED_MEMORY}, output_image,
+      input_image, {VideoFrame::STORAGE_OWNED_MEMORY}, &output_image,
       {VideoFrame::STORAGE_OWNED_MEMORY});
 
   ip_client->Process(input_image, output_image);
@@ -181,7 +195,7 @@ TEST_P(ImageProcessorParamTest, ConvertOneTime_DmabufToMem) {
   }
 
   auto ip_client = CreateImageProcessorClient(
-      input_image, {VideoFrame::STORAGE_DMABUFS}, output_image,
+      input_image, {VideoFrame::STORAGE_DMABUFS}, &output_image,
       {VideoFrame::STORAGE_OWNED_MEMORY});
 
   ip_client->Process(input_image, output_image);
@@ -202,7 +216,7 @@ TEST_P(ImageProcessorParamTest, ConvertOneTime_DmabufToDmabuf) {
 
   auto ip_client =
       CreateImageProcessorClient(input_image, {VideoFrame::STORAGE_DMABUFS},
-                                 output_image, {VideoFrame::STORAGE_DMABUFS});
+                                 &output_image, {VideoFrame::STORAGE_DMABUFS});
 
   ip_client->Process(input_image, output_image);
 

media/gpu/test/video_frame_helpers.cc

@@ -344,5 +344,37 @@ base::Optional<VideoFrameLayout> CreateVideoFrameLayout(VideoPixelFormat format,
   return VideoFrameLayout::CreateWithPlanes(format, size, std::move(planes));
 }
 
+size_t CompareFramesWithErrorDiff(const VideoFrame& frame1,
+                                  const VideoFrame& frame2,
+                                  uint8_t tolerance) {
+  LOG_ASSERT(frame1.format() == frame2.format());
+  LOG_ASSERT(frame1.visible_rect() == frame2.visible_rect());
+  LOG_ASSERT(frame1.visible_rect().origin() == gfx::Point(0, 0));
+  LOG_ASSERT(frame1.IsMappable() && frame2.IsMappable());
+
+  size_t diff_cnt = 0;
+
+  const VideoPixelFormat format = frame1.format();
+  const size_t num_planes = VideoFrame::NumPlanes(format);
+  const gfx::Size& visible_size = frame1.visible_rect().size();
+  for (size_t i = 0; i < num_planes; ++i) {
+    const uint8_t* data1 = frame1.data(i);
+    const int stride1 = frame1.stride(i);
+    const uint8_t* data2 = frame2.data(i);
+    const int stride2 = frame2.stride(i);
+    const size_t rows = VideoFrame::Rows(i, format, visible_size.height());
+    const int row_bytes = VideoFrame::RowBytes(i, format, visible_size.width());
+    for (size_t r = 0; r < rows; ++r) {
+      for (int c = 0; c < row_bytes; c++) {
+        uint8_t b1 = data1[(stride1 * r) + c];
+        uint8_t b2 = data2[(stride2 * r) + c];
+        uint8_t diff = std::max(b1, b2) - std::min(b1, b2);
+        diff_cnt += diff > tolerance;
+      }
+    }
+  }
+  return diff_cnt;
+}
+
 }  // namespace test
 }  // namespace media

media/gpu/test/video_frame_helpers.h

@@ -92,6 +92,13 @@ base::Optional<VideoFrameLayout> CreateVideoFrameLayout(
     VideoPixelFormat pixel_format,
     const gfx::Size& size);
 
+// Compare each byte of two VideoFrames, |frame1| and |frame2|, allowing the
+// error up to |tolerance|. Return number of bytes a difference of which is more
+// than |tolerance|.
+size_t CompareFramesWithErrorDiff(const VideoFrame& frame1,
+                                  const VideoFrame& frame2,
+                                  uint8_t tolerance);
+
 }  // namespace test
 }  // namespace media
 

media/gpu/test/video_frame_validator.cc

@@ -20,6 +20,24 @@
 namespace media {
 namespace test {
 
+VideoFrameValidator::MismatchedFrameInfo::MismatchedFrameInfo(
+    size_t frame_index,
+    std::string computed_md5,
+    std::string expected_md5)
+    : validate_mode(ValidateMode::MD5),
+      frame_index(frame_index),
+      computed_md5(std::move(computed_md5)),
+      expected_md5(std::move(expected_md5)) {}
+
+VideoFrameValidator::MismatchedFrameInfo::MismatchedFrameInfo(
+    size_t frame_index,
+    size_t diff_cnt)
+    : validate_mode(ValidateMode::RAW),
+      frame_index(frame_index),
+      diff_cnt(diff_cnt) {}
+
+VideoFrameValidator::MismatchedFrameInfo::~MismatchedFrameInfo() = default;
+
 // static
 std::unique_ptr<VideoFrameValidator> VideoFrameValidator::Create(
     const std::vector<std::string>& expected_frame_checksums,
@@ -36,11 +54,26 @@ std::unique_ptr<VideoFrameValidator> VideoFrameValidator::Create(
   return video_frame_validator;
 }
 
+std::unique_ptr<VideoFrameValidator> VideoFrameValidator::Create(
+    const std::vector<scoped_refptr<const VideoFrame>> model_frames,
+    const uint8_t tolerance,
+    std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor) {
+  auto video_frame_validator = base::WrapUnique(new VideoFrameValidator(
+      std::move(model_frames), tolerance, std::move(corrupt_frame_processor)));
+  if (!video_frame_validator->Initialize()) {
+    LOG(ERROR) << "Failed to initialize VideoFrameValidator.";
+    return nullptr;
+  }
+
+  return video_frame_validator;
+}
+
 VideoFrameValidator::VideoFrameValidator(
     std::vector<std::string> expected_frame_checksums,
     VideoPixelFormat validation_format,
     std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor)
-    : expected_frame_checksums_(std::move(expected_frame_checksums)),
+    : validate_mode_(ValidateMode::MD5),
+      expected_frame_checksums_(std::move(expected_frame_checksums)),
       validation_format_(validation_format),
       corrupt_frame_processor_(std::move(corrupt_frame_processor)),
       num_frames_validating_(0),
@@ -50,6 +83,21 @@ VideoFrameValidator::VideoFrameValidator(
   DETACH_FROM_SEQUENCE(validator_thread_sequence_checker_);
 }
 
+VideoFrameValidator::VideoFrameValidator(
+    const std::vector<scoped_refptr<const VideoFrame>> model_frames,
+    const uint8_t tolerance,
+    std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor)
+    : validate_mode_(ValidateMode::RAW),
+      model_frames_(std::move(model_frames)),
+      tolerance_(tolerance),
+      corrupt_frame_processor_(std::move(corrupt_frame_processor)),
+      num_frames_validating_(0),
+      frame_validator_thread_("FrameValidatorThread"),
+      frame_validator_cv_(&frame_validator_lock_) {
+  DETACH_FROM_SEQUENCE(validator_sequence_checker_);
+  DETACH_FROM_SEQUENCE(validator_thread_sequence_checker_);
+}
+
 VideoFrameValidator::~VideoFrameValidator() {
   Destroy();
 }
@@ -140,27 +188,30 @@ void VideoFrameValidator::ProcessVideoFrameTask(
 #endif  // BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
 
   LOG_ASSERT(validated_frame->IsMappable());
-  if (validated_frame->format() != validation_format_) {
-    validated_frame =
-        ConvertVideoFrame(validated_frame.get(), validation_format_);
-  }
 
-  ASSERT_TRUE(validated_frame);
-  std::string computed_md5 = ComputeMD5FromVideoFrame(validated_frame.get());
+  base::Optional<MismatchedFrameInfo> mismatched_info;
+  switch (validate_mode_) {
+    case ValidateMode::MD5: {
+      if (validated_frame->format() != validation_format_) {
+        validated_frame =
+            ConvertVideoFrame(validated_frame.get(), validation_format_);
+      }
+      ASSERT_TRUE(validated_frame);
+      mismatched_info = ValidateMD5(*validated_frame, frame_index);
+      break;
+    }
+    case ValidateMode::RAW:
+      mismatched_info = ValidateRaw(*validated_frame, frame_index);
+      break;
+  }
 
   base::AutoLock auto_lock(frame_validator_lock_);
 
-  if (expected_frame_checksums_.size() > 0) {
-    LOG_IF(FATAL, frame_index >= expected_frame_checksums_.size())
-        << "Frame number is over than the number of read md5 values in file.";
-    const auto& expected_md5 = expected_frame_checksums_[frame_index];
-    if (computed_md5 != expected_md5) {
-      mismatched_frames_.push_back(
-          MismatchedFrameInfo{frame_index, computed_md5, expected_md5});
-      // Perform additional processing on the corrupt video frame if requested.
-      if (corrupt_frame_processor_)
-        corrupt_frame_processor_->ProcessVideoFrame(video_frame, frame_index);
-    }
+  if (mismatched_info) {
+    mismatched_frames_.push_back(std::move(mismatched_info).value());
+    // Perform additional processing on the corrupt video frame if requested.
+    if (corrupt_frame_processor_)
+      corrupt_frame_processor_->ProcessVideoFrame(validated_frame, frame_index);
   }
 
   num_frames_validating_--;
@@ -168,15 +219,44 @@ void VideoFrameValidator::ProcessVideoFrameTask(
 }
 
 std::string VideoFrameValidator::ComputeMD5FromVideoFrame(
-    const VideoFrame* video_frame) const {
+    const VideoFrame& video_frame) const {
   DCHECK_CALLED_ON_VALID_SEQUENCE(validator_thread_sequence_checker_);
   base::MD5Context context;
   base::MD5Init(&context);
-  VideoFrame::HashFrameForTesting(&context, *video_frame);
+  VideoFrame::HashFrameForTesting(&context, video_frame);
   base::MD5Digest digest;
   base::MD5Final(&digest, &context);
   return MD5DigestToBase16(digest);
 }
 
+base::Optional<VideoFrameValidator::MismatchedFrameInfo>
+VideoFrameValidator::ValidateMD5(const VideoFrame& validated_frame,
+                                 size_t frame_index) {
+  std::string computed_md5 = ComputeMD5FromVideoFrame(validated_frame);
+
+  if (expected_frame_checksums_.size() > 0) {
+    LOG_IF(FATAL, frame_index >= expected_frame_checksums_.size())
+        << "Frame number is over than the number of read md5 values in file.";
+    const auto& expected_md5 = expected_frame_checksums_[frame_index];
+    if (computed_md5 != expected_md5) {
+      return MismatchedFrameInfo{frame_index, computed_md5, expected_md5};
+    }
+  }
+  return base::nullopt;
+}
+
+base::Optional<VideoFrameValidator::MismatchedFrameInfo>
+VideoFrameValidator::ValidateRaw(const VideoFrame& validated_frame,
+                                 size_t frame_index) {
+  if (model_frames_.size() > 0) {
+    LOG_IF(FATAL, frame_index >= model_frames_.size())
+        << "Frame number is over than the number of given frames.";
+    size_t diff_cnt = CompareFramesWithErrorDiff(
+        validated_frame, *model_frames_[frame_index], tolerance_);
+    if (diff_cnt > 0)
+      return MismatchedFrameInfo{frame_index, diff_cnt};
+  }
+  return base::nullopt;
+}
 }  // namespace test
 }  // namespace media

media/gpu/test/video_frame_validator.h

@@ -38,10 +38,32 @@ namespace test {
 // performance measurements.
 class VideoFrameValidator : public VideoFrameProcessor {
  public:
+  static constexpr uint8_t kDefaultTolerance = 4;
+  // TODO(hiroh): Support a validation by PSNR and SSIM.
+  enum class ValidateMode {
+    MD5,  // Check if md5sum value of coded area of a given VideoFrame on
+          // ProcessVideoFrame() matches the expected md5sum.
+    RAW,  // Compare each byte of visible area of a given VideoFrame on
+          // ProcessVideoFrame() with a expected VideoFrame. An absolute
+          // difference equal to or less than |tolerance_| is allowed on
+          // comparison.
+  };
+
   struct MismatchedFrameInfo {
-    size_t frame_index;
+    MismatchedFrameInfo(size_t frame_index,
+                        std::string computed_md5,
+                        std::string expected_md5);
+    MismatchedFrameInfo(size_t frame_index, size_t diff_cnt);
+    ~MismatchedFrameInfo();
+
+    ValidateMode validate_mode;
+    size_t frame_index = 0;
+
+    // variables for ValidateMode::MD5 mode.
     std::string computed_md5;
     std::string expected_md5;
+    // variables for ValidateMode::ERRORDIFF mode.
+    size_t diff_cnt = 0;
   };
 
   // Create an instance of the video frame validator. The calculated checksums
@@ -57,6 +79,11 @@ class VideoFrameValidator : public VideoFrameProcessor {
       const VideoPixelFormat validation_format = PIXEL_FORMAT_I420,
       std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor = nullptr);
 
+  static std::unique_ptr<VideoFrameValidator> Create(
+      const std::vector<scoped_refptr<const VideoFrame>> model_frames,
+      const uint8_t tolerance = kDefaultTolerance,
+      std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor = nullptr);
+
   ~VideoFrameValidator() override;
 
   // Returns information of frames that don't match golden md5 values.
@@ -82,6 +109,11 @@ class VideoFrameValidator : public VideoFrameProcessor {
       VideoPixelFormat validation_format,
       std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor);
 
+  VideoFrameValidator(
+      const std::vector<scoped_refptr<const VideoFrame>> model_frames,
+      const uint8_t tolerance,
+      std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor);
+
   // Start the frame validation thread.
   bool Initialize();
   // Stop the frame validation thread.
@@ -92,19 +124,29 @@ class VideoFrameValidator : public VideoFrameProcessor {
                              size_t frame_index);
 
   // Returns md5 values of video frame represented by |video_frame|.
-  std::string ComputeMD5FromVideoFrame(const VideoFrame* video_frame) const;
+  std::string ComputeMD5FromVideoFrame(const VideoFrame& video_frame) const;
 
-  // The results of invalid frame data.
-  std::vector<MismatchedFrameInfo> mismatched_frames_
-      GUARDED_BY(frame_validator_lock_);
+  base::Optional<MismatchedFrameInfo> ValidateMD5(
+      const VideoFrame& validated_frame,
+      size_t frame_index);
+  base::Optional<MismatchedFrameInfo> ValidateRaw(
+      const VideoFrame& validated_frame,
+      size_t frame_index);
+
+  const ValidateMode validate_mode_;
 
+  // Values used only if |validate_mode_| is MD5.
   // The list of expected MD5 frame checksums.
   const std::vector<std::string> expected_frame_checksums_;
+  // VideoPixelFormat the VideoFrame will be converted to for validation.
+  const VideoPixelFormat validation_format_ = PIXEL_FORMAT_UNKNOWN;
 
-  std::unique_ptr<VideoFrameMapper> video_frame_mapper_;
+  // Values used only if |validate_mode_| is RAW.
+  // The list of expected frames
+  const std::vector<scoped_refptr<const VideoFrame>> model_frames_;
+  const uint8_t tolerance_ = 0;
 
-  // VideoPixelFormat the VideoFrame will be converted to for validation.
-  const VideoPixelFormat validation_format_;
+  std::unique_ptr<VideoFrameMapper> video_frame_mapper_;
 
   // An optional video frame processor that all corrupted frames will be
   // forwarded to. This can be used to e.g. write corrupted frames to disk.
@@ -112,6 +154,9 @@ class VideoFrameValidator : public VideoFrameProcessor {
 
   // The number of frames currently queued for validation.
   size_t num_frames_validating_ GUARDED_BY(frame_validator_lock_);
+  // The results of invalid frame data.
+  std::vector<MismatchedFrameInfo> mismatched_frames_
+      GUARDED_BY(frame_validator_lock_);
 
   // Thread on which video frame validation is done.
   base::Thread frame_validator_thread_;

media/gpu/vaapi/accelerated_video_encoder.h

@@ -10,7 +10,7 @@
 #include "base/callback.h"
 #include "base/containers/queue.h"
 #include "base/macros.h"
-#include "base/memory/ref_counted.h"
+#include "base/memory/scoped_refptr.h"
 #include "base/time/time.h"
 #include "media/base/video_bitrate_allocation.h"
 #include "media/base/video_codecs.h"