media/gpu/{SSIM/PSNR}VideoFrameValidator: Add an option to evaluate with the average

Encoded bitstreams can contain frames that are worse than the
specified tolerance, while the quality of the overall stream is
good. To evaluate with the overall quality, this CL adds an option
to PSNR and SSIMVideoFrameValidator that the validation fails
when the average of the values are worse than their tolerances.

Bug: 1045825
Test: video_encode_accelerator_tests on rammus
Change-Id: I3d68616081d00a164d0495ed9b295bc6afa0e235
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2359876Reviewed-by: David Staessens <dstaessens@chromium.org>
Commit-Queue: Hirokazu Honda <hiroh@chromium.org>
Cr-Commit-Position: refs/heads/master@{#799946}

media/gpu/test/video_frame_validator.cc

@@ -52,6 +52,12 @@ void VideoFrameValidator::Destroy() {
   DCHECK_EQ(0u, num_frames_validating_);
 }
 
+void VideoFrameValidator::PrintMismatchedFramesInfo() const {
+  base::AutoLock auto_lock(frame_validator_lock_);
+  for (const auto& mismatched_frame_info : mismatched_frames_)
+    mismatched_frame_info->Print();
+}
+
 size_t VideoFrameValidator::GetMismatchedFramesCount() const {
   base::AutoLock auto_lock(frame_validator_lock_);
   return mismatched_frames_.size();
@@ -86,21 +92,28 @@ void VideoFrameValidator::ProcessVideoFrame(
 }
 
 bool VideoFrameValidator::WaitUntilDone() {
-  base::AutoLock auto_lock(frame_validator_lock_);
-  while (num_frames_validating_ > 0) {
-    frame_validator_cv_.Wait();
-  }
+  {
+    base::AutoLock auto_lock(frame_validator_lock_);
+    while (num_frames_validating_ > 0) {
+      frame_validator_cv_.Wait();
+    }
 
-  if (corrupt_frame_processor_ && !corrupt_frame_processor_->WaitUntilDone())
-    return false;
+    if (corrupt_frame_processor_ && !corrupt_frame_processor_->WaitUntilDone())
+      return false;
+  }
 
-  if (mismatched_frames_.size() > 0u) {
-    LOG(ERROR) << mismatched_frames_.size() << " frames failed to validate.";
+  if (!Passed()) {
+    LOG(ERROR) << GetMismatchedFramesCount() << " frames failed to validate.";
+    PrintMismatchedFramesInfo();
     return false;
   }
   return true;
 }
 
+bool VideoFrameValidator::Passed() const {
+  return GetMismatchedFramesCount() == 0u;
+}
+
 void VideoFrameValidator::ProcessVideoFrameTask(
     const scoped_refptr<const VideoFrame> video_frame,
     size_t frame_index) {
@@ -328,9 +341,11 @@ struct PSNRVideoFrameValidator::PSNRMismatchedFrameInfo
 std::unique_ptr<PSNRVideoFrameValidator> PSNRVideoFrameValidator::Create(
     const GetModelFrameCB& get_model_frame_cb,
     std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor,
+    ValidationMode validation_mode,
     double tolerance) {
   auto video_frame_validator = base::WrapUnique(new PSNRVideoFrameValidator(
-      get_model_frame_cb, std::move(corrupt_frame_processor), tolerance));
+      get_model_frame_cb, std::move(corrupt_frame_processor), validation_mode,
+      tolerance));
   if (!video_frame_validator->Initialize()) {
     LOG(ERROR) << "Failed to initialize PSNRVideoFrameValidator.";
     return nullptr;
@@ -342,10 +357,12 @@ std::unique_ptr<PSNRVideoFrameValidator> PSNRVideoFrameValidator::Create(
 PSNRVideoFrameValidator::PSNRVideoFrameValidator(
     const GetModelFrameCB& get_model_frame_cb,
     std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor,
+    ValidationMode validation_mode,
     double tolerance)
     : VideoFrameValidator(std::move(corrupt_frame_processor)),
       get_model_frame_cb_(get_model_frame_cb),
-      tolerance_(tolerance) {}
+      tolerance_(tolerance),
+      validation_mode_(validation_mode) {}
 
 PSNRVideoFrameValidator::~PSNRVideoFrameValidator() = default;
 
@@ -363,6 +380,24 @@ PSNRVideoFrameValidator::Validate(scoped_refptr<const VideoFrame> frame,
   return nullptr;
 }
 
+bool PSNRVideoFrameValidator::Passed() const {
+  if (validation_mode_ == ValidationMode::kThreshold)
+    return GetMismatchedFramesCount() == 0u;
+  if (psnr_.empty())
+    return true;
+
+  double average = 0;
+  for (const auto& psnr : psnr_) {
+    average += psnr.second;
+  }
+  average /= psnr_.size();
+  if (average < tolerance_) {
+    LOG(ERROR) << "Average PSNR is too low: " << average;
+    return false;
+  }
+  return true;
+}
+
 struct SSIMVideoFrameValidator::SSIMMismatchedFrameInfo
     : public VideoFrameValidator::MismatchedFrameInfo {
   SSIMMismatchedFrameInfo(size_t frame_index, double ssim)
@@ -379,9 +414,11 @@ struct SSIMVideoFrameValidator::SSIMMismatchedFrameInfo
 std::unique_ptr<SSIMVideoFrameValidator> SSIMVideoFrameValidator::Create(
     const GetModelFrameCB& get_model_frame_cb,
     std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor,
+    ValidationMode validation_mode,
     double tolerance) {
   auto video_frame_validator = base::WrapUnique(new SSIMVideoFrameValidator(
-      get_model_frame_cb, std::move(corrupt_frame_processor), tolerance));
+      get_model_frame_cb, std::move(corrupt_frame_processor), validation_mode,
+      tolerance));
   if (!video_frame_validator->Initialize()) {
     LOG(ERROR) << "Failed to initialize SSIMVideoFrameValidator.";
     return nullptr;
@@ -393,10 +430,12 @@ std::unique_ptr<SSIMVideoFrameValidator> SSIMVideoFrameValidator::Create(
 SSIMVideoFrameValidator::SSIMVideoFrameValidator(
     const GetModelFrameCB& get_model_frame_cb,
     std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor,
+    ValidationMode validation_mode,
     double tolerance)
     : VideoFrameValidator(std::move(corrupt_frame_processor)),
       get_model_frame_cb_(get_model_frame_cb),
-      tolerance_(tolerance) {}
+      tolerance_(tolerance),
+      validation_mode_(validation_mode) {}
 
 SSIMVideoFrameValidator::~SSIMVideoFrameValidator() = default;
 
@@ -413,5 +452,23 @@ SSIMVideoFrameValidator::Validate(scoped_refptr<const VideoFrame> frame,
     return std::make_unique<SSIMMismatchedFrameInfo>(frame_index, ssim);
   return nullptr;
 }
+
+bool SSIMVideoFrameValidator::Passed() const {
+  if (validation_mode_ == ValidationMode::kThreshold)
+    return GetMismatchedFramesCount() == 0u;
+  if (ssim_.empty())
+    return true;
+
+  double average = 0;
+  for (const auto& ssim : ssim_) {
+    average += ssim.second;
+  }
+  average /= ssim_.size();
+  if (average < tolerance_) {
+    LOG(ERROR) << "Average SSIM is too low: " << average;
+    return false;
+  }
+  return true;
+}
 }  // namespace test
 }  // namespace media

media/gpu/test/video_frame_validator.h

@@ -38,6 +38,12 @@ namespace test {
 // performance measurements.
 class VideoFrameValidator : public VideoFrameProcessor {
  public:
+  // This mode can be specified for PSNR and SSIM VideoFrameValidator.
+  enum class ValidationMode {
+    kThreshold,  // Each frame's quality need to pass the specified threshold.
+    kAverage,    // The average quality needs to pass the specified value.
+  };
+
   // Get the model frame from |frame_index|.
   using GetModelFrameCB =
       base::RepeatingCallback<scoped_refptr<const VideoFrame>(size_t)>;
@@ -88,6 +94,9 @@ class VideoFrameValidator : public VideoFrameProcessor {
       scoped_refptr<const VideoFrame> frame,
       size_t frame_index) = 0;
 
+  // Returns whether the overall validation passed.
+  virtual bool Passed() const;
+
   std::unique_ptr<VideoFrameMapper> video_frame_mapper_;
 
   // An optional video frame processor that all corrupted frames will be
@@ -184,6 +193,7 @@ class PSNRVideoFrameValidator : public VideoFrameValidator {
   static std::unique_ptr<PSNRVideoFrameValidator> Create(
       const GetModelFrameCB& get_model_frame_cb,
       std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor = nullptr,
+      ValidationMode validation_mode = ValidationMode::kThreshold,
       double tolerance = kDefaultTolerance);
   const std::map<size_t, double>& GetPSNRValues() { return psnr_; }
   ~PSNRVideoFrameValidator() override;
@@ -194,14 +204,18 @@ class PSNRVideoFrameValidator : public VideoFrameValidator {
   PSNRVideoFrameValidator(
       const GetModelFrameCB& get_model_frame_cb,
       std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor,
+      ValidationMode validation_mode,
       double tolerance);
 
   std::unique_ptr<MismatchedFrameInfo> Validate(
       scoped_refptr<const VideoFrame> frame,
       size_t frame_index) override;
 
+  bool Passed() const override;
+
   const GetModelFrameCB get_model_frame_cb_;
   const double tolerance_;
+  const ValidationMode validation_mode_;
   std::map<size_t, double> psnr_;
 };
 
@@ -215,6 +229,7 @@ class SSIMVideoFrameValidator : public VideoFrameValidator {
   static std::unique_ptr<SSIMVideoFrameValidator> Create(
       const GetModelFrameCB& get_model_frame_cb,
       std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor = nullptr,
+      ValidationMode validation_mode = ValidationMode::kThreshold,
       double tolerance = kDefaultTolerance);
   const std::map<size_t, double>& GetSSIMValues() { return ssim_; }
   ~SSIMVideoFrameValidator() override;
@@ -225,14 +240,18 @@ class SSIMVideoFrameValidator : public VideoFrameValidator {
   SSIMVideoFrameValidator(
       const GetModelFrameCB& get_model_frame_cb,
       std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor,
+      ValidationMode validation_mode,
       double tolerance);
 
   std::unique_ptr<MismatchedFrameInfo> Validate(
       scoped_refptr<const VideoFrame> frame,
       size_t frame_index) override;
 
+  bool Passed() const override;
+
   const GetModelFrameCB get_model_frame_cb_;
   const double tolerance_;
+  const ValidationMode validation_mode_;
   std::map<size_t, double> ssim_;
 };
 }  // namespace test