Commit e9781080 authored by Hirokazu Honda's avatar Hirokazu Honda Committed by Commit Bot

media/gpu/test/VFValidator: Add validation by comparing each byte value

VideoFrameValidator validates frames by comparing with expected md5sum. The
validation is appropriate for deterministic image/video processing like video
decoding and format conversion among YUVs. There are various non-deterministic
image/video processing like frame scaling. So that VideoFrameValidator can
verify resulted frames in such cases, this enables VideoFrameValidator to
validate by comparing each byte of a given buffer with one of an expected
buffer.

Bug: 917951
Test: image_processor_test on atlas (pass) and eve (fail due to b/141724493)
Change-Id: I19dac5df329d9c838edc184dedf2582cd72beead
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1866114
Commit-Queue: Hirokazu Honda <hiroh@chromium.org>
Reviewed-by: default avatarDavid Staessens <dstaessens@chromium.org>
Cr-Commit-Position: refs/heads/master@{#710226}
parent d852952b
......@@ -83,37 +83,51 @@ class ImageProcessorParamTest
std::unique_ptr<test::ImageProcessorClient> CreateImageProcessorClient(
const test::Image& input_image,
const std::vector<VideoFrame::StorageType>& input_storage_types,
const test::Image& output_image,
test::Image* const output_image,
const std::vector<VideoFrame::StorageType>& output_storage_types) {
Fourcc input_fourcc =
Fourcc::FromVideoPixelFormat(input_image.PixelFormat());
Fourcc output_fourcc =
Fourcc::FromVideoPixelFormat(output_image.PixelFormat());
Fourcc::FromVideoPixelFormat(output_image->PixelFormat());
auto input_layout = test::CreateVideoFrameLayout(input_image.PixelFormat(),
input_image.Size());
auto output_layout = test::CreateVideoFrameLayout(
output_image.PixelFormat(), output_image.Size());
output_image->PixelFormat(), output_image->Size());
LOG_ASSERT(input_layout && output_layout);
ImageProcessor::PortConfig input_config(
input_fourcc, input_image.Size(), input_layout->planes(),
input_image.Size(), input_storage_types);
ImageProcessor::PortConfig output_config(
output_fourcc, output_image.Size(), output_layout->planes(),
output_image.Size(), output_storage_types);
output_fourcc, output_image->Size(), output_layout->planes(),
output_image->Size(), output_storage_types);
// TODO(crbug.com/917951): Select more appropriate number of buffers.
constexpr size_t kNumBuffers = 1;
LOG_ASSERT(output_image.IsMetadataLoaded());
LOG_ASSERT(output_image->IsMetadataLoaded());
std::vector<std::unique_ptr<test::VideoFrameProcessor>> frame_processors;
// TODO(crbug.com/944823): Use VideoFrameValidator for RGB formats.
// TODO(crbug.com/917951): We should validate a scaled image with SSIM.
// Validating processed frames is currently not supported when a format is
// not YUV or when scaling images.
if (IsYuvPlanar(input_fourcc.ToVideoPixelFormat()) &&
IsYuvPlanar(output_fourcc.ToVideoPixelFormat()) &&
input_image.Size() == output_image.Size()) {
auto vf_validator = test::VideoFrameValidator::Create(
{output_image.Checksum()}, output_image.PixelFormat());
frame_processors.push_back(std::move(vf_validator));
IsYuvPlanar(output_fourcc.ToVideoPixelFormat())) {
if (input_image.Size() == output_image->Size()) {
auto vf_validator = test::VideoFrameValidator::Create(
{output_image->Checksum()}, output_image->PixelFormat());
frame_processors.push_back(std::move(vf_validator));
} else if (input_fourcc == output_fourcc) {
// Scaling case.
LOG_ASSERT(output_image->Load());
scoped_refptr<const VideoFrame> model_frame =
CreateVideoFrameFromImage(*output_image);
LOG_ASSERT(model_frame) << "Failed to create from image";
// Scaling is not deterministic process. There are various algorithms to
// scale images. We set a weaker tolerance value, 32, to avoid false
// negative.
constexpr uint32_t kImageProcessorTestTorelance = 32;
auto vf_validator = test::VideoFrameValidator::Create(
{model_frame}, kImageProcessorTestTorelance);
frame_processors.push_back(std::move(vf_validator));
}
}
if (g_save_images) {
......@@ -151,7 +165,7 @@ TEST_P(ImageProcessorParamTest, ConvertOneTime_MemToMem) {
}
auto ip_client = CreateImageProcessorClient(
input_image, {VideoFrame::STORAGE_OWNED_MEMORY}, output_image,
input_image, {VideoFrame::STORAGE_OWNED_MEMORY}, &output_image,
{VideoFrame::STORAGE_OWNED_MEMORY});
ip_client->Process(input_image, output_image);
......@@ -181,7 +195,7 @@ TEST_P(ImageProcessorParamTest, ConvertOneTime_DmabufToMem) {
}
auto ip_client = CreateImageProcessorClient(
input_image, {VideoFrame::STORAGE_DMABUFS}, output_image,
input_image, {VideoFrame::STORAGE_DMABUFS}, &output_image,
{VideoFrame::STORAGE_OWNED_MEMORY});
ip_client->Process(input_image, output_image);
......@@ -202,7 +216,7 @@ TEST_P(ImageProcessorParamTest, ConvertOneTime_DmabufToDmabuf) {
auto ip_client =
CreateImageProcessorClient(input_image, {VideoFrame::STORAGE_DMABUFS},
output_image, {VideoFrame::STORAGE_DMABUFS});
&output_image, {VideoFrame::STORAGE_DMABUFS});
ip_client->Process(input_image, output_image);
......
......@@ -344,5 +344,37 @@ base::Optional<VideoFrameLayout> CreateVideoFrameLayout(VideoPixelFormat format,
return VideoFrameLayout::CreateWithPlanes(format, size, std::move(planes));
}
size_t CompareFramesWithErrorDiff(const VideoFrame& frame1,
const VideoFrame& frame2,
uint8_t tolerance) {
LOG_ASSERT(frame1.format() == frame2.format());
LOG_ASSERT(frame1.visible_rect() == frame2.visible_rect());
LOG_ASSERT(frame1.visible_rect().origin() == gfx::Point(0, 0));
LOG_ASSERT(frame1.IsMappable() && frame2.IsMappable());
size_t diff_cnt = 0;
const VideoPixelFormat format = frame1.format();
const size_t num_planes = VideoFrame::NumPlanes(format);
const gfx::Size& visible_size = frame1.visible_rect().size();
for (size_t i = 0; i < num_planes; ++i) {
const uint8_t* data1 = frame1.data(i);
const int stride1 = frame1.stride(i);
const uint8_t* data2 = frame2.data(i);
const int stride2 = frame2.stride(i);
const size_t rows = VideoFrame::Rows(i, format, visible_size.height());
const int row_bytes = VideoFrame::RowBytes(i, format, visible_size.width());
for (size_t r = 0; r < rows; ++r) {
for (int c = 0; c < row_bytes; c++) {
uint8_t b1 = data1[(stride1 * r) + c];
uint8_t b2 = data2[(stride2 * r) + c];
uint8_t diff = std::max(b1, b2) - std::min(b1, b2);
diff_cnt += diff > tolerance;
}
}
}
return diff_cnt;
}
} // namespace test
} // namespace media
......@@ -92,6 +92,13 @@ base::Optional<VideoFrameLayout> CreateVideoFrameLayout(
VideoPixelFormat pixel_format,
const gfx::Size& size);
// Compare each byte of two VideoFrames, |frame1| and |frame2|, allowing the
// error up to |tolerance|. Return number of bytes a difference of which is more
// than |tolerance|.
size_t CompareFramesWithErrorDiff(const VideoFrame& frame1,
const VideoFrame& frame2,
uint8_t tolerance);
} // namespace test
} // namespace media
......
......@@ -20,6 +20,24 @@
namespace media {
namespace test {
VideoFrameValidator::MismatchedFrameInfo::MismatchedFrameInfo(
size_t frame_index,
std::string computed_md5,
std::string expected_md5)
: validate_mode(ValidateMode::MD5),
frame_index(frame_index),
computed_md5(std::move(computed_md5)),
expected_md5(std::move(expected_md5)) {}
VideoFrameValidator::MismatchedFrameInfo::MismatchedFrameInfo(
size_t frame_index,
size_t diff_cnt)
: validate_mode(ValidateMode::RAW),
frame_index(frame_index),
diff_cnt(diff_cnt) {}
VideoFrameValidator::MismatchedFrameInfo::~MismatchedFrameInfo() = default;
// static
std::unique_ptr<VideoFrameValidator> VideoFrameValidator::Create(
const std::vector<std::string>& expected_frame_checksums,
......@@ -36,11 +54,26 @@ std::unique_ptr<VideoFrameValidator> VideoFrameValidator::Create(
return video_frame_validator;
}
std::unique_ptr<VideoFrameValidator> VideoFrameValidator::Create(
const std::vector<scoped_refptr<const VideoFrame>> model_frames,
const uint8_t tolerance,
std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor) {
auto video_frame_validator = base::WrapUnique(new VideoFrameValidator(
std::move(model_frames), tolerance, std::move(corrupt_frame_processor)));
if (!video_frame_validator->Initialize()) {
LOG(ERROR) << "Failed to initialize VideoFrameValidator.";
return nullptr;
}
return video_frame_validator;
}
VideoFrameValidator::VideoFrameValidator(
std::vector<std::string> expected_frame_checksums,
VideoPixelFormat validation_format,
std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor)
: expected_frame_checksums_(std::move(expected_frame_checksums)),
: validate_mode_(ValidateMode::MD5),
expected_frame_checksums_(std::move(expected_frame_checksums)),
validation_format_(validation_format),
corrupt_frame_processor_(std::move(corrupt_frame_processor)),
num_frames_validating_(0),
......@@ -50,6 +83,21 @@ VideoFrameValidator::VideoFrameValidator(
DETACH_FROM_SEQUENCE(validator_thread_sequence_checker_);
}
VideoFrameValidator::VideoFrameValidator(
const std::vector<scoped_refptr<const VideoFrame>> model_frames,
const uint8_t tolerance,
std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor)
: validate_mode_(ValidateMode::RAW),
model_frames_(std::move(model_frames)),
tolerance_(tolerance),
corrupt_frame_processor_(std::move(corrupt_frame_processor)),
num_frames_validating_(0),
frame_validator_thread_("FrameValidatorThread"),
frame_validator_cv_(&frame_validator_lock_) {
DETACH_FROM_SEQUENCE(validator_sequence_checker_);
DETACH_FROM_SEQUENCE(validator_thread_sequence_checker_);
}
VideoFrameValidator::~VideoFrameValidator() {
Destroy();
}
......@@ -140,27 +188,30 @@ void VideoFrameValidator::ProcessVideoFrameTask(
#endif // BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
LOG_ASSERT(validated_frame->IsMappable());
if (validated_frame->format() != validation_format_) {
validated_frame =
ConvertVideoFrame(validated_frame.get(), validation_format_);
}
ASSERT_TRUE(validated_frame);
std::string computed_md5 = ComputeMD5FromVideoFrame(validated_frame.get());
base::Optional<MismatchedFrameInfo> mismatched_info;
switch (validate_mode_) {
case ValidateMode::MD5: {
if (validated_frame->format() != validation_format_) {
validated_frame =
ConvertVideoFrame(validated_frame.get(), validation_format_);
}
ASSERT_TRUE(validated_frame);
mismatched_info = ValidateMD5(*validated_frame, frame_index);
break;
}
case ValidateMode::RAW:
mismatched_info = ValidateRaw(*validated_frame, frame_index);
break;
}
base::AutoLock auto_lock(frame_validator_lock_);
if (expected_frame_checksums_.size() > 0) {
LOG_IF(FATAL, frame_index >= expected_frame_checksums_.size())
<< "Frame number is over than the number of read md5 values in file.";
const auto& expected_md5 = expected_frame_checksums_[frame_index];
if (computed_md5 != expected_md5) {
mismatched_frames_.push_back(
MismatchedFrameInfo{frame_index, computed_md5, expected_md5});
// Perform additional processing on the corrupt video frame if requested.
if (corrupt_frame_processor_)
corrupt_frame_processor_->ProcessVideoFrame(video_frame, frame_index);
}
if (mismatched_info) {
mismatched_frames_.push_back(std::move(mismatched_info).value());
// Perform additional processing on the corrupt video frame if requested.
if (corrupt_frame_processor_)
corrupt_frame_processor_->ProcessVideoFrame(validated_frame, frame_index);
}
num_frames_validating_--;
......@@ -168,15 +219,44 @@ void VideoFrameValidator::ProcessVideoFrameTask(
}
std::string VideoFrameValidator::ComputeMD5FromVideoFrame(
const VideoFrame* video_frame) const {
const VideoFrame& video_frame) const {
DCHECK_CALLED_ON_VALID_SEQUENCE(validator_thread_sequence_checker_);
base::MD5Context context;
base::MD5Init(&context);
VideoFrame::HashFrameForTesting(&context, *video_frame);
VideoFrame::HashFrameForTesting(&context, video_frame);
base::MD5Digest digest;
base::MD5Final(&digest, &context);
return MD5DigestToBase16(digest);
}
base::Optional<VideoFrameValidator::MismatchedFrameInfo>
VideoFrameValidator::ValidateMD5(const VideoFrame& validated_frame,
size_t frame_index) {
std::string computed_md5 = ComputeMD5FromVideoFrame(validated_frame);
if (expected_frame_checksums_.size() > 0) {
LOG_IF(FATAL, frame_index >= expected_frame_checksums_.size())
<< "Frame number is over than the number of read md5 values in file.";
const auto& expected_md5 = expected_frame_checksums_[frame_index];
if (computed_md5 != expected_md5) {
return MismatchedFrameInfo{frame_index, computed_md5, expected_md5};
}
}
return base::nullopt;
}
base::Optional<VideoFrameValidator::MismatchedFrameInfo>
VideoFrameValidator::ValidateRaw(const VideoFrame& validated_frame,
size_t frame_index) {
if (model_frames_.size() > 0) {
LOG_IF(FATAL, frame_index >= model_frames_.size())
<< "Frame number is over than the number of given frames.";
size_t diff_cnt = CompareFramesWithErrorDiff(
validated_frame, *model_frames_[frame_index], tolerance_);
if (diff_cnt > 0)
return MismatchedFrameInfo{frame_index, diff_cnt};
}
return base::nullopt;
}
} // namespace test
} // namespace media
......@@ -38,10 +38,32 @@ namespace test {
// performance measurements.
class VideoFrameValidator : public VideoFrameProcessor {
public:
static constexpr uint8_t kDefaultTolerance = 4;
// TODO(hiroh): Support a validation by PSNR and SSIM.
enum class ValidateMode {
MD5, // Check if md5sum value of coded area of a given VideoFrame on
// ProcessVideoFrame() matches the expected md5sum.
RAW, // Compare each byte of visible area of a given VideoFrame on
// ProcessVideoFrame() with a expected VideoFrame. An absolute
// difference equal to or less than |tolerance_| is allowed on
// comparison.
};
struct MismatchedFrameInfo {
size_t frame_index;
MismatchedFrameInfo(size_t frame_index,
std::string computed_md5,
std::string expected_md5);
MismatchedFrameInfo(size_t frame_index, size_t diff_cnt);
~MismatchedFrameInfo();
ValidateMode validate_mode;
size_t frame_index = 0;
// variables for ValidateMode::MD5 mode.
std::string computed_md5;
std::string expected_md5;
// variables for ValidateMode::ERRORDIFF mode.
size_t diff_cnt = 0;
};
// Create an instance of the video frame validator. The calculated checksums
......@@ -57,6 +79,11 @@ class VideoFrameValidator : public VideoFrameProcessor {
const VideoPixelFormat validation_format = PIXEL_FORMAT_I420,
std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor = nullptr);
static std::unique_ptr<VideoFrameValidator> Create(
const std::vector<scoped_refptr<const VideoFrame>> model_frames,
const uint8_t tolerance = kDefaultTolerance,
std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor = nullptr);
~VideoFrameValidator() override;
// Returns information of frames that don't match golden md5 values.
......@@ -82,6 +109,11 @@ class VideoFrameValidator : public VideoFrameProcessor {
VideoPixelFormat validation_format,
std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor);
VideoFrameValidator(
const std::vector<scoped_refptr<const VideoFrame>> model_frames,
const uint8_t tolerance,
std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor);
// Start the frame validation thread.
bool Initialize();
// Stop the frame validation thread.
......@@ -92,19 +124,29 @@ class VideoFrameValidator : public VideoFrameProcessor {
size_t frame_index);
// Returns md5 values of video frame represented by |video_frame|.
std::string ComputeMD5FromVideoFrame(const VideoFrame* video_frame) const;
std::string ComputeMD5FromVideoFrame(const VideoFrame& video_frame) const;
// The results of invalid frame data.
std::vector<MismatchedFrameInfo> mismatched_frames_
GUARDED_BY(frame_validator_lock_);
base::Optional<MismatchedFrameInfo> ValidateMD5(
const VideoFrame& validated_frame,
size_t frame_index);
base::Optional<MismatchedFrameInfo> ValidateRaw(
const VideoFrame& validated_frame,
size_t frame_index);
const ValidateMode validate_mode_;
// Values used only if |validate_mode_| is MD5.
// The list of expected MD5 frame checksums.
const std::vector<std::string> expected_frame_checksums_;
// VideoPixelFormat the VideoFrame will be converted to for validation.
const VideoPixelFormat validation_format_ = PIXEL_FORMAT_UNKNOWN;
std::unique_ptr<VideoFrameMapper> video_frame_mapper_;
// Values used only if |validate_mode_| is RAW.
// The list of expected frames
const std::vector<scoped_refptr<const VideoFrame>> model_frames_;
const uint8_t tolerance_ = 0;
// VideoPixelFormat the VideoFrame will be converted to for validation.
const VideoPixelFormat validation_format_;
std::unique_ptr<VideoFrameMapper> video_frame_mapper_;
// An optional video frame processor that all corrupted frames will be
// forwarded to. This can be used to e.g. write corrupted frames to disk.
......@@ -112,6 +154,9 @@ class VideoFrameValidator : public VideoFrameProcessor {
// The number of frames currently queued for validation.
size_t num_frames_validating_ GUARDED_BY(frame_validator_lock_);
// The results of invalid frame data.
std::vector<MismatchedFrameInfo> mismatched_frames_
GUARDED_BY(frame_validator_lock_);
// Thread on which video frame validation is done.
base::Thread frame_validator_thread_;
......
......@@ -10,7 +10,7 @@
#include "base/callback.h"
#include "base/containers/queue.h"
#include "base/macros.h"
#include "base/memory/ref_counted.h"
#include "base/memory/scoped_refptr.h"
#include "base/time/time.h"
#include "media/base/video_bitrate_allocation.h"
#include "media/base/video_codecs.h"
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment