Commit 568cc6a9 authored by Matt Wolenetz's avatar Matt Wolenetz Committed by Commit Bot

MSE: refactor mp4 AnnexB validation to also report keyframe-ness

In preparation for logging when there appears to be a mismatch between
MSE MP4 keyframe metadata and the encoded video bitstream's keyframe
metadata, this change incorporates basic analysis of the latter as part
of the existing Annex-B bitstream validation for MSE MP4 video.

Note that mp4::BitstreamConverter::AnalysisResult contains
base::Optional<bool> for each of conformance and keyframe-ness fields.
If a field doesn't have a value, that portion of the result was not
analyzed.

Note that keyframe analysis is implemented only for AVC currently, but
even that is skipped if the frame is detected as non-conformant before
enough indications of whether or not it is a keyframe are detected.
Neither AVC-DV nor HEVC AnnexB analyses do any actual keyframe analysis,
since such was either skipped or not yet implemented previously,
respectively.

A subsequent CL will use the newly reported MSE MP4 video bitstream
converter's keyframe analysis results, if that analysis was done, in
reporting to chrome://media-internals when the bitstream keyframe-ness
mismatches that of the mp4 container for a coded frame.

BUG=860420,584384

Cq-Include-Trybots: luci.chromium.try:android_optional_gpu_tests_rel;luci.chromium.try:linux_optional_gpu_tests_rel;luci.chromium.try:mac_optional_gpu_tests_rel;luci.chromium.try:win_optional_gpu_tests_rel
Change-Id: I169c7774070ad232c86658bcd7803160323993ad
Reviewed-on: https://chromium-review.googlesource.com/1144456
Commit-Queue: Matthew Wolenetz <wolenetz@chromium.org>
Reviewed-by: default avatarSergey Volk <servolk@chromium.org>
Reviewed-by: default avatarDan Sanders <sandersd@chromium.org>
Cr-Commit-Position: refs/heads/master@{#577717}
parent 76349e5b
...@@ -32,6 +32,7 @@ ...@@ -32,6 +32,7 @@
#include "media/filters/ffmpeg_demuxer.h" #include "media/filters/ffmpeg_demuxer.h"
#include "media/filters/file_data_source.h" #include "media/filters/file_data_source.h"
#include "media/formats/mp4/avc.h" #include "media/formats/mp4/avc.h"
#include "media/formats/mp4/bitstream_converter.h"
#include "media/media_buildflags.h" #include "media/media_buildflags.h"
#include "testing/gtest/include/gtest/gtest.h" #include "testing/gtest/include/gtest/gtest.h"
...@@ -1199,8 +1200,8 @@ static void ValidateAnnexB(DemuxerStream* stream, ...@@ -1199,8 +1200,8 @@ static void ValidateAnnexB(DemuxerStream* stream,
subsamples = buffer->decrypt_config()->subsamples(); subsamples = buffer->decrypt_config()->subsamples();
bool is_valid = bool is_valid =
mp4::AVC::IsValidAnnexB(buffer->data(), buffer->data_size(), mp4::AVC::AnalyzeAnnexB(buffer->data(), buffer->data_size(), subsamples)
subsamples); .is_conformant.value_or(false);
EXPECT_TRUE(is_valid); EXPECT_TRUE(is_valid);
if (!is_valid) { if (!is_valid) {
......
...@@ -171,15 +171,21 @@ bool AVC::ConvertConfigToAnnexB(const AVCDecoderConfigurationRecord& avc_config, ...@@ -171,15 +171,21 @@ bool AVC::ConvertConfigToAnnexB(const AVCDecoderConfigurationRecord& avc_config,
return true; return true;
} }
// Verifies AnnexB NALU order according to ISO/IEC 14496-10 Section 7.4.1.2.3 // static
bool AVC::IsValidAnnexB(const uint8_t* buffer, BitstreamConverter::AnalysisResult AVC::AnalyzeAnnexB(
size_t size, const uint8_t* buffer,
const std::vector<SubsampleEntry>& subsamples) { size_t size,
const std::vector<SubsampleEntry>& subsamples) {
DVLOG(3) << __func__; DVLOG(3) << __func__;
DCHECK(buffer); DCHECK(buffer);
if (size == 0) BitstreamConverter::AnalysisResult result;
return true; result.is_conformant = false; // Will change if needed before return.
if (size == 0) {
result.is_conformant = true;
return result;
}
H264Parser parser; H264Parser parser;
parser.SetEncryptedStream(buffer, size, subsamples); parser.SetEncryptedStream(buffer, size, subsamples);
...@@ -205,7 +211,7 @@ bool AVC::IsValidAnnexB(const uint8_t* buffer, ...@@ -205,7 +211,7 @@ bool AVC::IsValidAnnexB(const uint8_t* buffer,
case H264NALU::kAUD: case H264NALU::kAUD:
if (order_state > kAUDAllowed) { if (order_state > kAUDAllowed) {
DVLOG(1) << "Unexpected AUD in order_state " << order_state; DVLOG(1) << "Unexpected AUD in order_state " << order_state;
return false; return result;
} }
order_state = kBeforeFirstVCL; order_state = kBeforeFirstVCL;
break; break;
...@@ -221,7 +227,7 @@ bool AVC::IsValidAnnexB(const uint8_t* buffer, ...@@ -221,7 +227,7 @@ bool AVC::IsValidAnnexB(const uint8_t* buffer,
if (order_state > kBeforeFirstVCL) { if (order_state > kBeforeFirstVCL) {
DVLOG(1) << "Unexpected NALU type " << nalu.nal_unit_type DVLOG(1) << "Unexpected NALU type " << nalu.nal_unit_type
<< " in order_state " << order_state; << " in order_state " << order_state;
return false; return result;
} }
order_state = kBeforeFirstVCL; order_state = kBeforeFirstVCL;
break; break;
...@@ -229,7 +235,7 @@ bool AVC::IsValidAnnexB(const uint8_t* buffer, ...@@ -229,7 +235,7 @@ bool AVC::IsValidAnnexB(const uint8_t* buffer,
case H264NALU::kSPSExt: case H264NALU::kSPSExt:
if (last_nalu_type != H264NALU::kSPS) { if (last_nalu_type != H264NALU::kSPS) {
DVLOG(1) << "SPS extension does not follow an SPS."; DVLOG(1) << "SPS extension does not follow an SPS.";
return false; return result;
} }
break; break;
...@@ -240,22 +246,26 @@ bool AVC::IsValidAnnexB(const uint8_t* buffer, ...@@ -240,22 +246,26 @@ bool AVC::IsValidAnnexB(const uint8_t* buffer,
case H264NALU::kIDRSlice: case H264NALU::kIDRSlice:
if (order_state > kAfterFirstVCL) { if (order_state > kAfterFirstVCL) {
DVLOG(1) << "Unexpected VCL in order_state " << order_state; DVLOG(1) << "Unexpected VCL in order_state " << order_state;
return false; return result;
} }
if (!result.is_keyframe.has_value())
result.is_keyframe = nalu.nal_unit_type == H264NALU::kIDRSlice;
order_state = kAfterFirstVCL; order_state = kAfterFirstVCL;
break; break;
case H264NALU::kCodedSliceAux: case H264NALU::kCodedSliceAux:
if (order_state != kAfterFirstVCL) { if (order_state != kAfterFirstVCL) {
DVLOG(1) << "Unexpected extension in order_state " << order_state; DVLOG(1) << "Unexpected extension in order_state " << order_state;
return false; return result;
} }
break; break;
case H264NALU::kEOSeq: case H264NALU::kEOSeq:
if (order_state != kAfterFirstVCL) { if (order_state != kAfterFirstVCL) {
DVLOG(1) << "Unexpected EOSeq in order_state " << order_state; DVLOG(1) << "Unexpected EOSeq in order_state " << order_state;
return false; return result;
} }
order_state = kEOStreamAllowed; order_state = kEOStreamAllowed;
break; break;
...@@ -263,7 +273,7 @@ bool AVC::IsValidAnnexB(const uint8_t* buffer, ...@@ -263,7 +273,7 @@ bool AVC::IsValidAnnexB(const uint8_t* buffer,
case H264NALU::kEOStream: case H264NALU::kEOStream:
if (order_state < kAfterFirstVCL) { if (order_state < kAfterFirstVCL) {
DVLOG(1) << "Unexpected EOStream in order_state " << order_state; DVLOG(1) << "Unexpected EOStream in order_state " << order_state;
return false; return result;
} }
order_state = kNoMoreDataAllowed; order_state = kNoMoreDataAllowed;
break; break;
...@@ -274,7 +284,7 @@ bool AVC::IsValidAnnexB(const uint8_t* buffer, ...@@ -274,7 +284,7 @@ bool AVC::IsValidAnnexB(const uint8_t* buffer,
order_state < kEOStreamAllowed)) { order_state < kEOStreamAllowed)) {
DVLOG(1) << "Unexpected NALU type " << nalu.nal_unit_type DVLOG(1) << "Unexpected NALU type " << nalu.nal_unit_type
<< " in order_state " << order_state; << " in order_state " << order_state;
return false; return result;
} }
break; break;
...@@ -284,25 +294,30 @@ bool AVC::IsValidAnnexB(const uint8_t* buffer, ...@@ -284,25 +294,30 @@ bool AVC::IsValidAnnexB(const uint8_t* buffer,
order_state != kAfterFirstVCL) { order_state != kAfterFirstVCL) {
DVLOG(1) << "Unexpected NALU type " << nalu.nal_unit_type DVLOG(1) << "Unexpected NALU type " << nalu.nal_unit_type
<< " in order_state " << order_state; << " in order_state " << order_state;
return false; return result;
} }
} }
last_nalu_type = nalu.nal_unit_type; last_nalu_type = nalu.nal_unit_type;
break; break;
case H264Parser::kInvalidStream: case H264Parser::kInvalidStream:
return false; return result;
case H264Parser::kUnsupportedStream: case H264Parser::kUnsupportedStream:
NOTREACHED() << "AdvanceToNextNALU() returned kUnsupportedStream!"; NOTREACHED() << "AdvanceToNextNALU() returned kUnsupportedStream!";
return false; return result;
case H264Parser::kEOStream: case H264Parser::kEOStream:
done = true; done = true;
} }
} }
return order_state >= kAfterFirstVCL; if (order_state < kAfterFirstVCL)
return result;
result.is_conformant = true;
DCHECK(result.is_keyframe.has_value());
return result;
} }
AVCBitstreamConverter::AVCBitstreamConverter( AVCBitstreamConverter::AVCBitstreamConverter(
...@@ -338,14 +353,16 @@ bool AVCBitstreamConverter::ConvertFrame( ...@@ -338,14 +353,16 @@ bool AVCBitstreamConverter::ConvertFrame(
return true; return true;
} }
bool AVCBitstreamConverter::IsValid( BitstreamConverter::AnalysisResult AVCBitstreamConverter::Analyze(
std::vector<uint8_t>* frame_buf, std::vector<uint8_t>* frame_buf,
std::vector<SubsampleEntry>* subsamples) const { std::vector<SubsampleEntry>* subsamples) const {
#if BUILDFLAG(ENABLE_DOLBY_VISION_DEMUXING) #if BUILDFLAG(ENABLE_DOLBY_VISION_DEMUXING)
if (disable_validation_) if (disable_validation_) {
return true; BitstreamConverter::AnalysisResult result;
return result;
}
#endif // BUILDFLAG(ENABLE_DOLBY_VISION_DEMUXING) #endif // BUILDFLAG(ENABLE_DOLBY_VISION_DEMUXING)
return AVC::IsValidAnnexB(frame_buf->data(), frame_buf->size(), *subsamples); return AVC::AnalyzeAnnexB(frame_buf->data(), frame_buf->size(), *subsamples);
} }
} // namespace mp4 } // namespace mp4
......
...@@ -44,14 +44,17 @@ class MEDIA_EXPORT AVC { ...@@ -44,14 +44,17 @@ class MEDIA_EXPORT AVC {
const AVCDecoderConfigurationRecord& avc_config, const AVCDecoderConfigurationRecord& avc_config,
std::vector<uint8_t>* buffer); std::vector<uint8_t>* buffer);
// Verifies that the contents of |buffer| conform to // Analyzes the contents of |buffer| for conformance to Section 7.4.1.2.3 of
// Section 7.4.1.2.3 of ISO/IEC 14496-10. // ISO/IEC 14496-10. Also analyzes |buffer| and reports if it looks like a
// keyframe, if such can be determined. Determination of keyframe-ness is done
// only if |buffer| is conformant or if lack of conformance is detected after
// detecting keyframe-ness.
// |subsamples| contains the information about what parts of the buffer are // |subsamples| contains the information about what parts of the buffer are
// encrypted and which parts are clear. // encrypted and which parts are clear.
// Returns true if |buffer| contains conformant Annex B data static BitstreamConverter::AnalysisResult AnalyzeAnnexB(
static bool IsValidAnnexB(const uint8_t* buffer, const uint8_t* buffer,
size_t size, size_t size,
const std::vector<SubsampleEntry>& subsamples); const std::vector<SubsampleEntry>& subsamples);
// Given a |buffer| and |subsamples| information and |pts| pointer into the // Given a |buffer| and |subsamples| information and |pts| pointer into the
// |buffer| finds the index of the subsample |ptr| is pointing into. // |buffer| finds the index of the subsample |ptr| is pointing into.
...@@ -81,8 +84,9 @@ class AVCBitstreamConverter : public BitstreamConverter { ...@@ -81,8 +84,9 @@ class AVCBitstreamConverter : public BitstreamConverter {
bool is_keyframe, bool is_keyframe,
std::vector<SubsampleEntry>* subsamples) const override; std::vector<SubsampleEntry>* subsamples) const override;
bool IsValid(std::vector<uint8_t>* frame_buf, AnalysisResult Analyze(
std::vector<SubsampleEntry>* subsamples) const override; std::vector<uint8_t>* frame_buf,
std::vector<SubsampleEntry>* subsamples) const override;
private: private:
~AVCBitstreamConverter() override; ~AVCBitstreamConverter() override;
......
...@@ -6,12 +6,16 @@ ...@@ -6,12 +6,16 @@
#include <stdint.h> #include <stdint.h>
#include <string.h> #include <string.h>
#include <ostream>
#include "base/macros.h" #include "base/macros.h"
#include "base/optional.h"
#include "base/strings/string_split.h" #include "base/strings/string_split.h"
#include "base/strings/string_util.h" #include "base/strings/string_util.h"
#include "media/base/decrypt_config.h" #include "media/base/decrypt_config.h"
#include "media/base/stream_parser_buffer.h" #include "media/base/stream_parser_buffer.h"
#include "media/formats/mp4/avc.h" #include "media/formats/mp4/avc.h"
#include "media/formats/mp4/bitstream_converter.h"
#include "media/formats/mp4/box_definitions.h" #include "media/formats/mp4/box_definitions.h"
#include "media/video/h264_parser.h" #include "media/video/h264_parser.h"
#include "testing/gtest/include/gtest/gtest.h" #include "testing/gtest/include/gtest/gtest.h"
...@@ -36,6 +40,15 @@ static H264NALU::Type StringToNALUType(const std::string& name) { ...@@ -36,6 +40,15 @@ static H264NALU::Type StringToNALUType(const std::string& name) {
if (name == "I") if (name == "I")
return H264NALU::kIDRSlice; return H264NALU::kIDRSlice;
if (name == "SDA")
return H264NALU::kSliceDataA;
if (name == "SDB")
return H264NALU::kSliceDataB;
if (name == "SDC")
return H264NALU::kSliceDataC;
if (name == "SEI") if (name == "SEI")
return H264NALU::kSEIMessage; return H264NALU::kSEIMessage;
...@@ -132,9 +145,9 @@ static void WriteStartCodeAndNALUType(std::vector<uint8_t>* buffer, ...@@ -132,9 +145,9 @@ static void WriteStartCodeAndNALUType(std::vector<uint8_t>* buffer,
// The output buffer will contain a valid-looking Annex B (it's valid-looking in // The output buffer will contain a valid-looking Annex B (it's valid-looking in
// the sense that it has start codes and correct NALU types, but the actual NALU // the sense that it has start codes and correct NALU types, but the actual NALU
// payload is junk). // payload is junk).
void StringToAnnexB(const std::string& str, static void StringToAnnexB(const std::string& str,
std::vector<uint8_t>* buffer, std::vector<uint8_t>* buffer,
std::vector<SubsampleEntry>* subsamples) { std::vector<SubsampleEntry>* subsamples) {
DCHECK(!str.empty()); DCHECK(!str.empty());
std::vector<std::string> subsample_specs = base::SplitString( std::vector<std::string> subsample_specs = base::SplitString(
...@@ -176,8 +189,30 @@ void StringToAnnexB(const std::string& str, ...@@ -176,8 +189,30 @@ void StringToAnnexB(const std::string& str,
} }
} }
std::string AnnexBToString(const std::vector<uint8_t>& buffer, // Helper to compare two results of AVC::Analyze().
const std::vector<SubsampleEntry>& subsamples) { static bool AnalysesMatch(const BitstreamConverter::AnalysisResult& r1,
const BitstreamConverter::AnalysisResult& r2) {
return r1.is_conformant == r2.is_conformant &&
r1.is_keyframe == r2.is_keyframe;
}
// Helper output operator, for debugging/testability.
std::ostream& operator<<(std::ostream& os,
const BitstreamConverter::AnalysisResult& r) {
os << "{ is_conformant: "
<< (r.is_conformant.has_value()
? (r.is_conformant.value() ? "true" : "false")
: "nullopt/unknown")
<< ", is_keyframe: "
<< (r.is_keyframe.has_value() ? (r.is_keyframe.value() ? "true" : "false")
: "nullopt/unknown")
<< " }";
return os;
}
static std::string AnnexBToString(
const std::vector<uint8_t>& buffer,
const std::vector<SubsampleEntry>& subsamples) {
std::stringstream ss; std::stringstream ss;
H264Parser parser; H264Parser parser;
...@@ -230,7 +265,14 @@ TEST_P(AVCConversionTest, ParseCorrectly) { ...@@ -230,7 +265,14 @@ TEST_P(AVCConversionTest, ParseCorrectly) {
std::vector<SubsampleEntry> subsamples; std::vector<SubsampleEntry> subsamples;
MakeInputForLength(GetParam(), &buf); MakeInputForLength(GetParam(), &buf);
EXPECT_TRUE(AVC::ConvertFrameToAnnexB(GetParam(), &buf, &subsamples)); EXPECT_TRUE(AVC::ConvertFrameToAnnexB(GetParam(), &buf, &subsamples));
EXPECT_TRUE(AVC::IsValidAnnexB(buf.data(), buf.size(), subsamples));
BitstreamConverter::AnalysisResult expected;
expected.is_conformant = true;
expected.is_keyframe = false;
EXPECT_PRED2(AnalysesMatch,
AVC::AnalyzeAnnexB(buf.data(), buf.size(), subsamples),
expected);
EXPECT_EQ(buf.size(), sizeof(kExpected)); EXPECT_EQ(buf.size(), sizeof(kExpected));
EXPECT_EQ(0, memcmp(kExpected, &buf[0], sizeof(kExpected))); EXPECT_EQ(0, memcmp(kExpected, &buf[0], sizeof(kExpected)));
EXPECT_EQ("P,SDC", AnnexBToString(buf, subsamples)); EXPECT_EQ("P,SDC", AnnexBToString(buf, subsamples));
...@@ -350,63 +392,107 @@ TEST_F(AVCConversionTest, StringConversionFunctions) { ...@@ -350,63 +392,107 @@ TEST_F(AVCConversionTest, StringConversionFunctions) {
std::vector<uint8_t> buf; std::vector<uint8_t> buf;
std::vector<SubsampleEntry> subsamples; std::vector<SubsampleEntry> subsamples;
StringToAnnexB(str, &buf, &subsamples); StringToAnnexB(str, &buf, &subsamples);
EXPECT_TRUE(AVC::IsValidAnnexB(buf.data(), buf.size(), subsamples));
BitstreamConverter::AnalysisResult expected;
expected.is_conformant = true;
expected.is_keyframe = true;
EXPECT_PRED2(AnalysesMatch,
AVC::AnalyzeAnnexB(buf.data(), buf.size(), subsamples),
expected);
EXPECT_EQ(str, AnnexBToString(buf, subsamples)); EXPECT_EQ(str, AnnexBToString(buf, subsamples));
} }
TEST_F(AVCConversionTest, ValidAnnexBConstructs) { TEST_F(AVCConversionTest, ValidAnnexBConstructs) {
const char* test_cases[] = { struct {
"I", const char* case_string;
"I I I I", const bool is_keyframe;
"AUD I", } test_cases[] = {
"AUD SPS PPS I", {"I", true},
"I EOSeq", {"I I I I", true},
"I EOSeq EOStr", {"AUD I", true},
"I EOStr", {"AUD SPS PPS I", true},
"P", {"I EOSeq", true},
"P P P P", {"I EOSeq EOStr", true},
"AUD SPS PPS P", {"I EOStr", true},
"SEI SEI I", {"P", false},
"SEI SEI R14 I", {"P P P P", false},
"SPS SPSExt SPS PPS I P", {"AUD SPS PPS P", false},
"R14 SEI I", {"SEI SEI I", true},
"AUD,I", {"SEI SEI R14 I", true},
"AUD,SEI I", {"SPS SPSExt SPS PPS I P", true},
"AUD,SEI,SPS,PPS,I" {"R14 SEI I", true},
{"AUD,I", true},
{"AUD,SEI I", true},
{"AUD,SEI,SPS,PPS,I", true},
// In reality, these might not always be conformant/valid, but assuming
// they are, they're not keyframes because a non-IDR slice preceded the
// IDR slice, if any.
{"SDA SDB SDC", false},
{"P I", false},
{"SDA I", false},
{"SDB I", false},
{"SDC I", false},
}; };
for (size_t i = 0; i < arraysize(test_cases); ++i) { for (size_t i = 0; i < arraysize(test_cases); ++i) {
std::vector<uint8_t> buf; std::vector<uint8_t> buf;
std::vector<SubsampleEntry> subsamples; std::vector<SubsampleEntry> subsamples;
StringToAnnexB(test_cases[i], &buf, NULL); StringToAnnexB(test_cases[i].case_string, &buf, NULL);
EXPECT_TRUE(AVC::IsValidAnnexB(buf.data(), buf.size(), subsamples))
<< "'" << test_cases[i] << "' failed"; BitstreamConverter::AnalysisResult expected;
expected.is_conformant = true;
expected.is_keyframe = test_cases[i].is_keyframe;
EXPECT_PRED2(AnalysesMatch,
AVC::AnalyzeAnnexB(buf.data(), buf.size(), subsamples),
expected)
<< "'" << test_cases[i].case_string << "' failed";
} }
} }
TEST_F(AVCConversionTest, InvalidAnnexBConstructs) { TEST_F(AVCConversionTest, InvalidAnnexBConstructs) {
static const char* test_cases[] = { struct {
"AUD", // No VCL present. const char* case_string;
"AUD,SEI", // No VCL present. const base::Optional<bool> is_keyframe;
"SPS PPS", // No VCL present. } test_cases[] = {
"SPS PPS AUD I", // Parameter sets must come after AUD. // For these cases, lack of conformance is determined before detecting any
"SPSExt SPS P", // SPS must come before SPSExt. // IDR or non-IDR slices, so the non-conformant frames' keyframe analysis
"SPS PPS SPSExt P", // SPSExt must follow an SPS. // reports base::nullopt (which means undetermined analysis result).
"EOSeq", // EOSeq must come after a VCL. {"AUD", base::nullopt}, // No VCL present.
"EOStr", // EOStr must come after a VCL. {"AUD,SEI", base::nullopt}, // No VCL present.
"I EOStr EOSeq", // EOSeq must come before EOStr. {"SPS PPS", base::nullopt}, // No VCL present.
"I R14", // Reserved14-18 must come before first VCL. {"SPS PPS AUD I", base::nullopt}, // Parameter sets must come after AUD.
"I SEI", // SEI must come before first VCL. {"SPSExt SPS P", base::nullopt}, // SPS must come before SPSExt.
"P SPS P", // SPS after first VCL would indicate a new access unit. {"SPS PPS SPSExt P", base::nullopt}, // SPSExt must follow an SPS.
{"EOSeq", base::nullopt}, // EOSeq must come after a VCL.
{"EOStr", base::nullopt}, // EOStr must come after a VCL.
// For these cases, IDR slice is first VCL and is detected before
// conformance failure, so the non-conformant frame is reported as a
// keyframe.
{"I EOStr EOSeq", true}, // EOSeq must come before EOStr.
{"I R14", true}, // Reserved14-18 must come before first VCL.
{"I SEI", true}, // SEI must come before first VCL.
// For this case, P slice is first VCL and is detected before conformance
// failure, so the non-conformant frame is reported as a non-keyframe.
{"P SPS P",
false}, // SPS after first VCL would indicate a new access unit.
}; };
BitstreamConverter::AnalysisResult expected;
expected.is_conformant = false;
for (size_t i = 0; i < arraysize(test_cases); ++i) { for (size_t i = 0; i < arraysize(test_cases); ++i) {
std::vector<uint8_t> buf; std::vector<uint8_t> buf;
std::vector<SubsampleEntry> subsamples; std::vector<SubsampleEntry> subsamples;
StringToAnnexB(test_cases[i], &buf, NULL); StringToAnnexB(test_cases[i].case_string, &buf, NULL);
EXPECT_FALSE(AVC::IsValidAnnexB(buf.data(), buf.size(), subsamples)) expected.is_keyframe = test_cases[i].is_keyframe;
<< "'" << test_cases[i] << "' failed"; EXPECT_PRED2(AnalysesMatch,
AVC::AnalyzeAnnexB(buf.data(), buf.size(), subsamples),
expected)
<< "'" << test_cases[i].case_string << "' failed";
} }
} }
...@@ -442,6 +528,10 @@ TEST_F(AVCConversionTest, InsertParamSetsAnnexB) { ...@@ -442,6 +528,10 @@ TEST_F(AVCConversionTest, InsertParamSetsAnnexB) {
avc_config.pps_list[0].push_back(0x56); avc_config.pps_list[0].push_back(0x56);
avc_config.pps_list[0].push_back(0x78); avc_config.pps_list[0].push_back(0x78);
BitstreamConverter::AnalysisResult expected;
expected.is_conformant = true;
expected.is_keyframe = true;
for (size_t i = 0; i < arraysize(test_cases); ++i) { for (size_t i = 0; i < arraysize(test_cases); ++i) {
std::vector<uint8_t> buf; std::vector<uint8_t> buf;
std::vector<SubsampleEntry> subsamples; std::vector<SubsampleEntry> subsamples;
...@@ -450,7 +540,9 @@ TEST_F(AVCConversionTest, InsertParamSetsAnnexB) { ...@@ -450,7 +540,9 @@ TEST_F(AVCConversionTest, InsertParamSetsAnnexB) {
EXPECT_TRUE(AVC::InsertParamSetsAnnexB(avc_config, &buf, &subsamples)) EXPECT_TRUE(AVC::InsertParamSetsAnnexB(avc_config, &buf, &subsamples))
<< "'" << test_cases[i].input << "' insert failed."; << "'" << test_cases[i].input << "' insert failed.";
EXPECT_TRUE(AVC::IsValidAnnexB(buf.data(), buf.size(), subsamples)) EXPECT_PRED2(AnalysesMatch,
AVC::AnalyzeAnnexB(buf.data(), buf.size(), subsamples),
expected)
<< "'" << test_cases[i].input << "' created invalid AnnexB."; << "'" << test_cases[i].input << "' created invalid AnnexB.";
EXPECT_EQ(test_cases[i].expected, AnnexBToString(buf, subsamples)) EXPECT_EQ(test_cases[i].expected, AnnexBToString(buf, subsamples))
<< "'" << test_cases[i].input << "' generated unexpected output."; << "'" << test_cases[i].input << "' generated unexpected output.";
......
...@@ -7,6 +7,13 @@ ...@@ -7,6 +7,13 @@
namespace media { namespace media {
namespace mp4 { namespace mp4 {
BitstreamConverter::AnalysisResult::AnalysisResult(){};
BitstreamConverter::AnalysisResult::AnalysisResult(const AnalysisResult& other)
: is_conformant(other.is_conformant), is_keyframe(other.is_keyframe) {}
BitstreamConverter::AnalysisResult::~AnalysisResult() = default;
BitstreamConverter::~BitstreamConverter() = default; BitstreamConverter::~BitstreamConverter() = default;
} // namespace mp4 } // namespace mp4
......
...@@ -10,6 +10,8 @@ ...@@ -10,6 +10,8 @@
#include <vector> #include <vector>
#include "base/memory/ref_counted.h" #include "base/memory/ref_counted.h"
#include "base/optional.h"
#include "media/base/media_export.h"
namespace media { namespace media {
...@@ -20,9 +22,21 @@ namespace mp4 { ...@@ -20,9 +22,21 @@ namespace mp4 {
// BitstreamConverter provides a unified interface for performing some common // BitstreamConverter provides a unified interface for performing some common
// bitstream conversions (e.g. H.264 MP4 bitstream to Annex B, and elementary // bitstream conversions (e.g. H.264 MP4 bitstream to Annex B, and elementary
// AAC stream to ADTS). // AAC stream to ADTS).
class BitstreamConverter class MEDIA_EXPORT BitstreamConverter
: public base::RefCountedThreadSafe<BitstreamConverter> { : public base::RefCountedThreadSafe<BitstreamConverter> {
public: public:
// Describes the result of Analyze(). Not all analyses are implemented or
// enabled across mp4::BitstreamConverter implementations, hence the use of
// base::Optional<>.
struct AnalysisResult {
AnalysisResult();
AnalysisResult(const AnalysisResult&);
~AnalysisResult();
base::Optional<bool> is_conformant;
base::Optional<bool> is_keyframe;
};
// Converts a single frame/buffer |frame_buf| into the output format. // Converts a single frame/buffer |frame_buf| into the output format.
// Returns true iff the conversion was successful. // Returns true iff the conversion was successful.
// |frame_buf| is an input/output parameter, it contains input frame data and // |frame_buf| is an input/output parameter, it contains input frame data and
...@@ -37,11 +51,12 @@ class BitstreamConverter ...@@ -37,11 +51,12 @@ class BitstreamConverter
bool is_keyframe, bool is_keyframe,
std::vector<SubsampleEntry>* subsamples) const = 0; std::vector<SubsampleEntry>* subsamples) const = 0;
// Checks a converted frame for conformance. // Inspects an already converted frame for conformance. If conformant,
// Note: may return true even if the frame is not conformant; the checks may // inspects further to see if the converted frame appears to be a keyframe.
// not be exhaustive (or implemented at all). // Note, the checks may not be exhaustive (or implemented at all).
virtual bool IsValid(std::vector<uint8_t>* frame_buf, virtual AnalysisResult Analyze(
std::vector<SubsampleEntry>* subsamples) const = 0; std::vector<uint8_t>* frame_buf,
std::vector<SubsampleEntry>* subsamples) const = 0;
protected: protected:
friend class base::RefCountedThreadSafe<BitstreamConverter>; friend class base::RefCountedThreadSafe<BitstreamConverter>;
......
...@@ -137,11 +137,13 @@ VideoCodecProfile HEVCDecoderConfigurationRecord::GetVideoProfile() const { ...@@ -137,11 +137,13 @@ VideoCodecProfile HEVCDecoderConfigurationRecord::GetVideoProfile() const {
static const uint8_t kAnnexBStartCode[] = {0, 0, 0, 1}; static const uint8_t kAnnexBStartCode[] = {0, 0, 0, 1};
static const int kAnnexBStartCodeSize = 4; static const int kAnnexBStartCodeSize = 4;
// static
bool HEVC::InsertParamSetsAnnexB( bool HEVC::InsertParamSetsAnnexB(
const HEVCDecoderConfigurationRecord& hevc_config, const HEVCDecoderConfigurationRecord& hevc_config,
std::vector<uint8_t>* buffer, std::vector<uint8_t>* buffer,
std::vector<SubsampleEntry>* subsamples) { std::vector<SubsampleEntry>* subsamples) {
DCHECK(HEVC::IsValidAnnexB(buffer->data(), buffer->size(), *subsamples)); DCHECK(HEVC::AnalyzeAnnexB(buffer->data(), buffer->size(), *subsamples)
.is_conformant.value_or(true));
std::unique_ptr<H265Parser> parser(new H265Parser()); std::unique_ptr<H265Parser> parser(new H265Parser());
const uint8_t* start = buffer->data(); const uint8_t* start = buffer->data();
...@@ -179,10 +181,12 @@ bool HEVC::InsertParamSetsAnnexB( ...@@ -179,10 +181,12 @@ bool HEVC::InsertParamSetsAnnexB(
buffer->insert(config_insert_point, buffer->insert(config_insert_point,
param_sets.begin(), param_sets.end()); param_sets.begin(), param_sets.end());
DCHECK(HEVC::IsValidAnnexB(buffer->data(), buffer->size(), *subsamples)); DCHECK(HEVC::AnalyzeAnnexB(buffer->data(), buffer->size(), *subsamples)
.is_conformant.value_or(true));
return true; return true;
} }
// static
bool HEVC::ConvertConfigToAnnexB( bool HEVC::ConvertConfigToAnnexB(
const HEVCDecoderConfigurationRecord& hevc_config, const HEVCDecoderConfigurationRecord& hevc_config,
std::vector<uint8_t>* buffer) { std::vector<uint8_t>* buffer) {
...@@ -204,17 +208,23 @@ bool HEVC::ConvertConfigToAnnexB( ...@@ -204,17 +208,23 @@ bool HEVC::ConvertConfigToAnnexB(
return true; return true;
} }
// Verifies AnnexB NALU order according to section 7.4.2.4.4 of ISO/IEC 23008-2. // static
bool HEVC::IsValidAnnexB(const uint8_t* buffer, BitstreamConverter::AnalysisResult HEVC::AnalyzeAnnexB(
size_t size, const uint8_t* buffer,
const std::vector<SubsampleEntry>& subsamples) { size_t size,
const std::vector<SubsampleEntry>& subsamples) {
DCHECK(buffer); DCHECK(buffer);
if (size == 0) BitstreamConverter::AnalysisResult result;
return true;
// TODO(servolk): Implement this, see crbug.com/527595 if (size == 0) {
return true; result.is_conformant = true;
return result;
}
// TODO(servolk): Implement this, see https://crbug.com/527595. For now, we
// report that neither conformance nor keyframe analyses were performed.
return result;
} }
HEVCBitstreamConverter::HEVCBitstreamConverter( HEVCBitstreamConverter::HEVCBitstreamConverter(
...@@ -243,10 +253,10 @@ bool HEVCBitstreamConverter::ConvertFrame( ...@@ -243,10 +253,10 @@ bool HEVCBitstreamConverter::ConvertFrame(
return true; return true;
} }
bool HEVCBitstreamConverter::IsValid( BitstreamConverter::AnalysisResult HEVCBitstreamConverter::Analyze(
std::vector<uint8_t>* frame_buf, std::vector<uint8_t>* frame_buf,
std::vector<SubsampleEntry>* subsamples) const { std::vector<SubsampleEntry>* subsamples) const {
return HEVC::IsValidAnnexB(frame_buf->data(), frame_buf->size(), *subsamples); return HEVC::AnalyzeAnnexB(frame_buf->data(), frame_buf->size(), *subsamples);
} }
} // namespace mp4 } // namespace mp4
......
...@@ -78,14 +78,15 @@ class MEDIA_EXPORT HEVC { ...@@ -78,14 +78,15 @@ class MEDIA_EXPORT HEVC {
std::vector<uint8_t>* buffer, std::vector<uint8_t>* buffer,
std::vector<SubsampleEntry>* subsamples); std::vector<SubsampleEntry>* subsamples);
// Verifies that the contents of |buffer| conform to // Analyzes the contents of |buffer| for conformance to
// Section 7.4.2.4.4 of ISO/IEC 23008-2. // Section 7.4.2.4.4 of ISO/IEC 23008-2, and if conformant, further inspects
// |buffer| to report whether or not it looks like a keyframe.
// |subsamples| contains the information about what parts of the buffer are // |subsamples| contains the information about what parts of the buffer are
// encrypted and which parts are clear. // encrypted and which parts are clear.
// Returns true if |buffer| contains conformant Annex B data static BitstreamConverter::AnalysisResult AnalyzeAnnexB(
static bool IsValidAnnexB(const uint8_t* buffer, const uint8_t* buffer,
size_t size, size_t size,
const std::vector<SubsampleEntry>& subsamples); const std::vector<SubsampleEntry>& subsamples);
}; };
class HEVCBitstreamConverter : public BitstreamConverter { class HEVCBitstreamConverter : public BitstreamConverter {
...@@ -98,8 +99,9 @@ class HEVCBitstreamConverter : public BitstreamConverter { ...@@ -98,8 +99,9 @@ class HEVCBitstreamConverter : public BitstreamConverter {
bool is_keyframe, bool is_keyframe,
std::vector<SubsampleEntry>* subsamples) const override; std::vector<SubsampleEntry>* subsamples) const override;
bool IsValid(std::vector<uint8_t>* frame_buf, AnalysisResult Analyze(
std::vector<SubsampleEntry>* subsamples) const override; std::vector<uint8_t>* frame_buf,
std::vector<SubsampleEntry>* subsamples) const override;
private: private:
~HEVCBitstreamConverter() override; ~HEVCBitstreamConverter() override;
......
...@@ -802,12 +802,21 @@ ParseResult MP4StreamParser::EnqueueSample(BufferQueueMap* buffers) { ...@@ -802,12 +802,21 @@ ParseResult MP4StreamParser::EnqueueSample(BufferQueueMap* buffers) {
<< "Failed to prepare video sample for decode"; << "Failed to prepare video sample for decode";
return ParseResult::kError; return ParseResult::kError;
} }
if (!runs_->video_description().frame_bitstream_converter->IsValid( BitstreamConverter::AnalysisResult analysis =
&frame_buf, &subsamples)) { runs_->video_description().frame_bitstream_converter->Analyze(
&frame_buf, &subsamples);
// If conformance analysis was not actually performed, assume the frame is
// conformant. If it was performed and found to be non-conformant, log
// it.
if (!analysis.is_conformant.value_or(true)) {
LIMITED_MEDIA_LOG(DEBUG, media_log_, num_invalid_conversions_, LIMITED_MEDIA_LOG(DEBUG, media_log_, num_invalid_conversions_,
kMaxInvalidConversionLogs) kMaxInvalidConversionLogs)
<< "Prepared video sample is not conformant"; << "Prepared video sample is not conformant";
} }
// TODO(wolenetz): Use |analysis.is_keyframe|, if it was actually
// performed, for at least logging if the result mismatches container's
// keyframe metadata for |frame_buf|.
} }
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment