VAVDA: Switch to using max_num_reorder_frames.

H.264 standard allows for an optional max_num_reorder_frames parameter to be
specified in VUI parameters. When present, it indicates the maximum number
of frames that may precede a given frame in decode order, but come after it
in output order. This means, that if the number of decoded and not yet outputted
frames exceeds max_num_reorder_frames, we are allowed to output until this no
longer holds true.

In case max_num_reorder_frames is not present, the value is to be inferred
to be either 0 or equal to the size of DPB. In case of the latter, we would
always have to wait until the DPB is full before outputting anything. This means
we could incur visible stuttering whenever DPB had to be cleared (on IDRs,
etc.). max_num_reorder_frames seems to be very common in streams these days
however, so it should be a relatively rare occurrence.

Relying on max_num_reorder_frames also gives us an ability to handle streams
with negative POC values, which are produced by some camera models.

TEST=video playbacks, vdatest, switchtests
BUG=chromium:177692

Review URL: https://codereview.chromium.org/177583002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@255898 0039d316-1c4b-4281-b951-d872f2087c98

content/common/gpu/media/vaapi_h264_decoder.cc

@@ -7,6 +7,7 @@
 
 #include "base/bind.h"
 #include "base/bind_helpers.h"
+#include "base/numerics/safe_conversions.h"
 #include "base/stl_util.h"
 #include "content/common/gpu/media/vaapi_h264_decoder.h"
 
@@ -62,6 +63,7 @@ VaapiH264Decoder::VaapiH264Decoder(
       max_frame_num_(0),
       max_pic_num_(0),
       max_long_term_frame_idx_(0),
+      max_num_reorder_frames_(0),
       curr_sps_id_(-1),
       curr_pps_id_(-1),
       vaapi_wrapper_(vaapi_wrapper),
@@ -105,7 +107,7 @@ void VaapiH264Decoder::Reset() {
 
   dpb_.Clear();
   parser_.Reset();
-  last_output_poc_ = 0;
+  last_output_poc_ = std::numeric_limits<int>::min();
 
   // If we are in kDecoding, we can resume without processing an SPS.
   if (state_ == kDecoding)
@@ -1042,7 +1044,7 @@ void VaapiH264Decoder::ClearDPB() {
     UnassignSurfaceFromPoC((*it)->pic_order_cnt);
 
   dpb_.Clear();
-  last_output_poc_ = 0;
+  last_output_poc_ = std::numeric_limits<int>::min();
 }
 
 bool VaapiH264Decoder::OutputAllRemainingPics() {
@@ -1088,7 +1090,7 @@ bool VaapiH264Decoder::StartNewFrame(media::H264SliceHeader* slice_hdr) {
         return false;
     }
     dpb_.Clear();
-    last_output_poc_ = 0;
+    last_output_poc_ = std::numeric_limits<int>::min();
   }
 
   // curr_pic_ should have either been added to DPB or discarded when finishing
@@ -1329,20 +1331,18 @@ bool VaapiH264Decoder::FinishPicture() {
   dpb_.GetNotOutputtedPicsAppending(not_outputted);
   // Include the one we've just decoded.
   not_outputted.push_back(pic.get());
+
   // Sort in output order.
   std::sort(not_outputted.begin(), not_outputted.end(), POCAscCompare());
 
-  // Try to output as many pictures as we can. A picture can be output
-  // if its POC is next after the previously outputted one (which means
-  // last_output_poc_ + 2, because POCs are incremented by 2 to accommodate
-  // fields when decoding interleaved streams). POC can also be equal to
-  // last outputted picture's POC when it wraps around back to 0.
+  // Try to output as many pictures as we can. A picture can be output,
+  // if the number of decoded and not yet outputted pictures that would remain
+  // in DPB afterwards would at least be equal to max_num_reorder_frames.
   // If the outputted picture is not a reference picture, it doesn't have
   // to remain in the DPB and can be removed.
   H264Picture::PtrVector::iterator output_candidate = not_outputted.begin();
-  for (; output_candidate != not_outputted.end() &&
-      (*output_candidate)->pic_order_cnt <= last_output_poc_ + 2;
-      ++output_candidate) {
+  size_t num_remaining = not_outputted.size();
+  while (num_remaining > max_num_reorder_frames_) {
     int poc = (*output_candidate)->pic_order_cnt;
     DCHECK_GE(poc, last_output_poc_);
     if (!OutputPic(*output_candidate))
@@ -1356,6 +1356,9 @@ bool VaapiH264Decoder::FinishPicture() {
       // Mark as unused.
       UnassignSurfaceFromPoC(poc);
     }
+
+    ++output_candidate;
+    --num_remaining;
   }
 
   // If we haven't managed to output the picture that we just decoded, or if
@@ -1399,6 +1402,43 @@ static int LevelToMaxDpbMbs(int level) {
   }
 }
 
+bool VaapiH264Decoder::UpdateMaxNumReorderFrames(const media::H264SPS* sps) {
+  if (sps->vui_parameters_present_flag && sps->bitstream_restriction_flag) {
+    max_num_reorder_frames_ =
+        base::checked_cast<size_t>(sps->max_num_reorder_frames);
+    if (max_num_reorder_frames_ > dpb_.max_num_pics()) {
+      DVLOG(1)
+          << "max_num_reorder_frames present, but larger than MaxDpbFrames ("
+          << max_num_reorder_frames_ << " > " << dpb_.max_num_pics() << ")";
+      max_num_reorder_frames_ = 0;
+      return false;
+    }
+    return true;
+  }
+
+  // max_num_reorder_frames not present, infer from profile/constraints
+  // (see VUI semantics in spec).
+  if (sps->constraint_set3_flag) {
+    switch (sps->profile_idc) {
+      case 44:
+      case 86:
+      case 100:
+      case 110:
+      case 122:
+      case 244:
+        max_num_reorder_frames_ = 0;
+        break;
+      default:
+        max_num_reorder_frames_ = dpb_.max_num_pics();
+        break;
+    }
+  } else {
+    max_num_reorder_frames_ = dpb_.max_num_pics();
+  }
+
+  return true;
+}
+
 bool VaapiH264Decoder::ProcessSPS(int sps_id, bool* need_new_buffers) {
   const media::H264SPS* sps = parser_.GetSPS(sps_id);
   DCHECK(sps);
@@ -1459,6 +1499,10 @@ bool VaapiH264Decoder::ProcessSPS(int sps_id, bool* need_new_buffers) {
 
   dpb_.set_max_num_pics(max_dpb_size);
 
+  if (!UpdateMaxNumReorderFrames(sps))
+    return false;
+  DVLOG(1) << "max_num_reorder_frames: " << max_num_reorder_frames_;
+
   *need_new_buffers = true;
   return true;
 }

content/common/gpu/media/vaapi_h264_decoder.h

@@ -143,6 +143,8 @@ class CONTENT_EXPORT VaapiH264Decoder {
   // frame (see spec).
   void UpdatePicNums();
 
+  bool UpdateMaxNumReorderFrames(const media::H264SPS* sps);
+
   // Prepare reference picture lists (ref_pic_list[01]_).
   bool PrepareRefPicLists(media::H264SliceHeader* slice_hdr);
 
@@ -241,6 +243,7 @@ class CONTENT_EXPORT VaapiH264Decoder {
   int max_frame_num_;
   int max_pic_num_;
   int max_long_term_frame_idx_;
+  size_t max_num_reorder_frames_;
 
   int frame_num_;
   int prev_frame_num_;

media/filters/h264_parser.cc

@@ -579,6 +579,109 @@ H264Parser::Result H264Parser::ParsePPSScalingLists(const H264SPS& sps,
   return kOk;
 }
 
+H264Parser::Result H264Parser::ParseAndIgnoreHRDParameters(
+    bool* hrd_parameters_present) {
+  int data;
+  READ_BOOL_OR_RETURN(&data);  // {nal,vcl}_hrd_parameters_present_flag
+  if (!data)
+    return kOk;
+
+  *hrd_parameters_present = true;
+
+  int cpb_cnt_minus1;
+  READ_UE_OR_RETURN(&cpb_cnt_minus1);
+  IN_RANGE_OR_RETURN(cpb_cnt_minus1, 0, 31);
+  READ_BITS_OR_RETURN(8, &data);  // bit_rate_scale, cpb_size_scale
+  for (int i = 0; i <= cpb_cnt_minus1; ++i) {
+    READ_UE_OR_RETURN(&data);  // bit_rate_value_minus1[i]
+    READ_UE_OR_RETURN(&data);  // cpb_size_value_minus1[i]
+    READ_BOOL_OR_RETURN(&data);  // cbr_flag
+  }
+  READ_BITS_OR_RETURN(20, &data);  // cpb/dpb delays, etc.
+
+  return kOk;
+}
+
+H264Parser::Result H264Parser::ParseVUIParameters(H264SPS* sps) {
+  bool aspect_ratio_info_present_flag;
+  READ_BOOL_OR_RETURN(&aspect_ratio_info_present_flag);
+  if (aspect_ratio_info_present_flag) {
+    int aspect_ratio_idc;
+    READ_BITS_OR_RETURN(8, &aspect_ratio_idc);
+    if (aspect_ratio_idc == kExtendedSar) {
+      READ_BITS_OR_RETURN(16, &sps->sar_width);
+      READ_BITS_OR_RETURN(16, &sps->sar_height);
+    } else {
+      const int max_aspect_ratio_idc = arraysize(kTableSarWidth) - 1;
+      IN_RANGE_OR_RETURN(aspect_ratio_idc, 0, max_aspect_ratio_idc);
+      sps->sar_width = kTableSarWidth[aspect_ratio_idc];
+      sps->sar_height = kTableSarHeight[aspect_ratio_idc];
+    }
+  }
+
+  int data;
+  // Read and ignore overscan and video signal type info.
+  READ_BOOL_OR_RETURN(&data);  // overscan_info_present_flag
+  if (data)
+    READ_BOOL_OR_RETURN(&data);  // overscan_appropriate_flag
+
+  READ_BOOL_OR_RETURN(&data);  // video_signal_type_present_flag
+  if (data) {
+    READ_BITS_OR_RETURN(3, &data);  // video_format
+    READ_BOOL_OR_RETURN(&data);  // video_full_range_flag
+    READ_BOOL_OR_RETURN(&data);  // colour_description_present_flag
+    if (data)
+      READ_BITS_OR_RETURN(24, &data);  // color description syntax elements
+  }
+
+  READ_BOOL_OR_RETURN(&data);  // chroma_loc_info_present_flag
+  if (data) {
+    READ_UE_OR_RETURN(&data);  // chroma_sample_loc_type_top_field
+    READ_UE_OR_RETURN(&data);  // chroma_sample_loc_type_bottom_field
+  }
+
+  // Read and ignore timing info.
+  READ_BOOL_OR_RETURN(&data);  // timing_info_present_flag
+  if (data) {
+    READ_BITS_OR_RETURN(16, &data);  // num_units_in_tick
+    READ_BITS_OR_RETURN(16, &data);  // num_units_in_tick
+    READ_BITS_OR_RETURN(16, &data);  // time_scale
+    READ_BITS_OR_RETURN(16, &data);  // time_scale
+    READ_BOOL_OR_RETURN(&data);  // fixed_frame_rate_flag
+  }
+
+  // Read and ignore NAL HRD parameters, if present.
+  bool hrd_parameters_present = false;
+  Result res = ParseAndIgnoreHRDParameters(&hrd_parameters_present);
+  if (res != kOk)
+    return res;
+
+  // Read and ignore VCL HRD parameters, if present.
+  res = ParseAndIgnoreHRDParameters(&hrd_parameters_present);
+  if (res != kOk)
+    return res;
+
+  if (hrd_parameters_present)  // One of NAL or VCL params present is enough.
+    READ_BOOL_OR_RETURN(&data);  // low_delay_hrd_flag
+
+  READ_BOOL_OR_RETURN(&data);  // pic_struct_present_flag
+  READ_BOOL_OR_RETURN(&sps->bitstream_restriction_flag);
+  if (sps->bitstream_restriction_flag) {
+    READ_BOOL_OR_RETURN(&data);  // motion_vectors_over_pic_boundaries_flag
+    READ_UE_OR_RETURN(&data);  // max_bytes_per_pic_denom
+    READ_UE_OR_RETURN(&data);  // max_bits_per_mb_denom
+    READ_UE_OR_RETURN(&data);  // log2_max_mv_length_horizontal
+    READ_UE_OR_RETURN(&data);  // log2_max_mv_length_vertical
+    READ_UE_OR_RETURN(&sps->max_num_reorder_frames);
+    READ_UE_OR_RETURN(&sps->max_dec_frame_buffering);
+    TRUE_OR_RETURN(sps->max_dec_frame_buffering >= sps->max_num_ref_frames);
+    IN_RANGE_OR_RETURN(
+        sps->max_num_reorder_frames, 0, sps->max_dec_frame_buffering);
+  }
+
+  return kOk;
+}
+
 static void FillDefaultSeqScalingLists(H264SPS* sps) {
   for (int i = 0; i < 6; ++i)
     for (int j = 0; j < kH264ScalingList4x4Length; ++j)
@@ -599,8 +702,13 @@ H264Parser::Result H264Parser::ParseSPS(int* sps_id) {
   scoped_ptr<H264SPS> sps(new H264SPS());
 
   READ_BITS_OR_RETURN(8, &sps->profile_idc);
-  READ_BITS_OR_RETURN(6, &sps->constraint_setx_flag);
-  READ_BITS_OR_RETURN(2, &data);
+  READ_BOOL_OR_RETURN(&sps->constraint_set0_flag);
+  READ_BOOL_OR_RETURN(&sps->constraint_set1_flag);
+  READ_BOOL_OR_RETURN(&sps->constraint_set2_flag);
+  READ_BOOL_OR_RETURN(&sps->constraint_set3_flag);
+  READ_BOOL_OR_RETURN(&sps->constraint_set4_flag);
+  READ_BOOL_OR_RETURN(&sps->constraint_set5_flag);
+  READ_BITS_OR_RETURN(2, &data);  // reserved_zero_2bits
   READ_BITS_OR_RETURN(8, &sps->level_idc);
   READ_UE_OR_RETURN(&sps->seq_parameter_set_id);
   TRUE_OR_RETURN(sps->seq_parameter_set_id < 32);
@@ -692,21 +800,10 @@ H264Parser::Result H264Parser::ParseSPS(int* sps_id) {
 
   READ_BOOL_OR_RETURN(&sps->vui_parameters_present_flag);
   if (sps->vui_parameters_present_flag) {
-    bool aspect_ratio_info_present_flag;
-    READ_BOOL_OR_RETURN(&aspect_ratio_info_present_flag);
-    if (aspect_ratio_info_present_flag) {
-      int aspect_ratio_idc;
-      READ_BITS_OR_RETURN(8, &aspect_ratio_idc);
-      if (aspect_ratio_idc == kExtendedSar) {
-        READ_BITS_OR_RETURN(16, &sps->sar_width);
-        READ_BITS_OR_RETURN(16, &sps->sar_height);
-      } else {
-        const int max_aspect_ratio_idc = arraysize(kTableSarWidth) - 1;
-        IN_RANGE_OR_RETURN(aspect_ratio_idc, 0, max_aspect_ratio_idc);
-        sps->sar_width = kTableSarWidth[aspect_ratio_idc];
-        sps->sar_height = kTableSarHeight[aspect_ratio_idc];
-      }
-    }
+    DVLOG(4) << "VUI parameters present";
+    res = ParseVUIParameters(sps.get());
+    if (res != kOk)
+      return res;
   }
 
   // If an SPS with the same id already exists, replace it.

media/filters/h264_parser.h

@@ -53,7 +53,12 @@ struct MEDIA_EXPORT H264SPS {
   H264SPS();
 
   int profile_idc;
-  int constraint_setx_flag;
+  bool constraint_set0_flag;
+  bool constraint_set1_flag;
+  bool constraint_set2_flag;
+  bool constraint_set3_flag;
+  bool constraint_set4_flag;
+  bool constraint_set5_flag;
   int level_idc;
   int seq_parameter_set_id;
 
@@ -88,10 +93,15 @@ struct MEDIA_EXPORT H264SPS {
   int frame_crop_right_offset;
   int frame_crop_top_offset;
   int frame_crop_bottom_offset;
+
   bool vui_parameters_present_flag;
-  int chroma_array_type;
   int sar_width;    // Set to 0 when not specified.
   int sar_height;   // Set to 0 when not specified.
+  bool bitstream_restriction_flag;
+  int max_num_reorder_frames;
+  int max_dec_frame_buffering;
+
+  int chroma_array_type;
 };
 
 struct MEDIA_EXPORT H264PPS {
@@ -344,6 +354,11 @@ class MEDIA_EXPORT H264Parser {
   Result ParseSPSScalingLists(H264SPS* sps);
   Result ParsePPSScalingLists(const H264SPS& sps, H264PPS* pps);
 
+  // Parse optional VUI parameters in SPS (see spec).
+  Result ParseVUIParameters(H264SPS* sps);
+  // Set |hrd_parameters_present| to true only if they are present.
+  Result ParseAndIgnoreHRDParameters(bool* hrd_parameters_present);
+
   // Parse reference picture lists' modifications (see spec).
   Result ParseRefPicListModifications(H264SliceHeader* shdr);
   Result ParseRefPicListModification(int num_ref_idx_active_minus1,