H265 decoder slice header and decode flow

This implements slice header parsing and the rest of the overall decode
flow. It does not handle POCs, DPB, ref pic lists, etc. yet, those will
be added next.

BUG=chromium:1141237,b:153111783
TEST=media_unittests

Change-Id: Ib0173316ff8b25db9222d791e9dc8660ec282dad
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2518132
Commit-Queue: Jeffrey Kardatzke <jkardatzke@google.com>
Reviewed-by: Sergey Volk <servolk@chromium.org>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Cr-Commit-Position: refs/heads/master@{#824690}

media/gpu/h265_decoder.cc

@@ -78,8 +78,19 @@ void H265Decoder::SetStream(int32_t id, const DecoderBuffer& decoder_buffer) {
 }
 
 void H265Decoder::Reset() {
+  curr_pic_ = nullptr;
   curr_nalu_ = nullptr;
+  curr_slice_hdr_ = nullptr;
+  last_slice_hdr_ = nullptr;
+  curr_sps_id_ = -1;
+  curr_pps_id_ = -1;
 
+  prev_tid0_pic_ = nullptr;
+  ref_pic_list_.clear();
+  ref_pic_list0_.clear();
+  ref_pic_list1_.clear();
+
+  dpb_.Clear();
   parser_.Reset();
   accelerator_->Reset();
 
@@ -124,6 +135,8 @@ H265Decoder::DecodeResult H265Decoder::Decode() {
       par_res = parser_.AdvanceToNextNALU(curr_nalu_.get());
       if (par_res == H265Parser::kEOStream) {
         curr_nalu_.reset();
+        // We receive one frame per buffer, so we can output the frame now.
+        CHECK_ACCELERATOR_RESULT(FinishPrevFrameIfPresent());
         return kRanOutOfStreamData;
       }
       if (par_res != H265Parser::kOk) {
@@ -142,9 +155,103 @@ H265Decoder::DecodeResult H265Decoder::Decode() {
       continue;
     }
 
-    bool need_new_buffers;
     switch (curr_nalu_->nal_unit_type) {
+      case H265NALU::BLA_W_LP:  // fallthrough
+      case H265NALU::BLA_W_RADL:
+      case H265NALU::BLA_N_LP:
+      case H265NALU::IDR_W_RADL:
+      case H265NALU::IDR_N_LP:
+      case H265NALU::TRAIL_N:
+      case H265NALU::TRAIL_R:
+      case H265NALU::TSA_N:
+      case H265NALU::TSA_R:
+      case H265NALU::STSA_N:
+      case H265NALU::STSA_R:
+      case H265NALU::RADL_N:
+      case H265NALU::RADL_R:
+      case H265NALU::RASL_N:
+      case H265NALU::RASL_R:
+      case H265NALU::CRA_NUT:
+        if (!curr_slice_hdr_) {
+          curr_slice_hdr_.reset(new H265SliceHeader());
+          if (last_slice_hdr_) {
+            // This is a multi-slice picture, so we should copy all of the prior
+            // slice header data to the new slice and use those as the default
+            // values that don't have syntax elements present.
+            memcpy(curr_slice_hdr_.get(), last_slice_hdr_.get(),
+                   sizeof(H265SliceHeader));
+            last_slice_hdr_.reset();
+          }
+          par_res =
+              parser_.ParseSliceHeader(*curr_nalu_, curr_slice_hdr_.get());
+          if (par_res == H265Parser::kMissingParameterSet) {
+            // We may still be able to recover if we skip until we find the
+            // SPS/PPS.
+            curr_slice_hdr_.reset();
+            last_slice_hdr_.reset();
+            break;
+          }
+          if (par_res != H265Parser::kOk)
+            SET_ERROR_AND_RETURN();
+          if (!curr_slice_hdr_->irap_pic && state_ == kAfterReset) {
+            // We can't resume from a non-IRAP picture.
+            curr_slice_hdr_.reset();
+            last_slice_hdr_.reset();
+            break;
+          }
+
+          state_ = kTryPreprocessCurrentSlice;
+          if (curr_slice_hdr_->slice_pic_parameter_set_id != curr_pps_id_) {
+            bool need_new_buffers = false;
+            if (!ProcessPPS(curr_slice_hdr_->slice_pic_parameter_set_id,
+                            &need_new_buffers)) {
+              SET_ERROR_AND_RETURN();
+            }
+
+            if (need_new_buffers) {
+              curr_pic_ = nullptr;
+              return kConfigChange;
+            }
+          }
+        }
+
+        if (state_ == kTryPreprocessCurrentSlice) {
+          CHECK_ACCELERATOR_RESULT(PreprocessCurrentSlice());
+          state_ = kEnsurePicture;
+        }
+
+        if (state_ == kEnsurePicture) {
+          if (curr_pic_) {
+            // |curr_pic_| already exists, so skip to ProcessCurrentSlice().
+            state_ = kTryCurrentSlice;
+          } else {
+            // New picture, try to start a new one or tell client we need more
+            // surfaces.
+            curr_pic_ = accelerator_->CreateH265Picture();
+            if (!curr_pic_)
+              return kRanOutOfSurfaces;
+            if (current_decrypt_config_)
+              curr_pic_->set_decrypt_config(current_decrypt_config_->Clone());
+
+            curr_pic_->first_picture_ = first_picture_;
+            first_picture_ = false;
+            state_ = kTryNewFrame;
+          }
+        }
+
+        if (state_ == kTryNewFrame) {
+          CHECK_ACCELERATOR_RESULT(StartNewFrame(curr_slice_hdr_.get()));
+          state_ = kTryCurrentSlice;
+        }
+
+        DCHECK_EQ(state_, kTryCurrentSlice);
+        CHECK_ACCELERATOR_RESULT(ProcessCurrentSlice());
+        state_ = kDecoding;
+        last_slice_hdr_.swap(curr_slice_hdr_);
+        curr_slice_hdr_.reset();
+        break;
       case H265NALU::SPS_NUT:
+        CHECK_ACCELERATOR_RESULT(FinishPrevFrameIfPresent());
         int sps_id;
         par_res = parser_.ParseSPS(&sps_id);
         if (par_res != H265Parser::kOk)
@@ -152,17 +259,31 @@ H265Decoder::DecodeResult H265Decoder::Decode() {
 
         break;
       case H265NALU::PPS_NUT:
+        CHECK_ACCELERATOR_RESULT(FinishPrevFrameIfPresent());
         int pps_id;
         par_res = parser_.ParsePPS(*curr_nalu_, &pps_id);
         if (par_res != H265Parser::kOk)
           SET_ERROR_AND_RETURN();
 
-        if (!ProcessPPS(pps_id, &need_new_buffers))
-          SET_ERROR_AND_RETURN();
-
-        if (need_new_buffers)
-          return kConfigChange;
-
+        break;
+      case H265NALU::EOS_NUT:
+        first_picture_ = true;
+        FALLTHROUGH;
+      case H265NALU::EOB_NUT:  // fallthrough
+      case H265NALU::AUD_NUT:
+      case H265NALU::RSV_NVCL41:
+      case H265NALU::RSV_NVCL42:
+      case H265NALU::RSV_NVCL43:
+      case H265NALU::RSV_NVCL44:
+      case H265NALU::UNSPEC48:
+      case H265NALU::UNSPEC49:
+      case H265NALU::UNSPEC50:
+      case H265NALU::UNSPEC51:
+      case H265NALU::UNSPEC52:
+      case H265NALU::UNSPEC53:
+      case H265NALU::UNSPEC54:
+      case H265NALU::UNSPEC55:
+        CHECK_ACCELERATOR_RESULT(FinishPrevFrameIfPresent());
         break;
       default:
         DVLOG(4) << "Skipping NALU type: " << curr_nalu_->nal_unit_type;
@@ -242,8 +363,109 @@ bool H265Decoder::ProcessPPS(int pps_id, bool* need_new_buffers) {
   return true;
 }
 
+H265Decoder::H265Accelerator::Status H265Decoder::PreprocessCurrentSlice() {
+  const H265SliceHeader* slice_hdr = curr_slice_hdr_.get();
+  DCHECK(slice_hdr);
+
+  if (slice_hdr->first_slice_segment_in_pic_flag) {
+    // New picture, so first finish the previous one before processing it.
+    H265Accelerator::Status result = FinishPrevFrameIfPresent();
+    if (result != H265Accelerator::Status::kOk)
+      return result;
+
+    DCHECK(!curr_pic_);
+  }
+
+  return H265Accelerator::Status::kOk;
+}
+
+H265Decoder::H265Accelerator::Status H265Decoder::ProcessCurrentSlice() {
+  DCHECK(curr_pic_);
+
+  const H265SliceHeader* slice_hdr = curr_slice_hdr_.get();
+  DCHECK(slice_hdr);
+
+  const H265SPS* sps = parser_.GetSPS(curr_sps_id_);
+  DCHECK(sps);
+
+  const H265PPS* pps = parser_.GetPPS(curr_pps_id_);
+  DCHECK(pps);
+  return accelerator_->SubmitSlice(sps, pps, slice_hdr, ref_pic_list0_,
+                                   ref_pic_list1_, curr_pic_.get(),
+                                   slice_hdr->nalu_data, slice_hdr->nalu_size,
+                                   parser_.GetCurrentSubsamples());
+}
+
+H265Decoder::H265Accelerator::Status H265Decoder::StartNewFrame(
+    const H265SliceHeader* slice_hdr) {
+  CHECK(curr_pic_.get());
+  DCHECK(slice_hdr);
+
+  curr_pps_id_ = slice_hdr->slice_pic_parameter_set_id;
+  const H265PPS* pps = parser_.GetPPS(curr_pps_id_);
+  // Slice header parsing already verified this should exist.
+  DCHECK(pps);
+
+  curr_sps_id_ = pps->pps_seq_parameter_set_id;
+  const H265SPS* sps = parser_.GetSPS(curr_sps_id_);
+  // Slice header parsing already verified this should exist.
+  DCHECK(sps);
+
+  // Copy slice/pps variables we need to the picture.
+  curr_pic_->nal_unit_type_ = curr_nalu_->nal_unit_type;
+  curr_pic_->irap_pic_ = slice_hdr->irap_pic;
+
+  curr_pic_->set_visible_rect(visible_rect_);
+  curr_pic_->set_bitstream_id(stream_id_);
+  if (sps->GetColorSpace().IsSpecified())
+    curr_pic_->set_colorspace(sps->GetColorSpace());
+  else
+    curr_pic_->set_colorspace(container_color_space_);
+
+  // TODO(jkardatzke): Add calculation of picture output flags, POC,
+  // ref pic POCs, building ref pic lists and dpb operations.
+
+  return accelerator_->SubmitFrameMetadata(sps, pps, slice_hdr, ref_pic_list_,
+                                           curr_pic_);
+}
+
+H265Decoder::H265Accelerator::Status H265Decoder::FinishPrevFrameIfPresent() {
+  // If we already have a frame waiting to be decoded, decode it and finish.
+  if (curr_pic_) {
+    H265Accelerator::Status result = DecodePicture();
+    if (result != H265Accelerator::Status::kOk)
+      return result;
+
+    scoped_refptr<H265Picture> pic = curr_pic_;
+    curr_pic_ = nullptr;
+    FinishPicture(pic);
+  }
+
+  return H265Accelerator::Status::kOk;
+}
+
+void H265Decoder::FinishPicture(scoped_refptr<H265Picture> pic) {
+  // 8.3.1
+  if (pic->valid_for_prev_tid0_pic_)
+    prev_tid0_pic_ = pic;
+
+  ref_pic_list_.clear();
+  ref_pic_list0_.clear();
+  ref_pic_list1_.clear();
+
+  last_slice_hdr_.reset();
+}
+
+H265Decoder::H265Accelerator::Status H265Decoder::DecodePicture() {
+  DCHECK(curr_pic_.get());
+  return accelerator_->SubmitDecode(curr_pic_);
+}
+
 bool H265Decoder::Flush() {
   DVLOG(2) << "Decoder flush";
+
+  dpb_.Clear();
+  prev_tid0_pic_ = nullptr;
   return true;
 }
 

media/gpu/h265_decoder.h

@@ -12,7 +12,9 @@
 #include <vector>
 
 #include "base/containers/span.h"
+#include "base/memory/ref_counted.h"
 #include "media/base/decrypt_config.h"
+#include "media/base/subsample_entry.h"
 #include "media/base/video_codecs.h"
 #include "media/gpu/accelerated_video_decoder.h"
 #include "media/gpu/h265_dpb.h"
@@ -69,6 +71,59 @@ class MEDIA_GPU_EXPORT H265Decoder final : public AcceleratedVideoDecoder {
 
     virtual ~H265Accelerator();
 
+    // Create a new H265Picture that the decoder client can use for decoding
+    // and pass back to this accelerator for decoding or reference.
+    // When the picture is no longer needed by decoder, it will just drop
+    // its reference to it, and it may do so at any time.
+    // Note that this may return nullptr if accelerator is not able to provide
+    // any new pictures at given time. The decoder is expected to handle
+    // this situation as normal and return from Decode() with kRanOutOfSurfaces.
+    virtual scoped_refptr<H265Picture> CreateH265Picture() = 0;
+
+    // Submit metadata for the current frame, providing the current |sps|, |pps|
+    // and |slice_hdr| for it. |ref_pic_list| contains the set of pictures as
+    // described in 8.3.2 from the lists RefPicSetLtCurr, RefPicSetLtFoll,
+    // RefPicSetStCurrBefore, RefPicSetStCurrAfter and RefPicSetStFoll.
+    // |pic| contains information about the picture for the current frame.
+    // Note that this does not run decode in the accelerator and the decoder
+    // is expected to follow this call with one or more SubmitSlice() calls
+    // before calling SubmitDecode().
+    // Returns kOk if successful, kFail if there are errors, or kTryAgain if
+    // the accelerator needs additional data before being able to proceed.
+    virtual Status SubmitFrameMetadata(const H265SPS* sps,
+                                       const H265PPS* pps,
+                                       const H265SliceHeader* slice_hdr,
+                                       const H265Picture::Vector& ref_pic_list,
+                                       scoped_refptr<H265Picture> pic) = 0;
+
+    // Submit one slice for the current frame, passing the current |pps| and
+    // |pic| (same as in SubmitFrameMetadata()), the parsed header for the
+    // current slice in |slice_hdr|, and the |ref_pic_listX|, as per H265 spec.
+    // |data| pointing to the full slice (including the unparsed header) of
+    // |size| in bytes.
+    // |subsamples| specifies which part of the slice data is encrypted.
+    // This must be called one or more times per frame, before SubmitDecode().
+    // Note that |data| does not have to remain valid after this call returns.
+    // Returns kOk if successful, kFail if there are errors, or kTryAgain if
+    // the accelerator needs additional data before being able to proceed.
+    virtual Status SubmitSlice(
+        const H265SPS* sps,
+        const H265PPS* pps,
+        const H265SliceHeader* slice_hdr,
+        const H265Picture::Vector& ref_pic_list0,
+        const H265Picture::Vector& ref_pic_list1,
+        scoped_refptr<H265Picture> pic,
+        const uint8_t* data,
+        size_t size,
+        const std::vector<SubsampleEntry>& subsamples) = 0;
+
+    // Execute the decode in hardware for |pic|, using all the slices and
+    // metadata submitted via SubmitFrameMetadata() and SubmitSlice() since
+    // the previous call to SubmitDecode().
+    // Returns kOk if successful, kFail if there are errors, or kTryAgain if
+    // the accelerator needs additional data before being able to proceed.
+    virtual Status SubmitDecode(scoped_refptr<H265Picture> pic) = 0;
+
     // Reset any current state that may be cached in the accelerator, dropping
     // any cached parameters/slices that have not been committed yet.
     virtual void Reset() = 0;
@@ -129,6 +184,27 @@ class MEDIA_GPU_EXPORT H265Decoder final : public AcceleratedVideoDecoder {
   // Process H265 stream structures.
   bool ProcessPPS(int pps_id, bool* need_new_buffers);
 
+  // Process current slice header to discover if we need to start a new picture,
+  // finishing up the current one.
+  H265Accelerator::Status PreprocessCurrentSlice();
+
+  // Process current slice as a slice of the current picture.
+  H265Accelerator::Status ProcessCurrentSlice();
+
+  // Start processing a new frame. This also generates all the POC and output
+  // variables for the frame, generates reference picture lists, performs
+  // reference picture marking, DPB management and picture output.
+  H265Accelerator::Status StartNewFrame(const H265SliceHeader* slice_hdr);
+
+  // All data for a frame received, process it and decode.
+  H265Accelerator::Status FinishPrevFrameIfPresent();
+
+  // Called after we are done processing |pic|.
+  void FinishPicture(scoped_refptr<H265Picture> pic);
+
+  // Commits all pending data for HW decoder and starts HW decoder.
+  H265Accelerator::Status DecodePicture();
+
   // Decoder state.
   State state_;
 
@@ -155,11 +231,31 @@ class MEDIA_GPU_EXPORT H265Decoder final : public AcceleratedVideoDecoder {
   // Current stream buffer id; to be assigned to pictures decoded from it.
   int32_t stream_id_ = -1;
 
+  // Picture currently being processed/decoded.
+  scoped_refptr<H265Picture> curr_pic_;
+
+  // Used to identify first picture in decoding order or first picture that
+  // follows an EOS NALU.
+  bool first_picture_ = true;
+
   // Global state values, needed in decoding. See spec.
+  scoped_refptr<H265Picture> prev_tid0_pic_;
   int max_pic_order_cnt_lsb_;
+  H265Picture::Vector ref_pic_list0_;
+  H265Picture::Vector ref_pic_list1_;
+
+  // |ref_pic_list_| is the collection of all pictures from StCurrBefore,
+  // StCurrAfter, StFoll, LtCurr and LtFoll.
+  H265Picture::Vector ref_pic_list_;
+
+  // Currently active SPS and PPS.
+  int curr_sps_id_ = -1;
+  int curr_pps_id_ = -1;
 
-  // Current NALU being processed.
+  // Current NALU and slice header being processed.
   std::unique_ptr<H265NALU> curr_nalu_;
+  std::unique_ptr<H265SliceHeader> curr_slice_hdr_;
+  std::unique_ptr<H265SliceHeader> last_slice_hdr_;
 
   // Output picture size.
   gfx::Size pic_size_;

media/gpu/h265_dpb.cc

@@ -10,12 +10,21 @@
 
 namespace media {
 
-H265DPB::H265DPB() : max_num_pics_(0) {}
+H265Picture::H265Picture() = default;
+H265Picture::~H265Picture() = default;
+
+H265DPB::H265DPB() = default;
 H265DPB::~H265DPB() = default;
 
 void H265DPB::set_max_num_pics(size_t max_num_pics) {
   DCHECK_LE(max_num_pics, static_cast<size_t>(kMaxDpbSize));
   max_num_pics_ = max_num_pics;
+  if (pics_.size() > max_num_pics_)
+    pics_.resize(max_num_pics_);
+}
+
+void H265DPB::Clear() {
+  pics_.clear();
 }
 
 }  // namespace media
\ No newline at end of file

media/gpu/h265_dpb.h

@@ -5,11 +5,80 @@
 #ifndef MEDIA_GPU_H265_DPB_H_
 #define MEDIA_GPU_H265_DPB_H_
 
+#include <vector>
+
+#include "base/memory/ref_counted.h"
+#include "media/base/video_color_space.h"
+#include "media/gpu/codec_picture.h"
+#include "media/gpu/media_gpu_export.h"
+
 namespace media {
 
+// A picture (a frame or a field) in the H.265 spec sense.
+// See spec at http://www.itu.int/rec/T-REC-H.265
+class MEDIA_GPU_EXPORT H265Picture : public CodecPicture {
+ public:
+  using Vector = std::vector<scoped_refptr<H265Picture>>;
+
+  H265Picture();
+
+  H265Picture(const H265Picture&) = delete;
+  H265Picture& operator=(const H265Picture&) = delete;
+
+  enum ReferenceType {
+    kUnused = 0,
+    kShortTermCurrBefore = 1,
+    kShortTermCurrAfter = 2,
+    kShortTermFoll = 3,
+    kLongTermCurr = 4,
+    kLongTermFoll = 5,
+  };
+
+  static std::string GetReferenceName(ReferenceType ref) {
+    if (ref == kUnused)
+      return "Unused";
+    else if (ref == kLongTermCurr || ref == kLongTermFoll)
+      return "LongTerm";
+    else
+      return "ShortTerm";
+  }
+
+  bool IsLongTermRef() const {
+    return ref_ == kLongTermCurr || ref_ == kLongTermFoll;
+  }
+  bool IsShortTermRef() const {
+    return ref_ == kShortTermCurrBefore || ref_ == kShortTermCurrAfter ||
+           ref_ == kShortTermFoll;
+  }
+  bool IsUnused() const { return ref_ == kUnused; }
+
+  // Values calculated per H.265 specification or taken from slice header.
+  // See spec for more details on each (some names have been converted from
+  // CamelCase in spec to Chromium-style names).
+  int nal_unit_type_;
+  bool no_rasl_output_flag_{false};
+  bool no_output_of_prior_pics_flag_{false};
+  bool pic_output_flag_{false};
+  bool valid_for_prev_tid0_pic_{false};
+  int slice_pic_order_cnt_lsb_{0};
+  int pic_order_cnt_msb_{0};
+  int pic_order_cnt_val_{0};
+
+  // Our own state variables.
+  bool irap_pic_;
+  bool first_picture_;
+
+  ReferenceType ref_{kUnused};
+
+  bool outputted_{false};
+
+ protected:
+  ~H265Picture() override;
+};
+
 // DPB - Decoded Picture Buffer.
-// Stores decoded pictures that will be used for future display
-// and/or reference.
+// Stores decoded pictures that will be used for future display and/or
+// reference.
 class H265DPB {
  public:
   H265DPB();
@@ -22,8 +91,15 @@ class H265DPB {
   void set_max_num_pics(size_t max_num_pics);
   size_t max_num_pics() const { return max_num_pics_; }
 
+  // Removes all entries from the DPB.
+  void Clear();
+
+  size_t size() const { return pics_.size(); }
+  bool IsFull() const { return pics_.size() >= max_num_pics_; }
+
  private:
-  size_t max_num_pics_;
+  H265Picture::Vector pics_;
+  size_t max_num_pics_{0};
 };
 
 }  // namespace media

media/test/data/README.md

@@ -896,8 +896,12 @@ Used by h265_parser_unittest.cc.
 Used by h265_parser_unittest.cc. Copied from bbb_hevc_176x144_176kbps_60fps.hevc
 in Android repo.
 
-#### bear-frame0.hevc
-SPS, PPS and single IDR frame from bear.hevc for h265_decoder_unittest.cc.
+#### bear-sps-pps.hevc
+SPS and PPS from bear.hevc for h265_decoder_unittest.cc.
+
+#### bear-frame\{0,1,2,3,4,5\}.hevc
+Single IDR, P, B, B, B, P frames respectively from bear.hevc for
+h265_decoder_unittest.cc.
 
 ###  WebM files for testing multiple tracks.