Allow video buffering headroom for high resolution frames.

This allows the video renderer to grow its frame queue beyond the minimum value of
four frames that we use today. This provides additional headroom to avoid glitches
when the system comes under load.

Nominally this should improve the rebuffering rates seen by Chrome and YouTube by
giving less capable systems more headroom for issues during decoding. This will
be enabled by experiment so we can compare and contrast how modifying buffering
levels affects rebuffering.

Capacity size is chosen based on the time to decode a frame and the expected playout
time of the minimum buffer. In a perfect world with absolute future knowledge we want
to have a buffer of (sum(decode_duration) - sum(frame_duration)) / frame_duration. We
don't know the duration though, so the best we can do is make an estimate based
on how long it would take to play out our minimum buffer; 4 frames normally.

In order to avoid wasting memory, we do not grow capacity if it's impossible for the
decoder to keep up in real time (average decode duration > frame duration), if
there's any memory pressure, if the playback has been ongoing for < 7s, or if the
video is not in the foreground.

BUG=648710, 734813
TEST=new unittests added.

Change-Id: If6c84dd8e77fd9d043037434926e452286e50429
Reviewed-on: https://chromium-review.googlesource.com/525135
Commit-Queue: Dale Curtis <dalecurtis@chromium.org>
Reviewed-by: Xiaohan Wang (OOO June 21-22) <xhwang@chromium.org>
Cr-Commit-Position: refs/heads/master@{#481647}

media/base/limits.h

@@ -42,6 +42,10 @@ enum {
   // This limit is used by ParamTraits<VideoCaptureParams>.
   kMaxFramesPerSecond = 1000,
 
+  // The minimum elapsed amount of time (in seconds) for a playback to be
+  // considered as having active engagement.
+  kMinimumElapsedWatchTimeSecs = 7,
+
   // Maximum lengths for various EME API parameters. These are checks to
   // prevent unnecessarily large parameters from being passed around, and the
   // lengths are somewhat arbitrary as the EME spec doesn't specify any limits.

media/base/media_switches.cc

@@ -213,6 +213,9 @@ const base::Feature kBackgroundVideoTrackOptimization{
 const base::Feature kBackgroundVideoPauseOptimization{
     "BackgroundVideoPauseOptimization", base::FEATURE_DISABLED_BY_DEFAULT};
 
+const base::Feature kComplexityBasedVideoBuffering{
+    "ComplexityBasedVideoBuffering", base::FEATURE_DISABLED_BY_DEFAULT};
+
 // Make MSE garbage collection algorithm more aggressive when we are under
 // moderate or critical memory pressure. This will relieve memory pressure by
 // releasing stale data from MSE buffers.

media/base/media_switches.h

@@ -104,6 +104,7 @@ namespace media {
 
 MEDIA_EXPORT extern const base::Feature kBackgroundVideoPauseOptimization;
 MEDIA_EXPORT extern const base::Feature kBackgroundVideoTrackOptimization;
+MEDIA_EXPORT extern const base::Feature kComplexityBasedVideoBuffering;
 MEDIA_EXPORT extern const base::Feature kExternalClearKeyForTesting;
 MEDIA_EXPORT extern const base::Feature kLowDelayVideoRenderingOnLiveStream;
 MEDIA_EXPORT extern const base::Feature kMediaCastOverlayButton;

media/base/moving_average.cc

@@ -8,11 +8,9 @@
 
 namespace media {
 
-MovingAverage::MovingAverage(size_t depth)
-    : depth_(depth), count_(0), samples_(depth_), square_sum_us_(0) {}
+MovingAverage::MovingAverage(size_t depth) : depth_(depth), samples_(depth_) {}
 
-MovingAverage::~MovingAverage() {
-}
+MovingAverage::~MovingAverage() {}
 
 void MovingAverage::AddSample(base::TimeDelta sample) {
   // |samples_| is zero-initialized, so |oldest| is also zero before |count_|
@@ -22,6 +20,8 @@ void MovingAverage::AddSample(base::TimeDelta sample) {
   square_sum_us_ += sample.InMicroseconds() * sample.InMicroseconds() -
                     oldest.InMicroseconds() * oldest.InMicroseconds();
   oldest = sample;
+  if (sample > max_)
+    max_ = sample;
 }
 
 base::TimeDelta MovingAverage::Average() const {
@@ -48,6 +48,7 @@ base::TimeDelta MovingAverage::Deviation() const {
 void MovingAverage::Reset() {
   count_ = 0;
   total_ = base::TimeDelta();
+  max_ = kNoTimestamp;
   square_sum_us_ = 0;
   std::fill(samples_.begin(), samples_.end(), base::TimeDelta());
 }

media/base/moving_average.h

@@ -13,6 +13,7 @@
 #include "base/macros.h"
 #include "base/time/time.h"
 #include "media/base/media_export.h"
+#include "media/base/timestamp_constants.h"
 
 namespace media {
 
@@ -38,16 +39,21 @@ class MEDIA_EXPORT MovingAverage {
 
   size_t count() const { return count_; }
 
+  base::TimeDelta max() const { return max_; }
+
  private:
   // Maximum number of elements allowed in the average.
   const size_t depth_;
+  std::vector<base::TimeDelta> samples_;
 
   // Number of elements seen thus far.
-  uint64_t count_;
+  uint64_t count_ = 0;
 
-  std::vector<base::TimeDelta> samples_;
   base::TimeDelta total_;
-  uint64_t square_sum_us_;
+  uint64_t square_sum_us_ = 0;
+
+  // Maximum value ever seen.
+  base::TimeDelta max_ = kNoTimestamp;
 
   DISALLOW_COPY_AND_ASSIGN(MovingAverage);
 };

media/base/null_video_sink.h

@@ -53,11 +53,13 @@ class MEDIA_EXPORT NullVideoSink : public VideoRendererSink {
     background_render_ = is_background_rendering;
   }
 
+  void set_clockless(bool clockless) { clockless_ = clockless; }
+
  private:
   // Task that periodically calls Render() to consume video data.
   void CallRender();
 
-  const bool clockless_;
+  bool clockless_;
   const base::TimeDelta interval_;
   const NewFrameCB new_frame_cb_;
   scoped_refptr<base::SingleThreadTaskRunner> task_runner_;

media/blink/watch_time_reporter.cc

@@ -5,6 +5,7 @@
 #include "media/blink/watch_time_reporter.h"
 
 #include "base/power_monitor/power_monitor.h"
+#include "media/base/limits.h"
 #include "media/base/watch_time_keys.h"
 
 namespace media {
@@ -12,7 +13,7 @@ namespace media {
 // The minimum amount of media playback which can elapse before we'll report
 // watch time metrics for a playback.
 constexpr base::TimeDelta kMinimumElapsedWatchTime =
-    base::TimeDelta::FromSeconds(7);
+    base::TimeDelta::FromSeconds(limits::kMinimumElapsedWatchTimeSecs);
 
 // The minimum width and height of videos to report watch time metrics for.
 constexpr gfx::Size kMinimumVideoSize = gfx::Size(200, 200);

media/filters/video_renderer_algorithm.cc

@@ -13,10 +13,6 @@ namespace media {
 
 const int kMaxOutOfOrderFrameLogs = 10;
 
-// The number of frames to store for moving average calculations.  Value picked
-// after experimenting with playback of various local media and YouTube clips.
-const int kMovingAverageSamples = 32;
-
 VideoRendererAlgorithm::ReadyFrame::ReadyFrame(
     const scoped_refptr<VideoFrame>& ready_frame)
     : frame(ready_frame),

media/filters/video_renderer_algorithm.h

@@ -149,6 +149,13 @@ class MEDIA_EXPORT VideoRendererAlgorithm {
   // for display at least once.
   void disable_frame_dropping() { frame_dropping_disabled_ = true; }
 
+  enum : int {
+    // The number of frames to store for moving average calculations.  Value
+    // picked after experimenting with playback of various local media and
+    // YouTube clips.
+    kMovingAverageSamples = 32
+  };
+
  private:
   friend class VideoRendererAlgorithmTest;
 

media/renderers/video_renderer_impl.cc

@@ -11,6 +11,8 @@
 #include "base/callback_helpers.h"
 #include "base/feature_list.h"
 #include "base/location.h"
+#include "base/memory/memory_pressure_listener.h"
+#include "base/memory/memory_pressure_monitor.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/single_thread_task_runner.h"
 #include "base/strings/string_util.h"
@@ -137,7 +139,12 @@ VideoRendererImpl::VideoRendererImpl(
       have_renderered_frames_(false),
       last_frame_opaque_(false),
       painted_first_frame_(false),
-      max_buffered_frames_(limits::kMaxVideoFrames),
+      min_buffered_frames_(limits::kMaxVideoFrames),
+      max_buffered_frames_(min_buffered_frames_),
+      read_durations_(VideoRendererAlgorithm::kMovingAverageSamples),
+      has_playback_met_watch_time_duration_requirement_(false),
+      use_complexity_based_buffering_(
+          base::FeatureList::IsEnabled(kComplexityBasedVideoBuffering)),
       weak_factory_(this),
       frame_callback_weak_factory_(this) {
   DCHECK(create_video_decoders_cb_);
@@ -193,9 +200,8 @@ void VideoRendererImpl::Flush(const base::Closure& callback) {
   painted_first_frame_ = false;
 
   // Reset preroll capacity so seek time is not penalized.
-  // TODO(dalecurtis): Not sure if this is the right decision, but it's what we
-  // do for audio, so carry over that behavior for now.
-  max_buffered_frames_ = limits::kMaxVideoFrames;
+  min_buffered_frames_ = max_buffered_frames_ = limits::kMaxVideoFrames;
+  read_durations_.Reset();
 }
 
 void VideoRendererImpl::StartPlayingFrom(base::TimeDelta timestamp) {
@@ -209,6 +215,7 @@ void VideoRendererImpl::StartPlayingFrom(base::TimeDelta timestamp) {
   state_ = kPlaying;
   start_timestamp_ = timestamp;
   painted_first_frame_ = false;
+  has_playback_met_watch_time_duration_requirement_ = false;
   AttemptRead_Locked();
 }
 
@@ -432,27 +439,39 @@ void VideoRendererImpl::OnTimeStopped() {
     // If we've underflowed, increase the number of frames required to reach
     // BUFFERING_HAVE_ENOUGH upon resume; this will help prevent us from
     // repeatedly underflowing.
-    const size_t kMaxBufferedFrames = 2 * limits::kMaxVideoFrames;
-    if (max_buffered_frames_ < kMaxBufferedFrames)
-      ++max_buffered_frames_;
+    if (use_complexity_based_buffering_) {
+      if (min_buffered_frames_ < max_buffered_frames_) {
+        min_buffered_frames_ = max_buffered_frames_;
+        DVLOG(2) << "Increased min buffered frames to " << min_buffered_frames_;
+      }
+    } else {
+      const size_t kMaxBufferedFrames = 2 * limits::kMaxVideoFrames;
+      if (min_buffered_frames_ < kMaxBufferedFrames) {
+        ++min_buffered_frames_;
+        DVLOG(2) << "Increased min buffered frames to " << min_buffered_frames_;
+      }
+    }
   }
 }
 
 void VideoRendererImpl::FrameReadyForCopyingToGpuMemoryBuffers(
+    base::TimeTicks read_time,
     VideoFrameStream::Status status,
     const scoped_refptr<VideoFrame>& frame) {
   if (status != VideoFrameStream::OK || IsBeforeStartTime(frame->timestamp())) {
-    VideoRendererImpl::FrameReady(status, frame);
+    VideoRendererImpl::FrameReady(read_time, status, frame);
     return;
   }
 
   DCHECK(frame);
   gpu_memory_buffer_pool_->MaybeCreateHardwareFrame(
-      frame, base::Bind(&VideoRendererImpl::FrameReady,
-                        frame_callback_weak_factory_.GetWeakPtr(), status));
+      frame,
+      base::Bind(&VideoRendererImpl::FrameReady,
+                 frame_callback_weak_factory_.GetWeakPtr(), read_time, status));
 }
 
-void VideoRendererImpl::FrameReady(VideoFrameStream::Status status,
+void VideoRendererImpl::FrameReady(base::TimeTicks read_time,
+                                   VideoFrameStream::Status status,
                                    const scoped_refptr<VideoFrame>& frame) {
   DCHECK(task_runner_->BelongsToCurrentThread());
   base::AutoLock auto_lock(lock_);
@@ -475,6 +494,8 @@ void VideoRendererImpl::FrameReady(VideoFrameStream::Status status,
     return;
   }
 
+  read_durations_.AddSample(tick_clock_->NowTicks() - read_time);
+
   UMA_HISTOGRAM_ENUMERATION("Media.VideoFrame.ColorSpace",
                             ColorSpaceUMAHelper(frame->ColorSpace()),
                             static_cast<int>(VideoFrameColorSpaceUMA::MAX) + 1);
@@ -496,7 +517,15 @@ void VideoRendererImpl::FrameReady(VideoFrameStream::Status status,
     if (!sink_started_ && frame->timestamp() <= start_timestamp_)
       algorithm_->Reset();
 
+    if (!has_playback_met_watch_time_duration_requirement_ &&
+        frame->timestamp() - start_timestamp_ >
+            base::TimeDelta::FromSeconds(
+                limits::kMinimumElapsedWatchTimeSecs)) {
+      has_playback_met_watch_time_duration_requirement_ = true;
+    }
+
     AddReadyFrame_Locked(frame);
+    UpdateMaxBufferedFrames();
   }
 
   // Attempt to purge bad frames in case of underflow or backgrounding.
@@ -546,8 +575,12 @@ bool VideoRendererImpl::HaveEnoughData_Locked() {
   if (received_end_of_stream_)
     return true;
 
-  if (HaveReachedBufferingCap())
+  if (use_complexity_based_buffering_) {
+    if (algorithm_->effective_frames_queued() >= min_buffered_frames_)
+      return true;
+  } else if (HaveReachedBufferingCap()) {
     return true;
+  }
 
   if (was_background_rendering_ && frames_decoded_)
     return true;
@@ -619,11 +652,13 @@ void VideoRendererImpl::AttemptRead_Locked() {
       if (gpu_memory_buffer_pool_) {
         video_frame_stream_->Read(base::Bind(
             &VideoRendererImpl::FrameReadyForCopyingToGpuMemoryBuffers,
-            frame_callback_weak_factory_.GetWeakPtr()));
+            frame_callback_weak_factory_.GetWeakPtr(),
+            tick_clock_->NowTicks()));
       } else {
         video_frame_stream_->Read(
             base::Bind(&VideoRendererImpl::FrameReady,
-                       frame_callback_weak_factory_.GetWeakPtr()));
+                       frame_callback_weak_factory_.GetWeakPtr(),
+                       tick_clock_->NowTicks()));
       }
       return;
     case kUninitialized:
@@ -673,11 +708,14 @@ void VideoRendererImpl::UpdateStats_Locked() {
 bool VideoRendererImpl::HaveReachedBufferingCap() {
   DCHECK(task_runner_->BelongsToCurrentThread());
 
+  if (use_complexity_based_buffering_)
+    return algorithm_->effective_frames_queued() >= max_buffered_frames_;
+
   // When the display rate is less than the frame rate, the effective frames
   // queued may be much smaller than the actual number of frames queued.  Here
   // we ensure that frames_queued() doesn't get excessive.
-  return algorithm_->effective_frames_queued() >= max_buffered_frames_ ||
-         algorithm_->frames_queued() >= 3 * max_buffered_frames_;
+  return algorithm_->effective_frames_queued() >= min_buffered_frames_ ||
+         algorithm_->frames_queued() >= 3 * min_buffered_frames_;
 }
 
 void VideoRendererImpl::StartSink() {
@@ -822,4 +860,77 @@ void VideoRendererImpl::AttemptReadAndCheckForMetadataChanges(
   AttemptRead_Locked();
 }
 
+void VideoRendererImpl::UpdateMaxBufferedFrames() {
+  if (!use_complexity_based_buffering_)
+    return;
+
+  // Only allow extended buffering if we can compute the number frames cover the
+  // duration of a read and playback is actually progressing.
+  const base::TimeDelta frame_duration = algorithm_->average_frame_duration();
+  if (frame_duration.is_zero() || !time_progressing_)
+    return;
+
+  DCHECK(read_durations_.count());
+
+  // If we're background rendering or reads are faster than the frame duration,
+  // our maximum doesn't matter.
+  if (was_background_rendering_ || read_durations_.max() <= frame_duration) {
+    max_buffered_frames_ = min_buffered_frames_;
+    return;
+  }
+
+  // Conversely if the reads are always longer than the frame duration, there's
+  // no point in trying to buffer more, we will not be able to play the video in
+  // real time. If we've already buffered more though, assume this is momentary
+  // and avoid changing |max_buffered_frames_| for now.
+  if (min_buffered_frames_ == max_buffered_frames_ &&
+      frame_duration < read_durations_.Average()) {
+    DVLOG(3) << "Decoding is not fast enough for real time playback.";
+    return;
+  }
+
+  // Only allow extended buffering when there's no memory pressure.
+  if (auto* monitor = base::MemoryPressureMonitor::Get()) {
+    if (monitor->GetCurrentPressureLevel() !=
+        base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE) {
+      max_buffered_frames_ = min_buffered_frames_;
+      return;
+    }
+  }
+
+  // Only allow extended buffering for playbacks which have been running long
+  // enough to be considered as having user engagement.
+  if (!has_playback_met_watch_time_duration_requirement_)
+    return;
+
+  // Maximum number of buffered frames, regardless of the resolution.
+  const size_t kMaxBufferedFrames = 16;
+
+  // Choose a maximum that ensures we have enough frames to cover the length of
+  // the longest seen read duration.
+  //
+  // In a perfect world with absolute future knowledge we want to have a buffer
+  // of (sum(decode_duration) - sum(frame_duration)) / frame_duration. We
+  // don't know the duration though, so the best we can do is make an estimate
+  // based on how long it would take to play out |min_buffered_frames_|.
+  const size_t max_buffered_frames = std::min(
+      min_buffered_frames_ *
+          static_cast<size_t>(std::ceil(
+              (read_durations_.max() - frame_duration).InMillisecondsF() /
+              frame_duration.InMillisecondsF())),
+      kMaxBufferedFrames);
+
+  if (max_buffered_frames_ != max_buffered_frames) {
+    MEDIA_LOG(INFO, media_log_)
+        << "Updating max buffered frames to " << max_buffered_frames
+        << ", average frame duration: " << frame_duration.InMillisecondsF()
+        << "ms, average read duration: " << read_durations_.Average()
+        << "ms, max read duration: " << read_durations_.max().InMillisecondsF()
+        << "ms. [" << min_buffered_frames_ << ", " << max_buffered_frames_
+        << "]";
+  }
+
+  max_buffered_frames_ = max_buffered_frames;
+}
+
 }  // namespace media

media/renderers/video_renderer_impl.h

@@ -82,6 +82,12 @@ class MEDIA_EXPORT VideoRendererImpl
   size_t effective_frames_queued_for_testing() const {
     return algorithm_->effective_frames_queued();
   }
+  size_t min_buffered_frames_for_testing() const {
+    return min_buffered_frames_;
+  }
+  size_t max_buffered_frames_for_testing() const {
+    return max_buffered_frames_;
+  }
 
   // VideoRendererSink::RenderCallback implementation.
   scoped_refptr<VideoFrame> Render(base::TimeTicks deadline_min,
@@ -103,13 +109,17 @@ class MEDIA_EXPORT VideoRendererImpl
   // Callback for |video_frame_stream_| to deliver decoded video frames and
   // report video decoding status. If a frame is available the planes will be
   // copied asynchronously and FrameReady will be called once finished copying.
+  // |read_time| is the time at which this read was started.
   void FrameReadyForCopyingToGpuMemoryBuffers(
+      base::TimeTicks read_time,
       VideoFrameStream::Status status,
       const scoped_refptr<VideoFrame>& frame);
 
   // Callback for |video_frame_stream_| to deliver decoded video frames and
-  // report video decoding status.
-  void FrameReady(VideoFrameStream::Status status,
+  // report video decoding status. |read_time| is the time at which this read
+  // was started.
+  void FrameReady(base::TimeTicks read_time,
+                  VideoFrameStream::Status status,
                   const scoped_refptr<VideoFrame>& frame);
 
   // Helper method for enqueueing a frame to |alogorithm_|.
@@ -186,6 +196,10 @@ class MEDIA_EXPORT VideoRendererImpl
   void AttemptReadAndCheckForMetadataChanges(VideoPixelFormat pixel_format,
                                              const gfx::Size& natural_size);
 
+  // Updates |max_buffered_frames_| based on the current memory pressure level,
+  // |max_read_duration_|, and |time_progressing_|.
+  void UpdateMaxBufferedFrames();
+
   scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
 
   // Sink which calls into VideoRendererImpl via Render() for video frames.  Do
@@ -300,9 +314,27 @@ class MEDIA_EXPORT VideoRendererImpl
   // Indicates if we've painted the first valid frame after StartPlayingFrom().
   bool painted_first_frame_;
 
-  // Current maximum for buffered frames, increases up to a limit upon each
-  // call to OnTimeStopped() when we're in the BUFFERING_HAVE_NOTHING state.
+  // Current minimum and maximum for buffered frames. |min_buffered_frames_| is
+  // the number of frames required to transition from BUFFERING_HAVE_NOTHING to
+  // BUFFERING_HAVE_ENOUGH. |max_buffered_frames_| is the maximum number of
+  // frames the algorithm may queue.
+  //
+  // The maximum is determined by the observed time to decode a frame relative
+  // to the average frame duration. Specifically the maximum observed time for a
+  // call to VideoFrameStream::Read() to yield a new frame.
+  //
+  // During an underflow event, the minimum is set to the maximum. Any increases
+  // are reset upon Flush() to avoid Seek() penalties.
+  size_t min_buffered_frames_;
   size_t max_buffered_frames_;
+  MovingAverage read_durations_;
+
+  // Indicates that the playback has been ongoing for at least
+  // limits::kMinimumElapsedWatchTimeSecs.
+  bool has_playback_met_watch_time_duration_requirement_;
+
+  // Controls enrollment in the complexity based buffering experiment.
+  const bool use_complexity_based_buffering_;
 
   // NOTE: Weak pointers must be invalidated before all other member variables.
   base::WeakPtrFactory<VideoRendererImpl> weak_factory_;