Add support for generic display rates to WebRTC/low-latency renderer

The initial implementation of the low-latency renderer assumed
that the display rate was fixed at 60Hz. This CL adds support
for generic frame rates as well as missed render intervals.

The low-latency renderer algorithm is enabled from WebRTC by
setting the metadata field maximum_composition_delay_in_frames.

This is an experimental feature that is active if and only if the
RTP header extension playout-delay is set to min=0ms and max>0ms.

The feature can be completely disabled by specifying the field trial
WebRTC-LowLatencyRenderer/enabled:false/

Bug: 1138888
Change-Id: Ic64b26331d31aaf8fb8ed4fb8f1056b227cb6eaa
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2527424Reviewed-by: Guido Urdaneta <guidou@chromium.org>
Reviewed-by: Jesse Doherty <jwd@chromium.org>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Commit-Queue: Johannes Kron <kron@chromium.org>
Cr-Commit-Position: refs/heads/master@{#830225}

third_party/blink/renderer/modules/mediastream/low_latency_video_renderer_algorithm.cc

@@ -4,6 +4,9 @@
 
 #include "third_party/blink/renderer/modules/mediastream/low_latency_video_renderer_algorithm.h"
 
+#include <algorithm>
+
+#include "base/metrics/histogram_functions.h"
 #include "media/base/media_log.h"
 
 namespace blink {
@@ -20,98 +23,224 @@ scoped_refptr<media::VideoFrame> LowLatencyVideoRendererAlgorithm::Render(
     base::TimeTicks deadline_max,
     size_t* frames_dropped) {
   DCHECK_LE(deadline_min, deadline_max);
-  // TODO(crbug.com/1138888): Handle the case where the screen refresh rate and
-  // the video frame rate are not the same as well as occasional skips of
-  // rendering intervals.
-
   if (frames_dropped) {
     *frames_dropped = 0;
   }
 
-  if (frame_queue_.size() > 1) {
-    constexpr size_t kMaxQueueSize = 30;
-    if (frame_queue_.size() > kMaxQueueSize) {
-      // The queue has grown too big. Clear all but the last enqueued frame and
-      // enter normal mode.
-      if (frames_dropped) {
-        *frames_dropped += frame_queue_.size() - 1;
-      }
+  stats_.accumulated_queue_length += frame_queue_.size();
+  ++stats_.accumulated_queue_length_count;
+
+  // Determine how many fractional frames that should be rendered based on how
+  // much time has passed since the last renderer deadline.
+  double fractional_frames_to_render = 1.0;
+  if (last_render_deadline_min_) {
+    base::TimeDelta elapsed_time = deadline_min - *last_render_deadline_min_;
+    fractional_frames_to_render =
+        elapsed_time.InMillisecondsF() /
+            average_frame_duration().InMillisecondsF() +
+        unrendered_fractional_frames_;
+  }
 
-      while (frame_queue_.size() > 1) {
-        frame_queue_.pop_front();
-      }
+  size_t number_of_frames_to_render =
+      DetermineModeAndNumberOfFramesToRender(fractional_frames_to_render);
+
+  if (mode_ == Mode::kDrain) {
+    // Render twice as many frames in drain mode.
+    fractional_frames_to_render *= 2.0;
+    stats_.drained_frames +=
+        (fractional_frames_to_render - number_of_frames_to_render);
+    number_of_frames_to_render = fractional_frames_to_render;
+  } else if (ReduceSteadyStateQueue(number_of_frames_to_render)) {
+    // Increment counters to drop one extra frame.
+    ++fractional_frames_to_render;
+    ++number_of_frames_to_render;
+    ++stats_.reduce_steady_state;
+  }
+
+  // Limit |number_of_frames_to_render| to a valid number. +1 in the min
+  // operation to make sure that number_of_frames_to_render is not set to zero
+  // unless it already was zero. |number_of_frames_to_render| > 0 signals that
+  // enough time has passed so that a new frame should be rendered if possible.
+  number_of_frames_to_render =
+      std::min<size_t>(number_of_frames_to_render, frame_queue_.size() + 1);
+
+  // Pop frames that should be dropped.
+  for (size_t i = 1; i < number_of_frames_to_render; ++i) {
+    frame_queue_.pop_front();
+    if (frames_dropped) {
+      ++(*frames_dropped);
+    }
+  }
+
+  if (number_of_frames_to_render > 0) {
+    SelectNextAvailableFrameAndUpdateLastDeadline(deadline_min);
+    unrendered_fractional_frames_ =
+        fractional_frames_to_render - number_of_frames_to_render;
+    stats_.dropped_frames += number_of_frames_to_render - 1;
+    ++stats_.render_frame;
+  }
+
+  if (last_deadline_min_stats_recorded_) {
+    // Record stats for every 100 s, corresponding to roughly 6000 frames in
+    // normal conditions.
+    if (deadline_min - *last_deadline_min_stats_recorded_ >
+        base::TimeDelta::FromSeconds(100)) {
+      RecordAndResetStats();
+      last_deadline_min_stats_recorded_ = deadline_min;
+    }
+  } else {
+    last_deadline_min_stats_recorded_ = deadline_min;
+  }
+  return current_frame_;
+}
+
+void LowLatencyVideoRendererAlgorithm::Reset() {
+  last_render_deadline_min_.reset();
+  current_frame_.reset();
+  frame_queue_.clear();
+  mode_ = Mode::kNormal;
+  unrendered_fractional_frames_ = 0;
+  consecutive_frames_with_back_up_ = 0;
+  stats_ = {};
+}
+
+void LowLatencyVideoRendererAlgorithm::EnqueueFrame(
+    scoped_refptr<media::VideoFrame> frame) {
+  DCHECK(frame);
+  DCHECK(!frame->metadata()->end_of_stream);
+  frame_queue_.push_back(std::move(frame));
+  ++stats_.total_frames;
+}
+
+size_t LowLatencyVideoRendererAlgorithm::DetermineModeAndNumberOfFramesToRender(
+    double fractional_frames_to_render) {
+  // Determine number of entire frames that should be rendered and update
+  // mode_.
+  size_t number_of_frames_to_render = fractional_frames_to_render;
+  if (number_of_frames_to_render < frame_queue_.size()) {
+    // |kMaxQueueSize| is a safety mechanism that should be activated only in
+    // rare circumstances. The drain mode should normally take care of high
+    // queue levels. |kMaxQueueSize| should be set to the lowest possible value
+    // that doesn't shortcut the drain mode. If the number of frames in the
+    // queue is too high, we may run out of buffers in the HW decoder resulting
+    // in a fallback to SW decoder.
+    constexpr size_t kMaxQueueSize = 7;
+    if (frame_queue_.size() > kMaxQueueSize) {
+      // Clear all but the last enqueued frame and enter normal mode.
+      number_of_frames_to_render = frame_queue_.size();
       mode_ = Mode::kNormal;
+      ++stats_.max_size_drop_queue;
     } else {
       // There are several frames in the queue, determine if we should enter
       // drain mode based on queue length and the maximum composition delay that
       // is provided for the last enqueued frame.
-      constexpr size_t kDefaultMaxCompositionDelayInFrames = 10;
-      int max_queue_length = frame_queue_.back()
-                                 ->metadata()
-                                 ->maximum_composition_delay_in_frames.value_or(
-                                     kDefaultMaxCompositionDelayInFrames);
-
-      // The number of frames in the queue is in the range [2, kMaxQueueSize]
-      // due to the conditions that lead up to this point. This means that the
-      // active range of |max_queue_length| is [1, kMaxQueueSize].
-      if (max_queue_length < static_cast<int>(frame_queue_.size()))
+      constexpr size_t kDefaultMaxCompositionDelayInFrames = 6;
+      int max_remaining_queue_length =
+          frame_queue_.back()
+              ->metadata()
+              ->maximum_composition_delay_in_frames.value_or(
+                  kDefaultMaxCompositionDelayInFrames);
+
+      // The number of frames in the queue is in the range
+      // [number_of_frames_to_render + 1, kMaxQueueSize] due to the conditions
+      // that lead up to this point. This means that the active range of
+      // |max_queue_length| is [1, kMaxQueueSize].
+      if (max_remaining_queue_length <
+          static_cast<int>(frame_queue_.size() - number_of_frames_to_render +
+                           1)) {
         mode_ = Mode::kDrain;
-
-      if (mode_ == Mode::kDrain) {
-        // Drop one frame if we're in drain moide.
-        frame_queue_.pop_front();
-        if (frames_dropped) {
-          ++(*frames_dropped);
-        }
+        ++stats_.enter_drain_mode;
       }
     }
   } else if (mode_ == Mode::kDrain) {
     // At most one frame in the queue, exit drain mode.
     mode_ = Mode::kNormal;
   }
+  return number_of_frames_to_render;
+}
 
-  // Reduce steady-state queue length.
-  // Drop one frame if we have observed 10 consecutive rendered frames where
-  // there was a newer frame in the queue that could have been selected.
+bool LowLatencyVideoRendererAlgorithm::ReduceSteadyStateQueue(
+    size_t number_of_frames_to_render) {
+  // Reduce steady state queue if we have observed 10 consecutive rendered
+  // frames where there was a newer frame in the queue that could have been
+  // selected.
+  bool reduce_steady_state_queue = false;
   constexpr int kReduceSteadyStateQueueSizeThreshold = 10;
-  if (mode_ == Mode::kNormal && frame_queue_.size() >= 2) {
-    if (++consecutive_frames_with_back_up_ >
-        kReduceSteadyStateQueueSizeThreshold) {
-      frame_queue_.pop_front();
-      if (frames_dropped) {
-        ++(*frames_dropped);
+  // Has enough time passed so that at least one frame should be rendered?
+  if (number_of_frames_to_render > 0) {
+    // Is there a newer frame in the queue that could have been rendered?
+    if (frame_queue_.size() >= number_of_frames_to_render + 1) {
+      if (++consecutive_frames_with_back_up_ >
+          kReduceSteadyStateQueueSizeThreshold) {
+        reduce_steady_state_queue = true;
+        consecutive_frames_with_back_up_ = 0;
       }
+    } else {
       consecutive_frames_with_back_up_ = 0;
     }
-  } else {
-    consecutive_frames_with_back_up_ = 0;
   }
+  return reduce_steady_state_queue;
+}
 
-  // Select the first frame in the queue to be rendered.
-  if (!frame_queue_.empty()) {
+void LowLatencyVideoRendererAlgorithm::
+    SelectNextAvailableFrameAndUpdateLastDeadline(
+        base::TimeTicks deadline_min) {
+  if (frame_queue_.empty()) {
+    // No frame to render, reset |last_render_deadline_min_| so that the next
+    // available frame is rendered immediately.
+    last_render_deadline_min_.reset();
+    ++stats_.no_new_frame_to_render;
+  } else {
+    // Select the first frame in the queue to be rendered.
     current_frame_.swap(frame_queue_.front());
     frame_queue_.pop_front();
+    last_render_deadline_min_ = deadline_min;
   }
-
-  // Update the current render interval for subroutines.
-  render_interval_ = deadline_max - deadline_min;
-
-  return current_frame_;
 }
 
-void LowLatencyVideoRendererAlgorithm::Reset() {
-  render_interval_ = base::TimeDelta();
-  current_frame_.reset();
-  frame_queue_.clear();
-  mode_ = Mode::kNormal;
-  consecutive_frames_with_back_up_ = 0;
-}
+void LowLatencyVideoRendererAlgorithm::RecordAndResetStats() {
+  // Record UMA stats for sanity check and tuning of the algorithm if needed.
+  std::string uma_prefix = "Media.RtcLowLatencyVideoRenderer";
+  // Total frames count.
+  base::UmaHistogramCounts10000(uma_prefix + ".TotalFrames",
+                                stats_.total_frames);
+  if (stats_.total_frames > 0) {
+    // Dropped frames per mille (=percentage scaled by 10 to get an integer
+    // between 0-1000).
+    base::UmaHistogramCounts1000(
+        uma_prefix + ".DroppedFramesPermille",
+        1000 * stats_.dropped_frames / stats_.total_frames);
+    // Drained frames per mille.
+    base::UmaHistogramCounts1000(
+        uma_prefix + ".DrainedFramesPermille",
+        1000 * stats_.drained_frames / stats_.total_frames);
+  }
 
-void LowLatencyVideoRendererAlgorithm::EnqueueFrame(
-    scoped_refptr<media::VideoFrame> frame) {
-  DCHECK(frame);
-  DCHECK(!frame->metadata()->end_of_stream);
-  frame_queue_.push_back(std::move(frame));
+  // Render frame count.
+  base::UmaHistogramCounts10000(uma_prefix + ".TryToRenderFrameCount",
+                                stats_.render_frame);
+  if (stats_.render_frame > 0) {
+    // No new frame to render per mille.
+    base::UmaHistogramCounts1000(
+        uma_prefix + ".NoNewFrameToRenderPermille",
+        1000 * stats_.no_new_frame_to_render / stats_.render_frame);
+  }
+  // Average queue length x 10 since this is expected to be in the range 1-3
+  // frames.
+  CHECK_GT(stats_.accumulated_queue_length_count, 0);
+  base::UmaHistogramCounts1000(uma_prefix + ".AverageQueueLengthX10",
+                               10 * stats_.accumulated_queue_length /
+                                   stats_.accumulated_queue_length_count);
+  // Enter drain mode count.
+  base::UmaHistogramCounts10000(uma_prefix + ".EnterDrainModeCount",
+                                stats_.enter_drain_mode);
+  // Reduce steady state count.
+  base::UmaHistogramCounts1000(uma_prefix + ".ReduceSteadyStateCount",
+                               stats_.reduce_steady_state);
+  // Max size drop queue count.
+  base::UmaHistogramCounts1000(uma_prefix + ".MaxSizeDropQueueCount",
+                               stats_.max_size_drop_queue);
+  // Clear all stats.
+  stats_ = {};
 }
-
 }  // namespace blink

third_party/blink/renderer/modules/mediastream/low_latency_video_renderer_algorithm.h

@@ -64,15 +64,27 @@ class MODULES_EXPORT LowLatencyVideoRendererAlgorithm {
   }
 
  private:
+  size_t DetermineModeAndNumberOfFramesToRender(
+      double fractional_frames_to_render);
+
+  bool ReduceSteadyStateQueue(size_t number_of_frames_to_render);
+
+  void SelectNextAvailableFrameAndUpdateLastDeadline(
+      base::TimeTicks deadline_min);
+
   scoped_refptr<media::VideoFrame> current_frame_;
 
   // Queue of incoming frames waiting for rendering.
   using VideoFrameQueue = WTF::Deque<scoped_refptr<media::VideoFrame>>;
   VideoFrameQueue frame_queue_;
 
-  // The length of the last deadline interval given to Render(), updated at the
-  // start of Render().
-  base::TimeDelta render_interval_;
+  // Render deadline min for when the last frame was rendered.
+  base::Optional<base::TimeTicks> last_render_deadline_min_;
+
+  // Stores the number of fractional frames that were not rendered as of
+  // |last_render_deadline_min_|. This is needed in case the display refresh
+  // rate is not a multiple of the video stream frame rate.
+  double unrendered_fractional_frames_;
 
   enum class Mode {
     // Render frames at their intended rate.
@@ -87,6 +99,22 @@ class MODULES_EXPORT LowLatencyVideoRendererAlgorithm {
   // The number of consecutive render frames with a post-decode queue back-up
   // (defined as greater than one frame).
   int consecutive_frames_with_back_up_;
+
+  struct Stats {
+    int total_frames;
+    int dropped_frames;
+    int drained_frames;
+    int render_frame;
+    int no_new_frame_to_render;
+    int accumulated_queue_length;
+    int accumulated_queue_length_count;
+    int enter_drain_mode;
+    int reduce_steady_state;
+    int max_size_drop_queue;
+  };
+  Stats stats_;
+  base::Optional<base::TimeTicks> last_deadline_min_stats_recorded_;
+  void RecordAndResetStats();
 };
 
 }  // namespace blink

third_party/blink/renderer/modules/mediastream/low_latency_video_renderer_algorithm_unittest.cc

@@ -30,17 +30,55 @@ class LowLatencyVideoRendererAlgorithmTest : public testing::Test {
     return frame;
   }
 
+  int CreateAndEnqueueFrame(int max_composition_delay_in_frames) {
+    scoped_refptr<media::VideoFrame> frame =
+        CreateFrame(max_composition_delay_in_frames);
+    int unique_id = frame->unique_id();
+    algorithm_.EnqueueFrame(std::move(frame));
+    return unique_id;
+  }
+
   size_t frames_queued() const { return algorithm_.frames_queued(); }
 
   scoped_refptr<media::VideoFrame> RenderAndStep(size_t* frames_dropped) {
     constexpr base::TimeDelta kRenderInterval =
         base::TimeDelta::FromMillisecondsD(1000.0 / 60.0);  // 60fps.
+    return RenderAndStep(frames_dropped, kRenderInterval);
+  }
+
+  scoped_refptr<media::VideoFrame> RenderAndStep(
+      size_t* frames_dropped,
+      base::TimeDelta render_interval) {
     const base::TimeTicks start = current_render_time_;
-    current_render_time_ += kRenderInterval;
+    current_render_time_ += render_interval;
     const base::TimeTicks end = current_render_time_;
     return algorithm_.Render(start, end, frames_dropped);
   }
 
+  void StepUntilJustBeforeNextFrameIsRendered(
+      base::TimeDelta render_interval,
+      base::Optional<int> expected_id = base::nullopt) {
+    // No frame will be rendered until the total render time that has passed is
+    // greater than the frame duration of a frame.
+    base::TimeTicks start_time = current_render_time_;
+    while (current_render_time_ - start_time + render_interval <
+           FrameDuration()) {
+      scoped_refptr<media::VideoFrame> rendered_frame =
+          RenderAndStep(nullptr, render_interval);
+      if (expected_id) {
+        ASSERT_TRUE(rendered_frame);
+        EXPECT_EQ(rendered_frame->unique_id(), *expected_id);
+      } else {
+        EXPECT_FALSE(rendered_frame);
+      }
+    }
+  }
+
+  base::TimeDelta FrameDuration() const {
+    // Assume 60 Hz video content.
+    return base::TimeDelta::FromMillisecondsD(1000.0 / 60.0);
+  }
+
  protected:
   media::VideoFramePool frame_pool_;
   LowLatencyVideoRendererAlgorithm algorithm_;
@@ -58,15 +96,12 @@ TEST_F(LowLatencyVideoRendererAlgorithmTest, Empty) {
   EXPECT_EQ(0u, frames_queued());
 }
 
-TEST_F(LowLatencyVideoRendererAlgorithmTest, NormalMode) {
+TEST_F(LowLatencyVideoRendererAlgorithmTest, NormalMode60Hz) {
   // Every frame rendered.
   constexpr int kNumberOfFrames = 100;
   constexpr int kMaxCompositionDelayInFrames = 6;
   for (int i = 0; i < kNumberOfFrames; ++i) {
-    scoped_refptr<media::VideoFrame> frame =
-        CreateFrame(kMaxCompositionDelayInFrames);
-    int frame_id = frame->unique_id();
-    algorithm_.EnqueueFrame(std::move(frame));
+    int frame_id = CreateAndEnqueueFrame(kMaxCompositionDelayInFrames);
     size_t frames_dropped = 0u;
     scoped_refptr<media::VideoFrame> rendered_frame =
         RenderAndStep(&frames_dropped);
@@ -76,16 +111,150 @@ TEST_F(LowLatencyVideoRendererAlgorithmTest, NormalMode) {
   }
 }
 
-TEST_F(LowLatencyVideoRendererAlgorithmTest, EnterDrainMode) {
+// Half frame rate (30Hz playing back 60Hz video)
+TEST_F(LowLatencyVideoRendererAlgorithmTest, NormalMode30Hz) {
+  constexpr base::TimeDelta kRenderInterval =
+      base::TimeDelta::FromMillisecondsD(1000.0 / 30.0);  // 30Hz.
+  constexpr int kMaxCompositionDelayInFrames = 6;
+
+  constexpr size_t kNumberOfFrames = 120;
+  for (size_t i = 0; i < kNumberOfFrames; ++i) {
+    scoped_refptr<media::VideoFrame> frame;
+    size_t expected_frames_dropped = 0;
+    if (i > 0) {
+      // This frame will be dropped.
+      CreateAndEnqueueFrame(kMaxCompositionDelayInFrames);
+      ++expected_frames_dropped;
+    }
+
+    int last_id = CreateAndEnqueueFrame(kMaxCompositionDelayInFrames);
+
+    size_t frames_dropped = 0;
+    scoped_refptr<media::VideoFrame> rendered_frame =
+        RenderAndStep(&frames_dropped, kRenderInterval);
+    ASSERT_TRUE(rendered_frame);
+    EXPECT_EQ(rendered_frame->unique_id(), last_id);
+    EXPECT_EQ(frames_dropped, expected_frames_dropped);
+  }
+  // Only the currently rendered frame is in the queue.
+  EXPECT_EQ(frames_queued(), 1u);
+}
+
+// Fractional frame rate (90Hz playing back 60Hz video)
+TEST_F(LowLatencyVideoRendererAlgorithmTest, NormalMode90Hz) {
+  constexpr base::TimeDelta kRenderInterval =
+      base::TimeDelta::FromMillisecondsD(1000.0 / 90.0);  // 90Hz.
+  constexpr int kMaxCompositionDelayInFrames = 6;
+
+  CreateAndEnqueueFrame(kMaxCompositionDelayInFrames);
+
+  constexpr size_t kNumberOfFramesToSubmit = 100;
+  size_t submitted_frames = 0;
+  while (submitted_frames < kNumberOfFramesToSubmit) {
+    // In each while iteration: Enqueue two new frames (60Hz) and render three
+    // times (90Hz).
+    for (int i = 0; i < 2; ++i) {
+      size_t frames_dropped = 0;
+      scoped_refptr<media::VideoFrame> rendered_frame =
+          RenderAndStep(&frames_dropped, kRenderInterval);
+      ASSERT_TRUE(rendered_frame);
+      EXPECT_EQ(frames_dropped, 0u);
+      // Enqueue a new frame.
+      CreateAndEnqueueFrame(kMaxCompositionDelayInFrames);
+      ++submitted_frames;
+    }
+    size_t frames_dropped = 0;
+    scoped_refptr<media::VideoFrame> rendered_frame =
+        RenderAndStep(&frames_dropped, kRenderInterval);
+    ASSERT_TRUE(rendered_frame);
+    EXPECT_EQ(frames_dropped, 0u);
+  }
+}
+
+// Double frame rate (120Hz playing back 60Hz video)
+TEST_F(LowLatencyVideoRendererAlgorithmTest, NormalMode120Hz) {
+  constexpr base::TimeDelta kRenderInterval =
+      base::TimeDelta::FromMillisecondsD(1000.0 / 120.0);  // 120Hz.
+  constexpr int kMaxCompositionDelayInFrames = 6;
+
+  // Add one initial frame.
+  int last_id = CreateAndEnqueueFrame(kMaxCompositionDelayInFrames);
+
+  constexpr size_t kNumberOfFrames = 120;
+  for (size_t i = 0; i < kNumberOfFrames; ++i) {
+    size_t frames_dropped = 0;
+    scoped_refptr<media::VideoFrame> rendered_frame =
+        RenderAndStep(&frames_dropped, kRenderInterval);
+    ASSERT_TRUE(rendered_frame);
+    int rendered_frame_id = last_id;
+    EXPECT_EQ(rendered_frame->unique_id(), rendered_frame_id);
+
+    last_id = CreateAndEnqueueFrame(kMaxCompositionDelayInFrames);
+
+    // The same frame should be rendered.
+    rendered_frame = RenderAndStep(&frames_dropped, kRenderInterval);
+    ASSERT_TRUE(rendered_frame);
+    EXPECT_EQ(rendered_frame->unique_id(), rendered_frame_id);
+  }
+  // Two frames in the queue including the last rendered frame.
+  EXPECT_EQ(frames_queued(), 2u);
+}
+
+// Super high display rate (600Hz playing back 60Hz video)
+TEST_F(LowLatencyVideoRendererAlgorithmTest, NormalMode600Hz) {
+  constexpr base::TimeDelta kRenderInterval =
+      base::TimeDelta::FromMillisecondsD(1000.0 / 600.0 + 1.0e-3);  // 600Hz.
+  constexpr int kMaxCompositionDelayInFrames = 6;
+
+  // Add one initial frame.
+  int last_id = CreateAndEnqueueFrame(kMaxCompositionDelayInFrames);
+
+  constexpr size_t kNumberOfFrames = 120;
+  for (size_t i = 0; i < kNumberOfFrames; ++i) {
+    size_t frames_dropped = 0;
+    scoped_refptr<media::VideoFrame> rendered_frame =
+        RenderAndStep(&frames_dropped, kRenderInterval);
+    ASSERT_TRUE(rendered_frame);
+    int rendered_frame_id = last_id;
+    EXPECT_EQ(rendered_frame->unique_id(), rendered_frame_id);
+
+    last_id = CreateAndEnqueueFrame(kMaxCompositionDelayInFrames);
+
+    // The same frame should be rendered 9 times.
+    StepUntilJustBeforeNextFrameIsRendered(kRenderInterval, rendered_frame_id);
+  }
+  // Two frames in the queue including the last rendered frame.
+  EXPECT_EQ(frames_queued(), 2u);
+}
+
+TEST_F(LowLatencyVideoRendererAlgorithmTest,
+       DropAllFramesIfQueueExceedsMaxSize) {
+  // Create an initial queue of 60 frames.
+  constexpr int kMaxCompositionDelayInFrames = 6;
+  constexpr size_t kInitialQueueSize = 60;
+  int last_id = 0;
+  for (size_t i = 0; i < kInitialQueueSize; ++i) {
+    last_id = CreateAndEnqueueFrame(kMaxCompositionDelayInFrames);
+  }
+  EXPECT_EQ(frames_queued(), kInitialQueueSize);
+
+  // Last submitted frame should be rendered.
+  size_t frames_dropped = 0;
+  scoped_refptr<media::VideoFrame> rendered_frame =
+      RenderAndStep(&frames_dropped);
+  ASSERT_TRUE(rendered_frame);
+  EXPECT_EQ(frames_dropped, kInitialQueueSize - 1);
+  EXPECT_EQ(rendered_frame->unique_id(), last_id);
+}
+
+TEST_F(LowLatencyVideoRendererAlgorithmTest, EnterDrainMode60Hz) {
   // Enter drain mode when more than 6 frames are in the queue.
   constexpr int kMaxCompositionDelayInFrames = 6;
   constexpr int kNumberOfFramesSubmitted = kMaxCompositionDelayInFrames + 1;
   std::queue<int> enqueued_frame_ids;
   for (int i = 0; i < kNumberOfFramesSubmitted; ++i) {
-    scoped_refptr<media::VideoFrame> frame =
-        CreateFrame(kMaxCompositionDelayInFrames);
-    enqueued_frame_ids.push(frame->unique_id());
-    algorithm_.EnqueueFrame(std::move(frame));
+    enqueued_frame_ids.push(
+        CreateAndEnqueueFrame(kMaxCompositionDelayInFrames));
   }
   // Every other frame will be rendered until there's one frame in the queue.
   int processed_frames_count = 0;
@@ -112,16 +281,14 @@ TEST_F(LowLatencyVideoRendererAlgorithmTest, EnterDrainMode) {
   EXPECT_EQ(enqueued_frame_ids.size(), 0u);
 }
 
-TEST_F(LowLatencyVideoRendererAlgorithmTest, ExitDrainMode) {
+TEST_F(LowLatencyVideoRendererAlgorithmTest, ExitDrainMode60Hz) {
   // Enter drain mode when more than 6 frames are in the queue.
   constexpr int kMaxCompositionDelayInFrames = 6;
   int number_of_frames_submitted = kMaxCompositionDelayInFrames + 1;
   std::queue<int> enqueued_frame_ids;
   for (int i = 0; i < number_of_frames_submitted; ++i) {
-    scoped_refptr<media::VideoFrame> frame =
-        CreateFrame(kMaxCompositionDelayInFrames);
-    enqueued_frame_ids.push(frame->unique_id());
-    algorithm_.EnqueueFrame(std::move(frame));
+    enqueued_frame_ids.push(
+        CreateAndEnqueueFrame(kMaxCompositionDelayInFrames));
   }
 
   // Every other frame will be rendered until there's one frame in the queue.
@@ -136,10 +303,8 @@ TEST_F(LowLatencyVideoRendererAlgorithmTest, ExitDrainMode) {
     EXPECT_EQ(rendered_frame->unique_id(), enqueued_frame_ids.front());
     enqueued_frame_ids.pop();
     // Enqueue a new frame.
-    scoped_refptr<media::VideoFrame> frame =
-        CreateFrame(kMaxCompositionDelayInFrames);
-    enqueued_frame_ids.push(frame->unique_id());
-    algorithm_.EnqueueFrame(std::move(frame));
+    enqueued_frame_ids.push(
+        CreateAndEnqueueFrame(kMaxCompositionDelayInFrames));
     ++number_of_frames_submitted;
     processed_frames_count += 1 + frames_dropped;
   }
@@ -154,23 +319,61 @@ TEST_F(LowLatencyVideoRendererAlgorithmTest, ExitDrainMode) {
     EXPECT_EQ(frames_dropped, 0u);
     EXPECT_EQ(rendered_frame->unique_id(), enqueued_frame_ids.front());
     enqueued_frame_ids.pop();
-    scoped_refptr<media::VideoFrame> frame =
-        CreateFrame(kMaxCompositionDelayInFrames);
-    enqueued_frame_ids.push(frame->unique_id());
-    algorithm_.EnqueueFrame(std::move(frame));
+    enqueued_frame_ids.push(
+        CreateAndEnqueueFrame(kMaxCompositionDelayInFrames));
+  }
+}
+
+// Double Rate Drain (120Hz playing back 60Hz video in DRAIN mode)
+TEST_F(LowLatencyVideoRendererAlgorithmTest, EnterDrainMode120Hz) {
+  constexpr base::TimeDelta kRenderInterval =
+      base::TimeDelta::FromMillisecondsD(1000.0 / 120.0);  // 120Hz.
+  // Enter drain mode when more than 6 frames are in the queue.
+  constexpr int kMaxCompositionDelayInFrames = 6;
+
+  // Process one frame to initialize the algorithm.
+  CreateAndEnqueueFrame(kMaxCompositionDelayInFrames);
+  EXPECT_TRUE(RenderAndStep(nullptr, kRenderInterval));
+
+  constexpr int kNumberOfFramesSubmitted = kMaxCompositionDelayInFrames + 1;
+  std::queue<int> enqueued_frame_ids;
+  for (int i = 0; i < kNumberOfFramesSubmitted; ++i) {
+    enqueued_frame_ids.push(
+        CreateAndEnqueueFrame(kMaxCompositionDelayInFrames));
+  }
+  // Every frame will be rendered at double rate until there's one frame in the
+  // queue.
+  int processed_frames_count = 0;
+  while (processed_frames_count < kNumberOfFramesSubmitted - 1) {
+    size_t frames_dropped = 0;
+    scoped_refptr<media::VideoFrame> rendered_frame =
+        RenderAndStep(&frames_dropped, kRenderInterval);
+    ASSERT_TRUE(rendered_frame);
+    EXPECT_EQ(frames_dropped, 0u);
+    EXPECT_EQ(rendered_frame->unique_id(), enqueued_frame_ids.front());
+    enqueued_frame_ids.pop();
+    processed_frames_count += 1 + frames_dropped;
   }
+
+  // One more frame to render.
+  size_t frames_dropped = 0;
+  scoped_refptr<media::VideoFrame> rendered_frame =
+      RenderAndStep(&frames_dropped, kRenderInterval);
+  ASSERT_TRUE(rendered_frame);
+  EXPECT_EQ(frames_dropped, 0u);
+  EXPECT_EQ(rendered_frame->unique_id(), enqueued_frame_ids.front());
+  enqueued_frame_ids.pop();
+  EXPECT_EQ(enqueued_frame_ids.size(), 0u);
 }
 
-TEST_F(LowLatencyVideoRendererAlgorithmTest, SteadyStateQueueReduction) {
-  // Create an initial queue of 8 frames.
-  constexpr int kMaxCompositionDelayInFrames = 10;
-  constexpr size_t kInitialQueueSize = 8;
+TEST_F(LowLatencyVideoRendererAlgorithmTest, SteadyStateQueueReduction60Hz) {
+  // Create an initial queue of 5 frames.
+  constexpr int kMaxCompositionDelayInFrames = 6;
+  constexpr size_t kInitialQueueSize = 5;
   std::queue<int> enqueued_frame_ids;
   for (size_t i = 0; i < kInitialQueueSize; ++i) {
-    scoped_refptr<media::VideoFrame> frame =
-        CreateFrame(kMaxCompositionDelayInFrames);
-    enqueued_frame_ids.push(frame->unique_id());
-    algorithm_.EnqueueFrame(std::move(frame));
+    enqueued_frame_ids.push(
+        CreateAndEnqueueFrame(kMaxCompositionDelayInFrames));
   }
   EXPECT_EQ(frames_queued(), kInitialQueueSize);
 
@@ -197,11 +400,8 @@ TEST_F(LowLatencyVideoRendererAlgorithmTest, SteadyStateQueueReduction) {
 
     EXPECT_EQ(rendered_frame->unique_id(), enqueued_frame_ids.front());
     enqueued_frame_ids.pop();
-    // Enqueue a new frame.
-    scoped_refptr<media::VideoFrame> frame =
-        CreateFrame(kMaxCompositionDelayInFrames);
-    enqueued_frame_ids.push(frame->unique_id());
-    algorithm_.EnqueueFrame(std::move(frame));
+    enqueued_frame_ids.push(
+        CreateAndEnqueueFrame(kMaxCompositionDelayInFrames));
   }
 
   // Steady state queue should now have been reduced to one frame + the current
@@ -209,27 +409,106 @@ TEST_F(LowLatencyVideoRendererAlgorithmTest, SteadyStateQueueReduction) {
   EXPECT_EQ(frames_queued(), 2u);
 }
 
-TEST_F(LowLatencyVideoRendererAlgorithmTest,
-       DropAllFramesIfQueueExceedsMaxSize) {
-  // Create an initial queue of 60 frames.
-  constexpr int kMaxCompositionDelayInFrames = 10;
-  constexpr size_t kInitialQueueSize = 60;
-  int last_id = 0;
+// Fractional rate, steady state queue reduction.
+TEST_F(LowLatencyVideoRendererAlgorithmTest, SteadyStateReduction90Hz) {
+  constexpr base::TimeDelta kRenderInterval =
+      base::TimeDelta::FromMillisecondsD(1000.0 / 90.0);  // 90Hz.
+
+  // Create an initial queue of 5 frames.
+  constexpr int kMaxCompositionDelayInFrames = 6;
+  constexpr size_t kInitialQueueSize = 5;
   for (size_t i = 0; i < kInitialQueueSize; ++i) {
-    scoped_refptr<media::VideoFrame> frame =
-        CreateFrame(kMaxCompositionDelayInFrames);
-    last_id = frame->unique_id();
-    algorithm_.EnqueueFrame(std::move(frame));
+    CreateAndEnqueueFrame(kMaxCompositionDelayInFrames);
   }
   EXPECT_EQ(frames_queued(), kInitialQueueSize);
 
-  // Last submitted frame should be rendered.
-  size_t frames_dropped = 0;
-  scoped_refptr<media::VideoFrame> rendered_frame =
-      RenderAndStep(&frames_dropped);
-  ASSERT_TRUE(rendered_frame);
-  EXPECT_EQ(frames_dropped, kInitialQueueSize - 1);
-  EXPECT_EQ(rendered_frame->unique_id(), last_id);
+  constexpr size_t kNumberOfFramesToSubmit = 100;
+  constexpr int kMinimumNumberOfFramesBetweenDrops = 8;
+  int processed_frames_since_last_frame_drop = 0;
+  size_t submitted_frames = kInitialQueueSize;
+  while (submitted_frames < kNumberOfFramesToSubmit) {
+    // Every frame will be rendered with occasional frame drops to reduce the
+    // steady state queue.
+
+    // In each while iteration: Enqueue two new frames (60Hz) and render three
+    // times (90Hz).
+    for (int i = 0; i < 2; ++i) {
+      size_t frames_dropped = 0;
+      scoped_refptr<media::VideoFrame> rendered_frame =
+          RenderAndStep(&frames_dropped, kRenderInterval);
+      ASSERT_TRUE(rendered_frame);
+      if (frames_dropped > 0) {
+        ASSERT_EQ(frames_dropped, 1u);
+        EXPECT_GE(processed_frames_since_last_frame_drop,
+                  kMinimumNumberOfFramesBetweenDrops);
+        processed_frames_since_last_frame_drop = 0;
+      } else {
+        ++processed_frames_since_last_frame_drop;
+      }
+      CreateAndEnqueueFrame(kMaxCompositionDelayInFrames);
+      ++submitted_frames;
+    }
+    size_t frames_dropped = 0;
+    scoped_refptr<media::VideoFrame> rendered_frame =
+        RenderAndStep(&frames_dropped, kRenderInterval);
+    ASSERT_TRUE(rendered_frame);
+    if (frames_dropped > 0) {
+      ASSERT_EQ(frames_dropped, 1u);
+      EXPECT_GE(processed_frames_since_last_frame_drop,
+                kMinimumNumberOfFramesBetweenDrops);
+      processed_frames_since_last_frame_drop = 0;
+    } else {
+      ++processed_frames_since_last_frame_drop;
+    }
+  }
+
+  // Steady state queue should now have been reduced to one frame + the current
+  // frame that is also counted.
+  EXPECT_EQ(frames_queued(), 2u);
+}
+
+TEST_F(LowLatencyVideoRendererAlgorithmTest,
+       RenderFrameImmediatelyAfterOutage) {
+  constexpr base::TimeDelta kRenderInterval =
+      base::TimeDelta::FromMillisecondsD(1000.0 / 600.0 + 1.0e-3);  // 600Hz.
+  constexpr int kMaxCompositionDelayInFrames = 6;
+
+  for (int outage_length = 0; outage_length < 100; ++outage_length) {
+    algorithm_.Reset();
+
+    // Process one frame to get the algorithm initialized.
+    CreateAndEnqueueFrame(kMaxCompositionDelayInFrames);
+    scoped_refptr<media::VideoFrame> rendered_frame =
+        RenderAndStep(nullptr, kRenderInterval);
+    ASSERT_TRUE(rendered_frame);
+    int frame_id_0 = rendered_frame->unique_id();
+    StepUntilJustBeforeNextFrameIsRendered(kRenderInterval,
+                                           rendered_frame->unique_id());
+
+    for (int i = 0; i < outage_length; ++i) {
+      // Try to render, but no new frame has been enqueued so the last frame
+      // will be rendered again.
+      rendered_frame = RenderAndStep(nullptr, kRenderInterval);
+      ASSERT_TRUE(rendered_frame);
+      EXPECT_EQ(rendered_frame->unique_id(), frame_id_0);
+    }
+
+    // Enqueue two frames.
+    int frame_id_1 = CreateAndEnqueueFrame(kMaxCompositionDelayInFrames);
+    int frame_id_2 = CreateAndEnqueueFrame(kMaxCompositionDelayInFrames);
+
+    // The first submitted frame should be rendered.
+    rendered_frame = RenderAndStep(nullptr, kRenderInterval);
+    ASSERT_TRUE(rendered_frame);
+    EXPECT_EQ(rendered_frame->unique_id(), frame_id_1);
+    // The same frame is rendered for 9 more render intervals.
+    StepUntilJustBeforeNextFrameIsRendered(kRenderInterval, frame_id_1);
+
+    // The next frame is rendered.
+    rendered_frame = RenderAndStep(nullptr, kRenderInterval);
+    ASSERT_TRUE(rendered_frame);
+    EXPECT_EQ(rendered_frame->unique_id(), frame_id_2);
+  }
 }
 
 }  // namespace blink

tools/metrics/histograms/histograms_xml/media/histograms.xml

@@ -3042,6 +3042,103 @@ reviews. Googlers can read more about this at go/gwsq-gerrit.
   <summary>Video width while remoting content.</summary>
 </histogram>
 
+<histogram name="Media.RtcLowLatencyVideoRenderer.AverageQueueLengthX10"
+    units="frames" expires_after="2021-05-31">
+  <owner>kron@chromium.org</owner>
+  <owner>webrtc-video@google.com</owner>
+  <summary>
+    10 times the average queue length when Render() is called in the RTC
+    low-latency video renderer. Repeatedly measured with a period of 100 s for
+    as long as the stream is active.
+  </summary>
+</histogram>
+
+<histogram name="Media.RtcLowLatencyVideoRenderer.DrainedFramesPermille"
+    units="permille" expires_after="2021-05-31">
+  <owner>kron@chromium.org</owner>
+  <owner>webrtc-video@google.com</owner>
+  <summary>
+    Per mille of frames drained in the RTC low-latency video renderer.
+    Repeatedly measured with a period of 100 s for as long as the stream is
+    active.
+  </summary>
+</histogram>
+
+<histogram name="Media.RtcLowLatencyVideoRenderer.DroppedFramesPermille"
+    units="permille" expires_after="2021-05-31">
+  <owner>kron@chromium.org</owner>
+  <owner>webrtc-video@google.com</owner>
+  <summary>
+    Per mille of frames dropped in the RTC low-latency video renderer.
+    Repeatedly measured with a period of 100 s for as long as the stream is
+    active.
+  </summary>
+</histogram>
+
+<histogram name="Media.RtcLowLatencyVideoRenderer.EnterDrainModeCount"
+    units="count" expires_after="2021-05-31">
+  <owner>kron@chromium.org</owner>
+  <owner>webrtc-video@google.com</owner>
+  <summary>
+    Number of times drain mode is entered. Repeatedly measured with a period of
+    100 s for as long as the stream is active.
+  </summary>
+</histogram>
+
+<histogram name="Media.RtcLowLatencyVideoRenderer.MaxSizeDropQueueCount"
+    units="count" expires_after="2021-05-31">
+  <owner>kron@chromium.org</owner>
+  <owner>webrtc-video@google.com</owner>
+  <summary>
+    Number of times the maximum queue size is exceeded and the entire queue is
+    dropped. Repeatedly measured with a period of 100 s for as long as the
+    stream is active.
+  </summary>
+</histogram>
+
+<histogram name="Media.RtcLowLatencyVideoRenderer.NoNewFrameToRenderPermille"
+    units="permille" expires_after="2021-05-31">
+  <owner>kron@chromium.org</owner>
+  <owner>webrtc-video@google.com</owner>
+  <summary>
+    Per mille of render times where the queue is empty and there is no new frame
+    to render. Repeatedly measured with a period of 100 s for as long as the
+    stream is active.
+  </summary>
+</histogram>
+
+<histogram name="Media.RtcLowLatencyVideoRenderer.ReduceSteadyStateCount"
+    units="count" expires_after="2021-05-31">
+  <owner>kron@chromium.org</owner>
+  <owner>webrtc-video@google.com</owner>
+  <summary>
+    Number of times the reduce steady state queue feature is activated.
+    Repeatedly measured with a period of 100 s for as long as the stream is
+    active.
+  </summary>
+</histogram>
+
+<histogram name="Media.RtcLowLatencyVideoRenderer.TotalFrames" units="frames"
+    expires_after="2021-05-31">
+  <owner>kron@chromium.org</owner>
+  <owner>webrtc-video@google.com</owner>
+  <summary>
+    Total number of frames enqueued to the RTC low-latency video renderer.
+    Repeatedly measured with a period of 100 s for as long as the stream is
+    active.
+  </summary>
+</histogram>
+
+<histogram name="Media.RtcLowLatencyVideoRenderer.TryToRenderFrameCount"
+    units="count" expires_after="2021-05-31">
+  <owner>kron@chromium.org</owner>
+  <owner>webrtc-video@google.com</owner>
+  <summary>
+    Number of times we try to render a new frame. Repeatedly measured with a
+    period of 100 s for as long as the stream is active.
+  </summary>
+</histogram>
+
 <histogram name="Media.RTCVideoDecoderError" enum="VideoDecodeAcceleratorError"
     expires_after="M82">
   <owner>posciak@chromium.org</owner>