Bresenham video frame cadence algorithm

Choosing how many render intervals a given frame needs to be shown
is very much like drawing a line on a 2D pixel grid.
That’s why this algorithm is inspired by Bresenham's line algorithm,
although it is simpler for using FP arithmetic.

It's gated by a feature switch and intended for lab tests and
later maybe a canary\beta experiment.

Bug: 1042111
Change-Id: I1d4e54252d904dd80f37e4542aa4b6950a527c4c
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1994546Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Reviewed-by: Frank Liberato <liberato@chromium.org>
Commit-Queue: Eugene Zemtsov <eugene@chromium.org>
Cr-Commit-Position: refs/heads/master@{#732281}

media/base/media_switches.cc

@@ -652,6 +652,11 @@ const base::Feature kInternalMediaSession {
 const base::Feature kUseFakeDeviceForMediaStream{
     "use-fake-device-for-media-stream", base::FEATURE_DISABLED_BY_DEFAULT};
 
+// Makes VideoCadenceEstimator use Bresenham-like algorithm for frame cadence
+// estimations.
+const base::Feature kBresenhamCadence{"BresenhamCadence",
+                                      base::FEATURE_DISABLED_BY_DEFAULT};
+
 bool IsVideoCaptureAcceleratedJpegDecodingEnabled() {
   if (base::CommandLine::ForCurrentProcess()->HasSwitch(
           switches::kDisableAcceleratedMjpegDecode)) {

media/base/media_switches.h

@@ -105,6 +105,7 @@ MEDIA_EXPORT extern const base::Feature kAutoplayIgnoreWebAudio;
 MEDIA_EXPORT extern const base::Feature kAutoplayDisableSettings;
 MEDIA_EXPORT extern const base::Feature kAutoplayWhitelistSettings;
 MEDIA_EXPORT extern const base::Feature kBackgroundVideoPauseOptimization;
+MEDIA_EXPORT extern const base::Feature kBresenhamCadence;
 MEDIA_EXPORT extern const base::Feature kD3D11LimitTo11_0;
 MEDIA_EXPORT extern const base::Feature kD3D11PrintCodecOnCrash;
 MEDIA_EXPORT extern const base::Feature kD3D11VideoDecoder;

media/filters/video_cadence_estimator.cc

@@ -8,9 +8,11 @@
 #include <cmath>
 #include <iterator>
 #include <limits>
+#include <numeric>
 #include <string>
 
 #include "base/metrics/histogram_macros.h"
+#include "media/base/media_switches.h"
 
 namespace media {
 
@@ -87,6 +89,11 @@ void VideoCadenceEstimator::Reset() {
   pending_cadence_.clear();
   cadence_changes_ = render_intervals_cadence_held_ = 0;
   first_update_call_ = true;
+
+  bm_.use_bresenham_cadence_ =
+      base::FeatureList::IsEnabled(media::kBresenhamCadence);
+  bm_.perfect_cadence_.reset();
+  bm_.frame_index_shift_ = 0;
 }
 
 bool VideoCadenceEstimator::UpdateCadenceEstimate(
@@ -103,6 +110,9 @@ bool VideoCadenceEstimator::UpdateCadenceEstimate(
     is_variable_frame_rate_ = false;
   }
 
+  if (bm_.use_bresenham_cadence_)
+    return UpdateBresenhamCadenceEstimate(render_interval, frame_duration);
+
   // Variable FPS detected, turn off Cadence by force.
   if (is_variable_frame_rate_) {
     render_intervals_cadence_held_ = 0;
@@ -171,9 +181,91 @@ bool VideoCadenceEstimator::UpdateCadenceEstimate(
 
 int VideoCadenceEstimator::GetCadenceForFrame(uint64_t frame_number) const {
   DCHECK(has_cadence());
+  if (bm_.use_bresenham_cadence_) {
+    double cadence = *bm_.perfect_cadence_;
+    auto index = frame_number + bm_.frame_index_shift_;
+    auto result = static_cast<uint64_t>(cadence * (index + 1)) -
+                  static_cast<uint64_t>(cadence * index);
+    DCHECK(frame_number > 0 || result > 0);
+    return result;
+  }
   return cadence_[frame_number % cadence_.size()];
 }
 
+/* List of tests that are expected to fail when media::kBresenhamCadence
+   is enabled.
+   - VideoRendererAlgorithmTest.BestFrameByCadenceOverdisplayedForDrift
+     Reason: Bresenham cadence does not exhibit innate drift.
+   - VideoRendererAlgorithmTest.CadenceCalculations
+     Reason: The test inspects an internal data structures of the current alg.
+   - VideoRendererAlgorithmTest.VariablePlaybackRateCadence
+     Reason: The test assumes that cadence algorithm should fail for playback
+     rate of 3.15. Bresenham alg works fine.
+
+   - VideoCadenceEstimatorTest.CadenceCalculationWithLargeDeviation
+   - VideoCadenceEstimatorTest.CadenceCalculationWithLargeDrift
+   - VideoCadenceEstimatorTest.CadenceCalculations
+   - VideoCadenceEstimatorTest.CadenceHystersisPreventsOscillation
+   - VideoCadenceEstimatorTest.CadenceVariesWithAcceptableDrift
+   - VideoCadenceEstimatorTest.CadenceVariesWithAcceptableGlitchTime
+     Reason: These tests inspects an internal data structures of the current
+     algorithm.
+*/
+bool VideoCadenceEstimator::UpdateBresenhamCadenceEstimate(
+    base::TimeDelta render_interval,
+    base::TimeDelta frame_duration) {
+  if (is_variable_frame_rate_) {
+    if (bm_.perfect_cadence_.has_value()) {
+      bm_.perfect_cadence_.reset();
+      return true;
+    }
+    return false;
+  }
+
+  if (++render_intervals_cadence_held_ * render_interval <
+      cadence_hysteresis_threshold_) {
+    return false;
+  }
+
+  double current_cadence = bm_.perfect_cadence_.value_or(0.0);
+  double new_cadence =
+      frame_duration.InMicrosecondsF() / render_interval.InMicrosecondsF();
+  DCHECK(new_cadence >= 0.0);
+
+  double cadence_relative_diff = std::abs(current_cadence - new_cadence) /
+                                 std::max(current_cadence, new_cadence);
+
+  // Ignore tiny changes in cadence, as they are most likely just noise.
+  // Let's use a threshold of 0.08%, which is slightly less than NTSC frame
+  // rate adjustment coefficient.
+  constexpr double kCadenceRoundingError = 0.0008;
+  if (cadence_relative_diff <= kCadenceRoundingError)
+    return false;
+
+  bm_.perfect_cadence_ = new_cadence;
+  if (render_interval > frame_duration) {
+    // When display refresh rate is lower than the video frame rate,
+    // not all frames can be shown. But we want to make sure that the very
+    // first frame is shown. That's why frame indexes are shifted by this
+    // much to make sure that the cadence sequence always has 1 in the
+    // beginning.
+    bm_.frame_index_shift_ = (render_interval.InMicroseconds() - 1) /
+                             frame_duration.InMicroseconds();
+  } else {
+    // It can be 0 (or anything), but it makes the output look more like
+    // an output of the current cadence algorithm.
+    bm_.frame_index_shift_ = 1;
+  }
+
+  DVLOG(1) << "Cadence switch"
+           << " perfect_cadence: " << new_cadence
+           << " frame_index_shift: " << bm_.frame_index_shift_
+           << " cadence_relative_diff: " << cadence_relative_diff
+           << " cadence_held: " << render_intervals_cadence_held_;
+  render_intervals_cadence_held_ = 0;
+  return true;
+}
+
 VideoCadenceEstimator::Cadence VideoCadenceEstimator::CalculateCadence(
     base::TimeDelta render_interval,
     base::TimeDelta frame_duration,
@@ -259,4 +351,14 @@ std::string VideoCadenceEstimator::CadenceToString(
   return os.str();
 }
 
+double VideoCadenceEstimator::avg_cadence_for_testing() const {
+  if (!has_cadence())
+    return 0.0;
+  if (bm_.use_bresenham_cadence_)
+    return bm_.perfect_cadence_.value();
+
+  int sum = std::accumulate(begin(cadence_), end(cadence_), 0);
+  return static_cast<double>(sum) / cadence_.size();
+}
+
 }  // namespace media

media/filters/video_cadence_estimator.h

@@ -11,6 +11,7 @@
 #include <vector>
 
 #include "base/macros.h"
+#include "base/optional.h"
 #include "base/time/time.h"
 #include "media/base/media_export.h"
 
@@ -96,7 +97,10 @@ class MEDIA_EXPORT VideoCadenceEstimator {
                              base::TimeDelta max_acceptable_drift);
 
   // Returns true if a useful cadence was found.
-  bool has_cadence() const { return !cadence_.empty(); }
+  bool has_cadence() const {
+    return bm_.use_bresenham_cadence_ ? bm_.perfect_cadence_.has_value()
+                                      : !cadence_.empty();
+  }
 
   // Given a |frame_number|, where zero is the most recently rendered frame,
   // returns the ideal cadence for that frame.
@@ -113,6 +117,7 @@ class MEDIA_EXPORT VideoCadenceEstimator {
     cadence_hysteresis_threshold_ = threshold;
   }
 
+  double avg_cadence_for_testing() const;
   size_t cadence_size_for_testing() const { return cadence_.size(); }
   std::string GetCadenceForTesting() const { return CadenceToString(cadence_); }
 
@@ -131,6 +136,9 @@ class MEDIA_EXPORT VideoCadenceEstimator {
   // "[a: b: ...: z]".
   std::string CadenceToString(const Cadence& cadence) const;
 
+  bool UpdateBresenhamCadenceEstimate(base::TimeDelta render_interval,
+                                      base::TimeDelta frame_duration);
+
   // The approximate best N-frame cadence for all frames seen thus far; updated
   // by UpdateCadenceEstimate().  Empty when no cadence has been detected.
   Cadence cadence_;
@@ -160,6 +168,20 @@ class MEDIA_EXPORT VideoCadenceEstimator {
 
   bool is_variable_frame_rate_;
 
+  // Data members related to Bresenham cadence algorithm.
+  // No technical reason to have this struct except for grouping related fields.
+  struct {
+    bool use_bresenham_cadence_ = false;
+
+    // By how much to shift frame index before calculating Bresenham cadence.
+    int frame_index_shift_ = 0;
+
+    // In an ideal world, each video frame would be shown for this many display
+    // intervals. It equals (display frequency) divided by (video frame rate).
+    // Absent when a video has variable frame rate.
+    base::Optional<double> perfect_cadence_;
+  } bm_;
+
   DISALLOW_COPY_AND_ASSIGN(VideoCadenceEstimator);
 };
 

media/filters/video_cadence_estimator_unittest.cc

@@ -11,6 +11,8 @@
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/string_split.h"
 #include "base/strings/stringprintf.h"
+#include "base/test/scoped_feature_list.h"
+#include "media/base/media_switches.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 namespace media {
@@ -296,4 +298,86 @@ TEST(VideoCadenceEstimatorTest, CadenceHystersisPreventsOscillation) {
   EXPECT_FALSE(estimator->has_cadence());
 }
 
+void VerifyCadenceSequence(VideoCadenceEstimator* estimator,
+                           double frame_rate,
+                           double display_rate,
+                           std::vector<int> expected_cadence) {
+  SCOPED_TRACE(base::StringPrintf("Checking %.03f fps into %0.03f", frame_rate,
+                                  display_rate));
+
+  const base::TimeDelta render_interval = Interval(display_rate);
+  const base::TimeDelta frame_interval = Interval(frame_rate);
+  const base::TimeDelta acceptable_drift =
+      frame_interval < render_interval ? render_interval : frame_interval;
+  const base::TimeDelta test_runtime = base::TimeDelta::FromSeconds(10 * 60);
+  const int test_frames = test_runtime / frame_interval;
+
+  estimator->Reset();
+  EXPECT_TRUE(estimator->UpdateCadenceEstimate(
+      render_interval, frame_interval, base::TimeDelta(), acceptable_drift));
+  EXPECT_TRUE(estimator->has_cadence());
+  for (auto i = 0u; i < expected_cadence.size(); i++) {
+    ASSERT_EQ(expected_cadence[i], estimator->GetCadenceForFrame(i))
+        << " i=" << i;
+  }
+
+  int total_display_cycles = 0;
+  for (int i = 0; i < test_frames; i++) {
+    total_display_cycles += estimator->GetCadenceForFrame(i);
+    base::TimeDelta drift =
+        (total_display_cycles * render_interval) - ((i + 1) * frame_interval);
+    EXPECT_LE(drift.magnitude(), acceptable_drift)
+        << " i=" << i << " time=" << (total_display_cycles * render_interval);
+    if (drift.magnitude() > acceptable_drift)
+      break;
+  }
+}
+
+TEST(VideoCadenceEstimatorTest, BresenhamTest) {
+  base::test::ScopedFeatureList scoped_feature_list;
+  scoped_feature_list.InitAndEnableFeature(media::kBresenhamCadence);
+  VideoCadenceEstimator estimator(base::TimeDelta::FromSeconds(1));
+  estimator.set_cadence_hysteresis_threshold_for_testing(base::TimeDelta());
+
+  VerifyCadenceSequence(&estimator, 30, 60,
+                        {2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2});
+  VerifyCadenceSequence(&estimator, NTSC(30), 60,
+                        {2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2});
+  VerifyCadenceSequence(&estimator, 30, NTSC(60),
+                        {2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2});
+
+  VerifyCadenceSequence(&estimator, 25, 60, {2, 3, 2, 3, 2, 2, 3, 2});
+
+  VerifyCadenceSequence(&estimator, 24, 60, {3, 2, 3, 2, 3, 2, 3, 2});
+  VerifyCadenceSequence(&estimator, NTSC(24), 60, {3, 2, 3, 2, 3, 2, 3, 2});
+  VerifyCadenceSequence(&estimator, 24, NTSC(60), {2, 3, 2, 3, 2, 3, 2, 3, 2});
+
+  VerifyCadenceSequence(&estimator, 24, 50,
+                        {2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 2});
+  VerifyCadenceSequence(&estimator, NTSC(24), 50,
+                        {2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 2, 2});
+
+  VerifyCadenceSequence(&estimator, 30, 50, {2, 1, 2, 2, 1, 2, 2});
+  VerifyCadenceSequence(&estimator, NTSC(30), 50, {2, 2, 1, 2, 2});
+  VerifyCadenceSequence(&estimator, 120, 24, {1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1});
+  VerifyCadenceSequence(&estimator, 60, 50, {1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0});
+  VerifyCadenceSequence(&estimator, 25, 50, {2, 2, 2, 2, 2, 2, 2, 2, 2});
+
+  VerifyCadenceSequence(&estimator, 50, 25, {1, 0, 1, 0, 1, 0, 1, 0});
+  VerifyCadenceSequence(&estimator, 120, 60, {1, 0, 1, 0, 1, 0, 1, 0});
+
+  // Frame rate deviation is too high, refuse to provide cadence.
+  EXPECT_TRUE(estimator.UpdateCadenceEstimate(
+      Interval(60), Interval(30), base::TimeDelta::FromMilliseconds(20),
+      base::TimeDelta::FromSeconds(100)));
+  EXPECT_FALSE(estimator.has_cadence());
+
+  // No cadence change for neglegable rate changes
+  EXPECT_TRUE(estimator.UpdateCadenceEstimate(
+      Interval(60), Interval(30), base::TimeDelta(), base::TimeDelta()));
+  EXPECT_FALSE(estimator.UpdateCadenceEstimate(Interval(60 * 1.0001),
+                                               Interval(30), base::TimeDelta(),
+                                               base::TimeDelta()));
+}
+
 }  // namespace media

media/filters/video_renderer_algorithm_unittest.cc

@@ -116,24 +116,11 @@ class VideoRendererAlgorithmTest : public testing::Test {
     if (!is_using_cadence())
       return false;
 
-    size_t size = algorithm_.cadence_estimator_.cadence_size_for_testing();
-    for (size_t i = 0; i < size; ++i) {
-      if (!algorithm_.cadence_estimator_.GetCadenceForFrame(i))
-        return true;
-    }
-
-    return false;
+    return algorithm_.cadence_estimator_.avg_cadence_for_testing() < 1.0;
   }
 
   double CadenceValue() const {
-    int num_render_intervals = 0;
-    size_t size = algorithm_.cadence_estimator_.cadence_size_for_testing();
-    for (size_t i = 0; i < size; ++i) {
-      num_render_intervals +=
-          algorithm_.cadence_estimator_.GetCadenceForFrame(i);
-    }
-
-    return (num_render_intervals + 0.0) / size;
+    return algorithm_.cadence_estimator_.avg_cadence_for_testing();
   }
 
   size_t frames_queued() const { return algorithm_.frame_queue_.size(); }