Count maximum consecutive per-playback NNRs for ML.

Records the maximum number of consecutive NNRs observed during
playback.  "Consecutive" means "separated by at most 60s between
each NNR".  Adds one training example to the newly-introduced
learning task per playback.

Notes:
 NNRs are ignored if we haven't started the smoothness helper yet,
 and i have no idea if this will happen much.  Ideally, not.

 NNR counts are reset when playback is paused, or we adapt, or any
 of the other things that reset the smoothness helper.

Change-Id: I43a36cd053c7e37134dd8d6a6e83a8d3d27f44b6
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2016197
Commit-Queue: Chrome Cunningham <chcunningham@chromium.org>
Reviewed-by: Chrome Cunningham <chcunningham@chromium.org>
Cr-Commit-Position: refs/heads/master@{#735258}

media/blink/smoothness_helper.cc

@@ -14,6 +14,10 @@ namespace {
 static constexpr base::TimeDelta kSegmentSize =
     base::TimeDelta::FromSeconds(5);
 
+// Maximum distance between NNRs for them to be consecutive.
+static constexpr base::TimeDelta kMaxNNRDistance =
+    base::TimeDelta::FromSeconds(60);
+
 // Max proportion of dropped frames in a window before we call it "not smooth".
 static constexpr float kMaxDroppedFramesPerWindow = 0.2;
 }
@@ -76,10 +80,12 @@ class SmoothnessHelperImpl : public SmoothnessHelper {
  public:
   SmoothnessHelperImpl(
       std::unique_ptr<LearningTaskController> consecutive_controller,
+      std::unique_ptr<LearningTaskController> nnr_controller,
       const FeatureVector& features,
       Client* player)
       : SmoothnessHelper(features),
         consecutive_bad_(std::move(consecutive_controller)),
+        consecutive_nnr_(std::move(nnr_controller)),
         player_(player) {
     monitor_ = std::make_unique<SmoothnessWindowMonitor>(
         player_, base::BindRepeating(&SmoothnessHelperImpl::OnWindow,
@@ -91,9 +97,51 @@ class SmoothnessHelperImpl : public SmoothnessHelper {
   // with the default value if we've gotten enough data to set one.
   ~SmoothnessHelperImpl() override = default;
 
+  // See if we've exceeded the intra-NNR distance, and reset everything.  Note
+  // that this can be called even when there isn't an NNR.
+  void UpdateNNRWindow() {
+    if (!most_recent_nnr_)
+      return;
+
+    auto now = base::TimeTicks::Now();
+    auto delta = now - *most_recent_nnr_;
+    if (delta >= kMaxNNRDistance) {
+      most_recent_nnr_.reset();
+      num_consecutive_nnrs_ = 0;
+    }
+  }
+
+  void NotifyNNR() override {
+    UpdateNNRWindow();
+    most_recent_nnr_ = base::TimeTicks::Now();
+    num_consecutive_nnrs_++;
+
+    if (num_consecutive_nnrs_ > max_num_consecutive_nnrs_) {
+      max_num_consecutive_nnrs_ = num_consecutive_nnrs_;
+
+      // Insist that we've started the NNR instance, so that we enforce a
+      // minimum amount of playback time before recording anything.  Though
+      // it's possible that an NNR is interesting enough to record it anyway,
+      // and we only want to elide zero-NNR observations for short playbacks.
+      if (consecutive_nnr_.is_started()) {
+        consecutive_nnr_.UpdateObservation(
+            features(), TargetValue(max_num_consecutive_nnrs_));
+      }
+    }
+  }
+
   // Split playback into segments of length |kSegmentSize|, and update the
   // default value of the current playback.
   void OnWindow(int64_t dropped_frames, int64_t decoded_frames) {
+    // After the first window, start the NNR observation.  We want to ignore any
+    // short playback windows.  We might want to require more than one window.
+    // TODO(liberato): How many windows count as a playback for NNR?
+    if (!consecutive_nnr_.is_started()) {
+      UpdateNNRWindow();
+      consecutive_nnr_.UpdateObservation(
+          features(), TargetValue(max_num_consecutive_nnrs_));
+    }
+
     // Compute the percentage of dropped frames for this window.
     double pct = (static_cast<double>(dropped_frames)) / decoded_frames;
 
@@ -156,6 +204,17 @@ class SmoothnessHelperImpl : public SmoothnessHelper {
   int consecutive_bad_windows_ = 0;
   int max_consecutive_bad_windows_ = 0;
 
+  struct Task consecutive_nnr_;
+
+  // Time of the most recent nnr.
+  base::Optional<base::TimeTicks> most_recent_nnr_;
+
+  // Number of NNRs that have occurred within |kMaxNNRDistance|.
+  int num_consecutive_nnrs_ = 0;
+
+  // Maximum value of |num_consecutive_nnrs_| that we've observed.
+  int max_num_consecutive_nnrs_ = 0;
+
   // WebMediaPlayer which will tell us about the decoded / dropped frame counts.
   Client* player_;
 
@@ -164,11 +223,12 @@ class SmoothnessHelperImpl : public SmoothnessHelper {
 
 // static
 std::unique_ptr<SmoothnessHelper> SmoothnessHelper::Create(
-    std::unique_ptr<LearningTaskController> consecutive_controller,
+    std::unique_ptr<LearningTaskController> bad_controller,
+    std::unique_ptr<LearningTaskController> nnr_controller,
     const FeatureVector& features,
     Client* player) {
   return std::make_unique<SmoothnessHelperImpl>(
-      std::move(consecutive_controller), features, player);
+      std::move(bad_controller), std::move(nnr_controller), features, player);
 }
 
 // static

media/blink/smoothness_helper.h

@@ -39,11 +39,15 @@ class MEDIA_BLINK_EXPORT SmoothnessHelper {
   // Return the features that we were constructed with.
   const learning::FeatureVector& features() const { return features_; }
 
+  // Notify us that an NNR has occurred.
+  virtual void NotifyNNR() = 0;
+
   // |features| are the features that we'll use for any labelled examples that
   // we create.  They should be features that could be captured at the time a
   // prediction would be needed.
   static std::unique_ptr<SmoothnessHelper> Create(
-      std::unique_ptr<learning::LearningTaskController> consecutive_controller,
+      std::unique_ptr<learning::LearningTaskController> bad_controller,
+      std::unique_ptr<learning::LearningTaskController> nnr_controller,
       const learning::FeatureVector& features,
       Client* player);
 

media/blink/smoothness_helper_unittest.cc

@@ -76,11 +76,13 @@ class SmoothnessHelperTest : public testing::Test {
 
  public:
   void SetUp() override {
-    auto consecutive_ltc = std::make_unique<MockLearningTaskController>();
-    consecutive_ltc_ = consecutive_ltc.get();
+    auto bad_ltc = std::make_unique<MockLearningTaskController>();
+    bad_ltc_ = bad_ltc.get();
+    auto nnr_ltc = std::make_unique<MockLearningTaskController>();
+    nnr_ltc_ = nnr_ltc.get();
     features_.push_back(FeatureValue(123));
-    helper_ = SmoothnessHelper::Create(std::move(consecutive_ltc), features_,
-                                       &client_);
+    helper_ = SmoothnessHelper::Create(std::move(bad_ltc), std::move(nnr_ltc),
+                                       features_, &client_);
     segment_size_ = SmoothnessHelper::SegmentSizeForTesting();
   }
 
@@ -103,7 +105,11 @@ class SmoothnessHelperTest : public testing::Test {
   // Helper under test
   std::unique_ptr<SmoothnessHelper> helper_;
 
-  MockLearningTaskController* consecutive_ltc_;
+  // Max bad consecutive windows by frame drop LTC.
+  MockLearningTaskController* bad_ltc_;
+
+  // Max consecutive NNRs LTC.
+  MockLearningTaskController* nnr_ltc_;
 
   MockClient client_;
   FeatureVector features_;
@@ -111,6 +117,10 @@ class SmoothnessHelperTest : public testing::Test {
   base::TimeDelta segment_size_;
 };
 
+TEST_F(SmoothnessHelperTest, FeaturesAreReturned) {
+  EXPECT_EQ(features_, helper_->features());
+}
+
 TEST_F(SmoothnessHelperTest, MaxBadWindowsRecordsTrue) {
   // Record three bad segments, and verify that it records 'true'.
   SetFrameCounters(0, 0);
@@ -119,46 +129,80 @@ TEST_F(SmoothnessHelperTest, MaxBadWindowsRecordsTrue) {
   int total_frames = 0;
 
   // First segment has no dropped frames.  Should record 0.
-  EXPECT_CALL(*consecutive_ltc_, BeginObservation(_, _, OPT_TARGET(0.0), _))
-      .Times(1);
+  EXPECT_CALL(*bad_ltc_, BeginObservation(_, _, OPT_TARGET(0.0), _)).Times(1);
   SetFrameCounters(dropped_frames += 0, total_frames += 1000);
   FastForwardBy(segment_size_);
   base::RunLoop().RunUntilIdle();
-  testing::Mock::VerifyAndClearExpectations(consecutive_ltc_);
+  testing::Mock::VerifyAndClearExpectations(bad_ltc_);
 
   // Second segment has a lot of dropped frames, so the target should increase.
-  EXPECT_CALL(*consecutive_ltc_, UpdateDefaultTarget(_, OPT_TARGET(1.0)))
-      .Times(1);
+  EXPECT_CALL(*bad_ltc_, UpdateDefaultTarget(_, OPT_TARGET(1.0))).Times(1);
   SetFrameCounters(dropped_frames += 999, total_frames += 1000);
   FastForwardBy(segment_size_);
   base::RunLoop().RunUntilIdle();
-  testing::Mock::VerifyAndClearExpectations(consecutive_ltc_);
+  testing::Mock::VerifyAndClearExpectations(bad_ltc_);
 
   // Third segment looks nice, so nothing should update.
-  EXPECT_CALL(*consecutive_ltc_, UpdateDefaultTarget(_, OPT_TARGET(_)))
-      .Times(0);
+  EXPECT_CALL(*bad_ltc_, UpdateDefaultTarget(_, OPT_TARGET(_))).Times(0);
   SetFrameCounters(dropped_frames += 0, total_frames += 1000);
   FastForwardBy(segment_size_);
   base::RunLoop().RunUntilIdle();
-  testing::Mock::VerifyAndClearExpectations(consecutive_ltc_);
+  testing::Mock::VerifyAndClearExpectations(bad_ltc_);
 
   // Fourth segment has dropped frames, but the default shouldn't change.
   // It's okay if it changes to the same value, but we just memorize that it
   // won't change at all.
-  EXPECT_CALL(*consecutive_ltc_, UpdateDefaultTarget(_, OPT_TARGET(_)))
-      .Times(0);
+  EXPECT_CALL(*bad_ltc_, UpdateDefaultTarget(_, OPT_TARGET(_))).Times(0);
   SetFrameCounters(dropped_frames += 999, total_frames += 1000);
   FastForwardBy(segment_size_);
   base::RunLoop().RunUntilIdle();
-  testing::Mock::VerifyAndClearExpectations(consecutive_ltc_);
+  testing::Mock::VerifyAndClearExpectations(bad_ltc_);
 
   // The last segment is also bad, and should increase the max.
-  EXPECT_CALL(*consecutive_ltc_, UpdateDefaultTarget(_, OPT_TARGET(2.0)))
-      .Times(1);
+  EXPECT_CALL(*bad_ltc_, UpdateDefaultTarget(_, OPT_TARGET(2.0))).Times(1);
   SetFrameCounters(dropped_frames += 999, total_frames += 1000);
   FastForwardBy(segment_size_);
   base::RunLoop().RunUntilIdle();
-  testing::Mock::VerifyAndClearExpectations(consecutive_ltc_);
+  testing::Mock::VerifyAndClearExpectations(bad_ltc_);
+}
+
+TEST_F(SmoothnessHelperTest, NNRTaskRecordsMaxNNRs) {
+  // We should get the first target once a window has elapsed.  We need some
+  // decoded frames before anything happens.
+  SetFrameCounters(0, 1);
+  EXPECT_CALL(*nnr_ltc_, BeginObservation(_, _, OPT_TARGET(0.0), _)).Times(1);
+  FastForwardBy(segment_size_);
+  base::RunLoop().RunUntilIdle();
+  FastForwardBy(segment_size_);
+  base::RunLoop().RunUntilIdle();
+  testing::Mock::VerifyAndClearExpectations(nnr_ltc_);
+
+  // Add some NNRs, which should be reported immediately now that a segment
+  // has started.  Note that we don't care if NNRs are reported before a segment
+  // is started, because it's not really clear which behavior is right anyway.
+  EXPECT_CALL(*nnr_ltc_, UpdateDefaultTarget(_, OPT_TARGET(1))).Times(1);
+  helper_->NotifyNNR();
+  testing::Mock::VerifyAndClearExpectations(nnr_ltc_);
+
+  // Advance time by one window, and add an NNR.  It's close enough that we
+  // should be notified that the max went up.
+  FastForwardBy(segment_size_);
+  EXPECT_CALL(*nnr_ltc_, UpdateDefaultTarget(_, OPT_TARGET(2))).Times(1);
+  helper_->NotifyNNR();
+  testing::Mock::VerifyAndClearExpectations(nnr_ltc_);
+
+  // Fast forward by a lot, so that the next NNR isn't consecutive.  Nothing
+  // should be reported, because it's less than the current maximum.
+  EXPECT_CALL(*nnr_ltc_, UpdateDefaultTarget(_, OPT_TARGET(_))).Times(0);
+  FastForwardBy(base::TimeDelta::FromSeconds(1000));
+  helper_->NotifyNNR();
+  // It might be okay if this reported 2, since it's a tie.
+  helper_->NotifyNNR();
+  testing::Mock::VerifyAndClearExpectations(nnr_ltc_);
+
+  // The next NNR should advance the maximum to 3.
+  EXPECT_CALL(*nnr_ltc_, UpdateDefaultTarget(_, OPT_TARGET(3))).Times(1);
+  helper_->NotifyNNR();
 }
 
 }  // namespace media

media/blink/webmediaplayer_impl.cc

@@ -52,7 +52,6 @@
 #include "media/filters/chunk_demuxer.h"
 #include "media/filters/ffmpeg_demuxer.h"
 #include "media/filters/memory_data_source.h"
-#include "media/learning/common/media_learning_tasks.h"
 #include "media/learning/mojo/public/cpp/mojo_learning_task_controller.h"
 #include "media/media_buildflags.h"
 #include "mojo/public/cpp/bindings/pending_remote.h"
@@ -687,6 +686,10 @@ void WebMediaPlayerImpl::DoLoad(LoadType load_type,
   media_log_->AddEvent<MediaLogEvent::kLoad>(url.GetString().Utf8());
   load_start_time_ = base::TimeTicks::Now();
 
+  // If we're adapting, then restart the smoothness experiment.
+  if (smoothness_helper_)
+    smoothness_helper_.reset();
+
   media_metrics_provider_->Initialize(
       load_type == kLoadTypeMediaSource,
       load_type == kLoadTypeURL ? blink::GetMediaURLScheme(loaded_url_)
@@ -2119,6 +2122,14 @@ void WebMediaPlayerImpl::OnBufferingStateChangeInternal(
     SetReadyState(WebMediaPlayer::kReadyStateHaveCurrentData);
   }
 
+  // If this is an NNR, then notify the smoothness helper about it.  Note that
+  // it's unclear what we should do if there is no smoothness helper yet.  As it
+  // is, we just discard the NNR.
+  if (state == BUFFERING_HAVE_NOTHING && reason == DECODER_UNDERFLOW &&
+      smoothness_helper_) {
+    smoothness_helper_->NotifyNNR();
+  }
+
   UpdatePlayState();
 }
 
@@ -2837,6 +2848,7 @@ void WebMediaPlayerImpl::UpdatePlayState() {
         !paused_ && !seeking_ && !IsHidden() && !state.is_suspended &&
         ready_state_ == kReadyStateHaveEnoughData);
   }
+  UpdateSmoothnessHelper();
 }
 
 void WebMediaPlayerImpl::UpdateMediaPositionState() {
@@ -3714,18 +3726,25 @@ void WebMediaPlayerImpl::UpdateSmoothnessHelper() {
     return;
 
   // Create or restart the smoothness helper with |features|.
-  // Get the LearningTaskController for |task|.
-  learning::LearningTask task = learning::MediaLearningTasks::Get(
-      learning::MediaLearningTasks::Id::kConsecutiveBadWindows);
+  smoothness_helper_ = SmoothnessHelper::Create(
+      GetLearningTaskController(
+          learning::MediaLearningTasks::Id::kConsecutiveBadWindows),
+      GetLearningTaskController(
+          learning::MediaLearningTasks::Id::kConsecutiveNNRs),
+      features, this);
+}
+
+std::unique_ptr<learning::LearningTaskController>
+WebMediaPlayerImpl::GetLearningTaskController(
+    learning::MediaLearningTasks::Id task_id) {
+  // Get the LearningTaskController for |task_id|.
+  learning::LearningTask task = learning::MediaLearningTasks::Get(task_id);
 
   mojo::Remote<media::learning::mojom::LearningTaskController> remote_ltc;
   media_metrics_provider_->AcquireLearningTaskController(
       task.name, remote_ltc.BindNewPipeAndPassReceiver());
-  auto mojo_ltc = std::make_unique<learning::MojoLearningTaskController>(
+  return std::make_unique<learning::MojoLearningTaskController>(
       task, std::move(remote_ltc));
-
-  smoothness_helper_ =
-      SmoothnessHelper::Create(std::move(mojo_ltc), features, this);
 }
 
 }  // namespace media

media/blink/webmediaplayer_impl.h

@@ -44,6 +44,7 @@
 #include "media/blink/video_frame_compositor.h"
 #include "media/blink/webmediaplayer_params.h"
 #include "media/filters/pipeline_controller.h"
+#include "media/learning/common/media_learning_tasks.h"
 #include "media/renderers/paint_canvas_video_renderer.h"
 #include "mojo/public/cpp/bindings/remote.h"
 #include "services/media_session/public/cpp/media_position.h"
@@ -639,6 +640,10 @@ class MEDIA_BLINK_EXPORT WebMediaPlayerImpl
   // smoothness right now.
   void UpdateSmoothnessHelper();
 
+  // Get the LearningTaskController for |task_id|.
+  std::unique_ptr<learning::LearningTaskController> GetLearningTaskController(
+      learning::MediaLearningTasks::Id task_id);
+
   blink::WebLocalFrame* const frame_;
 
   // The playback state last reported to |delegate_|, to avoid setting duplicate

media/learning/common/media_learning_tasks.cc

@@ -17,6 +17,20 @@ static const LearningTask& GetWillPlayTask() {
   return task_;
 }
 
+// Add some features to |task| that WMPI knows how to add.
+static void PushWMPIFeatures(LearningTask& task) {
+  // TODO: Be sure to update webmediaplayer_impl if you change these, since it
+  // memorizes them.
+  task.feature_descriptions.push_back(
+      {"codec", LearningTask::Ordering::kUnordered});
+  task.feature_descriptions.push_back(
+      {"profile", LearningTask::Ordering::kUnordered});
+  task.feature_descriptions.push_back(
+      {"width", LearningTask::Ordering::kNumeric});
+  task.feature_descriptions.push_back(
+      {"fps", LearningTask::Ordering::kNumeric});
+}
+
 static const LearningTask& GetConsecutiveBadWindowsTask() {
   static LearningTask task_;
   if (!task_.feature_descriptions.size()) {
@@ -27,16 +41,7 @@ static const LearningTask& GetConsecutiveBadWindowsTask() {
     task_.target_description = {"max_bad_windows",
                                 LearningTask::Ordering::kNumeric};
 
-    // TODO: Be sure to update webmediaplayer_impl if you change these, since it
-    // memorizes them.
-    task_.feature_descriptions.push_back(
-        {"codec", LearningTask::Ordering::kUnordered});
-    task_.feature_descriptions.push_back(
-        {"profile", LearningTask::Ordering::kUnordered});
-    task_.feature_descriptions.push_back(
-        {"width", LearningTask::Ordering::kNumeric});
-    task_.feature_descriptions.push_back(
-        {"fps", LearningTask::Ordering::kNumeric});
+    PushWMPIFeatures(task_);
 
     // Report via UKM, but allow up to 100 bad windows, since it'll auto-scale
     // to two digits of precision.  Might as well use all of it, even if 100
@@ -49,6 +54,26 @@ static const LearningTask& GetConsecutiveBadWindowsTask() {
   return task_;
 }
 
+static const LearningTask& GetConsecutiveNNRsTask() {
+  static LearningTask task_;
+  if (!task_.feature_descriptions.size()) {
+    task_.name = "MediaLearningConsecutiveNNRs";
+    task_.model = LearningTask::Model::kExtraTrees;
+
+    // Target is max number of consecutive bad windows.
+    task_.target_description = {"total_playback_nnrs",
+                                LearningTask::Ordering::kNumeric};
+
+    PushWMPIFeatures(task_);
+
+    task_.report_via_ukm = true;
+    task_.ukm_min_input_value = 0.0;
+    task_.ukm_max_input_value = 100.0;
+  }
+
+  return task_;
+}
+
 // static
 const LearningTask& MediaLearningTasks::Get(Id id) {
   switch (id) {
@@ -56,6 +81,8 @@ const LearningTask& MediaLearningTasks::Get(Id id) {
       return GetWillPlayTask();
     case Id::kConsecutiveBadWindows:
       return GetConsecutiveBadWindowsTask();
+    case Id::kConsecutiveNNRs:
+      return GetConsecutiveNNRsTask();
   }
 }
 
@@ -64,6 +91,7 @@ void MediaLearningTasks::Register(
     base::RepeatingCallback<void(const LearningTask&)> cb) {
   cb.Run(Get(Id::kWillPlay));
   cb.Run(Get(Id::kConsecutiveBadWindows));
+  cb.Run(Get(Id::kConsecutiveNNRs));
 }
 
 }  // namespace learning

media/learning/common/media_learning_tasks.h

@@ -22,6 +22,7 @@ class COMPONENT_EXPORT(LEARNING_COMMON) MediaLearningTasks {
   enum class Id {
     kWillPlay,
     kConsecutiveBadWindows,
+    kConsecutiveNNRs,
   };
 
   // Return the LearningTask for |id|.

media/learning/common/media_learning_tasks_unittest.cc

@@ -31,13 +31,20 @@ TEST_F(MediaLearningTasksTest, ConsecutiveBadWindowsTask) {
   EXPECT_EQ(task.name, "MediaLearningConsecutiveBadWindows");
 }
 
+TEST_F(MediaLearningTasksTest, ConsecutiveNNRsTask) {
+  LearningTask task =
+      MediaLearningTasks::Get(MediaLearningTasks::Id::kConsecutiveNNRs);
+  // Make sure the name is correct, mostly to reduce cut-and-paste errors.
+  EXPECT_EQ(task.name, "MediaLearningConsecutiveNNRs");
+}
+
 TEST_F(MediaLearningTasksTest, EnumeratesAllTasks) {
   int count = 0;
   auto cb = base::BindRepeating(
       [](int* count, const LearningTask& task) { (*count)++; },
       base::Unretained(&count));
   MediaLearningTasks::Register(std::move(cb));
-  EXPECT_EQ(count, 2);
+  EXPECT_EQ(count, 3);
 }
 
 }  // namespace learning