[Power ML] Change smart dim model interface to return ModelPrediction.

Model could error out so we need to record this response in the log.
Also, the logged inactivity_score and decision_threshold are quantized
version of the actual values used in the inference. Hence the
model now returns it decision as part of the prediction.

Bug: 862461
Change-Id: I14c889d97bebd93b88a86027ff94b2ce7fad5bfb
Reviewed-on: https://chromium-review.googlesource.com/1154243
Commit-Queue: Jia Meng <jiameng@chromium.org>
Reviewed-by: Steven Holte <holte@chromium.org>
Reviewed-by: Andrew Moylan <amoylan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#580740}

chrome/browser/chromeos/power/ml/smart_dim_model.h

@@ -17,16 +17,10 @@ class SmartDimModel {
  public:
   virtual ~SmartDimModel() = default;
 
-  // Returns whether an upcoming dim should go ahead based on input |features|.
-  // If |inactive_probability_out| and |threshold_out| are non-null, also
-  // returns model confidence (probability that user will remain inactive if
-  // screen is dimmed now) and threshold: if probability >= threshold then model
-  // will return true for this function. Both |inactive_probability_out| and
-  // |threshold_out| are expected to be in the range of [0, 1.0] so that they
-  // can be logged as model results.
-  virtual bool ShouldDim(const UserActivityEvent::Features& features,
-                         float* inactive_probability_out,
-                         float* threshold_out) = 0;
+  // Returns a prediction whether an upcoming dim should go ahead based on input
+  // |features|.
+  virtual UserActivityEvent::ModelPrediction ShouldDim(
+      const UserActivityEvent::Features& features) = 0;
 };
 
 }  // namespace ml

chrome/browser/chromeos/power/ml/smart_dim_model_impl.cc

@@ -13,17 +13,14 @@ SmartDimModelImpl::SmartDimModelImpl() = default;
 SmartDimModelImpl::~SmartDimModelImpl() = default;
 
 // TODO(jiameng): add impl.
-bool SmartDimModelImpl::ShouldDim(const UserActivityEvent::Features& features,
-                                  float* inactive_probability_out,
-                                  float* threshold_out) {
+UserActivityEvent::ModelPrediction SmartDimModelImpl::ShouldDim(
+    const UserActivityEvent::Features& features) {
   // Let dim go ahead before we have a model implementation in place.
-  if (inactive_probability_out) {
-    *inactive_probability_out = 1.0;
-  }
-  if (threshold_out) {
-    *threshold_out = 0.0;
-  }
-  return true;
+  UserActivityEvent::ModelPrediction prediction;
+  prediction.set_decision_threshold(0);
+  prediction.set_inactivity_score(100);
+  prediction.set_response(UserActivityEvent::ModelPrediction::DIM);
+  return prediction;
 }
 
 }  // namespace ml

chrome/browser/chromeos/power/ml/smart_dim_model_impl.h

@@ -20,9 +20,8 @@ class SmartDimModelImpl : public SmartDimModel {
   ~SmartDimModelImpl() override;
 
   // chromeos::power::ml::SmartDimModel overrides:
-  bool ShouldDim(const UserActivityEvent::Features& features,
-                 float* inactive_probability_out,
-                 float* threshold_out) override;
+  UserActivityEvent::ModelPrediction ShouldDim(
+      const UserActivityEvent::Features& features) override;
 
  private:
   DISALLOW_COPY_AND_ASSIGN(SmartDimModelImpl);

chrome/browser/chromeos/power/ml/user_activity_event.proto

@@ -179,14 +179,33 @@ message UserActivityEvent {
     optional int32 previous_positive_actions_count = 30;
   }  // next id = 31
 
+  // All fields except |model_applied| in ModelPrediction are populated by the
+  // model. |model_applied| is populated by the UserActivityManager when
+  // deciding whether to apply the model decision.
   message ModelPrediction {
-    // If |inactivity_score| < |decision_threshold| then dim will be deferred.
+    enum Response {
+      // Dim should go ahead.
+      DIM = 0;
+      // Dim should be deferred.
+      NO_DIM = 1;
+      // Model could fail to make a prediction due to various reasons, e.g. it
+      // could fail to load the preprocessor or process the features for
+      // inference.
+      MODEL_ERROR = 2;
+    }
+
+    // Both |inactivity_score| and |decision_threshold| are in the range of
+    // [0,100]. These values are the quantized versions of actual values used in
+    // making a model prediction, so that they can be logged later. If
+    // |inactivity_score| < |decision_threshold| then dim will be deferred.
     optional int32 decision_threshold = 1;
     // How likely user will remain inactive if screen is dimmed.
     optional int32 inactivity_score = 2;
     // Whether model decision (regardless if dim is to be deferred) is
-    // taken by powerd.
+    // taken by powerd. It is false if model response is MODEL_ERROR or if dim
+    // was deferred last time dim imminent occurred.
     optional bool model_applied = 3;
+    optional Response response = 4;
   }
 
   optional ModelParams params = 1;

chrome/browser/chromeos/power/ml/user_activity_manager.cc

@@ -193,30 +193,27 @@ void UserActivityManager::OnIdleEventObserved(
   ExtractFeatures(activity_data);
   if (base::FeatureList::IsEnabled(features::kUserActivityPrediction) &&
       smart_dim_model_) {
-    float inactivity_probability = -1;
-    float threshold = -1;
-    const bool should_dim = smart_dim_model_->ShouldDim(
-        features_, &inactivity_probability, &threshold);
-    DCHECK(inactivity_probability >= 0 && inactivity_probability <= 1.0)
-        << inactivity_probability;
-    DCHECK(threshold >= 0 && threshold <= 1.0) << threshold;
-
-    UserActivityEvent::ModelPrediction model_prediction;
-    // If previous dim was deferred, then model decision will not be applied
-    // to this event.
-    model_prediction.set_model_applied(!dim_deferred_);
-    model_prediction.set_decision_threshold(round(threshold * 100));
-    model_prediction.set_inactivity_score(round(inactivity_probability * 100));
-    model_prediction_ = model_prediction;
+    // Decide whether to defer the imminent screen dim.
+    UserActivityEvent::ModelPrediction model_prediction =
+        smart_dim_model_->ShouldDim(features_);
 
     // Only defer the dim if the model predicts so and also if the dim was not
     // previously deferred.
-    if (should_dim || dim_deferred_) {
-      dim_deferred_ = false;
-    } else {
+    if (model_prediction.response() ==
+            UserActivityEvent::ModelPrediction::NO_DIM &&
+        !dim_deferred_) {
       power_manager_client_->DeferScreenDim();
       dim_deferred_ = true;
+      model_prediction.set_model_applied(true);
+    } else {
+      // Either model predicts dim or model fails, or it was previously dimmed.
+      dim_deferred_ = false;
+      model_prediction.set_model_applied(
+          model_prediction.response() ==
+              UserActivityEvent::ModelPrediction::DIM &&
+          !dim_deferred_);
     }
+    model_prediction_ = model_prediction;
   }
   waiting_for_final_action_ = true;
 }

chrome/browser/chromeos/power/ml/user_activity_manager_unittest.cc

@@ -63,12 +63,19 @@ void EqualEvent(const UserActivityEvent::Event& expected_event,
 void EqualModelPrediction(
     const UserActivityEvent::ModelPrediction& expected_prediction,
     const UserActivityEvent::ModelPrediction& result_prediction) {
-  EXPECT_EQ(expected_prediction.decision_threshold(),
-            result_prediction.decision_threshold());
-  EXPECT_EQ(expected_prediction.inactivity_score(),
-            result_prediction.inactivity_score());
   EXPECT_EQ(expected_prediction.model_applied(),
             result_prediction.model_applied());
+  EXPECT_EQ(expected_prediction.response(), result_prediction.response());
+  if (expected_prediction.response() !=
+      UserActivityEvent::ModelPrediction::MODEL_ERROR) {
+    EXPECT_EQ(expected_prediction.decision_threshold(),
+              result_prediction.decision_threshold());
+    EXPECT_EQ(expected_prediction.inactivity_score(),
+              result_prediction.inactivity_score());
+  } else {
+    EXPECT_FALSE(result_prediction.has_decision_threshold());
+    EXPECT_FALSE(result_prediction.has_inactivity_score());
+  }
 }
 
 // Testing UKM logger.
@@ -96,26 +103,42 @@ class FakeSmartDimModel : public SmartDimModel {
   FakeSmartDimModel() = default;
   ~FakeSmartDimModel() override = default;
 
-  void set_inactive_probability(const float inactive_probability) {
-    inactive_probability_ = inactive_probability;
+  void set_inactivity_score(const int inactivity_score) {
+    inactivity_score_ = inactivity_score;
   }
 
-  void set_threshold(const float threshold) { threshold_ = threshold; }
+  void set_decision_threshold(const int decision_threshold) {
+    decision_threshold_ = decision_threshold;
+  }
 
   // SmartDimModel overrides:
-  bool ShouldDim(const UserActivityEvent::Features& features,
-                 float* inactive_probability_out,
-                 float* threshold_out) override {
-    DCHECK(inactive_probability_out);
-    DCHECK(threshold_out);
-    *inactive_probability_out = inactive_probability_;
-    *threshold_out = threshold_;
-    return inactive_probability_ >= threshold_;
+  UserActivityEvent::ModelPrediction ShouldDim(
+      const UserActivityEvent::Features& features) override {
+    UserActivityEvent::ModelPrediction model_prediction;
+    // If either of these two values are set outside of the legal range [0,100],
+    // return an error code.
+    // The |model_applied| field is not filled by the model but by
+    // UserActivityManager.
+    if (inactivity_score_ < 0 || inactivity_score_ > 100 ||
+        decision_threshold_ < 0 || decision_threshold_ > 100) {
+      model_prediction.set_response(
+          UserActivityEvent::ModelPrediction::MODEL_ERROR);
+      return model_prediction;
+    }
+    model_prediction.set_decision_threshold(decision_threshold_);
+    model_prediction.set_inactivity_score(inactivity_score_);
+    if (inactivity_score_ < decision_threshold_) {
+      model_prediction.set_response(UserActivityEvent::ModelPrediction::NO_DIM);
+    } else {
+      model_prediction.set_response(UserActivityEvent::ModelPrediction::DIM);
+    }
+    return model_prediction;
   }
 
  private:
-  float inactive_probability_ = 0;
-  float threshold_ = 0;
+  int inactivity_score_ = -1;
+  int decision_threshold_ = -1;
+
   DISALLOW_COPY_AND_ASSIGN(FakeSmartDimModel);
 };
 
@@ -853,8 +876,8 @@ TEST_F(UserActivityManagerTest, ScreenDimDeferredWithFinalEvent) {
   scoped_feature_list.InitAndEnableFeatureWithParameters(
       features::kUserActivityPrediction, params);
 
-  model_.set_inactive_probability(0.6);
-  model_.set_threshold(0.651);
+  model_.set_inactivity_score(60);
+  model_.set_decision_threshold(65);
 
   const IdleEventNotifier::ActivityData data;
   ReportIdleEvent(data);
@@ -877,6 +900,7 @@ TEST_F(UserActivityManagerTest, ScreenDimDeferredWithFinalEvent) {
   expected_prediction.set_decision_threshold(65);
   expected_prediction.set_inactivity_score(60);
   expected_prediction.set_model_applied(true);
+  expected_prediction.set_response(UserActivityEvent::ModelPrediction::NO_DIM);
   EqualModelPrediction(expected_prediction, events[0].model_prediction());
 }
 
@@ -887,8 +911,8 @@ TEST_F(UserActivityManagerTest, ScreenDimDeferredWithoutFinalEvent) {
   scoped_feature_list.InitAndEnableFeatureWithParameters(
       features::kUserActivityPrediction, params);
 
-  model_.set_inactive_probability(0.6);
-  model_.set_threshold(0.651);
+  model_.set_inactivity_score(60);
+  model_.set_decision_threshold(65);
 
   const IdleEventNotifier::ActivityData data;
   ReportIdleEvent(data);
@@ -905,8 +929,8 @@ TEST_F(UserActivityManagerTest, ScreenDimNotDeferred) {
   scoped_feature_list.InitAndEnableFeatureWithParameters(
       features::kUserActivityPrediction, params);
 
-  model_.set_inactive_probability(0.6);
-  model_.set_threshold(0.5);
+  model_.set_inactivity_score(60);
+  model_.set_decision_threshold(50);
 
   const IdleEventNotifier::ActivityData data;
   ReportIdleEvent(data);
@@ -920,6 +944,8 @@ TEST_F(UserActivityManagerTest, ScreenDimNotDeferred) {
   expected_prediction.set_decision_threshold(50);
   expected_prediction.set_inactivity_score(60);
   expected_prediction.set_model_applied(true);
+  expected_prediction.set_response(UserActivityEvent::ModelPrediction::DIM);
+
   EqualModelPrediction(expected_prediction, events[0].model_prediction());
 }
 
@@ -929,10 +955,10 @@ TEST_F(UserActivityManagerTest, TwoScreenDimImminentWithEventInBetween) {
   base::test::ScopedFeatureList scoped_feature_list;
   scoped_feature_list.InitAndEnableFeatureWithParameters(
       features::kUserActivityPrediction, params);
-  model_.set_threshold(0.5);
+  model_.set_decision_threshold(50);
 
   // 1st ScreenDimImminent gets deferred
-  model_.set_inactive_probability(0.4);
+  model_.set_inactivity_score(40);
 
   const IdleEventNotifier::ActivityData data;
   ReportIdleEvent(data);
@@ -943,7 +969,7 @@ TEST_F(UserActivityManagerTest, TwoScreenDimImminentWithEventInBetween) {
                 base::TimeDelta::FromSeconds(3));
 
   // 2nd ScreenDimImminent is not deferred despite model score says so.
-  model_.set_inactive_probability(0.2);
+  model_.set_inactivity_score(20);
   GetTaskRunner()->FastForwardBy(base::TimeDelta::FromSeconds(10));
   ReportIdleEvent(data);
   EXPECT_EQ(1, GetNumberOfDeferredDims());
@@ -970,6 +996,8 @@ TEST_F(UserActivityManagerTest, TwoScreenDimImminentWithEventInBetween) {
   expected_prediction1.set_decision_threshold(50);
   expected_prediction1.set_inactivity_score(40);
   expected_prediction1.set_model_applied(true);
+  expected_prediction1.set_response(UserActivityEvent::ModelPrediction::NO_DIM);
+
   EqualModelPrediction(expected_prediction1, events[0].model_prediction());
 
   // The second screen dim imminent event.
@@ -986,6 +1014,7 @@ TEST_F(UserActivityManagerTest, TwoScreenDimImminentWithEventInBetween) {
   expected_prediction2.set_decision_threshold(50);
   expected_prediction2.set_inactivity_score(20);
   expected_prediction2.set_model_applied(false);
+  expected_prediction2.set_response(UserActivityEvent::ModelPrediction::NO_DIM);
   EqualModelPrediction(expected_prediction2, events[1].model_prediction());
 }
 
@@ -995,16 +1024,16 @@ TEST_F(UserActivityManagerTest, TwoScreenDimImminentWithoutEventInBetween) {
   base::test::ScopedFeatureList scoped_feature_list;
   scoped_feature_list.InitAndEnableFeatureWithParameters(
       features::kUserActivityPrediction, params);
-  model_.set_threshold(0.5);
+  model_.set_decision_threshold(50);
 
   // 1st ScreenDimImminent gets deferred
-  model_.set_inactive_probability(0.4);
+  model_.set_inactivity_score(40);
   const IdleEventNotifier::ActivityData data;
   ReportIdleEvent(data);
   EXPECT_EQ(1, GetNumberOfDeferredDims());
 
   // 2nd ScreenDimImminent is not deferred despite model score says so.
-  model_.set_inactive_probability(0.2);
+  model_.set_inactivity_score(20);
   GetTaskRunner()->FastForwardBy(base::TimeDelta::FromSeconds(10));
   ReportIdleEvent(data);
   EXPECT_EQ(1, GetNumberOfDeferredDims());
@@ -1031,6 +1060,8 @@ TEST_F(UserActivityManagerTest, TwoScreenDimImminentWithoutEventInBetween) {
   expected_prediction1.set_decision_threshold(50);
   expected_prediction1.set_inactivity_score(20);
   expected_prediction1.set_model_applied(false);
+  expected_prediction1.set_response(UserActivityEvent::ModelPrediction::NO_DIM);
+
   EqualModelPrediction(expected_prediction1, events[0].model_prediction());
 
   // The earlier idle event is logged afterwards.
@@ -1042,9 +1073,46 @@ TEST_F(UserActivityManagerTest, TwoScreenDimImminentWithoutEventInBetween) {
   expected_prediction2.set_decision_threshold(50);
   expected_prediction2.set_inactivity_score(40);
   expected_prediction2.set_model_applied(true);
+  expected_prediction2.set_response(UserActivityEvent::ModelPrediction::NO_DIM);
+
   EqualModelPrediction(expected_prediction2, events[1].model_prediction());
 }
 
+TEST_F(UserActivityManagerTest, ModelError) {
+  const std::map<std::string, std::string> params = {
+      {"dim_threshold", "0.651"}};
+  base::test::ScopedFeatureList scoped_feature_list;
+  scoped_feature_list.InitAndEnableFeatureWithParameters(
+      features::kUserActivityPrediction, params);
+
+  // This value will trigger a model error.
+  model_.set_inactivity_score(160);
+  model_.set_decision_threshold(65);
+
+  const IdleEventNotifier::ActivityData data;
+  ReportIdleEvent(data);
+  ReportUserActivity(nullptr);
+  EXPECT_EQ(0, GetNumberOfDeferredDims());
+
+  const std::vector<UserActivityEvent>& events = delegate_.events();
+  ASSERT_EQ(1U, events.size());
+
+  UserActivityEvent::Event expected_event;
+  expected_event.set_type(UserActivityEvent::Event::REACTIVATE);
+  expected_event.set_reason(UserActivityEvent::Event::USER_ACTIVITY);
+  expected_event.set_log_duration_sec(0);
+  expected_event.set_screen_dim_occurred(false);
+  expected_event.set_screen_off_occurred(false);
+  expected_event.set_screen_lock_occurred(false);
+  EqualEvent(expected_event, events[0].event());
+
+  UserActivityEvent::ModelPrediction expected_prediction;
+  expected_prediction.set_model_applied(false);
+  expected_prediction.set_response(
+      UserActivityEvent::ModelPrediction::MODEL_ERROR);
+  EqualModelPrediction(expected_prediction, events[0].model_prediction());
+}
+
 TEST_F(UserActivityManagerTest, BasicTabs) {
   std::unique_ptr<Browser> browser =
       CreateTestBrowser(true /* is_visible */, true /* is_focused */);

chrome/browser/chromeos/power/ml/user_activity_ukm_logger_impl.cc

@@ -156,9 +156,16 @@ void UserActivityUkmLoggerImpl::LogActivity(const UserActivityEvent& event) {
   if (event.has_model_prediction()) {
     const UserActivityEvent::ModelPrediction& model_prediction =
         event.model_prediction();
-    user_activity.SetModelApplied(model_prediction.model_applied())
-        .SetModelDecisionThreshold(model_prediction.decision_threshold())
-        .SetModelInactivityScore(model_prediction.inactivity_score());
+    user_activity.SetModelResponse(model_prediction.response())
+        .SetModelApplied(model_prediction.model_applied());
+    if (model_prediction.response() ==
+            UserActivityEvent::ModelPrediction::DIM ||
+        model_prediction.response() ==
+            UserActivityEvent::ModelPrediction::NO_DIM) {
+      user_activity
+          .SetModelDecisionThreshold(model_prediction.decision_threshold())
+          .SetModelInactivityScore(model_prediction.inactivity_score());
+    }
   }
 
   user_activity.Record(ukm_recorder_);

chrome/browser/chromeos/power/ml/user_activity_ukm_logger_unittest.cc

@@ -64,6 +64,7 @@ class UserActivityUkmLoggerTest : public testing::Test {
     prediction->set_decision_threshold(50);
     prediction->set_inactivity_score(60);
     prediction->set_model_applied(true);
+    prediction->set_response(UserActivityEvent::ModelPrediction::NO_DIM);
 
     user_activity_logger_delegate_ukm_.ukm_recorder_ = &recorder_;
   }
@@ -99,6 +100,7 @@ class UserActivityUkmLoggerTest : public testing::Test {
                                 UserActivity::kModelDecisionThresholdName, 50);
     recorder_.ExpectEntryMetric(entry, UserActivity::kModelInactivityScoreName,
                                 60);
+    recorder_.ExpectEntryMetric(entry, UserActivity::kModelResponseName, 1);
     recorder_.ExpectEntryMetric(entry, UserActivity::kMouseEventsInLastHourName,
                                 89);
     EXPECT_FALSE(recorder_.EntryHasMetric(entry, UserActivity::kOnBatteryName));
@@ -238,6 +240,11 @@ TEST_F(UserActivityUkmLoggerTest, TwoUserActivityEvents) {
   features->set_dim_to_screen_off_sec(20);
   features->set_time_since_last_mouse_sec(200);
 
+  UserActivityEvent::ModelPrediction* prediction =
+      user_activity_event2.mutable_model_prediction();
+  prediction->set_model_applied(false);
+  prediction->set_response(UserActivityEvent::ModelPrediction::MODEL_ERROR);
+
   LogActivity(user_activity_event_);
   LogActivity(user_activity_event2);
 
@@ -277,6 +284,12 @@ TEST_F(UserActivityUkmLoggerTest, TwoUserActivityEvents) {
       recorder_.EntryHasMetric(entry1, UserActivity::kTimeSinceLastKeyName));
   recorder_.ExpectEntryMetric(entry1, UserActivity::kTimeSinceLastMouseName,
                               200);
+  recorder_.ExpectEntryMetric(entry1, UserActivity::kModelResponseName, 2);
+  recorder_.ExpectEntryMetric(entry1, UserActivity::kModelAppliedName, 0);
+  EXPECT_FALSE(recorder_.EntryHasMetric(
+      entry1, UserActivity::kModelDecisionThresholdName));
+  EXPECT_FALSE(recorder_.EntryHasMetric(
+      entry1, UserActivity::kModelInactivityScoreName));
 
   EXPECT_EQ(0u, recorder_.GetEntriesByName(UserActivityId::kEntryName).size());
 }

tools/metrics/ukm/ukm.xml

@@ -4212,6 +4212,12 @@ be describing additional metrics about the same event.
       dim will be deferred.
     </summary>
   </metric>
+  <metric name="ModelResponse">
+    <summary>
+      Whether the model recommends the screen dim should be deferred (NO_DIM) or
+      allowed (DIM), or encountered an error (MODEL_ERROR).
+    </summary>
+  </metric>
   <metric name="MouseEventsInLastHour">
     <summary>
       The number of mouse events reported as user activity in the last hour.