Add more features to user activity logger

* last_user_activity_time
* last_activity_day
* last_mouse_time
* last_key_time
* recent_time_active_sec
* device_mode
* device_type
* battery_percent
* on_battery

Bug: 784232
Change-Id: Ic7eddd66b8fdd3f1680358842d135e0a45b857b0
Reviewed-on: https://chromium-review.googlesource.com/787010
Commit-Queue: Renjie Liu <renjieliu@chromium.org>
Reviewed-by: Michael Giuffrida <michaelpg@chromium.org>
Reviewed-by: Dan Erat <derat@chromium.org>
Reviewed-by: Jia Meng <jiameng@chromium.org>
Cr-Commit-Position: refs/heads/master@{#519935}

chrome/browser/chromeos/power/ml/user_activity_event.proto

@@ -62,6 +62,7 @@ message UserActivityEvent {
     optional int32 last_activity_time_sec = 3;
     optional int32 last_user_activity_time_sec = 4;
 
+    // This must match the week range in base::Time::Exploded.
     enum DayOfWeek {
       SUN = 0;
       MON = 1;

chrome/browser/chromeos/power/ml/user_activity_logger.cc

@@ -5,16 +5,40 @@
 #include "chrome/browser/chromeos/power/ml/user_activity_logger.h"
 
 #include "base/timer/timer.h"
-#include "chrome/browser/chromeos/power/ml/user_activity_event.pb.h"
-#include "ui/base/user_activity/user_activity_observer.h"
+#include "chromeos/dbus/power_manager/power_supply_properties.pb.h"
+#include "chromeos/system/devicetype.h"
 
 namespace chromeos {
 namespace power {
 namespace ml {
 
 UserActivityLogger::UserActivityLogger(
-    UserActivityLoggerDelegate* const delegate)
-    : logger_delegate_(delegate) {}
+    UserActivityLoggerDelegate* delegate,
+    IdleEventNotifier* idle_event_notifier,
+    ui::UserActivityDetector* detector,
+    chromeos::PowerManagerClient* power_manager_client)
+    : logger_delegate_(delegate),
+      idle_event_observer_(this),
+      user_activity_observer_(this),
+      power_manager_client_observer_(this) {
+  DCHECK(idle_event_notifier);
+  idle_event_observer_.Add(idle_event_notifier);
+
+  DCHECK(detector);
+  user_activity_observer_.Add(detector);
+
+  DCHECK(power_manager_client);
+  power_manager_client_observer_.Add(power_manager_client);
+  power_manager_client->RequestStatusUpdate();
+  power_manager_client->GetSwitchStates(base::BindOnce(
+      &UserActivityLogger::OnReceiveSwitchStates, base::Unretained(this)));
+
+  if (chromeos::GetDeviceType() == chromeos::DeviceType::kChromebook) {
+    device_type_ = UserActivityEvent::Features::CHROMEBOOK;
+  } else {
+    device_type_ = UserActivityEvent::Features::UNKNOWN_DEVICE;
+  }
+}
 
 UserActivityLogger::~UserActivityLogger() = default;
 
@@ -28,21 +52,105 @@ void UserActivityLogger::OnUserActivity(const ui::Event* /* event */) {
   event->set_type(UserActivityEvent::Event::REACTIVATE);
   event->set_reason(UserActivityEvent::Event::USER_ACTIVITY);
 
-  UserActivityEvent::Features* features = activity_event.mutable_features();
-  features->set_last_activity_time_sec(
-      (activity_data_.last_activity_time -
-       activity_data_.last_activity_time.LocalMidnight())
-          .InSeconds());
+  *activity_event.mutable_features() = features_;
 
   // Log to metrics.
   logger_delegate_->LogActivity(activity_event);
   idle_event_observed_ = false;
 }
 
+void UserActivityLogger::LidEventReceived(
+    chromeos::PowerManagerClient::LidState state,
+    const base::TimeTicks& /* timestamp */) {
+  lid_state_ = state;
+}
+
+// TODO(renjieliu): Log power changes activity here by checking whether the
+// battery status has changed.
+void UserActivityLogger::PowerChanged(
+    const power_manager::PowerSupplyProperties& proto) {
+  on_battery_ = (proto.external_power() ==
+                 power_manager::PowerSupplyProperties::DISCONNECTED);
+
+  if (proto.has_battery_percent()) {
+    battery_percent_ = proto.battery_percent();
+  }
+}
+
+void UserActivityLogger::TabletModeEventReceived(
+    chromeos::PowerManagerClient::TabletMode mode,
+    const base::TimeTicks& /* timestamp */) {
+  tablet_mode_ = mode;
+}
+
 void UserActivityLogger::OnIdleEventObserved(
     const IdleEventNotifier::ActivityData& activity_data) {
   idle_event_observed_ = true;
-  activity_data_ = activity_data;
+  ExtractFeatures(activity_data);
+}
+
+void UserActivityLogger::OnReceiveSwitchStates(
+    base::Optional<chromeos::PowerManagerClient::SwitchStates> switch_states) {
+  if (switch_states.has_value()) {
+    lid_state_ = switch_states->lid_state;
+    tablet_mode_ = switch_states->tablet_mode;
+  }
+}
+
+void UserActivityLogger::ExtractFeatures(
+    const IdleEventNotifier::ActivityData& activity_data) {
+  features_.Clear();
+
+  // Set time related features.
+  features_.set_last_activity_time_sec(
+      (activity_data.last_activity_time -
+       activity_data.last_activity_time.LocalMidnight())
+          .InSeconds());
+  if (activity_data.last_user_activity_time != base::Time()) {
+    features_.set_last_user_activity_time_sec(
+        (activity_data.last_user_activity_time -
+         activity_data.last_user_activity_time.LocalMidnight())
+            .InSeconds());
+  }
+
+  base::Time::Exploded exploded;
+  activity_data.last_activity_time.LocalExplode(&exploded);
+  features_.set_last_activity_day(
+      static_cast<chromeos::power::ml::UserActivityEvent_Features_DayOfWeek>(
+          exploded.day_of_week));
+
+  if (activity_data.last_mouse_time != base::Time()) {
+    features_.set_time_since_last_mouse_sec(
+        (base::Time::Now() - activity_data.last_mouse_time).InSeconds());
+  }
+  if (activity_data.last_key_time != base::Time()) {
+    features_.set_time_since_last_key_sec(
+        (base::Time::Now() - activity_data.last_key_time).InSeconds());
+  }
+
+  features_.set_recent_time_active_sec(
+      (activity_data.last_activity_time - activity_data.earliest_activity_time)
+          .InSeconds());
+
+  // Set device mode.
+  if (lid_state_ == chromeos::PowerManagerClient::LidState::CLOSED) {
+    features_.set_device_mode(UserActivityEvent::Features::CLOSED_LID);
+  } else if (lid_state_ == chromeos::PowerManagerClient::LidState::OPEN) {
+    if (tablet_mode_ == chromeos::PowerManagerClient::TabletMode::ON) {
+      features_.set_device_mode(UserActivityEvent::Features::TABLET);
+    } else {
+      features_.set_device_mode(UserActivityEvent::Features::CLAMSHELL);
+    }
+  } else {
+    features_.set_device_mode(UserActivityEvent::Features::UNKNOWN_MODE);
+  }
+
+  features_.set_device_type(device_type_);
+
+  if (battery_percent_.has_value()) {
+    features_.set_battery_percent(*battery_percent_);
+  }
+  features_.set_on_battery(on_battery_);
 }
 
 }  // namespace ml

chrome/browser/chromeos/power/ml/user_activity_logger.h

@@ -6,8 +6,14 @@
 #define CHROME_BROWSER_CHROMEOS_POWER_ML_USER_ACTIVITY_LOGGER_H_
 
 #include "base/macros.h"
+#include "base/optional.h"
+#include "base/scoped_observer.h"
+#include "base/time/time.h"
 #include "chrome/browser/chromeos/power/ml/idle_event_notifier.h"
+#include "chrome/browser/chromeos/power/ml/user_activity_event.pb.h"
 #include "chrome/browser/chromeos/power/ml/user_activity_logger_delegate.h"
+#include "chromeos/dbus/power_manager_client.h"
+#include "ui/base/user_activity/user_activity_detector.h"
 #include "ui/base/user_activity/user_activity_observer.h"
 
 namespace chromeos {
@@ -17,13 +23,21 @@ namespace ml {
 // Logs user activity after an idle event is observed.
 // TODO(renjieliu): Add power-related activity as well.
 class UserActivityLogger : public ui::UserActivityObserver,
-                           public IdleEventNotifier::Observer {
+                           public IdleEventNotifier::Observer,
+                           public PowerManagerClient::Observer {
  public:
-  explicit UserActivityLogger(UserActivityLoggerDelegate* delegate);
+  UserActivityLogger(UserActivityLoggerDelegate* delegate,
+                     IdleEventNotifier* idle_event_notifier,
+                     ui::UserActivityDetector* detector,
+                     chromeos::PowerManagerClient* power_manager_client);
   ~UserActivityLogger() override;
 
- private:
-  friend class UserActivityLoggerTest;
+  // chromeos::PowerManagerClient::Observer overrides:
+  void LidEventReceived(chromeos::PowerManagerClient::LidState state,
+                        const base::TimeTicks& timestamp) override;
+  void PowerChanged(const power_manager::PowerSupplyProperties& proto) override;
+  void TabletModeEventReceived(chromeos::PowerManagerClient::TabletMode mode,
+                               const base::TimeTicks& timestamp) override;
 
   // ui::UserActivityObserver overrides.
   void OnUserActivity(const ui::Event* event) override;
@@ -32,15 +46,49 @@ class UserActivityLogger : public ui::UserActivityObserver,
   void OnIdleEventObserved(
       const IdleEventNotifier::ActivityData& data) override;
 
+ private:
+  friend class UserActivityLoggerTest;
+
+  // Updates lid state and tablet mode from received switch states.
+  void OnReceiveSwitchStates(
+      base::Optional<chromeos::PowerManagerClient::SwitchStates> switch_states);
+
+  // Extracts features from last known activity data and from device states.
+  void ExtractFeatures(const IdleEventNotifier::ActivityData& activity_data);
+
   // Flag indicating whether an idle event has been observed.
   bool idle_event_observed_ = false;
 
-  // UserActivity data derived from idle event.
-  IdleEventNotifier::ActivityData activity_data_;
+  chromeos::PowerManagerClient::LidState lid_state_ =
+      chromeos::PowerManagerClient::LidState::NOT_PRESENT;
+
+  chromeos::PowerManagerClient::TabletMode tablet_mode_ =
+      chromeos::PowerManagerClient::TabletMode::UNSUPPORTED;
+
+  UserActivityEvent::Features::DeviceType device_type_ =
+      UserActivityEvent::Features::UNKNOWN_DEVICE;
+
+  // Flag indicating whether the device is on battery.
+  bool on_battery_ = false;
+
+  // Battery percent. This is in the range [0.0, 100.0].
+  base::Optional<float> battery_percent_;
+
+  // Features extracted when receives an idle event.
+  UserActivityEvent::Features features_;
 
-  // Logger delegate.
   UserActivityLoggerDelegate* const logger_delegate_;
 
+  ScopedObserver<IdleEventNotifier, IdleEventNotifier::Observer>
+      idle_event_observer_;
+
+  ScopedObserver<ui::UserActivityDetector, ui::UserActivityObserver>
+      user_activity_observer_;
+
+  ScopedObserver<chromeos::PowerManagerClient,
+                 chromeos::PowerManagerClient::Observer>
+      power_manager_client_observer_;
+
   DISALLOW_COPY_AND_ASSIGN(UserActivityLogger);
 };
 

chrome/browser/chromeos/power/ml/user_activity_logger_unittest.cc

@@ -7,11 +7,17 @@
 #include <memory>
 #include <vector>
 
+#include "base/single_thread_task_runner.h"
+#include "base/test/scoped_task_environment.h"
 #include "base/timer/timer.h"
 #include "chrome/browser/chromeos/power/ml/idle_event_notifier.h"
 #include "chrome/browser/chromeos/power/ml/user_activity_event.pb.h"
 #include "chrome/browser/chromeos/power/ml/user_activity_logger_delegate.h"
+#include "chromeos/dbus/fake_power_manager_client.h"
+#include "chromeos/dbus/power_manager/power_supply_properties.pb.h"
+#include "chromeos/dbus/power_manager_client.h"
 #include "testing/gtest/include/gtest/gtest.h"
+#include "ui/base/user_activity/user_activity_detector.h"
 
 namespace chromeos {
 namespace power {
@@ -44,16 +50,41 @@ class TestingUserActivityLoggerDelegate : public UserActivityLoggerDelegate {
 
 class UserActivityLoggerTest : public testing::Test {
  public:
-  UserActivityLoggerTest() : activity_logger_(&delegate_) {}
+  UserActivityLoggerTest() : task_runner_(base::ThreadTaskRunnerHandle::Get()) {
+    viz::mojom::VideoDetectorObserverPtr observer;
+    idle_event_notifier_ = std::make_unique<IdleEventNotifier>(
+        &fake_power_manager_client_, mojo::MakeRequest(&observer));
+    activity_logger_ = std::make_unique<UserActivityLogger>(
+        &delegate_, idle_event_notifier_.get(), &user_activity_detector_,
+        &fake_power_manager_client_);
+  }
+
   ~UserActivityLoggerTest() override = default;
 
  protected:
   void ReportUserActivity(const ui::Event* event) {
-    activity_logger_.OnUserActivity(event);
+    activity_logger_->OnUserActivity(event);
   }
 
   void ReportIdleEvent(const IdleEventNotifier::ActivityData& data) {
-    activity_logger_.OnIdleEventObserved(data);
+    activity_logger_->OnIdleEventObserved(data);
+  }
+
+  void ReportLidEvent(chromeos::PowerManagerClient::LidState state) {
+    fake_power_manager_client_.SetLidState(state, base::TimeTicks::Now());
+  }
+
+  void ReportPowerChangeEvent(
+      power_manager::PowerSupplyProperties::ExternalPower power,
+      float battery_percent) {
+    power_manager::PowerSupplyProperties proto;
+    proto.set_external_power(power);
+    proto.set_battery_percent(battery_percent);
+    fake_power_manager_client_.UpdatePowerProperties(proto);
+  }
+
+  void ReportTabletModeEvent(chromeos::PowerManagerClient::TabletMode mode) {
+    fake_power_manager_client_.SetTabletMode(mode, base::TimeTicks::Now());
   }
 
   const std::vector<UserActivityEvent>& GetEvents() {
@@ -61,8 +92,14 @@ class UserActivityLoggerTest : public testing::Test {
   }
 
  private:
+  base::test::ScopedTaskEnvironment scoped_task_environment_;
+  scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
+
+  ui::UserActivityDetector user_activity_detector_;
   TestingUserActivityLoggerDelegate delegate_;
-  UserActivityLogger activity_logger_;
+  std::unique_ptr<IdleEventNotifier> idle_event_notifier_;
+  chromeos::FakePowerManagerClient fake_power_manager_client_;
+  std::unique_ptr<UserActivityLogger> activity_logger_;
 };
 
 // After an idle event, we have a ui::Event, we should expect one
@@ -135,6 +172,24 @@ TEST_F(UserActivityLoggerTest, LogMultipleEvents) {
   EqualEvent(expected_event, events[1].event());
 }
 
+// Test feature extraction.
+TEST_F(UserActivityLoggerTest, FeatureExtraction) {
+  ReportLidEvent(chromeos::PowerManagerClient::LidState::OPEN);
+  ReportTabletModeEvent(chromeos::PowerManagerClient::TabletMode::UNSUPPORTED);
+  ReportPowerChangeEvent(power_manager::PowerSupplyProperties::AC, 23.0f);
+
+  ReportIdleEvent({});
+  ReportUserActivity(nullptr);
+
+  const auto& events = GetEvents();
+  ASSERT_EQ(1U, events.size());
+
+  const UserActivityEvent::Features& features = events[0].features();
+  EXPECT_EQ(UserActivityEvent::Features::CLAMSHELL, features.device_mode());
+  EXPECT_EQ(23.0f, features.battery_percent());
+  EXPECT_DOUBLE_EQ(false, features.on_battery());
+}
+
 }  // namespace ml
 }  // namespace power
 }  // namespace chromeos