Commit afdf8dfd authored by Tao Bai's avatar Tao Bai Committed by Commit Bot

Save the unsent log info for metrics

- Add the optional dedicated preference for the unsent log info.
- Save the unsent log info at each time the unsent log persisted.
- Read the unsent log info at next browser startup.
- Define the record method in UnsentLogStoreMetrics to record
  the count of the unsent samples, the count of the sent samples
  the percentage of unsent samples and the persisted logs size.

Bug: 1076564
Change-Id: I1ba3018ae819ff6f350fbf6a1fe8d7dbf0d17e26
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2224314
Commit-Queue: Tao Bai <michaelbai@chromium.org>
Reviewed-by: default avatarAlexei Svitkine <asvitkine@chromium.org>
Cr-Commit-Position: refs/heads/master@{#775841}
parent 21da79ea
...@@ -124,6 +124,7 @@ jumbo_static_library("metrics") { ...@@ -124,6 +124,7 @@ jumbo_static_library("metrics") {
"ukm_demographic_metrics_provider.h", "ukm_demographic_metrics_provider.h",
"unsent_log_store.cc", "unsent_log_store.cc",
"unsent_log_store.h", "unsent_log_store.h",
"unsent_log_store_metrics.cc",
"unsent_log_store_metrics.h", "unsent_log_store_metrics.h",
"unsent_log_store_metrics_impl.cc", "unsent_log_store_metrics_impl.cc",
"unsent_log_store_metrics_impl.h", "unsent_log_store_metrics_impl.h",
......
...@@ -42,6 +42,10 @@ class LogStore { ...@@ -42,6 +42,10 @@ class LogStore {
// Discards the staged log. // Discards the staged log.
virtual void DiscardStagedLog() = 0; virtual void DiscardStagedLog() = 0;
// Marks the staged log as sent, DiscardStagedLog() shall still be called if
// the staged log needs discarded.
virtual void MarkStagedLogAsSent() = 0;
// Saves any unsent logs to persistent storage. // Saves any unsent logs to persistent storage.
virtual void PersistUnsentLogs() const = 0; virtual void PersistUnsentLogs() const = 0;
......
...@@ -38,6 +38,8 @@ const size_t kStorageByteLimitPerLogType = 300 * 1000; // ~300kB ...@@ -38,6 +38,8 @@ const size_t kStorageByteLimitPerLogType = 300 * 1000; // ~300kB
void MetricsLogStore::RegisterPrefs(PrefRegistrySimple* registry) { void MetricsLogStore::RegisterPrefs(PrefRegistrySimple* registry) {
registry->RegisterListPref(prefs::kMetricsInitialLogs); registry->RegisterListPref(prefs::kMetricsInitialLogs);
registry->RegisterListPref(prefs::kMetricsOngoingLogs); registry->RegisterListPref(prefs::kMetricsOngoingLogs);
registry->RegisterDictionaryPref(prefs::kMetricsInitialLogsMetadata);
registry->RegisterDictionaryPref(prefs::kMetricsOngoingLogsMetadata);
} }
MetricsLogStore::MetricsLogStore(PrefService* local_state, MetricsLogStore::MetricsLogStore(PrefService* local_state,
...@@ -47,6 +49,7 @@ MetricsLogStore::MetricsLogStore(PrefService* local_state, ...@@ -47,6 +49,7 @@ MetricsLogStore::MetricsLogStore(PrefService* local_state,
initial_log_queue_(std::make_unique<UnsentLogStoreMetricsImpl>(), initial_log_queue_(std::make_unique<UnsentLogStoreMetricsImpl>(),
local_state, local_state,
prefs::kMetricsInitialLogs, prefs::kMetricsInitialLogs,
prefs::kMetricsInitialLogsMetadata,
kInitialLogsSaveLimit, kInitialLogsSaveLimit,
kStorageByteLimitPerLogType, kStorageByteLimitPerLogType,
0, 0,
...@@ -54,6 +57,7 @@ MetricsLogStore::MetricsLogStore(PrefService* local_state, ...@@ -54,6 +57,7 @@ MetricsLogStore::MetricsLogStore(PrefService* local_state,
ongoing_log_queue_(std::make_unique<UnsentLogStoreMetricsImpl>(), ongoing_log_queue_(std::make_unique<UnsentLogStoreMetricsImpl>(),
local_state, local_state,
prefs::kMetricsOngoingLogs, prefs::kMetricsOngoingLogs,
prefs::kMetricsOngoingLogsMetadata,
kOngoingLogsSaveLimit, kOngoingLogsSaveLimit,
kStorageByteLimitPerLogType, kStorageByteLimitPerLogType,
max_ongoing_log_size, max_ongoing_log_size,
...@@ -71,11 +75,13 @@ void MetricsLogStore::StoreLog(const std::string& log_data, ...@@ -71,11 +75,13 @@ void MetricsLogStore::StoreLog(const std::string& log_data,
MetricsLog::LogType log_type) { MetricsLog::LogType log_type) {
switch (log_type) { switch (log_type) {
case MetricsLog::INITIAL_STABILITY_LOG: case MetricsLog::INITIAL_STABILITY_LOG:
initial_log_queue_.StoreLog(log_data); // TODO(crbug.com/1076564): Pass the log's sample count.
initial_log_queue_.StoreLog(log_data, base::nullopt);
break; break;
case MetricsLog::ONGOING_LOG: case MetricsLog::ONGOING_LOG:
case MetricsLog::INDEPENDENT_LOG: case MetricsLog::INDEPENDENT_LOG:
ongoing_log_queue_.StoreLog(log_data); // TODO(crbug.com/1076564): Pass the log's sample count.
ongoing_log_queue_.StoreLog(log_data, base::nullopt);
break; break;
} }
} }
...@@ -124,6 +130,14 @@ void MetricsLogStore::DiscardStagedLog() { ...@@ -124,6 +130,14 @@ void MetricsLogStore::DiscardStagedLog() {
DCHECK(!has_staged_log()); DCHECK(!has_staged_log());
} }
void MetricsLogStore::MarkStagedLogAsSent() {
DCHECK(has_staged_log());
if (initial_log_queue_.has_staged_log())
initial_log_queue_.MarkStagedLogAsSent();
else
ongoing_log_queue_.MarkStagedLogAsSent();
}
void MetricsLogStore::PersistUnsentLogs() const { void MetricsLogStore::PersistUnsentLogs() const {
DCHECK(unsent_logs_loaded_); DCHECK(unsent_logs_loaded_);
if (!unsent_logs_loaded_) if (!unsent_logs_loaded_)
......
...@@ -46,6 +46,7 @@ class MetricsLogStore : public LogStore { ...@@ -46,6 +46,7 @@ class MetricsLogStore : public LogStore {
const std::string& staged_log_signature() const override; const std::string& staged_log_signature() const override;
void StageNextLog() override; void StageNextLog() override;
void DiscardStagedLog() override; void DiscardStagedLog() override;
void MarkStagedLogAsSent() override;
void PersistUnsentLogs() const override; void PersistUnsentLogs() const override;
void LoadPersistedUnsentLogs() override; void LoadPersistedUnsentLogs() override;
......
...@@ -28,6 +28,12 @@ const char kMetricsDefaultOptIn[] = "user_experience_metrics.default_opt_in"; ...@@ -28,6 +28,12 @@ const char kMetricsDefaultOptIn[] = "user_experience_metrics.default_opt_in";
// count info, etc. // count info, etc.
const char kMetricsInitialLogs[] = "user_experience_metrics.initial_logs2"; const char kMetricsInitialLogs[] = "user_experience_metrics.initial_logs2";
// An dictionary of information about the unsent initial logs, it was
// recorded when the unsent log is persisted and will be written into the
// metrics at the next browser starts up.
const char kMetricsInitialLogsMetadata[] =
"user_experience_metrics.unsent_log_metadata.initial_logs";
// Low entropy source values. The new source (with suffix "3") was created // Low entropy source values. The new source (with suffix "3") was created
// because the old source (with suffix "2") is biased in the wild. Clients which // because the old source (with suffix "2") is biased in the wild. Clients which
// have an old source still incorporate it into the high entropy source, to // have an old source still incorporate it into the high entropy source, to
...@@ -50,6 +56,11 @@ const char kMetricsMachineId[] = "user_experience_metrics.machine_id"; ...@@ -50,6 +56,11 @@ const char kMetricsMachineId[] = "user_experience_metrics.machine_id";
// user activities. // user activities.
const char kMetricsOngoingLogs[] = "user_experience_metrics.ongoing_logs2"; const char kMetricsOngoingLogs[] = "user_experience_metrics.ongoing_logs2";
// An dictionary that is same as kUnsentLogMetkMetricsInitialLogsMetadata,
// but for the ongoing logs.
const char kMetricsOngoingLogsMetadata[] =
"user_experience_metrics.unsent_log_metadata.ongoing_logs";
// Boolean that indicates a cloned install has been detected and the metrics // Boolean that indicates a cloned install has been detected and the metrics
// client id and low entropy source should be reset. // client id and low entropy source should be reset.
const char kMetricsResetIds[] = "user_experience_metrics.reset_metrics_ids"; const char kMetricsResetIds[] = "user_experience_metrics.reset_metrics_ids";
......
...@@ -14,10 +14,12 @@ extern const char kInstallDate[]; ...@@ -14,10 +14,12 @@ extern const char kInstallDate[];
extern const char kMetricsClientID[]; extern const char kMetricsClientID[];
extern const char kMetricsDefaultOptIn[]; extern const char kMetricsDefaultOptIn[];
extern const char kMetricsInitialLogs[]; extern const char kMetricsInitialLogs[];
extern const char kMetricsInitialLogsMetadata[];
extern const char kMetricsLowEntropySource[]; extern const char kMetricsLowEntropySource[];
extern const char kMetricsOldLowEntropySource[]; extern const char kMetricsOldLowEntropySource[];
extern const char kMetricsMachineId[]; extern const char kMetricsMachineId[];
extern const char kMetricsOngoingLogs[]; extern const char kMetricsOngoingLogs[];
extern const char kMetricsOngoingLogsMetadata[];
extern const char kMetricsResetIds[]; extern const char kMetricsResetIds[];
// For finding out whether metrics and crash reporting is enabled use the // For finding out whether metrics and crash reporting is enabled use the
......
...@@ -192,6 +192,9 @@ void ReportingService::OnLogUploadComplete(int response_code, ...@@ -192,6 +192,9 @@ void ReportingService::OnLogUploadComplete(int response_code,
} }
if (upload_succeeded || discard_log) { if (upload_succeeded || discard_log) {
if (upload_succeeded)
log_store()->MarkStagedLogAsSent();
log_store()->DiscardStagedLog(); log_store()->DiscardStagedLog();
// Store the updated list to disk now that the removed log is uploaded. // Store the updated list to disk now that the removed log is uploaded.
log_store()->PersistUnsentLogs(); log_store()->PersistUnsentLogs();
......
...@@ -56,6 +56,7 @@ class TestLogStore : public LogStore { ...@@ -56,6 +56,7 @@ class TestLogStore : public LogStore {
logs_.pop_front(); logs_.pop_front();
staged_log_hash_.clear(); staged_log_hash_.clear();
} }
void MarkStagedLogAsSent() override {}
void PersistUnsentLogs() const override {} void PersistUnsentLogs() const override {}
void LoadPersistedUnsentLogs() override {} void LoadPersistedUnsentLogs() override {}
......
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
#include "components/metrics/unsent_log_store.h" #include "components/metrics/unsent_log_store.h"
#include <cmath>
#include <memory> #include <memory>
#include <string> #include <string>
#include <utility> #include <utility>
...@@ -28,6 +29,9 @@ const char kLogHashKey[] = "hash"; ...@@ -28,6 +29,9 @@ const char kLogHashKey[] = "hash";
const char kLogSignatureKey[] = "signature"; const char kLogSignatureKey[] = "signature";
const char kLogTimestampKey[] = "timestamp"; const char kLogTimestampKey[] = "timestamp";
const char kLogDataKey[] = "data"; const char kLogDataKey[] = "data";
const char kLogUnsentCountKey[] = "unsent_samples_count";
const char kLogSentCountKey[] = "sent_samples_count";
const char kLogPersistedSizeInKbKey[] = "unsent_persisted_size_in_kb";
std::string EncodeToBase64(const std::string& to_convert) { std::string EncodeToBase64(const std::string& to_convert) {
DCHECK(to_convert.data()); DCHECK(to_convert.data());
...@@ -44,15 +48,17 @@ std::string DecodeFromBase64(const std::string& to_convert) { ...@@ -44,15 +48,17 @@ std::string DecodeFromBase64(const std::string& to_convert) {
} // namespace } // namespace
UnsentLogStore::LogInfo::LogInfo() {} UnsentLogStore::LogInfo::LogInfo() = default;
UnsentLogStore::LogInfo::LogInfo( UnsentLogStore::LogInfo::LogInfo(const UnsentLogStore::LogInfo& other) =
const UnsentLogStore::LogInfo& other) = default; default;
UnsentLogStore::LogInfo::~LogInfo() {} UnsentLogStore::LogInfo::~LogInfo() = default;
void UnsentLogStore::LogInfo::Init(UnsentLogStoreMetrics* metrics, void UnsentLogStore::LogInfo::Init(
UnsentLogStoreMetrics* metrics,
const std::string& log_data, const std::string& log_data,
const std::string& log_timestamp, const std::string& log_timestamp,
const std::string& signing_key) { const std::string& signing_key,
base::Optional<base::HistogramBase::Count> samples_count) {
DCHECK(!log_data.empty()); DCHECK(!log_data.empty());
if (!compression::GzipCompress(log_data, &compressed_log_data)) { if (!compression::GzipCompress(log_data, &compressed_log_data)) {
...@@ -75,18 +81,21 @@ void UnsentLogStore::LogInfo::Init(UnsentLogStoreMetrics* metrics, ...@@ -75,18 +81,21 @@ void UnsentLogStore::LogInfo::Init(UnsentLogStoreMetrics* metrics,
} }
timestamp = log_timestamp; timestamp = log_timestamp;
this->samples_count = samples_count;
} }
UnsentLogStore::UnsentLogStore(std::unique_ptr<UnsentLogStoreMetrics> metrics, UnsentLogStore::UnsentLogStore(std::unique_ptr<UnsentLogStoreMetrics> metrics,
PrefService* local_state, PrefService* local_state,
const char* pref_name, const char* log_data_pref_name,
const char* metadata_pref_name,
size_t min_log_count, size_t min_log_count,
size_t min_log_bytes, size_t min_log_bytes,
size_t max_log_size, size_t max_log_size,
const std::string& signing_key) const std::string& signing_key)
: metrics_(std::move(metrics)), : metrics_(std::move(metrics)),
local_state_(local_state), local_state_(local_state),
pref_name_(pref_name), log_data_pref_name_(log_data_pref_name),
metadata_pref_name_(metadata_pref_name),
min_log_count_(min_log_count), min_log_count_(min_log_count),
min_log_bytes_(min_log_bytes), min_log_bytes_(min_log_bytes),
max_log_size_(max_log_size != 0 ? max_log_size : static_cast<size_t>(-1)), max_log_size_(max_log_size != 0 ? max_log_size : static_cast<size_t>(-1)),
...@@ -148,8 +157,15 @@ void UnsentLogStore::DiscardStagedLog() { ...@@ -148,8 +157,15 @@ void UnsentLogStore::DiscardStagedLog() {
staged_log_index_ = -1; staged_log_index_ = -1;
} }
void UnsentLogStore::MarkStagedLogAsSent() {
DCHECK(has_staged_log());
DCHECK_LT(static_cast<size_t>(staged_log_index_), list_.size());
if (list_[staged_log_index_].samples_count.has_value())
total_samples_sent_ += list_[staged_log_index_].samples_count.value();
}
void UnsentLogStore::PersistUnsentLogs() const { void UnsentLogStore::PersistUnsentLogs() const {
ListPrefUpdate update(local_state_, pref_name_); ListPrefUpdate update(local_state_, log_data_pref_name_);
// TODO(crbug.com/859477): Verify that the preference has been properly // TODO(crbug.com/859477): Verify that the preference has been properly
// registered. // registered.
CHECK(update.Get()); CHECK(update.Get());
...@@ -157,14 +173,17 @@ void UnsentLogStore::PersistUnsentLogs() const { ...@@ -157,14 +173,17 @@ void UnsentLogStore::PersistUnsentLogs() const {
} }
void UnsentLogStore::LoadPersistedUnsentLogs() { void UnsentLogStore::LoadPersistedUnsentLogs() {
ReadLogsFromPrefList(*local_state_->GetList(pref_name_)); ReadLogsFromPrefList(*local_state_->GetList(log_data_pref_name_));
RecordMetaDataMertics();
} }
void UnsentLogStore::StoreLog(const std::string& log_data) { void UnsentLogStore::StoreLog(
list_.push_back(LogInfo()); const std::string& log_data,
base::Optional<base::HistogramBase::Count> samples_count) {
list_.emplace_back();
list_.back().Init(metrics_.get(), log_data, list_.back().Init(metrics_.get(), log_data,
base::NumberToString(base::Time::Now().ToTimeT()), base::NumberToString(base::Time::Now().ToTimeT()),
signing_key_); signing_key_, samples_count);
} }
const std::string& UnsentLogStore::GetLogAtIndex(size_t index) { const std::string& UnsentLogStore::GetLogAtIndex(size_t index) {
...@@ -173,8 +192,10 @@ const std::string& UnsentLogStore::GetLogAtIndex(size_t index) { ...@@ -173,8 +192,10 @@ const std::string& UnsentLogStore::GetLogAtIndex(size_t index) {
return list_[index].compressed_log_data; return list_[index].compressed_log_data;
} }
std::string UnsentLogStore::ReplaceLogAtIndex(size_t index, std::string UnsentLogStore::ReplaceLogAtIndex(
const std::string& new_log_data) { size_t index,
const std::string& new_log_data,
base::Optional<base::HistogramBase::Count> samples_count) {
DCHECK_GE(index, 0U); DCHECK_GE(index, 0U);
DCHECK_LT(index, list_.size()); DCHECK_LT(index, list_.size());
...@@ -185,7 +206,8 @@ std::string UnsentLogStore::ReplaceLogAtIndex(size_t index, ...@@ -185,7 +206,8 @@ std::string UnsentLogStore::ReplaceLogAtIndex(size_t index,
old_timestamp.swap(list_[index].timestamp); old_timestamp.swap(list_[index].timestamp);
list_[index] = LogInfo(); list_[index] = LogInfo();
list_[index].Init(metrics_.get(), new_log_data, old_timestamp, signing_key_); list_[index].Init(metrics_.get(), new_log_data, old_timestamp, signing_key_,
samples_count);
return old_log_data; return old_log_data;
} }
...@@ -194,7 +216,11 @@ void UnsentLogStore::Purge() { ...@@ -194,7 +216,11 @@ void UnsentLogStore::Purge() {
DiscardStagedLog(); DiscardStagedLog();
} }
list_.clear(); list_.clear();
local_state_->ClearPref(pref_name_); local_state_->ClearPref(log_data_pref_name_);
// The |total_samples_sent_| isn't cleared intentionally because it is still
// meaningful.
if (metadata_pref_name_)
local_state_->ClearPref(metadata_pref_name_);
} }
void UnsentLogStore::ReadLogsFromPrefList(const base::ListValue& list_value) { void UnsentLogStore::ReadLogsFromPrefList(const base::ListValue& list_value) {
...@@ -257,6 +283,8 @@ void UnsentLogStore::WriteLogsToPrefList(base::ListValue* list_value) const { ...@@ -257,6 +283,8 @@ void UnsentLogStore::WriteLogsToPrefList(base::ListValue* list_value) const {
++saved_log_count; ++saved_log_count;
} }
int dropped_logs_num = start - 1; int dropped_logs_num = start - 1;
base::HistogramBase::Count unsent_samples_count = 0;
size_t unsent_persisted_size = 0;
for (size_t i = start; i < list_.size(); ++i) { for (size_t i = start; i < list_.size(); ++i) {
size_t log_size = list_[i].compressed_log_data.length(); size_t log_size = list_[i].compressed_log_data.length();
...@@ -273,9 +301,56 @@ void UnsentLogStore::WriteLogsToPrefList(base::ListValue* list_value) const { ...@@ -273,9 +301,56 @@ void UnsentLogStore::WriteLogsToPrefList(base::ListValue* list_value) const {
EncodeToBase64(list_[i].compressed_log_data)); EncodeToBase64(list_[i].compressed_log_data));
dict_value->SetString(kLogTimestampKey, list_[i].timestamp); dict_value->SetString(kLogTimestampKey, list_[i].timestamp);
list_value->Append(std::move(dict_value)); list_value->Append(std::move(dict_value));
if (list_[i].samples_count.has_value()) {
unsent_samples_count += list_[i].samples_count.value();
}
unsent_persisted_size += log_size;
} }
if (dropped_logs_num > 0) if (dropped_logs_num > 0)
metrics_->RecordDroppedLogsNum(dropped_logs_num); metrics_->RecordDroppedLogsNum(dropped_logs_num);
WriteToMetricsPref(unsent_samples_count, total_samples_sent_,
unsent_persisted_size);
}
void UnsentLogStore::WriteToMetricsPref(
base::HistogramBase::Count unsent_samples_count,
base::HistogramBase::Count sent_samples_count,
size_t unsent_persisted_size) const {
if (metadata_pref_name_ == nullptr)
return;
DictionaryPrefUpdate update(local_state_, metadata_pref_name_);
base::DictionaryValue* pref_data = update.Get();
pref_data->SetKey(kLogUnsentCountKey, base::Value(unsent_samples_count));
pref_data->SetKey(kLogSentCountKey, base::Value(sent_samples_count));
// Round up to kb.
pref_data->SetKey(
kLogPersistedSizeInKbKey,
base::Value(static_cast<int>(std::ceil(unsent_persisted_size / 1024.0))));
}
void UnsentLogStore::RecordMetaDataMertics() {
if (metadata_pref_name_ == nullptr)
return;
const base::DictionaryValue* value =
local_state_->GetDictionary(metadata_pref_name_);
if (!value)
return;
auto unsent_samples_count = value->FindIntKey(kLogUnsentCountKey);
auto sent_samples_count = value->FindIntKey(kLogSentCountKey);
auto unsent_persisted_size_in_kb =
value->FindIntKey(kLogPersistedSizeInKbKey);
if (unsent_samples_count && sent_samples_count &&
unsent_persisted_size_in_kb) {
metrics_->RecordLastUnsentLogMetadataMetrics(
unsent_samples_count.value(), sent_samples_count.value(),
unsent_persisted_size_in_kb.value());
}
} }
} // namespace metrics } // namespace metrics
...@@ -11,8 +11,11 @@ ...@@ -11,8 +11,11 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include "base/gtest_prod_util.h"
#include "base/logging.h" #include "base/logging.h"
#include "base/macros.h" #include "base/macros.h"
#include "base/metrics/histogram_base.h"
#include "base/optional.h"
#include "base/values.h" #include "base/values.h"
#include "components/metrics/log_store.h" #include "components/metrics/log_store.h"
...@@ -26,10 +29,14 @@ class UnsentLogStoreMetrics; ...@@ -26,10 +29,14 @@ class UnsentLogStoreMetrics;
class UnsentLogStore : public LogStore { class UnsentLogStore : public LogStore {
public: public:
// Constructs an UnsentLogStore that stores data in |local_state| under the // Constructs an UnsentLogStore that stores data in |local_state| under the
// preference |pref_name|. // preference |log_data_pref_name|.
// Calling code is responsible for ensuring that the lifetime of |local_state| // Calling code is responsible for ensuring that the lifetime of |local_state|
// is longer than the lifetime of UnsentLogStore. // is longer than the lifetime of UnsentLogStore.
// //
// The optional |metadata_pref_name| is the preference that is used to store
// the unsent logs info while the unset logs are persisted. That info will be
// recorded as UMA metrics in next browser startup.
//
// When saving logs to disk, stores either the first |min_log_count| logs, or // When saving logs to disk, stores either the first |min_log_count| logs, or
// at least |min_log_bytes| bytes of logs, whichever is greater. // at least |min_log_bytes| bytes of logs, whichever is greater.
// //
...@@ -41,7 +48,8 @@ class UnsentLogStore : public LogStore { ...@@ -41,7 +48,8 @@ class UnsentLogStore : public LogStore {
// integrity. // integrity.
UnsentLogStore(std::unique_ptr<UnsentLogStoreMetrics> metrics, UnsentLogStore(std::unique_ptr<UnsentLogStoreMetrics> metrics,
PrefService* local_state, PrefService* local_state,
const char* pref_name, const char* log_data_pref_name,
const char* metadata_pref_name,
size_t min_log_count, size_t min_log_count,
size_t min_log_bytes, size_t min_log_bytes,
size_t max_log_size, size_t max_log_size,
...@@ -56,18 +64,24 @@ class UnsentLogStore : public LogStore { ...@@ -56,18 +64,24 @@ class UnsentLogStore : public LogStore {
const std::string& staged_log_signature() const override; const std::string& staged_log_signature() const override;
void StageNextLog() override; void StageNextLog() override;
void DiscardStagedLog() override; void DiscardStagedLog() override;
void MarkStagedLogAsSent() override;
void PersistUnsentLogs() const override; void PersistUnsentLogs() const override;
void LoadPersistedUnsentLogs() override; void LoadPersistedUnsentLogs() override;
// Adds a log to the list. // Adds a UMA log to the list, |samples_count| is the total number of samples
void StoreLog(const std::string& log_data); // in the log (if available).
void StoreLog(const std::string& log_data,
base::Optional<base::HistogramBase::Count> samples_count);
// Gets log data at the given index in the list. // Gets log data at the given index in the list.
const std::string& GetLogAtIndex(size_t index); const std::string& GetLogAtIndex(size_t index);
// Replaces the compressed log at |index| in the store with given log data // Replaces the compressed log at |index| in the store with given log data
// reusing the same timestamp from the original log, and returns old log data. // reusing the same timestamp from the original log, and returns old log data.
std::string ReplaceLogAtIndex(size_t index, const std::string& new_log_data); std::string ReplaceLogAtIndex(
size_t index,
const std::string& new_log_data,
base::Optional<base::HistogramBase::Count> samples_count);
// Deletes all logs, in memory and on disk. // Deletes all logs, in memory and on disk.
void Purge(); void Purge();
...@@ -79,12 +93,22 @@ class UnsentLogStore : public LogStore { ...@@ -79,12 +93,22 @@ class UnsentLogStore : public LogStore {
size_t size() const { return list_.size(); } size_t size() const { return list_.size(); }
private: private:
FRIEND_TEST_ALL_PREFIXES(UnsentLogStoreTest, UnsentLogMetadataMetrics);
// Writes the list to the ListValue. // Writes the list to the ListValue.
void WriteLogsToPrefList(base::ListValue* list) const; void WriteLogsToPrefList(base::ListValue* list) const;
// Reads the list from the ListValue. // Reads the list from the ListValue.
void ReadLogsFromPrefList(const base::ListValue& list); void ReadLogsFromPrefList(const base::ListValue& list);
// Writes the unsent log info to the |metadata_pref_name_| preference.
void WriteToMetricsPref(base::HistogramBase::Count unsent_samples_count,
base::HistogramBase::Count sent_samples_count,
size_t persisted_size) const;
// Records the info in |metadata_pref_name_| as UMA metrics.
void RecordMetaDataMertics();
// An object for recording UMA metrics. // An object for recording UMA metrics.
std::unique_ptr<UnsentLogStoreMetrics> metrics_; std::unique_ptr<UnsentLogStoreMetrics> metrics_;
...@@ -94,7 +118,11 @@ class UnsentLogStore : public LogStore { ...@@ -94,7 +118,11 @@ class UnsentLogStore : public LogStore {
PrefService* local_state_; PrefService* local_state_;
// The name of the preference to serialize logs to/from. // The name of the preference to serialize logs to/from.
const char* pref_name_; const char* log_data_pref_name_;
// The name of the preference to store the unsent logs info, it could be
// nullptr if the metadata isn't desired.
const char* metadata_pref_name_;
// We will keep at least this |min_log_count_| logs or |min_log_bytes_| bytes // We will keep at least this |min_log_count_| logs or |min_log_bytes_| bytes
// of logs, whichever is greater, when writing to disk. These apply after // of logs, whichever is greater, when writing to disk. These apply after
...@@ -124,7 +152,8 @@ class UnsentLogStore : public LogStore { ...@@ -124,7 +152,8 @@ class UnsentLogStore : public LogStore {
void Init(UnsentLogStoreMetrics* metrics, void Init(UnsentLogStoreMetrics* metrics,
const std::string& log_data, const std::string& log_data,
const std::string& log_timestamp, const std::string& log_timestamp,
const std::string& signing_key); const std::string& signing_key,
base::Optional<base::HistogramBase::Count> samples_count);
// Compressed log data - a serialized protobuf that's been gzipped. // Compressed log data - a serialized protobuf that's been gzipped.
std::string compressed_log_data; std::string compressed_log_data;
...@@ -140,6 +169,9 @@ class UnsentLogStore : public LogStore { ...@@ -140,6 +169,9 @@ class UnsentLogStore : public LogStore {
// The timestamp of when the log was created as a time_t value. // The timestamp of when the log was created as a time_t value.
std::string timestamp; std::string timestamp;
// The total number of samples in this log if applicable.
base::Optional<base::HistogramBase::Count> samples_count;
}; };
// A list of all of the stored logs, stored with SHA1 hashes to check for // A list of all of the stored logs, stored with SHA1 hashes to check for
// corruption while they are stored in memory. // corruption while they are stored in memory.
...@@ -149,6 +181,9 @@ class UnsentLogStore : public LogStore { ...@@ -149,6 +181,9 @@ class UnsentLogStore : public LogStore {
// staged, the index will be -1. // staged, the index will be -1.
int staged_log_index_; int staged_log_index_;
// The total number of samples that have been sent from this LogStore.
base::HistogramBase::Count total_samples_sent_ = 0;
DISALLOW_COPY_AND_ASSIGN(UnsentLogStore); DISALLOW_COPY_AND_ASSIGN(UnsentLogStore);
}; };
......
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "components/metrics/unsent_log_store_metrics.h"
namespace metrics {
// static
const base::Feature UnsentLogStoreMetrics::kRecordLastUnsentLogMetadataMetrics =
{"RecordLastUnsentLogMetadataMetrics", base::FEATURE_DISABLED_BY_DEFAULT};
UnsentLogStoreMetrics::UnsentLogStoreMetrics() = default;
UnsentLogStoreMetrics::~UnsentLogStoreMetrics() = default;
void UnsentLogStoreMetrics::RecordLogReadStatus(LogReadStatus status) {}
void UnsentLogStoreMetrics::RecordCompressionRatio(size_t compressed_size,
size_t original_size) {}
void UnsentLogStoreMetrics::RecordDroppedLogSize(size_t size) {}
void UnsentLogStoreMetrics::RecordDroppedLogsNum(int dropped_logs_num) {}
void UnsentLogStoreMetrics::RecordLastUnsentLogMetadataMetrics(
int unsent_samples_count,
int sent_samples_count,
int persisted_size_in_kb) {}
} // namespace metrics
...@@ -5,6 +5,7 @@ ...@@ -5,6 +5,7 @@
#ifndef COMPONENTS_METRICS_UNSENT_LOG_STORE_METRICS_H_ #ifndef COMPONENTS_METRICS_UNSENT_LOG_STORE_METRICS_H_
#define COMPONENTS_METRICS_UNSENT_LOG_STORE_METRICS_H_ #define COMPONENTS_METRICS_UNSENT_LOG_STORE_METRICS_H_
#include "base/feature_list.h"
#include "base/macros.h" #include "base/macros.h"
#include "components/metrics/unsent_log_store.h" #include "components/metrics/unsent_log_store.h"
...@@ -31,17 +32,25 @@ class UnsentLogStoreMetrics { ...@@ -31,17 +32,25 @@ class UnsentLogStoreMetrics {
END_RECALL_STATUS // Number of bins to use to create the histogram. END_RECALL_STATUS // Number of bins to use to create the histogram.
}; };
UnsentLogStoreMetrics() {} UnsentLogStoreMetrics();
virtual ~UnsentLogStoreMetrics() {} virtual ~UnsentLogStoreMetrics();
virtual void RecordLogReadStatus(LogReadStatus status) {} virtual void RecordLogReadStatus(LogReadStatus status);
virtual void RecordCompressionRatio( virtual void RecordCompressionRatio(size_t compressed_size,
size_t compressed_size, size_t original_size) {} size_t original_size);
virtual void RecordDroppedLogSize(size_t size) {} virtual void RecordDroppedLogSize(size_t size);
virtual void RecordDroppedLogsNum(int dropped_logs_num) {} virtual void RecordDroppedLogsNum(int dropped_logs_num);
virtual void RecordLastUnsentLogMetadataMetrics(int unsent_samples_count,
int sent_samples_count,
int persisted_size_in_kb);
// The feature to record the unsent log info metrics, refer to
// UnsentLogStoreMetricsImpl::RecordLastUnsentLogMetadataMetrics.
static const base::Feature kRecordLastUnsentLogMetadataMetrics;
private: private:
DISALLOW_COPY_AND_ASSIGN(UnsentLogStoreMetrics); DISALLOW_COPY_AND_ASSIGN(UnsentLogStoreMetrics);
......
...@@ -64,6 +64,7 @@ UkmReportingService::UkmReportingService(metrics::MetricsServiceClient* client, ...@@ -64,6 +64,7 @@ UkmReportingService::UkmReportingService(metrics::MetricsServiceClient* client,
unsent_log_store_(std::make_unique<ukm::UnsentLogStoreMetricsImpl>(), unsent_log_store_(std::make_unique<ukm::UnsentLogStoreMetricsImpl>(),
local_state, local_state,
prefs::kUkmUnsentLogStore, prefs::kUkmUnsentLogStore,
nullptr,
kMinUnsentLogCount, kMinUnsentLogCount,
kMinUnsentLogBytes, kMinUnsentLogBytes,
kMaxLogRetransmitSize, kMaxLogRetransmitSize,
......
...@@ -156,7 +156,8 @@ void PurgeExtensionDataFromUnsentLogStore( ...@@ -156,7 +156,8 @@ void PurgeExtensionDataFromUnsentLogStore(
// Replace the compressed log in the store by its filtered version. // Replace the compressed log in the store by its filtered version.
const std::string old_compressed_log_data = const std::string old_compressed_log_data =
ukm_log_store->ReplaceLogAtIndex(index, reserialized_log_data); ukm_log_store->ReplaceLogAtIndex(index, reserialized_log_data,
base::nullopt);
// Reached here only if extensions were found in the log, so data should now // Reached here only if extensions were found in the log, so data should now
// be different after filtering. // be different after filtering.
...@@ -395,7 +396,7 @@ void UkmService::BuildAndStoreLog() { ...@@ -395,7 +396,7 @@ void UkmService::BuildAndStoreLog() {
std::string serialized_log; std::string serialized_log;
report.SerializeToString(&serialized_log); report.SerializeToString(&serialized_log);
reporting_service_.ukm_log_store()->StoreLog(serialized_log); reporting_service_.ukm_log_store()->StoreLog(serialized_log, base::nullopt);
} }
bool UkmService::ShouldRestrictToWhitelistedEntries() const { bool UkmService::ShouldRestrictToWhitelistedEntries() const {
......
...@@ -146,10 +146,10 @@ class UkmServiceTest : public testing::Test { ...@@ -146,10 +146,10 @@ class UkmServiceTest : public testing::Test {
EXPECT_GE(GetPersistedLogCount(), 1); EXPECT_GE(GetPersistedLogCount(), 1);
metrics::UnsentLogStore result_unsent_log_store( metrics::UnsentLogStore result_unsent_log_store(
std::make_unique<ukm::UnsentLogStoreMetricsImpl>(), &prefs_, std::make_unique<ukm::UnsentLogStoreMetricsImpl>(), &prefs_,
prefs::kUkmUnsentLogStore, prefs::kUkmUnsentLogStore, /* meta_data_pref_name= */ nullptr,
3, // log count limit /* min_log_count= */ 3, /* min_log_bytes= */ 1000,
1000, // byte limit /* max_log_size= */ 0,
0, std::string()); /* signing_key= */ std::string());
result_unsent_log_store.LoadPersistedUnsentLogs(); result_unsent_log_store.LoadPersistedUnsentLogs();
result_unsent_log_store.StageNextLog(); result_unsent_log_store.StageNextLog();
...@@ -313,7 +313,7 @@ TEST_F(UkmServiceTest, PurgeExtensionDataFromUnsentLogStore) { ...@@ -313,7 +313,7 @@ TEST_F(UkmServiceTest, PurgeExtensionDataFromUnsentLogStore) {
// Save the Report to the store. // Save the Report to the store.
std::string serialized_log; std::string serialized_log;
report.SerializeToString(&serialized_log); report.SerializeToString(&serialized_log);
unsent_log_store->StoreLog(serialized_log); unsent_log_store->StoreLog(serialized_log, base::nullopt);
// Do extension purging. // Do extension purging.
service.PurgeExtensions(); service.PurgeExtensions();
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment