Report services data use to data saver

When network service is enabled, data saver site-breakdown gets the
services and downloads data usage from DataUseMeasurement.

Bug: 808498

Change-Id: I9ec7c98e67b5e8db531a345de1f5f89824efa807
Reviewed-on: https://chromium-review.googlesource.com/c/1295729Reviewed-by: Tarun Bansal <tbansal@chromium.org>
Commit-Queue: rajendrant <rajendrant@chromium.org>
Cr-Commit-Position: refs/heads/master@{#602098}

chrome/browser/data_use_measurement/chrome_data_use_measurement.cc

@@ -95,18 +95,25 @@ void ChromeDataUseMeasurement::ReportNetworkServiceDataUse(
   DCHECK_GE(recv_bytes, 0);
   DCHECK_GE(sent_bytes, 0);
 
-  UpdateMetricsUsagePrefs(
-      recv_bytes, IsCurrentNetworkCellular(),
-      IsMetricsServiceRequest(network_traffic_annotation_id_hash));
-  UpdateMetricsUsagePrefs(
-      sent_bytes, IsCurrentNetworkCellular(),
-      IsMetricsServiceRequest(network_traffic_annotation_id_hash));
-  if (!DataUseMeasurement::IsUserRequest(network_traffic_annotation_id_hash)) {
+  bool is_user_request =
+      DataUseMeasurement::IsUserRequest(network_traffic_annotation_id_hash);
+  bool is_metrics_service_request =
+      IsMetricsServiceRequest(network_traffic_annotation_id_hash);
+  UpdateMetricsUsagePrefs(recv_bytes, IsCurrentNetworkCellular(),
+                          is_metrics_service_request);
+  UpdateMetricsUsagePrefs(sent_bytes, IsCurrentNetworkCellular(),
+                          is_metrics_service_request);
+  if (!is_user_request) {
     ReportDataUsageServices(network_traffic_annotation_id_hash, UPSTREAM,
                             CurrentAppState(), sent_bytes);
     ReportDataUsageServices(network_traffic_annotation_id_hash, DOWNSTREAM,
                             CurrentAppState(), recv_bytes);
   }
+  if (!is_user_request || DataUseMeasurement::IsUserDownloadsRequest(
+                              network_traffic_annotation_id_hash)) {
+    for (auto& observer : services_data_use_observer_list_)
+      observer.OnServicesDataUse(recv_bytes, sent_bytes);
+  }
 #if defined(OS_ANDROID)
   MaybeRecordNetworkBytesOS();
 #endif

chrome/browser/net/spdyproxy/data_reduction_proxy_chrome_settings.cc

@@ -16,6 +16,7 @@
 #include "base/time/time.h"
 #include "build/build_config.h"
 #include "chrome/browser/browser_process.h"
+#include "chrome/browser/data_use_measurement/chrome_data_use_measurement.h"
 #include "chrome/browser/metrics/chrome_metrics_service_accessor.h"
 #include "chrome/browser/previews/previews_service.h"
 #include "chrome/browser/previews/previews_service_factory.h"
@@ -220,7 +221,8 @@ void DataReductionProxyChromeSettings::InitDataReductionProxySettings(
           std::make_unique<
               data_reduction_proxy::DataReductionProxyPingbackClientImpl>(
               url_loader_factory, ui_task_runner),
-          g_browser_process->network_quality_tracker(), ui_task_runner,
+          g_browser_process->network_quality_tracker(),
+          g_browser_process->data_use_measurement(), ui_task_runner,
           io_data->io_task_runner(), db_task_runner, commit_delay);
   data_reduction_proxy::DataReductionProxySettings::
       InitDataReductionProxySettings(data_reduction_proxy_enabled_pref_name_,

components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate.cc

@@ -39,6 +39,7 @@
 #include "net/url_request/url_request.h"
 #include "net/url_request/url_request_context.h"
 #include "net/url_request/url_request_status.h"
+#include "services/network/public/cpp/features.h"
 #include "url/gurl.h"
 
 namespace data_reduction_proxy {
@@ -594,6 +595,7 @@ void DataReductionProxyNetworkDelegate::CalculateAndRecordDataUsage(
           ->IsNonContentInitiatedRequest(request)) {
     // Record non-content initiated traffic to the Other bucket for data saver
     // site-breakdown.
+    DCHECK(!base::FeatureList::IsEnabled(network::features::kNetworkService));
     data_reduction_proxy_io_data_->UpdateDataUseForHost(
         data_used, original_size, util::GetSiteBreakdownOtherHostName());
   }

components/data_reduction_proxy/core/browser/data_reduction_proxy_service.cc

@@ -20,12 +20,15 @@
 #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_pingback_client.h"
 #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_service_observer.h"
 #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_settings.h"
+#include "components/data_reduction_proxy/core/browser/data_reduction_proxy_util.h"
 #include "components/data_reduction_proxy/core/browser/data_store.h"
 #include "components/data_reduction_proxy/core/common/data_reduction_proxy_features.h"
 #include "components/data_reduction_proxy/core/common/data_reduction_proxy_params.h"
 #include "components/data_reduction_proxy/core/common/data_reduction_proxy_pref_names.h"
 #include "components/data_reduction_proxy/proto/data_store.pb.h"
+#include "components/data_use_measurement/core/data_use_measurement.h"
 #include "components/prefs/pref_service.h"
+#include "services/network/public/cpp/features.h"
 
 namespace data_reduction_proxy {
 
@@ -37,6 +40,7 @@ DataReductionProxyService::DataReductionProxyService(
     std::unique_ptr<DataStore> store,
     std::unique_ptr<DataReductionProxyPingbackClient> pingback_client,
     network::NetworkQualityTracker* network_quality_tracker,
+    data_use_measurement::DataUseMeasurement* data_use_measurement,
     const scoped_refptr<base::SequencedTaskRunner>& ui_task_runner,
     const scoped_refptr<base::SingleThreadTaskRunner>& io_task_runner,
     const scoped_refptr<base::SequencedTaskRunner>& db_task_runner,
@@ -51,6 +55,7 @@ DataReductionProxyService::DataReductionProxyService(
       db_task_runner_(db_task_runner),
       initialized_(false),
       network_quality_tracker_(network_quality_tracker),
+      data_use_measurement_(data_use_measurement),
       effective_connection_type_(net::EFFECTIVE_CONNECTION_TYPE_UNKNOWN),
       weak_factory_(this) {
   DCHECK(settings);
@@ -64,6 +69,8 @@ DataReductionProxyService::DataReductionProxyService(
   }
   network_quality_tracker_->AddEffectiveConnectionTypeObserver(this);
   network_quality_tracker_->AddRTTAndThroughputEstimatesObserver(this);
+  if (base::FeatureList::IsEnabled(network::features::kNetworkService))
+    data_use_measurement_->AddServicesDataUseObserver(this);
 }
 
 DataReductionProxyService::~DataReductionProxyService() {
@@ -72,6 +79,8 @@ DataReductionProxyService::~DataReductionProxyService() {
   network_quality_tracker_->RemoveRTTAndThroughputEstimatesObserver(this);
   compression_stats_.reset();
   db_task_runner_->DeleteSoon(FROM_HERE, db_data_owner_.release());
+  if (base::FeatureList::IsEnabled(network::features::kNetworkService))
+    data_use_measurement_->RemoveServicesDataUseObserver(this);
 }
 
 void DataReductionProxyService::SetIOData(
@@ -338,4 +347,20 @@ DataReductionProxyService::GetWeakPtr() {
   return weak_factory_.GetWeakPtr();
 }
 
+void DataReductionProxyService::OnServicesDataUse(int64_t recv_bytes,
+                                                  int64_t sent_bytes) {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+  if (compression_stats_) {
+    // Record non-content initiated traffic to the Other bucket for data saver
+    // site-breakdown.
+    DCHECK(base::FeatureList::IsEnabled(network::features::kNetworkService));
+    compression_stats_->RecordDataUseByHost(
+        util::GetSiteBreakdownOtherHostName(), sent_bytes, sent_bytes,
+        base::Time::Now());
+    compression_stats_->RecordDataUseByHost(
+        util::GetSiteBreakdownOtherHostName(), recv_bytes, recv_bytes,
+        base::Time::Now());
+  }
+}
+
 }  // namespace data_reduction_proxy

components/data_reduction_proxy/core/browser/data_reduction_proxy_service.h

@@ -19,6 +19,7 @@
 #include "base/sequence_checker.h"
 #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_metrics.h"
 #include "components/data_reduction_proxy/core/browser/db_data_owner.h"
+#include "components/data_use_measurement/core/data_use_measurement.h"
 #include "components/data_use_measurement/core/data_use_user_data.h"
 #include "net/nqe/effective_connection_type.h"
 #include "services/network/public/cpp/network_quality_tracker.h"
@@ -48,7 +49,8 @@ class DataReductionProxySettings;
 // Contains and initializes all Data Reduction Proxy objects that have a
 // lifetime based on the UI thread.
 class DataReductionProxyService
-    : public network::NetworkQualityTracker::EffectiveConnectionTypeObserver,
+    : public data_use_measurement::DataUseMeasurement::ServicesDataUseObserver,
+      public network::NetworkQualityTracker::EffectiveConnectionTypeObserver,
       public network::NetworkQualityTracker::RTTAndThroughputEstimatesObserver {
  public:
   // The caller must ensure that |settings|, |prefs|, |request_context|, and
@@ -65,6 +67,7 @@ class DataReductionProxyService
       std::unique_ptr<DataStore> store,
       std::unique_ptr<DataReductionProxyPingbackClient> pingback_client,
       network::NetworkQualityTracker* network_quality_tracker,
+      data_use_measurement::DataUseMeasurement* data_use_measurement,
       const scoped_refptr<base::SequencedTaskRunner>& ui_task_runner,
       const scoped_refptr<base::SingleThreadTaskRunner>& io_task_runner,
       const scoped_refptr<base::SequencedTaskRunner>& db_task_runner,
@@ -182,6 +185,8 @@ class DataReductionProxyService
   // Loads the Data Reduction Proxy configuration from |prefs_| and applies it.
   void ReadPersistedClientConfig();
 
+  void OnServicesDataUse(int64_t recv_bytes, int64_t sent_bytes) override;
+
   net::URLRequestContextGetter* url_request_context_getter_;
   scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory_;
 
@@ -216,6 +221,10 @@ class DataReductionProxyService
   // lifetime of |this|.
   network::NetworkQualityTracker* network_quality_tracker_;
 
+  // Must be accessed on UI thread. Guaranteed to be non-null during the
+  // lifetime of |this|.
+  data_use_measurement::DataUseMeasurement* data_use_measurement_;
+
   // Current network quality estimates.
   net::EffectiveConnectionType effective_connection_type_;
   base::Optional<base::TimeDelta> http_rtt_;

components/data_reduction_proxy/core/browser/data_reduction_proxy_test_utils.cc

@@ -244,6 +244,7 @@ MockDataReductionProxyService::MockDataReductionProxyService(
                                 std::make_unique<TestDataStore>(),
                                 nullptr,
                                 test_network_quality_tracker,
+                                nullptr,
                                 task_runner,
                                 task_runner,
                                 task_runner,
@@ -654,8 +655,8 @@ DataReductionProxyTestContext::CreateDataReductionProxyServiceInternal(
   return std::make_unique<DataReductionProxyService>(
       settings, simple_pref_service_.get(), request_context_getter_.get(),
       test_shared_url_loader_factory_, base::WrapUnique(new TestDataStore()),
-      nullptr, test_network_quality_tracker_.get(), task_runner_, task_runner_,
-      task_runner_, base::TimeDelta());
+      nullptr, test_network_quality_tracker_.get(), nullptr, task_runner_,
+      task_runner_, task_runner_, base::TimeDelta());
 }
 
 void DataReductionProxyTestContext::AttachToURLRequestContext(

components/data_use_measurement/core/data_use_measurement.cc

@@ -116,6 +116,7 @@ DataUseMeasurement::DataUseMeasurement(
 DataUseMeasurement::~DataUseMeasurement() {
   if (network_connection_tracker_)
     network_connection_tracker_->RemoveNetworkConnectionObserver(this);
+  DCHECK(!services_data_use_observer_list_.might_have_observers());
 }
 
 void DataUseMeasurement::OnBeforeURLRequest(net::URLRequest* request) {
@@ -471,6 +472,24 @@ bool DataUseMeasurement::IsUserRequest(
          kUserInitiatedTrafficAnnotations.end();
 }
 
+// static
+bool DataUseMeasurement::IsUserDownloadsRequest(
+    int32_t network_traffic_annotation_hash_id) {
+  static const std::set<int32_t> kUserDownloadsTrafficAnnotations = {
+      COMPUTE_NETWORK_TRAFFIC_ANNOTATION_ID_HASH("parallel_download_job"),
+      COMPUTE_NETWORK_TRAFFIC_ANNOTATION_ID_HASH("renderer_initiated_download"),
+      COMPUTE_NETWORK_TRAFFIC_ANNOTATION_ID_HASH("drag_download_file"),
+      COMPUTE_NETWORK_TRAFFIC_ANNOTATION_ID_HASH("download_web_contents_frame"),
+      COMPUTE_NETWORK_TRAFFIC_ANNOTATION_ID_HASH("downloads_api_run_async"),
+      COMPUTE_NETWORK_TRAFFIC_ANNOTATION_ID_HASH("resumed_downloads"),
+      COMPUTE_NETWORK_TRAFFIC_ANNOTATION_ID_HASH("download_via_context_menu"),
+      COMPUTE_NETWORK_TRAFFIC_ANNOTATION_ID_HASH("offline_pages_download_file"),
+  };
+  return kUserDownloadsTrafficAnnotations.find(
+             network_traffic_annotation_hash_id) !=
+         kUserDownloadsTrafficAnnotations.end();
+}
+
 // static
 bool DataUseMeasurement::IsMetricsServiceRequest(
     int32_t network_traffic_annotation_hash_id) {
@@ -498,4 +517,14 @@ void DataUseMeasurement::OnConnectionChanged(
   connection_type_ = type;
 }
 
+void DataUseMeasurement::AddServicesDataUseObserver(
+    ServicesDataUseObserver* observer) {
+  services_data_use_observer_list_.AddObserver(observer);
+}
+
+void DataUseMeasurement::RemoveServicesDataUseObserver(
+    ServicesDataUseObserver* observer) {
+  services_data_use_observer_list_.RemoveObserver(observer);
+}
+
 }  // namespace data_use_measurement

components/data_use_measurement/core/data_use_measurement.h

@@ -12,6 +12,7 @@
 
 #include "base/callback.h"
 #include "base/macros.h"
+#include "base/observer_list.h"
 #include "base/time/time.h"
 #include "build/build_config.h"
 #include "components/data_use_measurement/core/data_use_user_data.h"
@@ -43,9 +44,19 @@ class URLRequestClassifier;
 class DataUseMeasurement
     : public network::NetworkConnectionTracker::NetworkConnectionObserver {
  public:
+  class ServicesDataUseObserver {
+   public:
+    // Called when services data use is reported.
+    virtual void OnServicesDataUse(int64_t recv_bytes, int64_t sent_bytes) = 0;
+  };
+
   // Returns true if the NTA hash is initiated by user traffic.
   static bool IsUserRequest(int32_t network_traffic_annotation_hash_id);
 
+  // Returns true if the NTA hash is one used by Chrome downloads.
+  static bool IsUserDownloadsRequest(
+      int32_t network_traffic_annotation_hash_id);
+
   // Returns true if the NTA hash is one used by metrics (UMA, UKM) component.
   static bool IsMetricsServiceRequest(
       int32_t network_traffic_annotation_hash_id);
@@ -93,6 +104,9 @@ class DataUseMeasurement
                                              bool is_cellular,
                                              bool is_metrics_service_usage) = 0;
 
+  void AddServicesDataUseObserver(ServicesDataUseObserver* observer);
+  void RemoveServicesDataUseObserver(ServicesDataUseObserver* observer);
+
  protected:
   // Specifies that data is received or sent, respectively.
   enum TrafficDirection { DOWNSTREAM, UPSTREAM };
@@ -119,6 +133,9 @@ class DataUseMeasurement
   void MaybeRecordNetworkBytesOS();
 #endif
 
+  base::ObserverList<ServicesDataUseObserver>::Unchecked
+      services_data_use_observer_list_;
+
  private:
   friend class DataUseMeasurementTest;