Reland "NetworkService: Report data use of URLLoader to browser process"

This is a reland of adb22130

Original change's description:
> NetworkService: Report data use of URLLoader to browser process
>
> When the requests complete, the total received and sent bytes are sent
> to browser process. ChromeDataUseMeasurement records metrics on Chrome-services
> data usage and updates metrics service.
>
> Subsequent CLs will record more metrics and also report this to data
> reduction proxy settings page.
>
> When NetworkService is disabled, DataUseMeasurement will get network
> delegate callbacks and record metrics.
>
> Bug: 808498
> Cq-Include-Trybots: luci.chromium.try:linux_mojo
> Change-Id: I113e480e4e1c67a65ff7461eb5ae2166a6b038d2
> Reviewed-on: https://chromium-review.googlesource.com/c/1274202
> Commit-Queue: rajendrant <rajendrant@chromium.org>
> Reviewed-by: Tarun Bansal <tbansal@chromium.org>
> Reviewed-by: Kinuko Yasuda <kinuko@chromium.org>
> Reviewed-by: John Abd-El-Malek <jam@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#600928}

TBR=kinuko@chromium.org

Bug: 808498
Change-Id: I341012581527fefeb6fd23ff16af4ed3a8373aa7
Cq-Include-Trybots: luci.chromium.try:linux_mojo
Reviewed-on: https://chromium-review.googlesource.com/c/1290522
Commit-Queue: rajendrant <rajendrant@chromium.org>
Reviewed-by: rajendrant <rajendrant@chromium.org>
Reviewed-by: John Abd-El-Malek <jam@chromium.org>
Reviewed-by: Tarun Bansal <tbansal@chromium.org>
Cr-Commit-Position: refs/heads/master@{#601229}

chrome/browser/browser_process.h

@@ -61,6 +61,10 @@ class ComponentUpdateService;
 class SupervisedUserWhitelistInstaller;
 }
 
+namespace data_use_measurement {
+class ChromeDataUseMeasurement;
+}
+
 namespace extensions {
 class EventRouterForwarder;
 }
@@ -292,6 +296,9 @@ class BrowserProcess {
 
   virtual prefs::InProcessPrefServiceFactory* pref_service_factory() const = 0;
 
+  virtual data_use_measurement::ChromeDataUseMeasurement*
+  data_use_measurement() = 0;
+
  private:
   DISALLOW_COPY_AND_ASSIGN(BrowserProcess);
 };

chrome/browser/browser_process_impl.cc

@@ -41,6 +41,7 @@
 #include "chrome/browser/chrome_notification_types.h"
 #include "chrome/browser/component_updater/chrome_component_updater_configurator.h"
 #include "chrome/browser/component_updater/supervised_user_whitelist_installer.h"
+#include "chrome/browser/data_use_measurement/chrome_data_use_measurement.h"
 #include "chrome/browser/defaults.h"
 #include "chrome/browser/devtools/devtools_auto_opener.h"
 #include "chrome/browser/devtools/remote_debugging_server.h"
@@ -927,6 +928,16 @@ prefs::InProcessPrefServiceFactory* BrowserProcessImpl::pref_service_factory()
   return pref_service_factory_.get();
 }
 
+data_use_measurement::ChromeDataUseMeasurement*
+BrowserProcessImpl::data_use_measurement() {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+  if (!data_use_measurement_) {
+    data_use_measurement_ = data_use_measurement::ChromeDataUseMeasurement::
+        CreateForNetworkService();
+  }
+  return data_use_measurement_.get();
+}
+
 // static
 void BrowserProcessImpl::RegisterPrefs(PrefRegistrySimple* registry) {
   registry->RegisterBooleanPref(prefs::kDefaultBrowserSettingEnabled,

chrome/browser/browser_process_impl.h

@@ -200,6 +200,8 @@ class BrowserProcessImpl : public BrowserProcess,
   shell_integration::DefaultWebClientState CachedDefaultWebClientState()
       override;
   prefs::InProcessPrefServiceFactory* pref_service_factory() const override;
+  data_use_measurement::ChromeDataUseMeasurement* data_use_measurement()
+      override;
 
   static void RegisterPrefs(PrefRegistrySimple* registry);
 
@@ -427,6 +429,9 @@ class BrowserProcessImpl : public BrowserProcess,
   base::OnceClosure quit_closure_;
 #endif
 
+  std::unique_ptr<data_use_measurement::ChromeDataUseMeasurement>
+      data_use_measurement_;
+
   SEQUENCE_CHECKER(sequence_checker_);
 
   DISALLOW_COPY_AND_ASSIGN(BrowserProcessImpl);

chrome/browser/chrome_content_browser_client.cc

@@ -43,6 +43,7 @@
 #include "chrome/browser/content_settings/cookie_settings_factory.h"
 #include "chrome/browser/content_settings/host_content_settings_map_factory.h"
 #include "chrome/browser/content_settings/tab_specific_content_settings.h"
+#include "chrome/browser/data_use_measurement/chrome_data_use_measurement.h"
 #include "chrome/browser/defaults.h"
 #include "chrome/browser/download/download_prefs.h"
 #include "chrome/browser/external_protocol/external_protocol_handler.h"
@@ -4925,3 +4926,13 @@ bool ChromeContentBrowserClient::CanIgnoreCertificateErrorIfNeeded() {
   return base::CommandLine::ForCurrentProcess()->HasSwitch(
       switches::kUserDataDir);
 }
+
+void ChromeContentBrowserClient::OnNetworkServiceDataUseUpdate(
+    int32_t network_traffic_annotation_id_hash,
+    int64_t recv_bytes,
+    int64_t sent_bytes) {
+  if (g_browser_process->data_use_measurement()) {
+    g_browser_process->data_use_measurement()->ReportNetworkServiceDataUse(
+        network_traffic_annotation_id_hash, recv_bytes, sent_bytes);
+  }
+}

chrome/browser/chrome_content_browser_client.h

@@ -500,12 +500,14 @@ class ChromeContentBrowserClient : public content::ContentBrowserClient {
   void RegisterRendererPreferenceWatcherForWorkers(
       content::BrowserContext* browser_context,
       content::mojom::RendererPreferenceWatcherPtr watcher) override;
-
   base::Optional<std::string> GetOriginPolicyErrorPage(
       content::OriginPolicyErrorReason error_reason,
       const url::Origin& origin,
       const GURL& url) override;
   bool CanIgnoreCertificateErrorIfNeeded() override;
+  void OnNetworkServiceDataUseUpdate(int32_t network_traffic_annotation_id_hash,
+                                     int64_t recv_bytes,
+                                     int64_t sent_bytes) override;
 
  protected:
   static bool HandleWebUI(GURL* url, content::BrowserContext* browser_context);

chrome/browser/data_use_measurement/chrome_data_use_ascriber.cc

@@ -584,7 +584,7 @@ ChromeDataUseAscriber::CreateNetworkDelegate(
   return std::make_unique<data_use_measurement::DataUseNetworkDelegate>(
       std::move(wrapped_network_delegate), this,
       std::make_unique<ChromeDataUseMeasurement>(CreateURLRequestClassifier(),
-                                                 this));
+                                                 this, /*network_connection_tracker=*/nullptr));
 }
 
 std::unique_ptr<URLRequestClassifier>

chrome/browser/data_use_measurement/chrome_data_use_measurement.cc

@@ -7,14 +7,18 @@
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/macros.h"
+#include "base/numerics/safe_conversions.h"
 #include "base/task/post_task.h"
 #include "base/task/task_traits.h"
+#include "build/build_config.h"
 #include "chrome/browser/browser_process.h"
 #include "components/data_use_measurement/core/data_use_ascriber.h"
 #include "components/data_use_measurement/core/url_request_classifier.h"
 #include "components/metrics/metrics_service.h"
 #include "content/public/browser/browser_task_traits.h"
 #include "content/public/browser/browser_thread.h"
+#include "content/public/browser/network_service_instance.h"
+#include "services/network/public/cpp/features.h"
 
 namespace data_use_measurement {
 
@@ -27,8 +31,9 @@ void UpdateMetricsUsagePrefs(int64_t total_bytes,
   // case it's fine to skip the update.
   auto* metrics_service = g_browser_process->metrics_service();
   if (metrics_service) {
-    metrics_service->UpdateMetricsUsagePrefs(total_bytes, is_cellular,
-                                             is_metrics_service_usage);
+    metrics_service->UpdateMetricsUsagePrefs(
+        base::saturated_cast<int>(total_bytes), is_cellular,
+        is_metrics_service_usage);
   }
 }
 
@@ -47,10 +52,27 @@ void UpdateMetricsUsagePrefsOnUIThread(int64_t total_bytes,
 }
 }  // namespace
 
+// static
+std::unique_ptr<ChromeDataUseMeasurement>
+ChromeDataUseMeasurement::CreateForNetworkService() {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+
+  // Do not create when NetworkService is disabled, since data use of URLLoader
+  // is reported via the network delegate callbacks.
+  if (!base::FeatureList::IsEnabled(network::features::kNetworkService))
+    return nullptr;
+
+  return std::make_unique<ChromeDataUseMeasurement>(
+      nullptr, nullptr, content::GetNetworkConnectionTracker());
+}
+
 ChromeDataUseMeasurement::ChromeDataUseMeasurement(
     std::unique_ptr<URLRequestClassifier> url_request_classifier,
-    DataUseAscriber* ascriber)
-    : DataUseMeasurement(std::move(url_request_classifier), ascriber) {}
+    DataUseAscriber* ascriber,
+    network::NetworkConnectionTracker* network_connection_tracker)
+    : DataUseMeasurement(std::move(url_request_classifier),
+                         ascriber,
+                         network_connection_tracker) {}
 
 void ChromeDataUseMeasurement::UpdateDataUseToMetricsService(
     int64_t total_bytes,
@@ -62,4 +84,32 @@ void ChromeDataUseMeasurement::UpdateDataUseToMetricsService(
                                     is_metrics_service_usage);
 }
 
+void ChromeDataUseMeasurement::ReportNetworkServiceDataUse(
+    int32_t network_traffic_annotation_id_hash,
+    int64_t recv_bytes,
+    int64_t sent_bytes) {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+  DCHECK(base::FeatureList::IsEnabled(network::features::kNetworkService));
+  // Negative byte numbres is not a critical problem (i.e. should have no security implications) but
+  // is not expected. TODO(rajendrant): remove these DCHECKs or consider using uint in Mojo instead.
+  DCHECK_GE(recv_bytes, 0);
+  DCHECK_GE(sent_bytes, 0);
+
+  UpdateMetricsUsagePrefs(
+      recv_bytes, IsCurrentNetworkCellular(),
+      IsMetricsServiceRequest(network_traffic_annotation_id_hash));
+  UpdateMetricsUsagePrefs(
+      sent_bytes, IsCurrentNetworkCellular(),
+      IsMetricsServiceRequest(network_traffic_annotation_id_hash));
+  if (!DataUseMeasurement::IsUserRequest(network_traffic_annotation_id_hash)) {
+    ReportDataUsageServices(network_traffic_annotation_id_hash, UPSTREAM,
+                            CurrentAppState(), sent_bytes);
+    ReportDataUsageServices(network_traffic_annotation_id_hash, DOWNSTREAM,
+                            CurrentAppState(), recv_bytes);
+  }
+#if defined(OS_ANDROID)
+  MaybeRecordNetworkBytesOS();
+#endif
+}
+
 }  // namespace data_use_measurement

chrome/browser/data_use_measurement/chrome_data_use_measurement.h

@@ -17,14 +17,22 @@ class DataUseAscriber;
 
 class ChromeDataUseMeasurement : public DataUseMeasurement {
  public:
+  static std::unique_ptr<ChromeDataUseMeasurement> CreateForNetworkService();
+
   ChromeDataUseMeasurement(
       std::unique_ptr<URLRequestClassifier> url_request_classifier,
-      DataUseAscriber* ascriber);
+      DataUseAscriber* ascriber,
+      network::NetworkConnectionTracker* network_connection_tracker);
 
   void UpdateDataUseToMetricsService(int64_t total_bytes,
                                      bool is_cellular,
                                      bool is_metrics_service_usage) override;
 
+  // Called when requests complete from NetworkService.
+  void ReportNetworkServiceDataUse(int32_t network_traffic_annotation_id_hash,
+                                   int64_t recv_bytes,
+                                   int64_t sent_bytes);
+
  private:
   DISALLOW_COPY_AND_ASSIGN(ChromeDataUseMeasurement);
 };

chrome/browser/data_use_measurement/data_use_measurement_browsertest.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/test/metrics/histogram_tester.h"
+#include "chrome/browser/browser_process.h"
+#include "chrome/browser/data_use_measurement/chrome_data_use_measurement.h"
+#include "chrome/browser/profiles/profile.h"
+#include "chrome/browser/ui/browser.h"
+#include "chrome/common/pref_names.h"
+#include "chrome/test/base/in_process_browser_test.h"
+#include "chrome/test/base/ui_test_utils.h"
+#include "components/data_reduction_proxy/core/browser/data_reduction_proxy_config_service_client_test_utils.h"
+#include "components/data_reduction_proxy/core/common/data_reduction_proxy_features.h"
+#include "components/data_reduction_proxy/core/common/data_reduction_proxy_headers.h"
+#include "components/data_reduction_proxy/core/common/data_reduction_proxy_params.h"
+#include "components/data_reduction_proxy/core/common/data_reduction_proxy_switches.h"
+#include "components/data_reduction_proxy/proto/client_config.pb.h"
+#include "components/prefs/pref_service.h"
+#include "content/public/test/browser_test_base.h"
+#include "content/public/test/browser_test_utils.h"
+#include "net/dns/mock_host_resolver.h"
+#include "net/http/http_status_code.h"
+#include "net/test/embedded_test_server/embedded_test_server.h"
+#include "net/test/embedded_test_server/http_request.h"
+#include "net/test/embedded_test_server/http_response.h"
+#include "net/traffic_annotation/network_traffic_annotation.h"
+#include "services/network/public/cpp/features.h"
+#include "testing/gmock/include/gmock/gmock.h"
+
+namespace data_use_measurement {
+
+namespace {
+std::unique_ptr<net::test_server::HttpResponse>
+HandleDataReductionProxyWarmupRequest(
+    const net::test_server::HttpRequest& request) {
+  if (!base::StartsWith(request.relative_url,
+                        data_reduction_proxy::params::GetWarmupURL().path(),
+                        base::CompareCase::INSENSITIVE_ASCII))
+    return std::unique_ptr<net::test_server::HttpResponse>();
+
+  auto http_response = std::make_unique<net::test_server::BasicHttpResponse>();
+  http_response->set_content(std::string(1024, ' '));
+  http_response->set_code(net::HTTP_OK);
+  return std::move(http_response);
+}
+}  // namespace
+
+// Enables the data saver and checks data use is recorded for the
+// services.
+class DataUseMeasurementBrowserTestWithDataSaverEnabled
+    : public InProcessBrowserTest {
+ public:
+  void SetUpCommandLine(base::CommandLine* command_line) override {
+    net::HostPortPair host_port_pair = embedded_test_server()->host_port_pair();
+    std::string config = data_reduction_proxy::EncodeConfig(CreateConfig(
+        "TheSessionKeyYay!", 1000, 0,
+        data_reduction_proxy::ProxyServer_ProxyScheme_HTTP,
+        host_port_pair.host(), host_port_pair.port(),
+        data_reduction_proxy::ProxyServer::CORE,
+        data_reduction_proxy::ProxyServer_ProxyScheme_HTTP, "fallback.net", 80,
+        data_reduction_proxy::ProxyServer::UNSPECIFIED_TYPE, 0.5f, false));
+    command_line->AppendSwitchASCII(
+        data_reduction_proxy::switches::kDataReductionProxyServerClientConfig,
+        config);
+  }
+
+  void SetUp() override {
+    embedded_test_server()->RegisterRequestHandler(
+        base::BindRepeating(&HandleDataReductionProxyWarmupRequest));
+    ASSERT_TRUE(embedded_test_server()->Start());
+    scoped_feature_list_.InitAndEnableFeature(
+        data_reduction_proxy::features::
+            kDataReductionProxyEnabledWithNetworkService);
+    InProcessBrowserTest::SetUp();
+  }
+
+ protected:
+  void EnableDataSaver(bool enabled) {
+    PrefService* prefs = browser()->profile()->GetPrefs();
+    prefs->SetBoolean(::prefs::kDataSaverEnabled, enabled);
+    base::RunLoop().RunUntilIdle();
+  }
+
+  const base::HistogramTester& histogram_tester() { return histogram_tester_; }
+
+ private:
+  base::test::ScopedFeatureList scoped_feature_list_;
+  base::test::ScopedFeatureList param_feature_list_;
+  base::HistogramTester histogram_tester_;
+};
+
+// TODO(rajendrant): Fix the test. The test does not work on ChromeOS. The test is flaky in linux.
+IN_PROC_BROWSER_TEST_F(DataUseMeasurementBrowserTestWithDataSaverEnabled,
+                       DISABLED_CheckServicesDataUseRecorded) {
+  EnableDataSaver(true);
+  ui_test_utils::NavigateToURL(browser(), GURL("about:blank"));
+  size_t data_saver_warmup_usage_kb = 0;
+  for (const auto& bucket : histogram_tester().GetAllSamples(
+           "DataUse.AllServicesKB.Downstream.Foreground")) {
+    if (bucket.min == COMPUTE_NETWORK_TRAFFIC_ANNOTATION_ID_HASH(
+                          "data_reduction_proxy_warmup")) {
+      data_saver_warmup_usage_kb += bucket.count;
+    }
+  }
+  // Data use is probabilistically rounded as 1-2 KB.
+  EXPECT_THAT(data_saver_warmup_usage_kb,
+              testing::AllOf(testing::Ge(1U), testing::Le(2U)));
+}
+
+}  // namespace data_use_measurement

chrome/test/BUILD.gn

@@ -554,6 +554,7 @@ test("browser_tests") {
       "../browser/data_saver/data_saver_browsertest.cc",
       "../browser/data_saver/data_saver_holdback_browsertest.cc",
       "../browser/data_saver/data_saver_webapis_browsertest.cc",
+      "../browser/data_use_measurement/data_use_measurement_browsertest.cc",
       "../browser/data_use_measurement/page_load_capping/page_load_capping_browsertest.cc",
       "../browser/devtools/device/adb/adb_client_socket_browsertest.cc",
       "../browser/devtools/device/adb/mock_adb_server.cc",

chrome/test/base/testing_browser_process.cc

@@ -11,6 +11,7 @@
 #include "build/build_config.h"
 #include "chrome/browser/browser_process.h"
 #include "chrome/browser/browser_process_impl.h"
+#include "chrome/browser/data_use_measurement/chrome_data_use_measurement.h"
 #include "chrome/browser/download/download_request_limiter.h"
 #include "chrome/browser/lifetime/application_lifetime.h"
 #include "chrome/browser/notifications/notification_platform_bridge.h"
@@ -452,6 +453,11 @@ TestingBrowserProcess::pref_service_factory() const {
   return nullptr;
 }
 
+data_use_measurement::ChromeDataUseMeasurement*
+TestingBrowserProcess::data_use_measurement() {
+  return nullptr;
+}
+
 void TestingBrowserProcess::SetSystemRequestContext(
     net::URLRequestContextGetter* context_getter) {
   system_request_context_ = context_getter;

chrome/test/base/testing_browser_process.h

@@ -138,6 +138,8 @@ class TestingBrowserProcess : public BrowserProcess {
   shell_integration::DefaultWebClientState CachedDefaultWebClientState()
       override;
   prefs::InProcessPrefServiceFactory* pref_service_factory() const override;
+  data_use_measurement::ChromeDataUseMeasurement* data_use_measurement()
+      override;
 
   // Set the local state for tests. Consumer is responsible for cleaning it up
   // afterwards (using ScopedTestingLocalState, for example).

components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate.cc

@@ -581,7 +581,8 @@ void DataReductionProxyNetworkDelegate::CalculateAndRecordDataUsage(
 
   AccumulateDataUsage(
       data_used, original_size, request_type, mime_type,
-      data_use_measurement::DataUseMeasurement::IsUserRequest(request),
+      data_use_measurement::DataUseMeasurement::IsUserRequest(
+          request.traffic_annotation().unique_id_hash_code),
       data_use_measurement::DataUseMeasurement::GetContentTypeForRequest(
           request),
       request.traffic_annotation().unique_id_hash_code);

components/data_use_measurement/core/BUILD.gn

@@ -30,8 +30,8 @@ static_library("ascriber") {
     ":core",
     "//base",
     "//components/metrics",
-    "//google_apis",
     "//net",
+    "//services/network/public/cpp:cpp",
   ]
 }
 

components/data_use_measurement/core/DEPS

 include_rules = [
   "+net",
   "+components/metrics",
+  "+services/network/public/cpp",
 ]

components/data_use_measurement/core/data_use_measurement.cc

@@ -6,6 +6,7 @@
 
 #include <set>
 
+#include "base/feature_list.h"
 #include "base/lazy_instance.h"
 #include "base/memory/ptr_util.h"
 #include "base/metrics/histogram_macros.h"
@@ -22,6 +23,7 @@
 #include "net/http/http_status_code.h"
 #include "net/traffic_annotation/network_traffic_annotation.h"
 #include "net/url_request/url_request.h"
+#include "services/network/public/cpp/features.h"
 
 #if defined(OS_ANDROID)
 #include "net/android/traffic_stats.h"
@@ -73,11 +75,11 @@ void RecordFavIconDataUse(const net::URLRequest& request) {
 
 DataUseMeasurement::DataUseMeasurement(
     std::unique_ptr<URLRequestClassifier> url_request_classifier,
-    DataUseAscriber* ascriber)
+    DataUseAscriber* ascriber,
+    network::NetworkConnectionTracker* network_connection_tracker)
     : url_request_classifier_(std::move(url_request_classifier)),
-      ascriber_(ascriber)
+      ascriber_(ascriber),
 #if defined(OS_ANDROID)
-      ,
       app_state_(base::android::APPLICATION_STATE_HAS_RUNNING_ACTIVITIES),
       app_listener_(base::android::ApplicationStatusListener::New(
           base::BindRepeating(&DataUseMeasurement::OnApplicationStateChange,
@@ -85,11 +87,20 @@ DataUseMeasurement::DataUseMeasurement(
       rx_bytes_os_(0),
       tx_bytes_os_(0),
       bytes_transferred_since_last_traffic_stats_query_(0),
-      no_reads_since_background_(false)
+      no_reads_since_background_(false),
 #endif
-{
-  DCHECK(ascriber_);
-  DCHECK(url_request_classifier_);
+      network_connection_tracker_(network_connection_tracker),
+      connection_type_(network::mojom::ConnectionType::CONNECTION_UNKNOWN) {
+  DCHECK(ascriber_ ||
+         base::FeatureList::IsEnabled(network::features::kNetworkService));
+  DCHECK(url_request_classifier_ ||
+         base::FeatureList::IsEnabled(network::features::kNetworkService));
+  DCHECK(network_connection_tracker_ ||
+         !base::FeatureList::IsEnabled(network::features::kNetworkService));
+
+  if (network_connection_tracker_) {
+    network_connection_tracker_->AddNetworkConnectionObserver(this);
+  }
 
 #if defined(OS_ANDROID)
   int64_t bytes = 0;
@@ -102,7 +113,10 @@ DataUseMeasurement::DataUseMeasurement(
 #endif
 }
 
-DataUseMeasurement::~DataUseMeasurement(){};
+DataUseMeasurement::~DataUseMeasurement() {
+  if (network_connection_tracker_)
+    network_connection_tracker_->RemoveNetworkConnectionObserver(this);
+}
 
 void DataUseMeasurement::OnBeforeURLRequest(net::URLRequest* request) {
   DataUseUserData* data_use_user_data = reinterpret_cast<DataUseUserData*>(
@@ -122,8 +136,7 @@ void DataUseMeasurement::OnBeforeRedirect(const net::URLRequest& request,
   // TODO(rajendrant): May not be needed when http://crbug/651957 is fixed.
   UpdateDataUseToMetricsService(
       request.GetTotalSentBytes() + request.GetTotalReceivedBytes(),
-      net::NetworkChangeNotifier::IsConnectionCellular(
-          net::NetworkChangeNotifier::GetConnectionType()),
+      IsCurrentNetworkCellular(),
       IsMetricsServiceRequest(
           request.traffic_annotation().unique_id_hash_code));
   ReportServicesMessageSizeUMA(request);
@@ -166,8 +179,7 @@ void DataUseMeasurement::OnCompleted(const net::URLRequest& request,
   // of redirected requests.
   UpdateDataUseToMetricsService(
       request.GetTotalSentBytes() + request.GetTotalReceivedBytes(),
-      net::NetworkChangeNotifier::IsConnectionCellular(
-          net::NetworkChangeNotifier::GetConnectionType()),
+      IsCurrentNetworkCellular(),
       IsMetricsServiceRequest(
           request.traffic_annotation().unique_id_hash_code));
   ReportServicesMessageSizeUMA(request);
@@ -191,10 +203,8 @@ DataUseMeasurement::GetContentTypeForRequest(const net::URLRequest& request) {
 void DataUseMeasurement::ReportDataUseUMA(const net::URLRequest& request,
                                           TrafficDirection dir,
                                           int64_t bytes) {
-  bool is_user_traffic = IsUserRequest(request);
-  bool is_connection_cellular =
-      net::NetworkChangeNotifier::IsConnectionCellular(
-          net::NetworkChangeNotifier::GetConnectionType());
+  bool is_user_traffic =
+      IsUserRequest(request.traffic_annotation().unique_id_hash_code);
 
   DataUseUserData* attached_service_data = static_cast<DataUseUserData*>(
       request.GetUserData(DataUseUserData::kUserDataKey));
@@ -213,7 +223,7 @@ void DataUseMeasurement::ReportDataUseUMA(const net::URLRequest& request,
   RecordUMAHistogramCount(
       GetHistogramName(is_user_traffic ? "DataUse.TrafficSize.User"
                                        : "DataUse.TrafficSize.System",
-                       dir, new_app_state, is_connection_cellular),
+                       dir, new_app_state, IsCurrentNetworkCellular()),
       bytes);
 
 #if defined(OS_ANDROID)
@@ -351,7 +361,7 @@ void DataUseMeasurement::MaybeRecordNetworkBytesOS() {
 
 void DataUseMeasurement::ReportServicesMessageSizeUMA(
     const net::URLRequest& request) {
-  if (!IsUserRequest(request)) {
+  if (!IsUserRequest(request.traffic_annotation().unique_id_hash_code)) {
     ReportDataUsageServices(request.traffic_annotation().unique_id_hash_code,
                             UPSTREAM, CurrentAppState(),
                             request.GetTotalSentBytes());
@@ -365,14 +375,15 @@ void DataUseMeasurement::ReportDataUsageServices(
     int32_t traffic_annotation_hash,
     TrafficDirection dir,
     DataUseUserData::AppState app_state,
-    int64_t message_size) const {
-  if (message_size > 0) {
+    int64_t message_size_bytes) const {
+  if (message_size_bytes > 0) {
     // Conventional UMA histograms are not used because name is not static.
     base::HistogramBase* histogram = base::SparseHistogram::FactoryGet(
         GetHistogramNameWithConnectionType("DataUse.AllServicesKB", dir,
                                            app_state),
         base::HistogramBase::kUmaTargetedHistogramFlag);
-    histogram->AddKiB(traffic_annotation_hash, message_size);
+    histogram->AddKiB(traffic_annotation_hash,
+                      base::saturated_cast<int>(message_size_bytes));
   }
 }
 
@@ -424,7 +435,7 @@ void DataUseMeasurement::RecordContentTypeHistogram(
 
 void DataUseMeasurement::RecordPageTransitionUMA(
     const net::URLRequest& request) const {
-  if (!IsUserRequest(request))
+  if (!IsUserRequest(request.traffic_annotation().unique_id_hash_code))
     return;
 
   const DataUseRecorder* recorder = ascriber_->GetDataUseRecorder(request);
@@ -435,7 +446,8 @@ void DataUseMeasurement::RecordPageTransitionUMA(
 }
 
 // static
-bool DataUseMeasurement::IsUserRequest(const net::URLRequest& request) {
+bool DataUseMeasurement::IsUserRequest(
+    int32_t network_traffic_annotation_hash_id) {
   static const std::set<int32_t> kUserInitiatedTrafficAnnotations = {
       COMPUTE_NETWORK_TRAFFIC_ANNOTATION_ID_HASH(
           "blink_extension_resource_loader"),
@@ -455,7 +467,7 @@ bool DataUseMeasurement::IsUserRequest(const net::URLRequest& request) {
       COMPUTE_NETWORK_TRAFFIC_ANNOTATION_ID_HASH("resource_dispatcher_host"),
   };
   return kUserInitiatedTrafficAnnotations.find(
-             request.traffic_annotation().unique_id_hash_code) !=
+             network_traffic_annotation_hash_id) !=
          kUserInitiatedTrafficAnnotations.end();
 }
 
@@ -471,4 +483,19 @@ bool DataUseMeasurement::IsMetricsServiceRequest(
          kMetricsServiceTrafficAnnotations.end();
 }
 
+bool DataUseMeasurement::IsCurrentNetworkCellular() const {
+  if (network_connection_tracker_) {
+    DCHECK(base::FeatureList::IsEnabled(network::features::kNetworkService));
+    return network::NetworkConnectionTracker::IsConnectionCellular(
+        connection_type_);
+  }
+  return net::NetworkChangeNotifier::IsConnectionCellular(
+      net::NetworkChangeNotifier::GetConnectionType());
+}
+
+void DataUseMeasurement::OnConnectionChanged(
+    network::mojom::ConnectionType type) {
+  connection_type_ = type;
+}
+
 }  // namespace data_use_measurement

components/data_use_measurement/core/data_use_measurement.h

@@ -16,6 +16,7 @@
 #include "build/build_config.h"
 #include "components/data_use_measurement/core/data_use_user_data.h"
 #include "components/metrics/data_use_tracker.h"
+#include "services/network/public/cpp/network_connection_tracker.h"
 
 #if defined(OS_ANDROID)
 #include "base/android/application_status_listener.h"
@@ -39,10 +40,11 @@ class URLRequestClassifier;
 // background or foreground during the request.
 // TODO(amohammadkhan): Complete the layered architecture.
 // http://crbug.com/527460
-class DataUseMeasurement {
+class DataUseMeasurement
+    : public network::NetworkConnectionTracker::NetworkConnectionObserver {
  public:
-  // Returns true if the |request| is initiated by user traffic.
-  static bool IsUserRequest(const net::URLRequest& request);
+  // Returns true if the NTA hash is initiated by user traffic.
+  static bool IsUserRequest(int32_t network_traffic_annotation_hash_id);
 
   // Returns true if the NTA hash is one used by metrics (UMA, UKM) component.
   static bool IsMetricsServiceRequest(
@@ -55,8 +57,9 @@ class DataUseMeasurement {
 
   DataUseMeasurement(
       std::unique_ptr<URLRequestClassifier> url_request_classifier,
-      DataUseAscriber* ascriber);
-  virtual ~DataUseMeasurement();
+      DataUseAscriber* ascriber,
+      network::NetworkConnectionTracker* network_connection_tracker);
+  ~DataUseMeasurement() override;
 
   // Called before a request is sent.
   void OnBeforeURLRequest(net::URLRequest* request);
@@ -90,11 +93,7 @@ class DataUseMeasurement {
                                              bool is_cellular,
                                              bool is_metrics_service_usage) = 0;
 
- private:
-  friend class DataUseMeasurementTest;
-  FRIEND_TEST_ALL_PREFIXES(DataUseMeasurementTest,
-                           TimeOfBackgroundDownstreamBytes);
-
+ protected:
   // Specifies that data is received or sent, respectively.
   enum TrafficDirection { DOWNSTREAM, UPSTREAM };
 
@@ -102,6 +101,27 @@ class DataUseMeasurement {
   // returns Foreground if Chrome is not running on Android.
   DataUseUserData::AppState CurrentAppState() const;
 
+  // Records data use histograms of services. It gets the size of exchanged
+  // message, its direction (which is upstream or downstream) and reports to the
+  // histogram DataUse.Services.{Dimensions} with, services as the buckets.
+  // |app_state| indicates the app state which can be foreground, or background.
+  void ReportDataUsageServices(int32_t traffic_annotation_hash,
+                               TrafficDirection dir,
+                               DataUseUserData::AppState app_state,
+                               int64_t message_size_bytes) const;
+
+  // Returns if the current network connection type is cellular.
+  bool IsCurrentNetworkCellular() const;
+
+#if defined(OS_ANDROID)
+  // Records the count of bytes received and sent by Chrome on the network as
+  // reported by the operating system.
+  void MaybeRecordNetworkBytesOS();
+#endif
+
+ private:
+  friend class DataUseMeasurementTest;
+
   // Makes the full name of the histogram. It is made from |prefix| and suffix
   // which is made based on network and application status. suffix is a string
   // representing whether the data use was on the send ("Upstream") or receive
@@ -130,10 +150,6 @@ class DataUseMeasurement {
   // and vice versa.
   void OnApplicationStateChange(
       base::android::ApplicationState application_state);
-
-  // Records the count of bytes received and sent by Chrome on the network as
-  // reported by the operating system.
-  void MaybeRecordNetworkBytesOS();
 #endif
 
   // Records the data use of the |request|, thus |request| must be non-null.
@@ -146,15 +162,6 @@ class DataUseMeasurement {
   // Reports the message size of the service requests.
   void ReportServicesMessageSizeUMA(const net::URLRequest& request);
 
-  // Records data use histograms of services. It gets the size of exchanged
-  // message, its direction (which is upstream or downstream) and reports to the
-  // histogram DataUse.Services.{Dimensions} with, services as the buckets.
-  // |app_state| indicates the app state which can be foreground, or background.
-  void ReportDataUsageServices(int32_t traffic_annotation_hash,
-                               TrafficDirection dir,
-                               DataUseUserData::AppState app_state,
-                               int64_t message_size) const;
-
   // Records data use histograms split on TrafficDirection, AppState and
   // TabState.
   void RecordTabStateHistogram(TrafficDirection dir,
@@ -174,6 +181,10 @@ class DataUseMeasurement {
       bool is_tab_visible,
       int64_t bytes);
 
+  // NetworkConnectionObserver overrides
+  void OnConnectionChanged(
+      network::mojom::ConnectionType connection_type) override;
+
   // Classifier for identifying if an URL request is user initiated.
   std::unique_ptr<URLRequestClassifier> url_request_classifier_;
 
@@ -208,6 +219,13 @@ class DataUseMeasurement {
   bool no_reads_since_background_;
 #endif
 
+  // Watches for network connection changes. Global singleton object and
+  // outlives |this|
+  network::NetworkConnectionTracker* network_connection_tracker_;
+
+  // The current connection type.
+  network::mojom::ConnectionType connection_type_;
+
   DISALLOW_COPY_AND_ASSIGN(DataUseMeasurement);
 };
 

components/data_use_measurement/core/data_use_measurement_unittest.cc

@@ -92,7 +92,9 @@ class TestDataUseMeasurement : public DataUseMeasurement {
   TestDataUseMeasurement(
       std::unique_ptr<URLRequestClassifier> url_request_classifier,
       DataUseAscriber* ascriber)
-      : DataUseMeasurement(std::move(url_request_classifier), ascriber) {}
+      : DataUseMeasurement(std::move(url_request_classifier),
+                           ascriber,
+                           nullptr) {}
 
   void UpdateDataUseToMetricsService(int64_t total_bytes,
                                      bool is_cellular,
@@ -313,7 +315,7 @@ TEST_F(DataUseMeasurementTest, TimeOfBackgroundDownstreamBytes) {
     std::unique_ptr<net::URLRequest> request = CreateTestRequest(kUserRequest);
     data_use_measurement_.OnBeforeURLRequest(request.get());
     base::HistogramTester histogram_tester;
-    data_use_measurement()->OnApplicationStateChange(
+    data_use_measurement()->OnApplicationStateChangeForTesting(
         base::android::APPLICATION_STATE_HAS_RUNNING_ACTIVITIES);
     data_use_measurement_.OnNetworkBytesSent(*request, 100);
     data_use_measurement_.OnNetworkBytesReceived(*request, 1000);
@@ -344,7 +346,7 @@ TEST_F(DataUseMeasurementTest, TimeOfBackgroundDownstreamBytes) {
     std::unique_ptr<net::URLRequest> request = CreateTestRequest(kUserRequest);
     data_use_measurement_.OnBeforeURLRequest(request.get());
     base::HistogramTester histogram_tester;
-    data_use_measurement()->OnApplicationStateChange(
+    data_use_measurement()->OnApplicationStateChangeForTesting(
         base::android::APPLICATION_STATE_HAS_STOPPED_ACTIVITIES);
     data_use_measurement_.OnNetworkBytesSent(*request, 100);
     data_use_measurement_.OnNetworkBytesReceived(*request, 1000);
@@ -395,7 +397,7 @@ TEST_F(DataUseMeasurementTest, TimeOfBackgroundDownstreamBytes) {
     std::unique_ptr<net::URLRequest> request = CreateTestRequest(kUserRequest);
     data_use_measurement_.OnBeforeURLRequest(request.get());
     base::HistogramTester histogram_tester;
-    data_use_measurement()->OnApplicationStateChange(
+    data_use_measurement()->OnApplicationStateChangeForTesting(
         base::android::APPLICATION_STATE_HAS_RUNNING_ACTIVITIES);
     data_use_measurement_.OnNetworkBytesSent(*request, 100);
     data_use_measurement_.OnNetworkBytesReceived(*request, 1000);

components/data_use_measurement/core/data_use_network_delegate.cc

@@ -7,9 +7,11 @@
 #include <memory>
 #include <utility>
 
+#include "base/feature_list.h"
 #include "components/data_use_measurement/core/data_use_ascriber.h"
 #include "components/data_use_measurement/core/url_request_classifier.h"
 #include "net/url_request/url_request.h"
+#include "services/network/public/cpp/features.h"
 
 namespace data_use_measurement {
 DataUseNetworkDelegate::DataUseNetworkDelegate(
@@ -20,6 +22,7 @@ DataUseNetworkDelegate::DataUseNetworkDelegate(
       ascriber_(ascriber),
       data_use_measurement_(std::move(data_use_measurement)) {
   DCHECK(ascriber);
+  DCHECK(!base::FeatureList::IsEnabled(network::features::kNetworkService));
 }
 
 DataUseNetworkDelegate::~DataUseNetworkDelegate() {}

components/data_use_measurement/core/data_use_network_delegate_unittest.cc

@@ -70,7 +70,9 @@ class TestDataUseMeasurement : public DataUseMeasurement {
   TestDataUseMeasurement(
       std::unique_ptr<URLRequestClassifier> url_request_classifier,
       DataUseAscriber* ascriber)
-      : DataUseMeasurement(std::move(url_request_classifier), ascriber) {}
+      : DataUseMeasurement(std::move(url_request_classifier),
+                           ascriber,
+                           nullptr) {}
 
   void UpdateDataUseToMetricsService(int64_t total_bytes,
                                      bool is_cellular,

content/browser/network_service_client.cc

@@ -489,4 +489,12 @@ void NetworkServiceClient::OnApplicationStateChange(
 }
 #endif
 
+void NetworkServiceClient::OnDataUseUpdate(
+    int32_t network_traffic_annotation_id_hash,
+    int64_t recv_bytes,
+    int64_t sent_bytes) {
+  GetContentClient()->browser()->OnNetworkServiceDataUseUpdate(
+      network_traffic_annotation_id_hash, recv_bytes, sent_bytes);
+}
+
 }  // namespace content

content/browser/network_service_client.h

@@ -76,6 +76,9 @@ class CONTENT_EXPORT NetworkServiceClient
                        const std::string& header_value,
                        int load_flags,
                        OnClearSiteDataCallback callback) override;
+  void OnDataUseUpdate(int32_t network_traffic_annotation_id_hash,
+                       int64_t recv_bytes,
+                       int64_t sent_bytes) override;
 
 #if defined(OS_ANDROID)
   void OnApplicationStateChange(base::android::ApplicationState state);

content/public/browser/content_browser_client.cc

@@ -853,4 +853,9 @@ bool ContentBrowserClient::CanIgnoreCertificateErrorIfNeeded() {
   return false;
 }
 
+void ContentBrowserClient::OnNetworkServiceDataUseUpdate(
+    int32_t network_traffic_annotation_id_hash,
+    int64_t recv_bytes,
+    int64_t sent_bytes) {}
+
 }  // namespace content

content/public/browser/content_browser_client.h

@@ -1355,6 +1355,13 @@ class CONTENT_EXPORT ContentBrowserClient {
   // perform additional checks, such as requiring --user-data-dir flag too to
   // make sure that insecure contents will not persist accidentally.
   virtual bool CanIgnoreCertificateErrorIfNeeded();
+
+  // Called on every request completion to update the data use when network
+  // service is enabled.
+  virtual void OnNetworkServiceDataUseUpdate(
+      int32_t network_traffic_annotation_id_hash,
+      int64_t recv_bytes,
+      int64_t sent_bytes);
 };
 
 }  // namespace content

services/network/public/mojom/network_service.mojom

@@ -138,6 +138,13 @@ interface NetworkServiceClient {
                   url.mojom.Url url,
                   string header_value,
                   int32 load_flags) => ();
+
+  // Called on every request completion to update the network traffic annotation
+  // ID, and the total bytes received and sent.
+  // |network_traffic_annotation_id_hash| represents the hash of unique tag that
+  // identifies the annotation of the request.
+  OnDataUseUpdate(int32 network_traffic_annotation_id_hash, int64 recv_bytes,
+                  int64 sent_bytes);
 };
 
 // An HTTPS server to send DNS queries to, per the DNS Queries over HTTPS spec.

services/network/test/test_network_service_client.cc

@@ -118,4 +118,9 @@ void TestNetworkServiceClient::OnClearSiteData(
   NOTREACHED();
 }
 
+void TestNetworkServiceClient::OnDataUseUpdate(
+    int32_t network_traffic_annotation_id_hash,
+    int64_t recv_bytes,
+    int64_t sent_bytes) {}
+
 }  // namespace network

services/network/test/test_network_service_client.h

@@ -73,6 +73,9 @@ class TestNetworkServiceClient : public network::mojom::NetworkServiceClient {
                        const std::string& header_value,
                        int load_flags,
                        OnClearSiteDataCallback callback) override;
+  void OnDataUseUpdate(int32_t network_traffic_annotation_id_hash,
+                       int64_t recv_bytes,
+                       int64_t sent_bytes) override;
 
  private:
   bool enable_uploads_;

services/network/url_loader.cc

@@ -23,6 +23,7 @@
 #include "net/cert/symantec_certs.h"
 #include "net/ssl/client_cert_store.h"
 #include "net/ssl/ssl_private_key.h"
+#include "net/traffic_annotation/network_traffic_annotation.h"
 #include "net/url_request/url_request_context.h"
 #include "net/url_request/url_request_context_getter.h"
 #include "services/network/chunked_data_pipe_upload_data_stream.h"
@@ -955,6 +956,13 @@ void URLLoader::NotifyCompleted(int error_code) {
         url_request_->GetTotalReceivedBytes(),
         url_request_->GetTotalSentBytes());
   }
+  if (network_service_client_ && (url_request_->GetTotalReceivedBytes() > 0 ||
+                                  url_request_->GetTotalSentBytes() > 0)) {
+    network_service_client_->OnDataUseUpdate(
+        url_request_->traffic_annotation().unique_id_hash_code,
+        url_request_->GetTotalReceivedBytes(),
+        url_request_->GetTotalSentBytes());
+  }
 
   if (url_loader_client_) {
     if (consumer_handle_.is_valid())

services/network/url_loader_unittest.cc

@@ -2232,6 +2232,10 @@ class MockNetworkServiceClient : public mojom::NetworkServiceClient {
     NOTREACHED();
   }
 
+  void OnDataUseUpdate(int32_t network_traffic_annotation_id_hash,
+                       int64_t recv_bytes,
+                       int64_t sent_bytes) override {}
+
   void set_credentials_response(CredentialsResponse credentials_response) {
     credentials_response_ = credentials_response;
   }