Add connection type and cached percentage to DataSaver pingback

The number of bytes from cache are saved on each request and then
made into a fraction of the total bytes on the page (cache + network).
This fraction is bucketed for privacy sake and then sent in the
pingback.

Also adds connection type, which is also a per-page metric.

Bug: 830949
Change-Id: Ia17242cc9a54acf061a72f03b7f327e141d6f3c8
Reviewed-on: https://chromium-review.googlesource.com/1026522
Commit-Queue: Robert Ogden <robertogden@chromium.org>
Reviewed-by: Ryan Sturm <ryansturm@chromium.org>
Cr-Commit-Position: refs/heads/master@{#556065}

chrome/browser/page_load_metrics/observers/data_reduction_proxy_metrics_observer.cc

@@ -109,6 +109,8 @@ DataReductionProxyMetricsObserver::DataReductionProxyMetricsObserver()
       network_bytes_proxied_(0),
       insecure_network_bytes_(0),
       secure_network_bytes_(0),
+      insecure_cached_bytes_(0),
+      secure_cached_bytes_(0),
       process_id_(base::kNullProcessId),
       renderer_memory_usage_kb_(0),
       render_process_host_id_(content::ChildProcessHost::kInvalidUniqueID),
@@ -352,14 +354,35 @@ void DataReductionProxyMetricsObserver::SendPingback(
       ExponentiallyBucketBytes(secure_original_network_bytes_);
   const int64_t network_bytes =
       insecure_network_bytes_ + ExponentiallyBucketBytes(secure_network_bytes_);
+  const int64_t total_page_size_bytes =
+      insecure_network_bytes_ + insecure_cached_bytes_ +
+      ExponentiallyBucketBytes(secure_network_bytes_ + secure_cached_bytes_);
+
+  // Recording cached bytes can be done with raw data, but the end result must
+  // be bucketed in 50 linear buckets between 0% - 100%.
+  const int64_t cached_bytes = insecure_cached_bytes_ + secure_cached_bytes_;
+  const int64_t total_bytes =
+      cached_bytes + insecure_network_bytes_ + secure_network_bytes_;
+  int cached_percentage;
+  if (total_bytes <= 0) {
+    cached_percentage = 0;
+  } else {
+    cached_percentage =
+        static_cast<int>(std::lround(static_cast<float>(cached_bytes) /
+                                     static_cast<float>(total_bytes) * 100.0));
+  }
+  DCHECK_GE(cached_percentage, 0);
+  DCHECK_LE(cached_percentage, 100);
+  cached_percentage = cached_percentage - (cached_percentage % 2);
+  const float cached_fraction = static_cast<float>(cached_percentage) / 100.0;
 
   DataReductionProxyPageLoadTiming data_reduction_proxy_timing(
       timing.navigation_start, response_start, load_event_start,
       first_image_paint, first_contentful_paint,
       experimental_first_meaningful_paint,
       parse_blocked_on_script_load_duration, parse_stop, network_bytes,
-      original_network_bytes, app_background_occurred, opted_out_,
-      renderer_memory_usage_kb_, host_id);
+      original_network_bytes, total_page_size_bytes, cached_fraction,
+      app_background_occurred, opted_out_, renderer_memory_usage_kb_, host_id);
   GetPingbackClient()->SendPingback(*data_, data_reduction_proxy_timing);
 }
 
@@ -476,11 +499,19 @@ void DataReductionProxyMetricsObserver::OnLoadedResource(
       extra_request_complete_info.data_reduction_proxy_data->lofi_received()) {
     data_->set_lofi_received(true);
   }
-  if (extra_request_complete_info.was_cached)
-    return;
 
   const bool is_secure =
       extra_request_complete_info.url.SchemeIsCryptographic();
+
+  if (extra_request_complete_info.was_cached) {
+    if (is_secure) {
+      secure_cached_bytes_ += extra_request_complete_info.raw_body_bytes;
+    } else {
+      insecure_cached_bytes_ += extra_request_complete_info.raw_body_bytes;
+    }
+    return;
+  }
+
   if (is_secure) {
     secure_original_network_bytes_ +=
         extra_request_complete_info.original_network_content_length;

chrome/browser/page_load_metrics/observers/data_reduction_proxy_metrics_observer.h

@@ -160,6 +160,12 @@ class DataReductionProxyMetricsObserver
   // The total network bytes used for HTTPS resources.
   int64_t secure_network_bytes_;
 
+  // The total cached bytes used for HTTP resources.
+  int64_t insecure_cached_bytes_;
+
+  // The total cached bytes used for HTTPS resources.
+  int64_t secure_cached_bytes_;
+
   // The process ID of the main frame renderer during OnCommit.
   base::ProcessId process_id_;
 

components/data_reduction_proxy/content/browser/data_reduction_proxy_pingback_client_impl.cc

@@ -121,8 +121,13 @@ void AddDataToPageloadMetrics(const DataReductionProxyData& request_data,
   request->set_effective_connection_type(
       protobuf_parser::ProtoEffectiveConnectionTypeFromEffectiveConnectionType(
           request_data.effective_connection_type()));
+  request->set_connection_type(
+      protobuf_parser::ProtoConnectionTypeFromConnectionType(
+          request_data.connection_type()));
   request->set_compressed_page_size_bytes(timing.network_bytes);
   request->set_original_page_size_bytes(timing.original_network_bytes);
+  request->set_total_page_size_bytes(timing.total_page_size_bytes);
+  request->set_cached_fraction(timing.cached_fraction);
   request->set_renderer_memory_usage_kb(timing.renderer_memory_usage_kb);
 
   request->set_renderer_crash_type(crash_type);

components/data_reduction_proxy/content/browser/data_reduction_proxy_pingback_client_impl_unittest.cc

@@ -28,6 +28,7 @@
 #include "components/data_reduction_proxy/proto/pageload_metrics.pb.h"
 #include "content/public/common/child_process_host.h"
 #include "net/base/net_errors.h"
+#include "net/base/network_change_notifier.h"
 #include "net/nqe/effective_connection_type.h"
 #include "net/url_request/test_url_fetcher_factory.h"
 #include "net/url_request/url_request_context_getter.h"
@@ -47,6 +48,8 @@ static const char kSessionKey[] = "fake-session";
 static const char kFakeURL[] = "http://www.google.com/";
 static const int64_t kBytes = 10000;
 static const int64_t kBytesOriginal = 1000000;
+static const int64_t kTotalPageSizeBytes = 20000;
+static const float kCachedFraction = 0.5;
 static const int kCrashProcessId = 1;
 static const int64_t kRendererMemory = 1024;
 
@@ -138,7 +141,9 @@ class DataReductionProxyPingbackClientImplTest : public testing::Test {
         base::Optional<base::TimeDelta>(
             base::TimeDelta::FromMilliseconds(2000)) /* parse_stop */,
         kBytes /* network_bytes */, kBytesOriginal /* original_network_bytes */,
-        app_background_occurred, opt_out_occurred, kRendererMemory,
+        kTotalPageSizeBytes /* total_page_size_bytes */,
+        kCachedFraction /* cached_fraction */, app_background_occurred,
+        opt_out_occurred, kRendererMemory,
         crash ? kCrashProcessId : content::ChildProcessHost::kInvalidUniqueID);
 
     DataReductionProxyData request_data;
@@ -146,6 +151,8 @@ class DataReductionProxyPingbackClientImplTest : public testing::Test {
     request_data.set_request_url(GURL(kFakeURL));
     request_data.set_effective_connection_type(
         net::EFFECTIVE_CONNECTION_TYPE_OFFLINE);
+    request_data.set_connection_type(
+        net::NetworkChangeNotifier::CONNECTION_UNKNOWN);
     request_data.set_lofi_received(lofi_received);
     request_data.set_client_lofi_requested(client_lofi_requested);
     request_data.set_lite_page_received(lite_page_received);
@@ -156,7 +163,7 @@ class DataReductionProxyPingbackClientImplTest : public testing::Test {
     page_id_++;
   }
 
-  // Send a fake crash report frome breakpad.
+  // Send a fake crash report from breakpad.
   void ReportCrash(bool oom) {
 #if defined(OS_ANDROID)
     breakpad::CrashDumpManager::CrashDumpDetails details = {
@@ -240,6 +247,8 @@ TEST_F(DataReductionProxyPingbackClientImplTest, VerifyPingbackContent) {
   EXPECT_EQ(kFakeURL, pageload_metrics.first_request_url());
   EXPECT_EQ(kBytes, pageload_metrics.compressed_page_size_bytes());
   EXPECT_EQ(kBytesOriginal, pageload_metrics.original_page_size_bytes());
+  EXPECT_EQ(kTotalPageSizeBytes, pageload_metrics.total_page_size_bytes());
+  EXPECT_EQ(kCachedFraction, pageload_metrics.cached_fraction());
   EXPECT_EQ(data_page_id, pageload_metrics.page_id());
 
   EXPECT_EQ(PageloadMetrics_PreviewsType_NONE,
@@ -250,6 +259,8 @@ TEST_F(DataReductionProxyPingbackClientImplTest, VerifyPingbackContent) {
   EXPECT_EQ(
       PageloadMetrics_EffectiveConnectionType_EFFECTIVE_CONNECTION_TYPE_OFFLINE,
       pageload_metrics.effective_connection_type());
+  EXPECT_EQ(PageloadMetrics_ConnectionType_CONNECTION_UNKNOWN,
+            pageload_metrics.connection_type());
   EXPECT_EQ(kRendererMemory, pageload_metrics.renderer_memory_usage_kb());
   EXPECT_EQ(std::string(), pageload_metrics.holdback_group());
   EXPECT_EQ(PageloadMetrics_RendererCrashType_NO_CRASH,
@@ -369,11 +380,15 @@ TEST_F(DataReductionProxyPingbackClientImplTest,
     EXPECT_EQ(kFakeURL, pageload_metrics.first_request_url());
     EXPECT_EQ(kBytes, pageload_metrics.compressed_page_size_bytes());
     EXPECT_EQ(kBytesOriginal, pageload_metrics.original_page_size_bytes());
+    EXPECT_EQ(kTotalPageSizeBytes, pageload_metrics.total_page_size_bytes());
+    EXPECT_EQ(kCachedFraction, pageload_metrics.cached_fraction());
     EXPECT_EQ(page_ids.front(), pageload_metrics.page_id());
     page_ids.pop_front();
     EXPECT_EQ(
         PageloadMetrics_EffectiveConnectionType_EFFECTIVE_CONNECTION_TYPE_OFFLINE,
         pageload_metrics.effective_connection_type());
+    EXPECT_EQ(PageloadMetrics_ConnectionType_CONNECTION_UNKNOWN,
+              pageload_metrics.connection_type());
     EXPECT_EQ(kRendererMemory, pageload_metrics.renderer_memory_usage_kb());
   }
 

components/data_reduction_proxy/core/browser/data_reduction_proxy_data.cc

@@ -19,7 +19,8 @@ DataReductionProxyData::DataReductionProxyData()
       lite_page_received_(false),
       lofi_policy_received_(false),
       lofi_received_(false),
-      effective_connection_type_(net::EFFECTIVE_CONNECTION_TYPE_UNKNOWN) {}
+      effective_connection_type_(net::EFFECTIVE_CONNECTION_TYPE_UNKNOWN),
+      connection_type_(net::NetworkChangeNotifier::CONNECTION_UNKNOWN) {}
 
 DataReductionProxyData::~DataReductionProxyData() {}
 

components/data_reduction_proxy/core/browser/data_reduction_proxy_data.h

@@ -13,6 +13,7 @@
 #include "base/macros.h"
 #include "base/optional.h"
 #include "base/supports_user_data.h"
+#include "net/base/network_change_notifier.h"
 #include "net/nqe/effective_connection_type.h"
 #include "url/gurl.h"
 
@@ -95,6 +96,16 @@ class DataReductionProxyData : public base::SupportsUserData::Data {
     effective_connection_type_ = effective_connection_type;
   }
 
+  // The connection type (Wifi, 2G, 3G, 4G, None, etc) as reported by the
+  // NetworkChangeNotifier. Only set for main frame requests.
+  net::NetworkChangeNotifier::ConnectionType connection_type() const {
+    return connection_type_;
+  }
+  void set_connection_type(
+      const net::NetworkChangeNotifier::ConnectionType connection_type) {
+    connection_type_ = connection_type;
+  }
+
   // An identifier that is guaranteed to be unique to each page load during a
   // data saver session. Only present on main frame requests.
   const base::Optional<uint64_t>& page_id() const { return page_id_; }
@@ -153,6 +164,10 @@ class DataReductionProxyData : public base::SupportsUserData::Data {
   // set for main frame requests only.
   net::EffectiveConnectionType effective_connection_type_;
 
+  // The connection type (Wifi, 2G, 3G, 4G, None, etc) as reported by the
+  // NetworkChangeNotifier. Only set for main frame requests.
+  net::NetworkChangeNotifier::ConnectionType connection_type_;
+
   // An identifier that is guaranteed to be unique to each page load during a
   // data saver session. Only present on main frame requests.
   base::Optional<uint64_t> page_id_;

components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate.cc

@@ -26,6 +26,7 @@
 #include "components/data_reduction_proxy/core/common/lofi_decider.h"
 #include "net/base/load_flags.h"
 #include "net/base/mime_util.h"
+#include "net/base/network_change_notifier.h"
 #include "net/base/proxy_server.h"
 #include "net/http/http_request_headers.h"
 #include "net/http/http_response_headers.h"
@@ -391,6 +392,8 @@ void DataReductionProxyNetworkDelegate::OnBeforeSendHeadersInternal(
                                                 ->network_quality_estimator()
                                                 ->GetEffectiveConnectionType());
       }
+      data->set_connection_type(
+          net::NetworkChangeNotifier::GetConnectionType());
       // Generate a page ID for main frame requests that don't already have one.
       // TODO(ryansturm): remove LOAD_MAIN_FRAME_DEPRECATED from d_r_p.
       // crbug.com/709621

components/data_reduction_proxy/core/browser/data_reduction_proxy_util.cc

@@ -310,6 +310,28 @@ ProtoEffectiveConnectionTypeFromEffectiveConnectionType(
   }
 }
 
+PageloadMetrics_ConnectionType ProtoConnectionTypeFromConnectionType(
+    net::NetworkChangeNotifier::ConnectionType connection_type) {
+  switch (connection_type) {
+    case net::NetworkChangeNotifier::CONNECTION_UNKNOWN:
+      return PageloadMetrics_ConnectionType_CONNECTION_UNKNOWN;
+    case net::NetworkChangeNotifier::CONNECTION_ETHERNET:
+      return PageloadMetrics_ConnectionType_CONNECTION_ETHERNET;
+    case net::NetworkChangeNotifier::CONNECTION_WIFI:
+      return PageloadMetrics_ConnectionType_CONNECTION_WIFI;
+    case net::NetworkChangeNotifier::CONNECTION_2G:
+      return PageloadMetrics_ConnectionType_CONNECTION_2G;
+    case net::NetworkChangeNotifier::CONNECTION_3G:
+      return PageloadMetrics_ConnectionType_CONNECTION_3G;
+    case net::NetworkChangeNotifier::CONNECTION_4G:
+      return PageloadMetrics_ConnectionType_CONNECTION_4G;
+    case net::NetworkChangeNotifier::CONNECTION_NONE:
+      return PageloadMetrics_ConnectionType_CONNECTION_NONE;
+    case net::NetworkChangeNotifier::CONNECTION_BLUETOOTH:
+      return PageloadMetrics_ConnectionType_CONNECTION_BLUETOOTH;
+  }
+}
+
 net::ProxyServer::Scheme SchemeFromProxyScheme(
     ProxyServer_ProxyScheme proxy_scheme) {
   switch (proxy_scheme) {

components/data_reduction_proxy/core/browser/data_reduction_proxy_util.h

@@ -10,6 +10,7 @@
 
 #include "components/data_reduction_proxy/proto/client_config.pb.h"
 #include "components/data_reduction_proxy/proto/pageload_metrics.pb.h"
+#include "net/base/network_change_notifier.h"
 #include "net/base/proxy_server.h"
 #include "net/nqe/effective_connection_type.h"
 #include "net/proxy_resolution/proxy_retry_info.h"
@@ -128,12 +129,17 @@ static_assert(net::EFFECTIVE_CONNECTION_TYPE_LAST == 6,
               "If net::EFFECTIVE_CONNECTION_TYPE changes, "
               "PageloadMetrics_EffectiveConnectionType needs to be updated.");
 
-// Returns the PageloadMetrics_EffectiveConnection equivalent of
+// Returns the PageloadMetrics_EffectiveConnectionType equivalent of
 // |effective_connection_type|.
 PageloadMetrics_EffectiveConnectionType
 ProtoEffectiveConnectionTypeFromEffectiveConnectionType(
     net::EffectiveConnectionType effective_connection_type);
 
+// Returns the PageloadMetrics_ConnectionType equivalent of
+// |connection_type|.
+PageloadMetrics_ConnectionType ProtoConnectionTypeFromConnectionType(
+    net::NetworkChangeNotifier::ConnectionType connection_type);
+
 // Returns the |net::ProxyServer::Scheme| for a ProxyServer_ProxyScheme.
 net::ProxyServer::Scheme SchemeFromProxyScheme(
     ProxyServer_ProxyScheme proxy_scheme);

components/data_reduction_proxy/core/common/data_reduction_proxy_page_load_timing.cc

@@ -18,6 +18,8 @@ DataReductionProxyPageLoadTiming::DataReductionProxyPageLoadTiming(
     const base::Optional<base::TimeDelta>& parse_stop,
     int64_t network_bytes,
     int64_t original_network_bytes,
+    int64_t total_page_size_bytes,
+    float cached_fraction,
     bool app_background_occurred,
     bool opt_out_occurred,
     int64_t renderer_memory_usage_kb,
@@ -33,6 +35,8 @@ DataReductionProxyPageLoadTiming::DataReductionProxyPageLoadTiming(
       parse_stop(parse_stop),
       network_bytes(network_bytes),
       original_network_bytes(original_network_bytes),
+      total_page_size_bytes(total_page_size_bytes),
+      cached_fraction(cached_fraction),
       app_background_occurred(app_background_occurred),
       opt_out_occurred(opt_out_occurred),
       renderer_memory_usage_kb(renderer_memory_usage_kb),

components/data_reduction_proxy/core/common/data_reduction_proxy_page_load_timing.h

@@ -27,6 +27,8 @@ struct DataReductionProxyPageLoadTiming {
       const base::Optional<base::TimeDelta>& parse_stop,
       int64_t network_bytes,
       int64_t original_network_bytes,
+      int64_t total_page_size_bytes,
+      float cached_fraction,
       bool app_background_occurred,
       bool opt_out_occurred,
       int64_t renderer_memory_usage_kb,
@@ -61,6 +63,10 @@ struct DataReductionProxyPageLoadTiming {
   // The number of bytes that would have been served over the network if the
   // user were not using data reduction proxy, not including headers.
   const int64_t original_network_bytes;
+  // The total number of bytes loaded for the page content, including cache.
+  const int64_t total_page_size_bytes;
+  // The fraction of bytes that were served from the cache for this page load.
+  const float cached_fraction;
   // True when android app background occurred during the page load lifetime.
   const bool app_background_occurred;
   // True when the user clicks "Show Original" on the Previews infobar.

components/data_reduction_proxy/proto/pageload_metrics.proto

@@ -44,6 +44,19 @@ message PageloadMetrics {
     EFFECTIVE_CONNECTION_TYPE_4G = 5;
   };
 
+  // The possible conntion type values. See //net/base/network_change_notifier.h
+  // for a detailed description of the enum values.
+  enum ConnectionType {
+    CONNECTION_UNKNOWN = 0;
+    CONNECTION_ETHERNET = 1;
+    CONNECTION_WIFI = 2;
+    CONNECTION_2G = 3;
+    CONNECTION_3G = 4;
+    CONNECTION_4G = 5;
+    CONNECTION_NONE = 6;
+    CONNECTION_BLUETOOTH = 7;
+  };
+
   // The various opt out states seen by server previews.
   enum PreviewsOptOut {
     // Set for non-previews navigations and app background navigations.
@@ -101,9 +114,11 @@ message PageloadMetrics {
 
   // The sum of original-content-length values, over resources that were not
   // loaded from browser cache.
+  // Secure resources (i.e., HTTPS) that go into this sum are bucketed.
   optional int64 original_page_size_bytes = 10;
   // The sum of (compressed) content-length, over resources that were not loaded
   // from browser cache.
+  // Secure resources (i.e., HTTPS) that go into this sum are bucketed.
   optional int64 compressed_page_size_bytes = 11;
 
   // The effective connection type at the start of the navigation.
@@ -140,4 +155,17 @@ message PageloadMetrics {
 
   // The RendererCrashType for the page load.
   optional RendererCrashType renderer_crash_type = 21;
+
+  // The percent of total_page_size_bytes that were loaded from the cache. This
+  // value is in the range [0.0, 1.0].
+  optional float cached_fraction = 22;
+
+  // The connection type (Wifi, 2G, 3G, 4G, None, etc) reported by the
+  // NetworkChangeNotifier.
+  optional ConnectionType connection_type = 23;
+
+  // The sum of all bytes loaded on a page. This is compressed_page_size_bytes
+  // plus the sum of content-length for all resources loaded from the browser
+  // cache. Secure resources (i.e., HTTPS) that go into this sum are bucketed.
+  optional int64 total_page_size_bytes = 24;
 }