Report data use from renderer to browser page load metrics

Report the continuous data use to page load metrics in browser process
via the existing renderer -> browser PageLoadMetrics mojo.

Bug: 836029
Cq-Include-Trybots: luci.chromium.try:linux_mojo
Change-Id: I1c7b47eb41e13f9b72d087e3ca2c0a7096c11b29
Reviewed-on: https://chromium-review.googlesource.com/1042877Reviewed-by: Kinuko Yasuda <kinuko@chromium.org>
Reviewed-by: Charlie Harrison <csharrison@chromium.org>
Reviewed-by: Ryan Sturm <ryansturm@chromium.org>
Commit-Queue: rajendrant <rajendrant@chromium.org>
Cr-Commit-Position: refs/heads/master@{#574421}

chrome/browser/page_load_metrics/metrics_web_contents_observer.cc

@@ -602,7 +602,8 @@ void MetricsWebContentsObserver::OnTimingUpdated(
     content::RenderFrameHost* render_frame_host,
     const mojom::PageLoadTiming& timing,
     const mojom::PageLoadMetadata& metadata,
-    const mojom::PageLoadFeatures& new_features) {
+    const mojom::PageLoadFeatures& new_features,
+    const mojom::PageLoadDataUse& new_data_use) {
   // We may receive notifications from frames that have been navigated away
   // from. We simply ignore them.
   if (GetMainFrame(render_frame_host) != web_contents()->GetMainFrame()) {
@@ -635,17 +636,19 @@ void MetricsWebContentsObserver::OnTimingUpdated(
 
   if (committed_load_) {
     committed_load_->metrics_update_dispatcher()->UpdateMetrics(
-        render_frame_host, timing, metadata, new_features);
+        render_frame_host, timing, metadata, new_features, new_data_use);
   }
 }
 
 void MetricsWebContentsObserver::UpdateTiming(
     const mojom::PageLoadTimingPtr timing,
     const mojom::PageLoadMetadataPtr metadata,
-    const mojom::PageLoadFeaturesPtr new_features) {
+    const mojom::PageLoadFeaturesPtr new_features,
+    const mojom::PageLoadDataUsePtr new_data_use) {
   content::RenderFrameHost* render_frame_host =
       page_load_metrics_binding_.GetCurrentTargetFrame();
-  OnTimingUpdated(render_frame_host, *timing, *metadata, *new_features);
+  OnTimingUpdated(render_frame_host, *timing, *metadata, *new_features,
+                  *new_data_use);
 }
 
 bool MetricsWebContentsObserver::ShouldTrackNavigation(

chrome/browser/page_load_metrics/metrics_web_contents_observer.h

@@ -145,7 +145,8 @@ class MetricsWebContentsObserver
   void OnTimingUpdated(content::RenderFrameHost* render_frame_host,
                        const mojom::PageLoadTiming& timing,
                        const mojom::PageLoadMetadata& metadata,
-                       const mojom::PageLoadFeatures& new_features);
+                       const mojom::PageLoadFeatures& new_features,
+                       const mojom::PageLoadDataUse& new_data_use);
 
   // Informs the observers of the currently committed load that the event
   // corresponding to |event_key| has occurred. This should not be called within
@@ -157,9 +158,10 @@ class MetricsWebContentsObserver
   friend class content::WebContentsUserData<MetricsWebContentsObserver>;
 
   // page_load_metrics::mojom::PageLoadMetrics implementation.
-  void UpdateTiming(mojom::PageLoadTimingPtr timing,
-                    mojom::PageLoadMetadataPtr metadata,
-                    mojom::PageLoadFeaturesPtr new_features) override;
+  void UpdateTiming(const mojom::PageLoadTimingPtr timing,
+                    const mojom::PageLoadMetadataPtr metadata,
+                    const mojom::PageLoadFeaturesPtr new_features,
+                    const mojom::PageLoadDataUsePtr new_data_use) override;
 
   void HandleFailedNavigationForTrackedLoad(
       content::NavigationHandle* navigation_handle,

chrome/browser/page_load_metrics/metrics_web_contents_observer_unittest.cc

@@ -256,9 +256,9 @@ class MetricsWebContentsObserverTest : public ChromeRenderViewHostTestHarness {
   void SimulateTimingUpdateWithoutFiringDispatchTimer(
       const mojom::PageLoadTiming& timing,
       content::RenderFrameHost* render_frame_host) {
-    observer()->OnTimingUpdated(render_frame_host, timing,
-                                mojom::PageLoadMetadata(),
-                                mojom::PageLoadFeatures());
+    observer()->OnTimingUpdated(
+        render_frame_host, timing, mojom::PageLoadMetadata(),
+        mojom::PageLoadFeatures(), mojom::PageLoadDataUse());
   }
 
   void AttachObserver() {

chrome/browser/page_load_metrics/observers/page_load_metrics_observer_tester.cc

@@ -88,7 +88,7 @@ void PageLoadMetricsObserverTester::SimulatePageLoadTimingUpdate(
     const mojom::PageLoadMetadata& metadata,
     const mojom::PageLoadFeatures& new_features) {
   observer_->OnTimingUpdated(web_contents()->GetMainFrame(), timing, metadata,
-                             new_features);
+                             new_features, mojom::PageLoadDataUse());
   // If sending the timing update caused the PageLoadMetricsUpdateDispatcher to
   // schedule a buffering timer, then fire it now so metrics are dispatched to
   // observers.

chrome/browser/page_load_metrics/page_load_metrics_browsertest.cc

@@ -68,6 +68,7 @@
 #include "net/dns/mock_host_resolver.h"
 #include "net/http/failing_http_transaction_factory.h"
 #include "net/http/http_cache.h"
+#include "net/test/embedded_test_server/controllable_http_response.h"
 #include "net/test/embedded_test_server/embedded_test_server.h"
 #include "net/test/url_request/url_request_failed_job.h"
 #include "net/test/url_request/url_request_mock_http_job.h"
@@ -126,6 +127,11 @@ class PageLoadMetricsWaiter
       page_expected_fields_.Set(field);
   }
 
+  void AddMinimumPageLoadDataUseExpectation(
+      int expected_minimum_page_load_data_use) {
+    expected_minimum_page_load_data_use_ = expected_minimum_page_load_data_use;
+  }
+
   // Whether the given TimingField was observed in the page.
   bool DidObserveInPage(TimingField field) {
     return observed_page_fields_.IsSet(field);
@@ -189,6 +195,15 @@ class PageLoadMetricsWaiter
       run_loop_->Quit();
   }
 
+  void OnDataUseObserved(int64_t received_data_length,
+                         int64_t data_reduction_proxy_bytes_saved) {
+    current_page_load_data_use_ += received_data_length;
+    if (expectations_satisfied() && run_loop_)
+      run_loop_->Quit();
+  }
+
+  int64_t current_page_load_data_use() { return current_page_load_data_use_; }
+
  private:
   // PageLoadMetricsObserver used by the PageLoadMetricsWaiter to observe
   // metrics updates.
@@ -214,6 +229,13 @@ class PageLoadMetricsWaiter
         waiter_->OnLoadedResource(extra_request_complete_info);
     }
 
+    void OnDataUseObserved(int64_t received_data_length,
+                           int64_t data_reduction_proxy_bytes_saved) override {
+      if (waiter_)
+        waiter_->OnDataUseObserved(received_data_length,
+                                   data_reduction_proxy_bytes_saved);
+    }
+
    private:
     const base::WeakPtr<PageLoadMetricsWaiter> waiter_;
   };
@@ -305,7 +327,10 @@ class PageLoadMetricsWaiter
   }
 
   bool expectations_satisfied() const {
-    return subframe_expected_fields_.Empty() && page_expected_fields_.Empty();
+    return subframe_expected_fields_.Empty() && page_expected_fields_.Empty() &&
+           (expected_minimum_page_load_data_use_ == 0 ||
+            current_page_load_data_use_ >=
+                expected_minimum_page_load_data_use_);
   }
 
   std::unique_ptr<base::RunLoop> run_loop_;
@@ -315,6 +340,9 @@ class PageLoadMetricsWaiter
 
   TimingFieldBitSet observed_page_fields_;
 
+  int64_t expected_minimum_page_load_data_use_ = 0;
+  int64_t current_page_load_data_use_ = 0;
+
   bool attach_on_tracker_creation_ = false;
   bool did_add_observer_ = false;
 
@@ -1011,8 +1039,8 @@ IN_PROC_BROWSER_TEST_F(PageLoadMetricsBrowserTest,
   histogram_tester_.ExpectUniqueSample(
       internal::kHistogramFirstMeaningfulPaintStatus,
       internal::FIRST_MEANINGFUL_PAINT_RECORDED, 1);
-  histogram_tester_.ExpectTotalCount(
-      internal::kHistogramFirstMeaningfulPaint, 1);
+  histogram_tester_.ExpectTotalCount(internal::kHistogramFirstMeaningfulPaint,
+                                     1);
   histogram_tester_.ExpectTotalCount(
       internal::kHistogramParseStartToFirstMeaningfulPaint, 1);
 }
@@ -1037,8 +1065,8 @@ IN_PROC_BROWSER_TEST_F(PageLoadMetricsBrowserTest,
   histogram_tester_.ExpectUniqueSample(
       internal::kHistogramFirstMeaningfulPaintStatus,
       internal::FIRST_MEANINGFUL_PAINT_DID_NOT_REACH_NETWORK_STABLE, 1);
-  histogram_tester_.ExpectTotalCount(
-      internal::kHistogramFirstMeaningfulPaint, 0);
+  histogram_tester_.ExpectTotalCount(internal::kHistogramFirstMeaningfulPaint,
+                                     0);
   histogram_tester_.ExpectTotalCount(
       internal::kHistogramParseStartToFirstMeaningfulPaint, 0);
 }
@@ -1996,3 +2024,90 @@ IN_PROC_BROWSER_TEST_F(SessionRestorePageLoadMetricsBrowserTest,
   ASSERT_NO_FATAL_FAILURE(WaitForTabsToLoad(new_browser));
   ExpectFirstPaintMetricsTotalCount(1);
 }
+
+// TODO(rajendrant): Add tests for data reduction proxy savings, and tests for
+// page loads in brand new renderers that the navigation response is recorded
+// (e.g. do window.open('/some-cross-site-page')).
+IN_PROC_BROWSER_TEST_F(PageLoadMetricsBrowserTest, ReceivedDataLength) {
+  ASSERT_TRUE(embedded_test_server()->Start());
+
+  auto waiter = CreatePageLoadMetricsWaiter();
+  waiter->AddPageExpectation(TimingField::LOAD_EVENT);
+  ui_test_utils::NavigateToURL(browser(), embedded_test_server()->GetURL(
+                                              "/page_load_metrics/large.html"));
+  waiter->AddMinimumPageLoadDataUseExpectation(10000);
+  waiter->Wait();
+}
+
+IN_PROC_BROWSER_TEST_F(PageLoadMetricsBrowserTest,
+                       ReceivedDataLengthControlledLoad) {
+  const char kHttpResponseHeader[] =
+      "HTTP/1.1 200 OK\r\n"
+      "Content-Type: text/html; charset=utf-8\r\n"
+      "\r\n";
+
+  auto main_html_response =
+      std::make_unique<net::test_server::ControllableHttpResponse>(
+          embedded_test_server(), "/mock_page.html",
+          true /*relative_url_is_prefix*/);
+  auto css_response =
+      std::make_unique<net::test_server::ControllableHttpResponse>(
+          embedded_test_server(), "/mock_css.css",
+          true /*relative_url_is_prefix*/);
+  ASSERT_TRUE(embedded_test_server()->Start());
+
+  auto waiter = CreatePageLoadMetricsWaiter();
+
+  browser()->OpenURL(content::OpenURLParams(
+      embedded_test_server()->GetURL("/mock_page.html"), content::Referrer(),
+      WindowOpenDisposition::CURRENT_TAB, ui::PAGE_TRANSITION_TYPED, false));
+
+  main_html_response->WaitForRequest();
+  main_html_response->Send(kHttpResponseHeader);
+  main_html_response->Send(
+      "<html><link rel=\"stylesheet\" href=\"mock_css.css\">");
+  main_html_response->Send(std::string(1000, ' '));
+  // TODO(rajendrant): Verify that 1000 bytes are received at this point before
+  // the request completes. This is hard to verify now since
+  // MojoAsyncResourceHandler throttles the transfer size updates, and calls
+  // only when a read is called after 1 second from previous update.
+
+  main_html_response->Send("</html>");
+  main_html_response->Send(std::string(1000, ' '));
+  main_html_response->Done();
+  waiter->AddMinimumPageLoadDataUseExpectation(2000);
+  waiter->Wait();
+
+  css_response->WaitForRequest();
+  css_response->Send(kHttpResponseHeader);
+  css_response->Send(std::string(1000, ' '));
+  css_response->Done();
+  waiter->AddMinimumPageLoadDataUseExpectation(3000);
+  waiter->Wait();
+}
+
+IN_PROC_BROWSER_TEST_F(PageLoadMetricsBrowserTest,
+                       ReceivedDataLengthCrossSiteIframe) {
+  embedded_test_server()->ServeFilesFromSourceDirectory("content/test/data");
+  content::SetupCrossSiteRedirector(embedded_test_server());
+  ASSERT_TRUE(embedded_test_server()->Start());
+
+  auto waiter = CreatePageLoadMetricsWaiter();
+  waiter->AddPageExpectation(TimingField::LOAD_EVENT);
+  ui_test_utils::NavigateToURL(
+      browser(), embedded_test_server()->GetURL(
+                     "foo.com", "/cross_site_iframe_factory.html?foo"));
+  waiter->Wait();
+  int64_t one_frame_page_size = waiter->current_page_load_data_use();
+
+  waiter = CreatePageLoadMetricsWaiter();
+  waiter->AddPageExpectation(TimingField::LOAD_EVENT);
+  ui_test_utils::NavigateToURL(
+      browser(),
+      embedded_test_server()->GetURL(
+          "a.com", "/cross_site_iframe_factory.html?a(b,c,d(e,f,g))"));
+  // Verify that 7 iframes are fetched, with some amount of tolerance since
+  // favicon is fetched only once.
+  waiter->AddMinimumPageLoadDataUseExpectation(7 * (one_frame_page_size - 100));
+  waiter->Wait();
+}

chrome/browser/page_load_metrics/page_load_metrics_observer.h

@@ -420,6 +420,14 @@ class PageLoadMetricsObserver {
   virtual void OnFeaturesUsageObserved(const mojom::PageLoadFeatures& features,
                                        const PageLoadExtraInfo& extra_info) {}
 
+  // Invoked when data use is observed for the page load across all frames.
+  // These bytes are the additional bytes reported since the last call to
+  // OnDataUseObserved. |received_data_length| is the received network bytes.
+  // |data_reduction_proxy_bytes_saved| is the bytes saved by the data reduction
+  // proxy, which could be negative if the proxy had inflated the resource.
+  virtual void OnDataUseObserved(int64_t received_data_length,
+                                 int64_t data_reduction_proxy_bytes_saved) {}
+
   // Invoked when a media element starts playing.
   virtual void MediaStartedPlaying(
       const content::WebContentsObserver::MediaPlayerInfo& video_type,

chrome/browser/page_load_metrics/page_load_metrics_update_dispatcher.cc

@@ -430,7 +430,8 @@ void PageLoadMetricsUpdateDispatcher::UpdateMetrics(
     content::RenderFrameHost* render_frame_host,
     const mojom::PageLoadTiming& new_timing,
     const mojom::PageLoadMetadata& new_metadata,
-    const mojom::PageLoadFeatures& new_features) {
+    const mojom::PageLoadFeatures& new_features,
+    const mojom::PageLoadDataUse& new_data_use) {
   if (render_frame_host->GetLastCommittedURL().SchemeIs(
           extensions::kExtensionScheme)) {
     // Extensions can inject child frames into a page. We don't want to track
@@ -446,6 +447,7 @@ void PageLoadMetricsUpdateDispatcher::UpdateMetrics(
     UpdateSubFrameTiming(render_frame_host, new_timing);
   }
   client_->UpdateFeaturesUsage(new_features);
+  client_->UpdateDataUse(new_data_use);
 }
 
 void PageLoadMetricsUpdateDispatcher::UpdateFeatures(

chrome/browser/page_load_metrics/page_load_metrics_update_dispatcher.h

@@ -109,6 +109,7 @@ class PageLoadMetricsUpdateDispatcher {
     virtual void OnSubframeMetadataChanged() = 0;
     virtual void UpdateFeaturesUsage(
         const mojom::PageLoadFeatures& new_features) = 0;
+    virtual void UpdateDataUse(const mojom::PageLoadDataUse& new_data_use) = 0;
   };
 
   // The |client| instance must outlive this object.
@@ -121,7 +122,8 @@ class PageLoadMetricsUpdateDispatcher {
   void UpdateMetrics(content::RenderFrameHost* render_frame_host,
                      const mojom::PageLoadTiming& new_timing,
                      const mojom::PageLoadMetadata& new_metadata,
-                     const mojom::PageLoadFeatures& new_features);
+                     const mojom::PageLoadFeatures& new_features,
+                     const mojom::PageLoadDataUse& new_data_use);
 
   // This method is only intended to be called for PageLoadFeatures being
   // recorded directly from the browser process. Features coming from the
@@ -158,6 +160,8 @@ class PageLoadMetricsUpdateDispatcher {
   void MaybeDispatchTimingUpdates(bool did_merge_new_timing_value);
   void DispatchTimingUpdates();
 
+  void UpdateDataUse(const mojom::PageLoadDataUse& new_data_use);
+
   // The client is guaranteed to outlive this object.
   Client* const client_;
 

chrome/browser/page_load_metrics/page_load_tracker.cc

@@ -643,4 +643,12 @@ void PageLoadTracker::UpdateFeaturesUsage(
   }
 }
 
+void PageLoadTracker::UpdateDataUse(
+    const mojom::PageLoadDataUse& new_data_use) {
+  for (const auto& observer : observers_) {
+    observer->OnDataUseObserved(new_data_use.received_data_length,
+                                new_data_use.data_reduction_proxy_bytes_saved);
+  }
+}
+
 }  // namespace page_load_metrics

chrome/browser/page_load_metrics/page_load_tracker.h

@@ -180,6 +180,7 @@ class PageLoadTracker : public PageLoadMetricsUpdateDispatcher::Client {
   void OnSubframeMetadataChanged() override;
   void UpdateFeaturesUsage(
       const mojom::PageLoadFeatures& new_features) override;
+  void UpdateDataUse(const mojom::PageLoadDataUse& new_datause) override;
 
   void Redirect(content::NavigationHandle* navigation_handle);
   void WillProcessNavigationResponse(

chrome/common/page_load_metrics/page_load_metrics.mojom

@@ -151,10 +151,22 @@ struct PageLoadFeatures {
   array<int32> animated_css_properties;
 };
 
+// Data used for the page load.
+struct PageLoadDataUse {
+  // Network bytes received for the page load.
+  int64 received_data_length = 0;
+
+  // The number of bytes saved by the data reduction proxy. Will be zero when
+  // data reduction proxy is not used. Can be negative if the proxy inflated the
+  // resource.
+  int64 data_reduction_proxy_bytes_saved = 0;
+};
+
 // Sent from renderer to browser process when the PageLoadTiming for the
 // associated frame changed.
 interface PageLoadMetrics {
   UpdateTiming(PageLoadTiming page_load_timing,
                PageLoadMetadata page_load_metadata,
-               PageLoadFeatures new_features);
+               PageLoadFeatures new_features,
+               PageLoadDataUse data_use);
 };

chrome/renderer/BUILD.gn

@@ -73,6 +73,8 @@ static_library("renderer") {
     "net_benchmarking_extension.h",
     "page_load_metrics/metrics_render_frame_observer.cc",
     "page_load_metrics/metrics_render_frame_observer.h",
+    "page_load_metrics/page_resource_data_use.cc",
+    "page_load_metrics/page_resource_data_use.h",
     "page_load_metrics/page_timing_metrics_sender.cc",
     "page_load_metrics/page_timing_metrics_sender.h",
     "page_load_metrics/page_timing_sender.h",

chrome/renderer/page_load_metrics/fake_page_timing_sender.cc

@@ -17,8 +17,9 @@ FakePageTimingSender::~FakePageTimingSender() {}
 void FakePageTimingSender::SendTiming(
     const mojom::PageLoadTimingPtr& timing,
     const mojom::PageLoadMetadataPtr& metadata,
-    mojom::PageLoadFeaturesPtr new_features) {
-  validator_->UpdateTiming(timing, metadata, new_features);
+    mojom::PageLoadFeaturesPtr new_features,
+    mojom::PageLoadDataUsePtr new_data_use) {
+  validator_->UpdateTiming(timing, metadata, new_features, new_data_use);
 }
 
 FakePageTimingSender::PageTimingValidator::PageTimingValidator() {}
@@ -93,7 +94,8 @@ void FakePageTimingSender::PageTimingValidator::VerifyExpectedCssProperties()
 void FakePageTimingSender::PageTimingValidator::UpdateTiming(
     const mojom::PageLoadTimingPtr& timing,
     const mojom::PageLoadMetadataPtr& metadata,
-    const mojom::PageLoadFeaturesPtr& new_features) {
+    const mojom::PageLoadFeaturesPtr& new_features,
+    const mojom::PageLoadDataUsePtr& new_data_use) {
   actual_timings_.push_back(timing.Clone());
   for (const auto feature : new_features->features) {
     EXPECT_EQ(actual_features_.find(feature), actual_features_.end())

chrome/renderer/page_load_metrics/fake_page_timing_sender.h

@@ -70,7 +70,8 @@ class FakePageTimingSender : public PageTimingSender {
 
     void UpdateTiming(const mojom::PageLoadTimingPtr& timing,
                       const mojom::PageLoadMetadataPtr& metadata,
-                      const mojom::PageLoadFeaturesPtr& new_features);
+                      const mojom::PageLoadFeaturesPtr& new_features,
+                      const mojom::PageLoadDataUsePtr& new_data_use);
 
    private:
     std::vector<mojom::PageLoadTimingPtr> expected_timings_;
@@ -86,7 +87,8 @@ class FakePageTimingSender : public PageTimingSender {
   ~FakePageTimingSender() override;
   void SendTiming(const mojom::PageLoadTimingPtr& timing,
                   const mojom::PageLoadMetadataPtr& metadata,
-                  mojom::PageLoadFeaturesPtr new_features) override;
+                  mojom::PageLoadFeaturesPtr new_features,
+                  mojom::PageLoadDataUsePtr new_data_use) override;
 
  private:
   PageTimingValidator* const validator_;

chrome/renderer/page_load_metrics/metrics_render_frame_observer.cc

@@ -40,10 +40,12 @@ class MojoPageTimingSender : public PageTimingSender {
   ~MojoPageTimingSender() override {}
   void SendTiming(const mojom::PageLoadTimingPtr& timing,
                   const mojom::PageLoadMetadataPtr& metadata,
-                  mojom::PageLoadFeaturesPtr new_features) override {
+                  mojom::PageLoadFeaturesPtr new_features,
+                  mojom::PageLoadDataUsePtr new_data_use) override {
     DCHECK(page_load_metrics_);
     page_load_metrics_->UpdateTiming(timing->Clone(), metadata->Clone(),
-                                     std::move(new_features));
+                                     std::move(new_features),
+                                     std::move(new_data_use));
   }
 
  private:
@@ -88,22 +90,73 @@ void MetricsRenderFrameObserver::DidObserveNewCssPropertyUsage(
 void MetricsRenderFrameObserver::DidStartResponse(
     int request_id,
     const network::ResourceResponseHead& response_head,
-    content::ResourceType resource_type) {}
+    content::ResourceType resource_type) {
+  if (provisional_frame_resource_data_use_ &&
+      content::IsResourceTypeFrame(resource_type)) {
+    // TODO(rajendrant): This frame request might start before the provisional
+    // load starts, and data use of the frame request might be missed in that
+    // case. There should be a guarantee that DidStartProvisionalLoad be called
+    // before DidStartResponse for the frame request.
+    provisional_frame_resource_data_use_->DidStartResponse(request_id,
+                                                           response_head);
+  } else if (page_timing_metrics_sender_) {
+    page_timing_metrics_sender_->DidStartResponse(request_id, response_head);
+  }
+}
 
 void MetricsRenderFrameObserver::DidCompleteResponse(
     int request_id,
-    const network::URLLoaderCompletionStatus& status) {}
+    const network::URLLoaderCompletionStatus& status) {
+  if (provisional_frame_resource_data_use_ &&
+      provisional_frame_resource_data_use_->resource_id() == request_id) {
+    provisional_frame_resource_data_use_->DidCompleteResponse(
+        status, provisional_delta_data_use_.get());
+  } else if (page_timing_metrics_sender_) {
+    page_timing_metrics_sender_->DidCompleteResponse(request_id, status);
+  }
+}
 
-void MetricsRenderFrameObserver::DidCancelResponse(int request_id) {}
+void MetricsRenderFrameObserver::DidCancelResponse(int request_id) {
+  if (provisional_frame_resource_data_use_ &&
+      provisional_frame_resource_data_use_->resource_id() == request_id) {
+    provisional_frame_resource_data_use_.reset();
+  } else if (page_timing_metrics_sender_) {
+    page_timing_metrics_sender_->DidCancelResponse(request_id);
+  }
+}
 
 void MetricsRenderFrameObserver::DidReceiveTransferSizeUpdate(
     int request_id,
-    int received_data_length) {}
+    int received_data_length) {
+  if (provisional_frame_resource_data_use_ &&
+      provisional_frame_resource_data_use_->resource_id() == request_id) {
+    provisional_frame_resource_data_use_->DidReceiveTransferSizeUpdate(
+        received_data_length, provisional_delta_data_use_.get());
+  } else if (page_timing_metrics_sender_) {
+    page_timing_metrics_sender_->DidReceiveTransferSizeUpdate(
+        request_id, received_data_length);
+  }
+}
 
 void MetricsRenderFrameObserver::FrameDetached() {
   page_timing_metrics_sender_.reset();
 }
 
+void MetricsRenderFrameObserver::DidStartProvisionalLoad(
+    blink::WebDocumentLoader* document_loader) {
+  // Create a new data use tracker for the new provisional load.
+  provisional_frame_resource_data_use_ =
+      std::make_unique<PageResourceDataUse>();
+  provisional_delta_data_use_ = mojom::PageLoadDataUse::New();
+}
+
+void MetricsRenderFrameObserver::DidFailProvisionalLoad(
+    const blink::WebURLError& error) {
+  // Clear the data use tracker for the provisional navigation that started.
+  provisional_frame_resource_data_use_.reset();
+  provisional_delta_data_use_.reset();
+}
+
 void MetricsRenderFrameObserver::DidCommitProvisionalLoad(
     bool is_new_navigation,
     bool is_same_document_navigation) {
@@ -122,7 +175,9 @@ void MetricsRenderFrameObserver::DidCommitProvisionalLoad(
     return;
 
   page_timing_metrics_sender_ = std::make_unique<PageTimingMetricsSender>(
-      CreatePageTimingSender(), CreateTimer(), GetTiming());
+      CreatePageTimingSender(), CreateTimer(), GetTiming(),
+      std::move(provisional_frame_resource_data_use_),
+      std::move(provisional_delta_data_use_));
 }
 
 void MetricsRenderFrameObserver::SendMetrics() {

chrome/renderer/page_load_metrics/metrics_render_frame_observer.h

@@ -9,6 +9,7 @@
 
 #include "base/macros.h"
 #include "chrome/common/page_load_metrics/page_load_timing.h"
+#include "chrome/renderer/page_load_metrics/page_resource_data_use.h"
 #include "content/public/renderer/render_frame_observer.h"
 #include "third_party/blink/public/platform/web_loading_behavior_flag.h"
 
@@ -47,6 +48,9 @@ class MetricsRenderFrameObserver : public content::RenderFrameObserver {
       int request_id,
       const network::URLLoaderCompletionStatus& status) override;
   void DidCancelResponse(int request_id) override;
+  void DidStartProvisionalLoad(
+      blink::WebDocumentLoader* document_loader) override;
+  void DidFailProvisionalLoad(const blink::WebURLError& error) override;
   void DidCommitProvisionalLoad(bool is_new_navigation,
                                 bool is_same_document_navigation) override;
   void OnDestruct() override;
@@ -62,6 +66,15 @@ class MetricsRenderFrameObserver : public content::RenderFrameObserver {
   virtual std::unique_ptr<PageTimingSender> CreatePageTimingSender();
   virtual bool HasNoRenderFrame() const;
 
+  // Collects the data use of the frame request for a provisional load until the
+  // load is committed. We want to collect data use for completed navigations in
+  // this class, but the various navigation callbacks do not provide enough data
+  // for us to use them for data attribution. Instead, we try to get this
+  // information from ongoing resource requests on the previous page (or right
+  // before this page loads in a new renderer).
+  std::unique_ptr<PageResourceDataUse> provisional_frame_resource_data_use_;
+  mojom::PageLoadDataUsePtr provisional_delta_data_use_;
+
   // Will be null when we're not actively sending metrics.
   std::unique_ptr<PageTimingMetricsSender> page_timing_metrics_sender_;
 

chrome/renderer/page_load_metrics/metrics_render_frame_observer_unittest.cc

@@ -81,6 +81,7 @@ TEST_F(MetricsRenderFrameObserverTest, SingleMetric) {
   page_load_metrics::InitPageLoadTimingForTest(&timing);
   timing.navigation_start = nav_start;
   observer.ExpectPageLoadTiming(timing);
+  observer.DidStartProvisionalLoad(nullptr);
   observer.DidCommitProvisionalLoad(true, false);
   observer.GetMockTimer()->Fire();
 
@@ -103,6 +104,7 @@ TEST_F(MetricsRenderFrameObserverTest, MultipleMetrics) {
   page_load_metrics::InitPageLoadTimingForTest(&timing);
   timing.navigation_start = nav_start;
   observer.ExpectPageLoadTiming(timing);
+  observer.DidStartProvisionalLoad(nullptr);
   observer.DidCommitProvisionalLoad(true, false);
   observer.GetMockTimer()->Fire();
 
@@ -149,6 +151,7 @@ TEST_F(MetricsRenderFrameObserverTest, MultipleNavigations) {
   page_load_metrics::InitPageLoadTimingForTest(&timing);
   timing.navigation_start = nav_start;
   observer.ExpectPageLoadTiming(timing);
+  observer.DidStartProvisionalLoad(nullptr);
   observer.DidCommitProvisionalLoad(true, false);
   observer.GetMockTimer()->Fire();
 
@@ -175,6 +178,7 @@ TEST_F(MetricsRenderFrameObserverTest, MultipleNavigations) {
   observer.SetMockTimer(nullptr);
 
   observer.ExpectPageLoadTiming(timing_2);
+  observer.DidStartProvisionalLoad(nullptr);
   observer.DidCommitProvisionalLoad(true, false);
   observer.GetMockTimer()->Fire();
 

chrome/renderer/page_load_metrics/page_resource_data_use.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/renderer/page_load_metrics/page_resource_data_use.h"
+
+#include "base/stl_util.h"
+#include "components/data_reduction_proxy/core/common/data_reduction_proxy_headers.h"
+#include "services/network/public/cpp/resource_response.h"
+#include "services/network/public/cpp/url_loader_completion_status.h"
+
+namespace page_load_metrics {
+
+PageResourceDataUse::PageResourceDataUse()
+    : resource_id_(-1),
+      data_reduction_proxy_compression_ratio_estimate_(1.0),
+      total_received_bytes_(0) {}
+
+PageResourceDataUse::~PageResourceDataUse() = default;
+
+void PageResourceDataUse::DidStartResponse(
+    int resource_id,
+    const network::ResourceResponseHead& response_head) {
+  resource_id_ = resource_id;
+  data_reduction_proxy_compression_ratio_estimate_ =
+      data_reduction_proxy::EstimateCompressionRatioFromHeaders(&response_head);
+  total_received_bytes_ = 0;
+}
+
+void PageResourceDataUse::DidReceiveTransferSizeUpdate(
+    int received_data_length,
+    mojom::PageLoadDataUse* delta_data_use) {
+  delta_data_use->received_data_length += received_data_length;
+  delta_data_use->data_reduction_proxy_bytes_saved +=
+      received_data_length *
+      (data_reduction_proxy_compression_ratio_estimate_ - 1.0);
+  total_received_bytes_ += received_data_length;
+}
+
+bool PageResourceDataUse::DidCompleteResponse(
+    const network::URLLoaderCompletionStatus& status,
+    mojom::PageLoadDataUse* delta_data_use) {
+  // Report the difference in received bytes.
+  int64_t delta_bytes = status.encoded_data_length - total_received_bytes_;
+  if (delta_bytes > 0) {
+    delta_data_use->received_data_length += delta_bytes;
+    delta_data_use->data_reduction_proxy_bytes_saved +=
+        delta_bytes * (data_reduction_proxy_compression_ratio_estimate_ - 1.0);
+    return true;
+  }
+  return false;
+}
+
+}  // namespace page_load_metrics

chrome/renderer/page_load_metrics/page_resource_data_use.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_RENDERER_PAGE_LOAD_METRICS_PAGE_RESOURCE_DATA_USE_H_
+#define CHROME_RENDERER_PAGE_LOAD_METRICS_PAGE_RESOURCE_DATA_USE_H_
+
+#include "base/macros.h"
+#include "chrome/common/page_load_metrics/page_load_metrics.mojom.h"
+
+namespace network {
+struct ResourceResponseHead;
+struct URLLoaderCompletionStatus;
+}  // namespace network
+
+namespace page_load_metrics {
+
+// PageResourceDataUse contains the data use information of one resource. Data
+// use is updated when resource size updates are called.
+class PageResourceDataUse {
+ public:
+  PageResourceDataUse();
+  ~PageResourceDataUse();
+
+  void DidStartResponse(int resource_id,
+                        const network::ResourceResponseHead& response_head);
+
+  // Updates any additional bytes of data use to |delta_data_use|.
+  void DidReceiveTransferSizeUpdate(int received_data_length,
+                                    mojom::PageLoadDataUse* delta_data_use);
+
+  // Updates additional bytes to |delta_data_use| and returns whether it was
+  // updated.
+  bool DidCompleteResponse(const network::URLLoaderCompletionStatus& status,
+                           mojom::PageLoadDataUse* delta_data_use);
+
+  int resource_id() const { return resource_id_; }
+
+ private:
+  int resource_id_;
+
+  // Compression ratio estimated from the response headers if data saver was
+  // used.
+  double data_reduction_proxy_compression_ratio_estimate_;
+
+  uint64_t total_received_bytes_;
+
+  DISALLOW_ASSIGN(PageResourceDataUse);
+};
+
+}  // namespace page_load_metrics
+
+#endif  // CHROME_RENDERER_PAGE_LOAD_METRICS_PAGE_RESOURCE_DATA_USE_H_

chrome/renderer/page_load_metrics/page_timing_metrics_sender.cc

@@ -9,10 +9,13 @@
 #include "base/callback.h"
 #include "base/feature_list.h"
 #include "base/metrics/field_trial_params.h"
+#include "base/stl_util.h"
 #include "base/time/time.h"
 #include "base/timer/timer.h"
 #include "chrome/common/page_load_metrics/page_load_metrics_constants.h"
 #include "chrome/renderer/page_load_metrics/page_timing_sender.h"
+#include "services/network/public/cpp/resource_response.h"
+#include "services/network/public/cpp/url_loader_completion_status.h"
 
 namespace page_load_metrics {
 
@@ -25,13 +28,20 @@ const base::Feature kPageLoadMetricsTimerDelayFeature{
 PageTimingMetricsSender::PageTimingMetricsSender(
     std::unique_ptr<PageTimingSender> sender,
     std::unique_ptr<base::OneShotTimer> timer,
-    mojom::PageLoadTimingPtr initial_timing)
+    mojom::PageLoadTimingPtr initial_timing,
+    std::unique_ptr<PageResourceDataUse> initial_request,
+    mojom::PageLoadDataUsePtr initial_data_use)
     : sender_(std::move(sender)),
       timer_(std::move(timer)),
       last_timing_(std::move(initial_timing)),
       metadata_(mojom::PageLoadMetadata::New()),
       new_features_(mojom::PageLoadFeatures::New()),
+      new_data_use_(std::move(initial_data_use)),
       buffer_timer_delay_ms_(kBufferTimerDelayMillis) {
+  page_resource_data_use_.emplace(
+      std::piecewise_construct,
+      std::forward_as_tuple(initial_request->resource_id()),
+      std::forward_as_tuple(*initial_request));
   buffer_timer_delay_ms_ = base::GetFieldTrialParamByFeatureAsInt(
       kPageLoadMetricsTimerDelayFeature, "BufferTimerDelayMillis",
       kBufferTimerDelayMillis /* default value */);
@@ -80,6 +90,56 @@ void PageTimingMetricsSender::DidObserveNewCssPropertyUsage(int css_property,
   }
 }
 
+void PageTimingMetricsSender::DidStartResponse(
+    int resource_id,
+    const network::ResourceResponseHead& response_head) {
+  DCHECK(!base::ContainsKey(page_resource_data_use_, resource_id));
+
+  auto resource_it = page_resource_data_use_.emplace(
+      std::piecewise_construct, std::forward_as_tuple(resource_id),
+      std::forward_as_tuple());
+  resource_it.first->second.DidStartResponse(resource_id, response_head);
+}
+
+void PageTimingMetricsSender::DidReceiveTransferSizeUpdate(
+    int resource_id,
+    int received_data_length) {
+  // Transfer size updates are called in a throttled manner.
+  const auto& resource_it = page_resource_data_use_.find(resource_id);
+
+  // It is possible that resources are not in the map, if response headers were
+  // not received or for failed/cancelled resources.
+  if (resource_it == page_resource_data_use_.end())
+    return;
+
+  resource_it->second.DidReceiveTransferSizeUpdate(received_data_length,
+                                                   new_data_use_.get());
+  EnsureSendTimer();
+}
+
+void PageTimingMetricsSender::DidCompleteResponse(
+    int resource_id,
+    const network::URLLoaderCompletionStatus& status) {
+  auto resource_it = page_resource_data_use_.find(resource_id);
+
+  // It is possible that resources are not in the map, if response headers were
+  // not received or for failed/cancelled resources. For data reduction proxy
+  // purposes treat these as having no savings.
+  if (resource_it == page_resource_data_use_.end()) {
+    auto new_resource_it = page_resource_data_use_.emplace(
+        std::piecewise_construct, std::forward_as_tuple(resource_id),
+        std::forward_as_tuple());
+    resource_it = new_resource_it.first;
+  }
+
+  if (resource_it->second.DidCompleteResponse(status, new_data_use_.get()))
+    EnsureSendTimer();
+}
+
+void PageTimingMetricsSender::DidCancelResponse(int resource_id) {
+  page_resource_data_use_.erase(resource_id);
+}
+
 void PageTimingMetricsSender::Send(mojom::PageLoadTimingPtr timing) {
   if (last_timing_->Equals(*timing))
     return;
@@ -110,8 +170,10 @@ void PageTimingMetricsSender::EnsureSendTimer() {
 
 void PageTimingMetricsSender::SendNow() {
   have_sent_ipc_ = true;
-  sender_->SendTiming(last_timing_, metadata_, std::move(new_features_));
+  sender_->SendTiming(last_timing_, metadata_, std::move(new_features_),
+                      std::move(new_data_use_));
   new_features_ = mojom::PageLoadFeatures::New();
+  new_data_use_ = mojom::PageLoadDataUse::New();
 }
 
 }  // namespace page_load_metrics

chrome/renderer/page_load_metrics/page_timing_metrics_sender.h

@@ -8,8 +8,10 @@
 #include <bitset>
 #include <memory>
 
+#include "base/containers/small_map.h"
 #include "base/macros.h"
 #include "chrome/common/page_load_metrics/page_load_timing.h"
+#include "chrome/renderer/page_load_metrics/page_resource_data_use.h"
 #include "third_party/blink/public/mojom/use_counter/css_property_id.mojom.h"
 #include "third_party/blink/public/platform/web_feature.mojom-shared.h"
 #include "third_party/blink/public/platform/web_loading_behavior_flag.h"
@@ -18,6 +20,11 @@ namespace base {
 class OneShotTimer;
 }  // namespace base
 
+namespace network {
+struct ResourceResponseHead;
+struct URLLoaderCompletionStatus;
+}  // namespace network
+
 namespace page_load_metrics {
 
 class PageTimingSender;
@@ -29,12 +36,21 @@ class PageTimingMetricsSender {
  public:
   PageTimingMetricsSender(std::unique_ptr<PageTimingSender> sender,
                           std::unique_ptr<base::OneShotTimer> timer,
-                          mojom::PageLoadTimingPtr initial_timing);
+                          mojom::PageLoadTimingPtr initial_timing,
+                          std::unique_ptr<PageResourceDataUse> initial_request,
+                          mojom::PageLoadDataUsePtr initial_data_use);
   ~PageTimingMetricsSender();
 
   void DidObserveLoadingBehavior(blink::WebLoadingBehaviorFlag behavior);
   void DidObserveNewFeatureUsage(blink::mojom::WebFeature feature);
   void DidObserveNewCssPropertyUsage(int css_property, bool is_animated);
+  void DidStartResponse(int resource_id,
+                        const network::ResourceResponseHead& response_head);
+  void DidReceiveTransferSizeUpdate(int resource_id, int received_data_length);
+  void DidCompleteResponse(int resource_id,
+                           const network::URLLoaderCompletionStatus& status);
+  void DidCancelResponse(int resource_id);
+
   void Send(mojom::PageLoadTimingPtr timing);
 
  protected:
@@ -55,6 +71,9 @@ class PageTimingMetricsSender {
   // A list of newly observed features during page load, to be sent to the
   // browser.
   mojom::PageLoadFeaturesPtr new_features_;
+  // Additional data use observed during the page load.
+  mojom::PageLoadDataUsePtr new_data_use_;
+
   std::bitset<static_cast<size_t>(blink::mojom::WebFeature::kNumberOfFeatures)>
       features_sent_;
   std::bitset<static_cast<size_t>(blink::mojom::kMaximumCSSSampleId + 1)>
@@ -64,6 +83,9 @@ class PageTimingMetricsSender {
 
   bool have_sent_ipc_ = false;
 
+  base::small_map<std::map<int, PageResourceDataUse>, 16>
+      page_resource_data_use_;
+
   // Field trial for alternating page timing metrics sender buffer timer delay.
   // https://crbug.com/847269.
   int buffer_timer_delay_ms_;

chrome/renderer/page_load_metrics/page_timing_metrics_sender_unittest.cc

@@ -21,7 +21,9 @@ class TestPageTimingMetricsSender : public PageTimingMetricsSender {
       mojom::PageLoadTimingPtr initial_timing)
       : PageTimingMetricsSender(std::move(page_timing_sender),
                                 std::make_unique<base::MockOneShotTimer>(),
-                                std::move(initial_timing)) {}
+                                std::move(initial_timing),
+                                std::make_unique<PageResourceDataUse>(),
+                                mojom::PageLoadDataUse::New()) {}
 
   base::MockOneShotTimer* mock_timer() const {
     return static_cast<base::MockOneShotTimer*>(timer());

chrome/renderer/page_load_metrics/page_timing_sender.h

@@ -16,7 +16,8 @@ class PageTimingSender {
   virtual ~PageTimingSender() {}
   virtual void SendTiming(const mojom::PageLoadTimingPtr& timing,
                           const mojom::PageLoadMetadataPtr& metadata,
-                          mojom::PageLoadFeaturesPtr new_features) = 0;
+                          mojom::PageLoadFeaturesPtr new_features,
+                          mojom::PageLoadDataUsePtr new_data_use) = 0;
 };
 
 }  // namespace page_load_metrics

components/data_reduction_proxy/DEPS

@@ -7,6 +7,7 @@ include_rules = [
   "+crypto",
   "+google_apis",
   "+net",
+  "+services/network/public/cpp",
 
   # Data Reduction Proxy is a layered component; subdirectories must explicitly
   # introduce the ability to use the content layer as appropriate.

components/data_reduction_proxy/core/common/BUILD.gn

@@ -45,6 +45,7 @@ template("common_tmpl") {
       "//components/data_reduction_proxy/proto:data_reduction_proxy_proto",
       "//components/variations",
       "//google_apis",
+      "//services/network/public/cpp",
     ]
 
     defines = [ "USE_GOOGLE_API_KEYS" ]

components/data_reduction_proxy/core/common/data_reduction_proxy_headers.cc

@@ -24,6 +24,7 @@
 #include "net/http/http_status_code.h"
 #include "net/http/http_util.h"
 #include "net/url_request/url_request.h"
+#include "services/network/public/cpp/resource_response.h"
 
 namespace {
 
@@ -484,4 +485,20 @@ int64_t GetDataReductionProxyOFCL(const net::HttpResponseHeaders* headers) {
   return -1;
 }
 
+double EstimateCompressionRatioFromHeaders(
+    const network::ResourceResponseHead* response_head) {
+  if (!response_head->network_accessed || !response_head->headers ||
+      response_head->headers->GetContentLength() <= 0) {
+    return 1.0;  // No compression
+  }
+
+  int64_t original_content_length =
+      GetDataReductionProxyOFCL(response_head->headers.get());
+  if (original_content_length > 0) {
+    return static_cast<double>(original_content_length) /
+           static_cast<double>(response_head->headers->GetContentLength());
+  }
+  return 1.0;  // No compression
+}
+
 }  // namespace data_reduction_proxy

components/data_reduction_proxy/core/common/data_reduction_proxy_headers.h

@@ -21,6 +21,10 @@ namespace net {
 class HttpResponseHeaders;
 }  // namespace net
 
+namespace network {
+struct ResourceResponseHead;
+}  // namespace network
+
 namespace data_reduction_proxy {
 
 // Transform directives that may be parsed out of http headers.
@@ -176,9 +180,19 @@ bool ParseHeadersAndSetBypassDuration(const net::HttpResponseHeaders* headers,
                                       base::StringPiece action_prefix,
                                       base::TimeDelta* bypass_duration);
 
-// Returns the OFCL value in the Chrome-Proxy header. Returns -1 in case of
-// of error or if OFCL does not exist. |headers| must be non-null.
+// Returns the Original-Full-Content-Length(OFCL) value in the Chrome-Proxy
+// header. Returns -1 in case of of error or if OFCL does not exist. |headers|
+// must be non-null.
 int64_t GetDataReductionProxyOFCL(const net::HttpResponseHeaders* headers);
 
+// Returns an estimate of the compression ratio from the Content-Length and
+// Chrome-Proxy Original-Full-Content-Length(OFCL) response headers. These may
+// not be populated for responses which are streamed from the origin which will
+// be treated as a no compression case. Notably, only the response body size is
+// used to compute the ratio, and headers are excluded, since this is only an
+// estimate for response that is beginning to arrive.
+double EstimateCompressionRatioFromHeaders(
+    const network::ResourceResponseHead* response_head);
+
 }  // namespace data_reduction_proxy
 #endif  // COMPONENTS_DATA_REDUCTION_PROXY_CORE_COMMON_DATA_REDUCTION_PROXY_HEADERS_H_