Report ad resource sizes by mime type

Record the size of ad resources according to MIME type. This will help us
understand which resources are using up more network bytes and allow
us to better estimate the effect any ad intervention will have. Only
HTML/JS/CSS/Image/Video are recorded as they are expected to capture most
of the resources commonly seen in ads. It is expected that these metrics
will not be completely accurate because requests do not always contain
the correct content type. It is assumed that the MIME type in the response
header is a suitable proxy resource type.

Bug: 881049
Change-Id: I000466fb3fe5467c4f73527cb4cedbd7709d7c3f
Reviewed-on: https://chromium-review.googlesource.com/1219900
Commit-Queue: John Delaney <johnidel@chromium.org>
Reviewed-by: Josh Karlin <jkarlin@chromium.org>
Reviewed-by: Kinuko Yasuda <kinuko@chromium.org>
Reviewed-by: Ilya Sherman <isherman@chromium.org>
Cr-Commit-Position: refs/heads/master@{#593087}

chrome/browser/page_load_metrics/observers/ads_page_load_metrics_observer.cc

@@ -15,6 +15,8 @@
 #include "content/public/browser/navigation_handle.h"
 #include "content/public/browser/render_frame_host.h"
 #include "content/public/browser/web_contents.h"
+#include "net/base/mime_util.h"
+#include "third_party/blink/public/common/mime_util/mime_util.h"
 #include "url/gurl.h"
 
 namespace {
@@ -370,7 +372,7 @@ void AdsPageLoadMetricsObserver::UpdateResource(
 
   // Update resource map.
   if (resource->is_complete) {
-    RecordResourceHistogram(resource);
+    RecordResourceHistograms(resource);
     if (it != page_resources_.end())
       page_resources_.erase(it);
   } else {
@@ -386,7 +388,45 @@ void AdsPageLoadMetricsObserver::UpdateResource(
   }
 }
 
-void AdsPageLoadMetricsObserver::RecordResourceHistogram(
+void AdsPageLoadMetricsObserver::RecordResourceMimeHistograms(
+    const page_load_metrics::mojom::ResourceDataUpdatePtr& resource) {
+  if (blink::IsSupportedImageMimeType(resource->mime_type)) {
+    PAGE_BYTES_HISTOGRAM("Ads.ResourceUsage.Size.Mime.Image",
+                         resource->received_data_length);
+  } else if (blink::IsSupportedJavascriptMimeType(resource->mime_type)) {
+    PAGE_BYTES_HISTOGRAM("Ads.ResourceUsage.Size.Mime.JS",
+                         resource->received_data_length);
+  } else {
+    std::string top_level_type;
+    std::string subtype;
+
+    if (!net::ParseMimeTypeWithoutParameter(resource->mime_type,
+                                            &top_level_type, &subtype)) {
+      // Log invalid mime types as "Other".
+      PAGE_BYTES_HISTOGRAM("Ads.ResourceUsage.Size.Mime.Other",
+                           resource->received_data_length);
+      return;
+    }
+
+    if (top_level_type.compare("video") == 0) {
+      PAGE_BYTES_HISTOGRAM("Ads.ResourceUsage.Size.Mime.Video",
+                           resource->received_data_length);
+    } else if (top_level_type.compare("text") == 0 &&
+               subtype.compare("css") == 0) {
+      PAGE_BYTES_HISTOGRAM("Ads.ResourceUsage.Size.Mime.CSS",
+                           resource->received_data_length);
+    } else if (top_level_type.compare("text") == 0 &&
+               subtype.compare("html") == 0) {
+      PAGE_BYTES_HISTOGRAM("Ads.ResourceUsage.Size.Mime.HTML",
+                           resource->received_data_length);
+    } else {
+      PAGE_BYTES_HISTOGRAM("Ads.ResourceUsage.Size.Mime.Other",
+                           resource->received_data_length);
+    }
+  }
+}
+
+void AdsPageLoadMetricsObserver::RecordResourceHistograms(
     const page_load_metrics::mojom::ResourceDataUpdatePtr& resource) {
   if (resource->is_main_frame_resource && resource->reported_as_ad_resource) {
     PAGE_BYTES_HISTOGRAM("Ads.ResourceUsage.Size.Mainframe.AdResource",
@@ -401,6 +441,10 @@ void AdsPageLoadMetricsObserver::RecordResourceHistogram(
     PAGE_BYTES_HISTOGRAM("Ads.ResourceUsage.Size.Subframe.VanillaResource",
                          resource->received_data_length);
   }
+
+  // Only report sizes by mime type for ad resources.
+  if (resource->reported_as_ad_resource)
+    RecordResourceMimeHistograms(resource);
 }
 
 void AdsPageLoadMetricsObserver::RecordPageResourceTotalHistograms() {
@@ -426,7 +470,7 @@ void AdsPageLoadMetricsObserver::RecordHistograms() {
   RecordHistogramsForType(AD_TYPE_ALL);
   RecordPageResourceTotalHistograms();
   for (auto const& kv : page_resources_)
-    RecordResourceHistogram(kv.second);
+    RecordResourceHistograms(kv.second);
 }
 
 void AdsPageLoadMetricsObserver::RecordHistogramsForType(int ad_type) {

chrome/browser/page_load_metrics/observers/ads_page_load_metrics_observer.h

@@ -117,7 +117,12 @@ class AdsPageLoadMetricsObserver
   void UpdateResource(
       const page_load_metrics::mojom::ResourceDataUpdatePtr& resource);
 
-  void RecordResourceHistogram(
+  // Records size of resources by mime type.
+  void RecordResourceMimeHistograms(
+      const page_load_metrics::mojom::ResourceDataUpdatePtr& resource);
+
+  // Records per-resource histograms.
+  void RecordResourceHistograms(
       const page_load_metrics::mojom::ResourceDataUpdatePtr& resource);
   void RecordPageResourceTotalHistograms();
   void RecordHistograms();

chrome/browser/page_load_metrics/observers/ads_page_load_metrics_observer_browsertest.cc

@@ -371,3 +371,38 @@ IN_PROC_BROWSER_TEST_F(AdsPageLoadMetricsObserverResourceBrowserTest,
   histogram_tester.ExpectBucketCount(
       "PageLoad.Clients.Ads.Resources.Bytes.Unfinished", 1, 1);
 }
+
+// Verify that Mime type metrics are recorded correctly.
+IN_PROC_BROWSER_TEST_F(AdsPageLoadMetricsObserverResourceBrowserTest,
+                       RecordedMimeMetrics) {
+  base::HistogramTester histogram_tester;
+  embedded_test_server()->ServeFilesFromSourceDirectory(
+      "chrome/test/data/ad_tagging");
+  content::SetupCrossSiteRedirector(embedded_test_server());
+  ASSERT_TRUE(embedded_test_server()->Start());
+
+  auto waiter = CreateAdsPageLoadMetricsTestWaiter();
+
+  content::WebContents* contents =
+      browser()->tab_strip_model()->GetActiveWebContents();
+  ui_test_utils::NavigateToURL(
+      browser(),
+      embedded_test_server()->GetURL("foo.com", "/frame_factory.html"));
+  contents->GetMainFrame()->ExecuteJavaScriptForTests(
+      base::ASCIIToUTF16("createFrame('multiple_mimes.html', 'test');"));
+  waiter->AddMinimumAdResourceExpectation(8);
+  waiter->Wait();
+
+  // Close all tabs to log metrics, as the video resource request is incomplete.
+  browser()->tab_strip_model()->CloseAllTabs();
+
+  histogram_tester.ExpectTotalCount("Ads.ResourceUsage.Size.Mime.HTML", 1);
+  histogram_tester.ExpectTotalCount("Ads.ResourceUsage.Size.Mime.CSS", 1);
+  histogram_tester.ExpectTotalCount("Ads.ResourceUsage.Size.Mime.JS", 3);
+
+  // Note: png and video/webm mime types are not set explicitly by the
+  // embedded_test_server.
+  histogram_tester.ExpectTotalCount("Ads.ResourceUsage.Size.Mime.Image", 1);
+  histogram_tester.ExpectTotalCount("Ads.ResourceUsage.Size.Mime.Video", 1);
+  histogram_tester.ExpectTotalCount("Ads.ResourceUsage.Size.Mime.Other", 1);
+}

chrome/common/page_load_metrics/page_load_metrics.mojom

@@ -182,6 +182,9 @@ struct ResourceDataUpdate {
 
   // Whether this resource was loaded in the top-level frame.
   bool is_main_frame_resource;
+
+  // Mime type for the resource found in the network response header.
+  string mime_type;
 };
 
 // Sent from renderer to browser process when the PageLoadTiming for the

chrome/renderer/page_load_metrics/page_resource_data_use.cc

@@ -21,6 +21,8 @@ PageResourceDataUse::PageResourceDataUse()
       reported_as_ad_resource_(false),
       is_main_frame_resource_(false) {}
 
+PageResourceDataUse::PageResourceDataUse(const PageResourceDataUse& other) =
+    default;
 PageResourceDataUse::~PageResourceDataUse() = default;
 
 void PageResourceDataUse::DidStartResponse(
@@ -29,6 +31,7 @@ void PageResourceDataUse::DidStartResponse(
   resource_id_ = resource_id;
   data_reduction_proxy_compression_ratio_estimate_ =
       data_reduction_proxy::EstimateCompressionRatioFromHeaders(&response_head);
+  mime_type_ = response_head.mime_type;
   total_received_bytes_ = 0;
   last_update_bytes_ = 0;
 }
@@ -85,6 +88,7 @@ mojom::ResourceDataUpdatePtr PageResourceDataUse::GetResourceDataUpdate() {
       data_reduction_proxy_compression_ratio_estimate_;
   resource_data_update->reported_as_ad_resource = reported_as_ad_resource_;
   resource_data_update->is_main_frame_resource = is_main_frame_resource_;
+  resource_data_update->mime_type = mime_type_;
   return resource_data_update;
 }
 }  // namespace page_load_metrics

chrome/renderer/page_load_metrics/page_resource_data_use.h

@@ -20,6 +20,7 @@ namespace page_load_metrics {
 class PageResourceDataUse {
  public:
   PageResourceDataUse();
+  PageResourceDataUse(const PageResourceDataUse& other);
   ~PageResourceDataUse();
 
   void DidStartResponse(int resource_id,
@@ -67,6 +68,8 @@ class PageResourceDataUse {
   bool reported_as_ad_resource_;
   bool is_main_frame_resource_;
 
+  std::string mime_type_;
+
   DISALLOW_ASSIGN(PageResourceDataUse);
 };
 

chrome/test/data/ad_tagging/multiple_mimes.html

+<html>
+  <script src="ad_script.js"></script>
+  <link rel="stylesheet" type="text/css" href="style.css">
+  <img src="pixel.png"></img>
+  <video src="bear-320x240-video-only.webm" preload="auto"></video>
+  <iframe src="test.pdf"></iframe>
+</html>

chrome/test/data/ad_tagging/style.css

+/*
+ * Copyright 2014 The Chromium Authors. All rights reserved.
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+/* Fake style sheet */

net/test/embedded_test_server/request_handler_util.cc

@@ -32,6 +32,8 @@ const char kMockHttpHeadersExtension[] = "mock-http-headers";
 std::string GetContentType(const base::FilePath& path) {
   if (path.MatchesExtension(FILE_PATH_LITERAL(".crx")))
     return "application/x-chrome-extension";
+  if (path.MatchesExtension(FILE_PATH_LITERAL(".css")))
+    return "text/css";
   if (path.MatchesExtension(FILE_PATH_LITERAL(".exe")))
     return "application/octet-stream";
   if (path.MatchesExtension(FILE_PATH_LITERAL(".gif")))

tools/metrics/histograms/histograms.xml

@@ -673,6 +673,18 @@ uploading your change for review.
   </summary>
 </histogram>
 
+<histogram base="true" name="Ads.ResourceUsage.Size.Mime" units="KB"
+    expires_after="2019-09-10">
+  <owner>johnidel@chromium.org</owner>
+  <owner>jkarlin@chromium.org</owner>
+  <summary>
+    Size of resources tagged as ads, identified by the response header mime
+    type. This includes resources that did not complete loading. Recorded when
+    the resource request is completed. For incomplete requests, recorded when
+    the page is destroyed/navigated.
+  </summary>
+</histogram>
+
 <histogram name="Ads.ResourceUsage.Size.Subframe.AdResource" units="KB"
     expires_after="2019-09-05">
   <owner>jkarlin@chromium.org</owner>
@@ -120049,6 +120061,20 @@ uploading your change for review.
   <affected-histogram name="Startup.Android.Cold.TimeToFirstNavigationCommit"/>
 </histogram_suffixes>
 
+<histogram_suffixes name="AdResourceMimeTypes" separator=".">
+  <suffix name="CSS" label="Resources identified by text/css."/>
+  <suffix name="HTML" label="Resources identified by text/html."/>
+  <suffix name="Image"
+      label="Resources identified by any supported image mime type."/>
+  <suffix name="JS"
+      label="Resources identified by any supported javascript mime type."/>
+  <suffix name="Other"
+      label="Resources identified by any mime type that did match any
+             supported css, html, image, javascript, or video mime types."/>
+  <suffix name="Video" label="Resources identified by video/*."/>
+  <affected-histogram name="Ads.ResourceUsage.Size.Mime"/>
+</histogram_suffixes>
+
 <histogram_suffixes name="AdsPageLoadMetrics" separator="." ordering="prefix">
   <suffix name="Clients.Ads.All" label="Includes all discovered ads."/>
   <suffix name="Clients.Ads.Google" label="Includes only Google ads."/>