Send local network request information to UMA and UKM

We are implementing new UMA and UKM metric collection for local network requests by public and private pages.

This CL creates a new subclass of page_load_metrics::PageLoadMetricsObserver, LocalNetworkRequestsMetricsPageLoadObserver, to monitor pages for local network requests and report them to UMA and UKM. It also includes unit tests for the new metric collection classes.

Additionally, this CL adds the ability to specify the socket address for a navigation to content::NavigationSimulator, which is needed for LocalNetworkRequestsPageLoadMetricsObserver's unit tests, which require manipulation of the address of committed page loads to test the observer's functionality. (Note that this change is included in this CL because content/ policy requires that changes to the public API must have a caller outside content/.)

Concretely, this CL does the following:
(1) Creates the LocalNetworkRequestsPageLoadMetricsObserver class, which implements local network request UKM and UMA metric collection.
(2) Registers the LocalNetworkRequestsPageLoadMetricsObserver in PageLoadMetricsEmbedder to enable metric and histogram collection.
(3) Adds unit tests for LocalNetworkRequestsPageLoadMetricsObserver.
(4) Updates the Chrome browser and unit test BUILD files to build the LocalNetworkRequestsPageLoadMetricsObserver and its unit tests.
(5) Registers the new local network requests UMA histograms and UKM metrics in histograms.xml and ukm.xml, respectively.
(6) Adds the |SetSocketAddress| method to NavigationSimulator to support specification of the socket address of a navigation.

BUG=735085,728707

Change-Id: Ie176881a326ca7a856c1813bfc2d031c5ae4261b
Reviewed-on: https://chromium-review.googlesource.com/532254
Commit-Queue: U Thakore <uthakore@chromium.org>
Reviewed-by: Robert Kaplow <rkaplow@chromium.org>
Reviewed-by: Bryan McQuade <bmcquade@chromium.org>
Reviewed-by: Camille Lamy <clamy@chromium.org>
Cr-Commit-Position: refs/heads/master@{#485037}

chrome/browser/BUILD.gn

@@ -847,6 +847,8 @@ split_static_library("browser") {
     "page_load_metrics/observers/https_engagement_metrics/https_engagement_service_factory.h",
     "page_load_metrics/observers/loading_predictor_page_load_metrics_observer.cc",
     "page_load_metrics/observers/loading_predictor_page_load_metrics_observer.h",
+    "page_load_metrics/observers/local_network_requests_page_load_metrics_observer.cc",
+    "page_load_metrics/observers/local_network_requests_page_load_metrics_observer.h",
     "page_load_metrics/observers/lofi_page_load_metrics_observer.cc",
     "page_load_metrics/observers/lofi_page_load_metrics_observer.h",
     "page_load_metrics/observers/media_page_load_metrics_observer.cc",

chrome/browser/page_load_metrics/observers/local_network_requests_page_load_metrics_observer.h

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_PAGE_LOAD_METRICS_OBSERVERS_LOCAL_NETWORK_REQUESTS_PAGE_LOAD_METRICS_OBSERVER_H_
+#define CHROME_BROWSER_PAGE_LOAD_METRICS_OBSERVERS_LOCAL_NETWORK_REQUESTS_PAGE_LOAD_METRICS_OBSERVER_H_
+
+#include <map>
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "chrome/browser/page_load_metrics/page_load_metrics_observer.h"
+#include "net/base/ip_address.h"
+
+namespace internal {
+
+// The domain type of the IP address of the loaded page. We use these to
+// determine what classes of resource request metrics to collect.
+enum DomainType {
+  DOMAIN_TYPE_UNKNOWN = 0,
+  DOMAIN_TYPE_PUBLIC = 1,
+  DOMAIN_TYPE_PRIVATE = 2,
+  DOMAIN_TYPE_LOCALHOST = 4,
+};
+
+// The type of the IP address of the loaded resource.
+enum ResourceType {
+  RESOURCE_TYPE_PUBLIC = 0,
+  RESOURCE_TYPE_PRIVATE = 1,
+  RESOURCE_TYPE_LOCAL_SAME_SUBNET = 2,
+  RESOURCE_TYPE_LOCAL_DIFF_SUBNET = 4,
+  RESOURCE_TYPE_ROUTER = 8,
+  RESOURCE_TYPE_LOCALHOST = 16,
+};
+
+// The types of services to distinguish between when collecting local network
+// request metrics.
+enum PortType {
+  PORT_TYPE_WEB = 1,
+  PORT_TYPE_DB = 2,
+  PORT_TYPE_PRINT = 4,
+  PORT_TYPE_DEV = 8,
+  PORT_TYPE_OTHER = 0,
+};
+
+// Exposed for tests.
+extern const char kUkmPageDomainEventName[];
+extern const char kUkmLocalNetworkRequestsEventName[];
+
+extern const char kUkmDomainTypeName[];
+extern const char kUkmResourceTypeName[];
+extern const char kUkmPortTypeName[];
+extern const char kUkmSuccessfulCountName[];
+extern const char kUkmFailedCountName[];
+
+// For simple access during UMA histogram logging, the names are in a
+// multidimensional map indexed by [DomainType][ResourceType][Status].
+const std::map<DomainType, std::map<ResourceType, std::map<bool, std::string>>>&
+GetNonlocalhostHistogramNames();
+// For localhost histogram names, the map is indexed by
+// [DomainType][PortType][Status].
+const std::map<DomainType, std::map<PortType, std::map<bool, std::string>>>&
+GetLocalhostHistogramNames();
+
+}  // namespace internal
+
+// This observer is for observing local network requests.
+// TODO(uthakore): Add description.
+class LocalNetworkRequestsPageLoadMetricsObserver
+    : public page_load_metrics::PageLoadMetricsObserver {
+  using SuccessFailCounts = std::pair<uint32_t, uint32_t>;
+
+ public:
+  LocalNetworkRequestsPageLoadMetricsObserver();
+  ~LocalNetworkRequestsPageLoadMetricsObserver() override;
+
+  // page_load_metrics::PageLoadMetricsObserver
+  ObservePolicy OnCommit(content::NavigationHandle* navigation_handle,
+                         ukm::SourceId source_id) override;
+  ObservePolicy FlushMetricsOnAppEnterBackground(
+      const page_load_metrics::mojom::PageLoadTiming& timing,
+      const page_load_metrics::PageLoadExtraInfo& extra_info) override;
+  void OnLoadedResource(const page_load_metrics::ExtraRequestCompleteInfo&
+                            extra_request_info) override;
+  void OnComplete(const page_load_metrics::mojom::PageLoadTiming& timing,
+                  const page_load_metrics::PageLoadExtraInfo& info) override;
+
+ private:
+  // Clears all local resource request counts. Only used if we decide to log
+  // metrics but the observer may stay in scope and capture additional resource
+  // requests.
+  void ClearLocalState();
+  // Determines the resource type for the |ip_address| based on the page load
+  // type.
+  internal::ResourceType DetermineResourceType(net::IPAddress ip_address);
+  // Determines the port type for the localhost |port|.
+  internal::PortType DeterminePortType(int port);
+  // Resolves the resource types to report for all IP addresses in
+  // |resource_request_counts_|.
+  void ResolveResourceTypes();
+
+  void RecordUkmDomainType(ukm::SourceId source_id);
+  void RecordHistograms();
+  void RecordUkmMetrics(ukm::SourceId source_id);
+
+  // Stores the counts of resource requests for each non-localhost IP address as
+  // pairs of (successful, failed) request counts.
+  std::map<net::IPAddress, SuccessFailCounts> resource_request_counts_;
+
+  std::unique_ptr<std::map<net::IPAddress, internal::ResourceType>>
+      requested_resource_types_;
+
+  // Stores the counts of resource requests for each localhost port as
+  // pairs of (successful, failed) request counts.
+  std::map<int, SuccessFailCounts> localhost_request_counts_;
+
+  // The page load type. This is used to determine what resource requests to
+  // monitor while the page is committed and to determine the UMA histogram name
+  // to use.
+  internal::DomainType page_domain_type_ = internal::DOMAIN_TYPE_UNKNOWN;
+
+  // The IP address of the page that was loaded.
+  net::IPAddress page_ip_address_;
+
+  // For private page loads, the IP prefix defining the largest reserved subnet
+  // the page could belong to. Used to distinguish between same subnet and
+  // different subnet private network queries.
+  size_t page_ip_prefix_length_ = 0;
+
+  DISALLOW_COPY_AND_ASSIGN(LocalNetworkRequestsPageLoadMetricsObserver);
+};
+
+#endif  // CHROME_BROWSER_PAGE_LOAD_METRICS_OBSERVERS_LOCAL_NETWORK_REQUESTS_PAGE_LOAD_METRICS_OBSERVER_H_

chrome/browser/page_load_metrics/observers/page_load_metrics_observer_test_harness.cc

@@ -116,7 +116,7 @@ void PageLoadMetricsObserverTestHarness::SimulateLoadedResource(
           ? info.data_reduction_proxy_data->DeepCopy()
           : nullptr,
       info.raw_body_bytes, info.original_network_content_length,
-      base::TimeTicks::Now(), 0);
+      base::TimeTicks::Now(), info.net_error);
 }
 
 void PageLoadMetricsObserverTestHarness::SimulateInputEvent(

chrome/browser/page_load_metrics/page_load_metrics_initialize.cc

@@ -26,6 +26,7 @@
 #include "chrome/browser/page_load_metrics/observers/google_captcha_observer.h"
 #include "chrome/browser/page_load_metrics/observers/https_engagement_metrics/https_engagement_page_load_metrics_observer.h"
 #include "chrome/browser/page_load_metrics/observers/loading_predictor_page_load_metrics_observer.h"
+#include "chrome/browser/page_load_metrics/observers/local_network_requests_page_load_metrics_observer.h"
 #include "chrome/browser/page_load_metrics/observers/lofi_page_load_metrics_observer.h"
 #include "chrome/browser/page_load_metrics/observers/media_page_load_metrics_observer.h"
 #include "chrome/browser/page_load_metrics/observers/multi_tab_loading_page_load_metrics_observer.h"
@@ -132,6 +133,8 @@ void PageLoadMetricsEmbedder::RegisterObservers(
                 web_contents_);
     if (loading_predictor_observer)
       tracker->AddObserver(std::move(loading_predictor_observer));
+    tracker->AddObserver(
+        base::MakeUnique<LocalNetworkRequestsPageLoadMetricsObserver>());
   } else {
     std::unique_ptr<page_load_metrics::PageLoadMetricsObserver>
         prerender_observer =

chrome/browser/page_load_metrics/page_load_metrics_observer.h

@@ -255,7 +255,7 @@ struct ExtraRequestCompleteInfo {
   // The network error encountered by the request, as defined by
   // net/base/net_error_list.h. If no error was encountered, this value will be
   // 0.
-  int net_error;
+  const int net_error;
 };
 
 // Interface for PageLoadMetrics observers. All instances of this class are

chrome/test/BUILD.gn

@@ -3190,6 +3190,7 @@ test("unit_tests") {
     "../browser/page_load_metrics/observers/from_gws_page_load_metrics_observer_unittest.cc",
     "../browser/page_load_metrics/observers/google_captcha_observer_unittest.cc",
     "../browser/page_load_metrics/observers/loading_predictor_page_load_metrics_observer_unittest.cc",
+    "../browser/page_load_metrics/observers/local_network_requests_page_load_metrics_observer_unittest.cc",
     "../browser/page_load_metrics/observers/lofi_page_load_metrics_observer_unittest.cc",
     "../browser/page_load_metrics/observers/media_page_load_metrics_observer_unittest.cc",
     "../browser/page_load_metrics/observers/multi_tab_loading_page_load_metrics_observer_unittest.cc",

content/public/test/navigation_simulator.cc

@@ -16,6 +16,7 @@
 #include "content/public/common/resource_request_body.h"
 #include "content/test/test_navigation_url_loader.h"
 #include "content/test/test_render_frame_host.h"
+#include "net/base/host_port_pair.h"
 #include "net/base/load_flags.h"
 #include "net/url_request/redirect_info.h"
 
@@ -98,6 +99,7 @@ NavigationSimulator::NavigationSimulator(const GURL& original_url,
       render_frame_host_(render_frame_host),
       handle_(nullptr),
       navigation_url_(original_url),
+      socket_address_("2001:db8::1", 80),
       weak_factory_(this) {
   if (render_frame_host->GetParent()) {
     if (!render_frame_host->frame_tree_node()->has_committed_real_load())
@@ -318,8 +320,7 @@ void NavigationSimulator::Commit() {
   params.contents_mime_type = "text/html";
   params.method = "GET";
   params.http_status_code = 200;
-  params.socket_address.set_host("2001:db8::1");
-  params.socket_address.set_port(80);
+  params.socket_address = socket_address_;
   params.history_list_was_cleared = false;
   params.original_request_url = navigation_url_;
   params.was_within_same_document = false;
@@ -450,8 +451,7 @@ void NavigationSimulator::CommitSameDocument() {
   params.contents_mime_type = "text/html";
   params.method = "GET";
   params.http_status_code = 200;
-  params.socket_address.set_host("2001:db8::1");
-  params.socket_address.set_port(80);
+  params.socket_address = socket_address_;
   params.history_list_was_cleared = false;
   params.original_request_url = navigation_url_;
   params.was_within_same_document = true;
@@ -484,6 +484,13 @@ void NavigationSimulator::SetReferrer(const Referrer& referrer) {
   referrer_ = referrer;
 }
 
+void NavigationSimulator::SetSocketAddress(
+    const net::HostPortPair& socket_address) {
+  CHECK_LE(state_, STARTED) << "The socket address cannot be set after the "
+                               "navigation has committed or failed";
+  socket_address_ = socket_address;
+}
+
 NavigationThrottle::ThrottleCheckResult
 NavigationSimulator::GetLastThrottleCheckResult() {
   return last_throttle_check_result_.value();

content/public/test/navigation_simulator.h

@@ -14,6 +14,7 @@
 #include "content/public/browser/web_contents_observer.h"
 #include "content/public/common/referrer.h"
 #include "content/public/test/navigation_simulator.h"
+#include "net/base/host_port_pair.h"
 #include "ui/base/page_transition_types.h"
 
 class GURL;
@@ -129,9 +130,7 @@ class NavigationSimulator : public WebContentsObserver {
   // --------------------------------------------------------------------------
 
   // The following functions are used to specify the parameters of the
-  // navigation. Changes should be  made before calling |Start|, unless they are
-  // meant to apply to a redirect. In that case, they should be made before
-  // calling |Redirect|.
+  // navigation.
 
   // The following parameters are constant during the navigation and may only be
   // specified before calling |Start|.
@@ -143,6 +142,12 @@ class NavigationSimulator : public WebContentsObserver {
   // |Redirect|.
   virtual void SetReferrer(const Referrer& referrer);
 
+  // The following parameters can change at any point until the page fails or
+  // commits. They should be specified before calling |Fail| or |Commit|.
+  virtual void SetSocketAddress(const net::HostPortPair& socket_address);
+
+  // --------------------------------------------------------------------------
+
   // Gets the last throttle check result computed by the navigation throttles.
   // It is an error to call this before Start() is called.
   virtual NavigationThrottle::ThrottleCheckResult GetLastThrottleCheckResult();
@@ -201,6 +206,7 @@ class NavigationSimulator : public WebContentsObserver {
   NavigationHandleImpl* handle_;
 
   GURL navigation_url_;
+  net::HostPortPair socket_address_;
   Referrer referrer_;
   ui::PageTransition transition_ = ui::PAGE_TRANSITION_LINK;
 

services/metrics/public/cpp/ukm_recorder.h

@@ -21,6 +21,7 @@
 class ContextualSearchRankerLoggerImpl;
 class PluginInfoMessageFilter;
 class UkmPageLoadMetricsObserver;
+class LocalNetworkRequestsPageLoadMetricsObserver;
 
 namespace autofill {
 class AutofillMetrics;
@@ -90,6 +91,7 @@ class METRICS_EXPORT UkmRecorder {
   friend ContextualSearchRankerLoggerImpl;
   friend PluginInfoMessageFilter;
   friend UkmPageLoadMetricsObserver;
+  friend LocalNetworkRequestsPageLoadMetricsObserver;
   friend translate::TranslateRankerImpl;
   friend TestRecordingHelper;
   friend UkmInterface;

tools/metrics/histograms/histograms.xml

@@ -27796,6 +27796,21 @@ http://cs/file:chrome/histograms.xml - but prefer this file for new entries.
   <summary>Records events related to local discovery notifications.</summary>
 </histogram>
 
+<histogram name="LocalNetworkRequests.PrivatePage" units="requests">
+  <owner>uthakore@chromium.org</owner>
+  <summary>
+    Number of requests made by a private page to resources besides itself.
+  </summary>
+</histogram>
+
+<histogram name="LocalNetworkRequests.PublicPage" units="requests">
+  <owner>uthakore@chromium.org</owner>
+  <summary>
+    Number of requests made by a public page to resources with reserved IP
+    addresses.
+  </summary>
+</histogram>
+
 <histogram name="LocalStorage.BrowserLocalStorageCachePurgedInKB" units="KB">
   <owner>ssid@chromium.org</owner>
   <summary>
@@ -91398,6 +91413,71 @@ http://cs/file:chrome/histograms.xml - but prefer this file for new entries.
   <affected-histogram name="LevelDB.Open"/>
 </histogram_suffixes>
 
+<histogram_suffixes name="LocalNetReqsLocalhostResources" separator=".">
+  <suffix name="Localhost.DbRequests"
+      label="Requests made to localhost on a database server port."/>
+  <suffix name="Localhost.DevRequests"
+      label="Requests made to localhost on a development server port."/>
+  <suffix name="Localhost.OtherRequests"
+      label="Requests made to localhost on any port not otherwise monitored
+             by the other local network request metrics."/>
+  <suffix name="Localhost.PrinterRequests"
+      label="Requests made to localhost on a printer server port."/>
+  <suffix name="Localhost.WebRequests"
+      label="Requests made to localhost on a web server port."/>
+  <affected-histogram name="LocalNetworkRequests.PrivatePage"/>
+  <affected-histogram name="LocalNetworkRequests.PublicPage"/>
+</histogram_suffixes>
+
+<histogram_suffixes name="LocalNetReqsPrivatePage" separator=".">
+  <suffix name="DifferentSubnetRequests"
+      label="Requests made to local resources on a different subnet."/>
+  <suffix name="PublicRequests" label="Requests made to public resources."/>
+  <suffix name="SameSubnetRequests"
+      label="Requests made to local resources on the same reserved IP space
+             as the page."/>
+  <affected-histogram name="LocalNetworkRequests.PrivatePage"/>
+</histogram_suffixes>
+
+<histogram_suffixes name="LocalNetReqsPublicPage" separator=".">
+  <suffix name="PrivateRequests" label="Requests made to private resources."/>
+  <suffix name="RouterRequests"
+      label="Requests made to resources likely to be routers."/>
+  <affected-histogram name="LocalNetworkRequests.PublicPage"/>
+</histogram_suffixes>
+
+<histogram_suffixes name="LocalNetReqsStatuses" separator=".">
+  <suffix name="Failed" label="Failed requests."/>
+  <suffix name="Successful" label="Successful requests."/>
+  <affected-histogram
+      name="LocalNetworkRequests.PrivatePage.DifferentSubnetRequests"/>
+  <affected-histogram
+      name="LocalNetworkRequests.PrivatePage.Localhost.DbRequests"/>
+  <affected-histogram
+      name="LocalNetworkRequests.PrivatePage.Localhost.DevRequests"/>
+  <affected-histogram
+      name="LocalNetworkRequests.PrivatePage.Localhost.OtherRequests"/>
+  <affected-histogram
+      name="LocalNetworkRequests.PrivatePage.Localhost.PrinterRequests"/>
+  <affected-histogram
+      name="LocalNetworkRequests.PrivatePage.Localhost.WebRequests"/>
+  <affected-histogram name="LocalNetworkRequests.PrivatePage.PublicRequests"/>
+  <affected-histogram
+      name="LocalNetworkRequests.PrivatePage.SameSubnetRequests"/>
+  <affected-histogram
+      name="LocalNetworkRequests.PublicPage.Localhost.DbRequests"/>
+  <affected-histogram
+      name="LocalNetworkRequests.PublicPage.Localhost.DevRequests"/>
+  <affected-histogram
+      name="LocalNetworkRequests.PublicPage.Localhost.OtherRequests"/>
+  <affected-histogram
+      name="LocalNetworkRequests.PublicPage.Localhost.PrinterRequests"/>
+  <affected-histogram
+      name="LocalNetworkRequests.PublicPage.Localhost.WebRequests"/>
+  <affected-histogram name="LocalNetworkRequests.PublicPage.PrivateRequests"/>
+  <affected-histogram name="LocalNetworkRequests.PublicPage.RouterRequests"/>
+</histogram_suffixes>
+
 <histogram_suffixes name="LocalStorageCachePurgeReason" separator=".">
   <suffix name="AggressivePurgeTriggered"
       label="Aggressive purge was triggered on memory pressure."/>

tools/metrics/ukm/ukm.xml

@@ -395,6 +395,52 @@ be describing additional metrics about the same event.
   </metric>
 </event>
 
+<event name="LocalNetworkRequests">
+  <owner>uthakore@chromium.org</owner>
+  <summary>
+    Metrics that describe the resource request behavior of pages for which
+    navigation successfully commits. A separate entry is generated for every
+    unique IP address or localhost port number to which the loaded page makes a
+    resource request.
+  </summary>
+  <metric name="Count.Failed">
+    <summary>
+      The count of requests made by the page to the given resource for which a
+      response is not received or a network error occurs (i.e. |net_error| !=
+      |net::OK|) between the time navigation to the page commits and the time it
+      completes.
+    </summary>
+  </metric>
+  <metric name="Count.Successful">
+    <summary>
+      The count of requests made by the page to the given resource for which a
+      successful response is received (i.e. |net_error| == |net::OK|) between
+      the time navigation to the page commits and the time it completes.
+    </summary>
+  </metric>
+  <metric name="PortType">
+    <summary>
+      An enum value representing the type of port for requests to localhost. The
+      enum is defined in |LocalNetworkRequestsPageLoadMetricsObserver|. Possible
+      values are 1 for common web server ports, 2 for common database server
+      ports, 4 for common print server ports, 8 for common development server
+      ports, and 0 for all other ports.
+    </summary>
+  </metric>
+  <metric name="ResourceType">
+    <summary>
+      An enum value representing the type of resource requested. The enum is
+      defined in |LocalNetworkRequestsPageLoadMetricsObserver|. Possible values
+      are 0 for public resources requested by private pages, 1 for private
+      resources requested by public pages, 2 for private resources within the
+      same reserved IP space as the loaded private page, 4 for for private
+      resources within a different reserved IP space than the loaded private
+      page, 8 for resources requested by public pages that are suspected to be
+      routers, and 16 for localhost resources.
+    </summary>
+  </metric>
+</event>
+
 <event name="Media.WatchTime">
   <owner>dalecurtis@chromium.org</owner>
   <summary>
@@ -443,6 +489,24 @@ be describing additional metrics about the same event.
   <metric name="AudioVideo.SRC"/>
 </event>
 
+<event name="PageDomainInfo">
+  <owner>uthakore@chromium.org</owner>
+  <summary>
+    Metrics that describe the domain of pages that successfully commit. One
+    event is generated per page load that commits. Currently associated with
+    LocalNetworkRequests UKM metric collection.
+  </summary>
+  <metric name="DomainType">
+    <summary>
+      An enum value representing the type of domain of the loaded page. The enum
+      is defined in |LocalNetworkRequestsPageLoadMetricsObserver|. Possible
+      values are 0 for unknown (should never be logged), 1 for pages with public
+      domains, 2 for pages with private domains (IP addresses that are part of a
+      reserved IP space not including localhost), or 4 for localhost.
+    </summary>
+  </metric>
+</event>
+
 <event name="PageLoad" singular="True">
   <owner>bmcquade@chromium.org</owner>
   <summary>