Add referrer chain to Threat Details.

Includes a component-accessible interface for interacting with the
navigation observer manager.

Bug: 817377
Change-Id: I6074e9cfa53cbc7332c1f78b310124be99932abc
Reviewed-on: https://chromium-review.googlesource.com/1030776Reviewed-by: Jialiu Lin <jialiul@chromium.org>
Reviewed-by: Changwan Ryu <changwan@chromium.org>
Commit-Queue: Luke Z <lpz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#556776}

android_webview/browser/aw_browser_context.cc

@@ -179,7 +179,8 @@ void AwBrowserContext::PreMainMessageLoopRun(net::NetLog* net_log) {
       new safe_browsing::RemoteSafeBrowsingDatabaseManager();
   safe_browsing_trigger_manager_ =
       std::make_unique<safe_browsing::TriggerManager>(
-          safe_browsing_ui_manager_.get());
+          safe_browsing_ui_manager_.get(),
+          /*referrer_chain_provider=*/nullptr);
   safe_browsing_whitelist_manager_ = CreateSafeBrowsingWhitelistManager();
 
   content::WebUIControllerFactory::RegisterFactory(

chrome/browser/safe_browsing/safe_browsing_blocking_page_test.cc

@@ -253,11 +253,13 @@ class TestThreatDetailsFactory : public ThreatDetailsFactory {
       const security_interstitials::UnsafeResource& unsafe_resource,
       scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory,
       history::HistoryService* history_service,
+      ReferrerChainProvider* referrer_chain_provider,
       bool trim_to_ad_tags,
       ThreatDetailsDoneCallback done_callback) override {
     details_ = new ThreatDetails(delegate, web_contents, unsafe_resource,
                                  url_loader_factory, history_service,
-                                 trim_to_ad_tags, done_callback);
+                                 referrer_chain_provider, trim_to_ad_tags,
+                                 done_callback);
     return details_;
   }
 

chrome/browser/safe_browsing/safe_browsing_navigation_observer_manager.h

@@ -8,6 +8,7 @@
 #include "base/containers/circular_deque.h"
 #include "base/feature_list.h"
 #include "base/supports_user_data.h"
+#include "components/safe_browsing/browser/referrer_chain_provider.h"
 #include "components/safe_browsing/proto/csd.pb.h"
 #include "components/sessions/core/session_id.h"
 #include "content/public/browser/web_contents_observer.h"
@@ -22,9 +23,6 @@ class SafeBrowsingNavigationObserver;
 struct NavigationEvent;
 struct ResolvedIPAddress;
 
-typedef google::protobuf::RepeatedPtrField<safe_browsing::ReferrerChainEntry>
-    ReferrerChain;
-
 // User data stored in DownloadItem for referrer chain information.
 class ReferrerChainData : public base::SupportsUserData::Data {
  public:
@@ -116,20 +114,9 @@ struct NavigationEventList {
 // cleaning up stale navigation events, and identifying landing page/landing
 // referrer for a specific Safe Browsing event.
 class SafeBrowsingNavigationObserverManager
-    : public base::RefCountedThreadSafe<SafeBrowsingNavigationObserverManager> {
+    : public base::RefCountedThreadSafe<SafeBrowsingNavigationObserverManager>,
+      public ReferrerChainProvider {
  public:
-  // For UMA histogram counting. Do NOT change order.
-  enum AttributionResult {
-    SUCCESS = 1,                   // Identified referrer chain is not empty.
-    SUCCESS_LANDING_PAGE = 2,      // Successfully identified landing page.
-    SUCCESS_LANDING_REFERRER = 3,  // Successfully identified landing referrer.
-    INVALID_URL = 4,
-    NAVIGATION_EVENT_NOT_FOUND = 5,
-
-    // Always at the end.
-    ATTRIBUTION_FAILURE_TYPE_MAX
-  };
-
   // Helper function to check if user gesture is older than
   // kUserGestureTTLInSecond.
   static bool IsUserGestureExpired(const base::Time& timestamp);
@@ -189,7 +176,7 @@ class SafeBrowsingNavigationObserverManager
   AttributionResult IdentifyReferrerChainByWebContents(
       content::WebContents* web_contents,
       int user_gesture_count_limit,
-      ReferrerChain* out_referrer_chain);
+      ReferrerChain* out_referrer_chain) override;
 
   // Based on the |initiating_frame_url| and its associated |tab_id|, traces
   // back the observed NavigationEvents in navigation_event_list_ to identify

chrome/browser/safe_browsing/safe_browsing_service.cc

@@ -614,7 +614,8 @@ void SafeBrowsingService::ProcessResourceRequest(
 
 void SafeBrowsingService::CreateTriggerManager() {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
-  trigger_manager_ = std::make_unique<TriggerManager>(ui_manager_.get());
+  trigger_manager_ = std::make_unique<TriggerManager>(
+      ui_manager_.get(), navigation_observer_manager_.get());
 }
 
 void SafeBrowsingService::CreateURLLoaderFactoryForIO(

components/safe_browsing/browser/referrer_chain_provider.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_SAFE_BROWSING_BROWSER_REFERRER_CHAIN_PROVIDER_H_
+#define COMPONENTS_SAFE_BROWSING_BROWSER_REFERRER_CHAIN_PROVIDER_H_
+
+#include "components/safe_browsing/proto/csd.pb.h"
+
+namespace content {
+class WebContents;
+}
+
+namespace safe_browsing {
+using ReferrerChain =
+    google::protobuf::RepeatedPtrField<safe_browsing::ReferrerChainEntry>;
+
+class ReferrerChainProvider {
+ public:
+  // For UMA histogram counting. Do NOT change order.
+  enum AttributionResult {
+    SUCCESS = 1,                   // Identified referrer chain is not empty.
+    SUCCESS_LANDING_PAGE = 2,      // Successfully identified landing page.
+    SUCCESS_LANDING_REFERRER = 3,  // Successfully identified landing referrer.
+    INVALID_URL = 4,
+    NAVIGATION_EVENT_NOT_FOUND = 5,
+
+    // Always at the end.
+    ATTRIBUTION_FAILURE_TYPE_MAX
+  };
+
+  virtual AttributionResult IdentifyReferrerChainByWebContents(
+      content::WebContents* web_contents,
+      int user_gesture_count_limit,
+      ReferrerChain* out_referrer_chain) = 0;
+};
+}  // namespace safe_browsing
+
+#endif  // COMPONENTS_SAFE_BROWSING_BROWSER_REFERRER_CHAIN_PROVIDER_H_
\ No newline at end of file

components/safe_browsing/browser/threat_details.cc

@@ -18,6 +18,7 @@
 #include "base/strings/string_util.h"
 #include "components/history/core/browser/history_service.h"
 #include "components/safe_browsing/base_ui_manager.h"
+#include "components/safe_browsing/browser/referrer_chain_provider.h"
 #include "components/safe_browsing/browser/threat_details_cache.h"
 #include "components/safe_browsing/browser/threat_details_history.h"
 #include "components/safe_browsing/db/hit_report.h"
@@ -51,6 +52,9 @@ namespace {
 // An element ID indicating that an HTML Element has no parent.
 const int kElementIdNoParent = -1;
 
+// The number of user gestures to trace back for the referrer chain.
+const int kThreatDetailsUserGestureLimit = 2;
+
 typedef std::unordered_set<std::string> StringSet;
 // A set of HTTPS headers that are allowed to be collected. Contains both
 // request and response headers. All entries in this list should be lower-case
@@ -272,11 +276,13 @@ class ThreatDetailsFactoryImpl : public ThreatDetailsFactory {
       const security_interstitials::UnsafeResource& unsafe_resource,
       scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory,
       history::HistoryService* history_service,
+      ReferrerChainProvider* referrer_chain_provider,
       bool trim_to_ad_tags,
       ThreatDetailsDoneCallback done_callback) override {
     return new ThreatDetails(ui_manager, web_contents, unsafe_resource,
                              url_loader_factory, history_service,
-                             trim_to_ad_tags, done_callback);
+                             referrer_chain_provider, trim_to_ad_tags,
+                             done_callback);
   }
 
  private:
@@ -298,15 +304,16 @@ ThreatDetails* ThreatDetails::NewThreatDetails(
     const UnsafeResource& resource,
     scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory,
     history::HistoryService* history_service,
+    ReferrerChainProvider* referrer_chain_provider,
     bool trim_to_ad_tags,
     ThreatDetailsDoneCallback done_callback) {
   // Set up the factory if this has not been done already (tests do that
   // before this method is called).
   if (!factory_)
     factory_ = g_threat_details_factory_impl.Pointer();
-  return factory_->CreateThreatDetails(ui_manager, web_contents, resource,
-                                       url_loader_factory, history_service,
-                                       trim_to_ad_tags, done_callback);
+  return factory_->CreateThreatDetails(
+      ui_manager, web_contents, resource, url_loader_factory, history_service,
+      referrer_chain_provider, trim_to_ad_tags, done_callback);
 }
 
 // Create a ThreatDetails for the given tab. Runs in the UI thread.
@@ -316,12 +323,14 @@ ThreatDetails::ThreatDetails(
     const UnsafeResource& resource,
     scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory,
     history::HistoryService* history_service,
+    ReferrerChainProvider* referrer_chain_provider,
     bool trim_to_ad_tags,
     ThreatDetailsDoneCallback done_callback)
     : content::WebContentsObserver(web_contents),
       url_loader_factory_(url_loader_factory),
       ui_manager_(ui_manager),
       resource_(resource),
+      referrer_chain_provider_(referrer_chain_provider),
       cache_result_(false),
       did_proceed_(false),
       num_visits_(0),
@@ -753,6 +762,9 @@ void ThreatDetails::OnCacheCollectionReady() {
   report_->mutable_client_properties()->set_url_api_type(
       GetUrlApiTypeForThreatSource(resource_.threat_source));
 
+  // Fill the referrer chain if applicable.
+  MaybeFillReferrerChain();
+
   // Send the report, using the SafeBrowsingService.
   std::string serialized;
   if (!report_->SerializeToString(&serialized)) {
@@ -780,6 +792,20 @@ void ThreatDetails::OnCacheCollectionReady() {
   AllDone();
 }
 
+void ThreatDetails::MaybeFillReferrerChain() {
+  if (!referrer_chain_provider_)
+    return;
+
+  if (!report_ ||
+      report_->type() != ClientSafeBrowsingReportRequest::URL_SUSPICIOUS) {
+    return;
+  }
+
+  referrer_chain_provider_->IdentifyReferrerChainByWebContents(
+      web_contents(), kThreatDetailsUserGestureLimit,
+      report_->mutable_referrer_chain());
+}
+
 void ThreatDetails::AllDone() {
   is_all_done_ = true;
   BrowserThread::PostTask(

components/safe_browsing/browser/threat_details.h

@@ -38,6 +38,7 @@ class SharedURLLoaderFactory;
 namespace safe_browsing {
 
 class BaseUIManager;
+class ReferrerChainProvider;
 
 // Maps a URL to its Resource.
 class ThreatDetailsCacheCollector;
@@ -78,6 +79,7 @@ class ThreatDetails : public base::RefCounted<ThreatDetails>,
       const UnsafeResource& resource,
       scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory,
       history::HistoryService* history_service,
+      ReferrerChainProvider* referrer_chain_provider,
       bool trim_to_ad_tags,
       ThreatDetailsDoneCallback done_callback);
 
@@ -109,6 +111,7 @@ class ThreatDetails : public base::RefCounted<ThreatDetails>,
       const UnsafeResource& resource,
       scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory,
       history::HistoryService* history_service,
+      ReferrerChainProvider* referrer_chain_provider,
       bool trim_to_ad_tags,
       ThreatDetailsDoneCallback done_callback);
 
@@ -176,6 +179,11 @@ class ThreatDetails : public base::RefCounted<ThreatDetails>,
                      const std::vector<mojom::AttributeNameValuePtr> attributes,
                      const ClientSafeBrowsingReportRequest::Resource* resource);
 
+  // Populates the referrer chain data in |report_|. This may be skipped if the
+  // referrer chain provider isn't available, or the type of report doesn't
+  // include the referrer chain.
+  void MaybeFillReferrerChain();
+
   // Called when the report is complete. Runs |done_callback_|.
   void AllDone();
 
@@ -183,6 +191,8 @@ class ThreatDetails : public base::RefCounted<ThreatDetails>,
 
   const UnsafeResource resource_;
 
+  ReferrerChainProvider* referrer_chain_provider_;
+
   // For every Url we collect we create a Resource message. We keep
   // them in a map so we can avoid duplicates.
   ResourceMap resources_;
@@ -275,6 +285,7 @@ class ThreatDetailsFactory {
       const security_interstitials::UnsafeResource& unsafe_resource,
       scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory,
       history::HistoryService* history_service,
+      ReferrerChainProvider* referrer_chain_provider,
       bool trim_to_ad_tags,
       ThreatDetailsDoneCallback done_callback) = 0;
 };

components/safe_browsing/proto/csd.proto

@@ -974,7 +974,7 @@ message DownloadMetadata {
 // A Detailed Safebrowsing Report from clients. Chrome safebrowsing reports are
 // only sent by Chrome users who have opted into extended Safe Browsing.
 // This proto is replacing ClientMalwareReportRequest.
-// Next tag: 19
+// Next tag: 24
 message ClientSafeBrowsingReportRequest {
   // Note: A lot of the "optional" fields would make sense to be
   // "required" instead.  However, having them as optional allows the
@@ -1109,6 +1109,12 @@ message ClientSafeBrowsingReportRequest {
   // False means user directly executed this download via download shelf or
   // other download UIs.
   optional bool show_download_in_folder = 18;
+
+  // If we can find the complete referrer chain, this field will contains URLs
+  // transitions from landing referrer to event in reverse chronological
+  // order, i.e. event url comes first in this list, and landing referrer
+  // comes last.
+  repeated ReferrerChainEntry referrer_chain = 23;
 }
 
 // An HTML Element on the page (eg: iframe, div, script, etc).

components/safe_browsing/triggers/mock_trigger_manager.cc

@@ -6,7 +6,7 @@
 
 namespace safe_browsing {
 
-MockTriggerManager::MockTriggerManager() : TriggerManager(nullptr) {}
+MockTriggerManager::MockTriggerManager() : TriggerManager(nullptr, nullptr) {}
 
 MockTriggerManager::~MockTriggerManager() {}
 

components/safe_browsing/triggers/trigger_manager.cc

@@ -93,8 +93,10 @@ bool CanSendReport(const SBErrorOptions& error_display_options,
 DataCollectorsContainer::DataCollectorsContainer() {}
 DataCollectorsContainer::~DataCollectorsContainer() {}
 
-TriggerManager::TriggerManager(BaseUIManager* ui_manager)
+TriggerManager::TriggerManager(BaseUIManager* ui_manager,
+                               ReferrerChainProvider* referrer_chain_provider)
     : ui_manager_(ui_manager),
+      referrer_chain_provider_(referrer_chain_provider),
       trigger_throttler_(new TriggerThrottler()),
       weak_factory_(this) {}
 
@@ -205,7 +207,7 @@ bool TriggerManager::StartCollectingThreatDetailsWithReason(
   collectors->threat_details =
       scoped_refptr<ThreatDetails>(ThreatDetails::NewThreatDetails(
           ui_manager_, web_contents, resource, url_loader_factory,
-          history_service, should_trim_threat_details,
+          history_service, referrer_chain_provider_, should_trim_threat_details,
           base::Bind(&TriggerManager::ThreatDetailsDone,
                      weak_factory_.GetWeakPtr())));
   return true;

components/safe_browsing/triggers/trigger_manager.h

@@ -9,6 +9,7 @@
 
 #include "base/macros.h"
 #include "base/memory/ref_counted.h"
+#include "components/safe_browsing/browser/referrer_chain_provider.h"
 #include "components/safe_browsing/triggers/trigger_throttler.h"
 #include "components/security_interstitials/content/unsafe_resource.h"
 #include "components/security_interstitials/core/base_safe_browsing_error_ui.h"
@@ -84,7 +85,8 @@ enum class TriggerManagerReason {
 // tracking how often triggers fire and throttling them when necessary.
 class TriggerManager {
  public:
-  TriggerManager(BaseUIManager* ui_manager);
+  TriggerManager(BaseUIManager* ui_manager,
+                 ReferrerChainProvider* referrer_chain_provider);
   virtual ~TriggerManager();
 
   // Returns a SBErrorDisplayOptions struct containing user state that is
@@ -173,6 +175,10 @@ class TriggerManager {
   // TODO(lpz): we may only need a the PingManager here.
   BaseUIManager* ui_manager_;
 
+  // The Referrer Chain Provider is used to retrieve the referrer chain for
+  // reports that require it. Not owned.
+  ReferrerChainProvider* referrer_chain_provider_;
+
   // Map of the data collectors running on each tabs. New keys are added the
   // first time any trigger tries to collect data on a tab and are removed when
   // the tab is destroyed. The values can be null if a trigger has finished on

components/safe_browsing/triggers/trigger_manager_unittest.cc

@@ -46,6 +46,7 @@ class MockThreatDetailsFactory : public ThreatDetailsFactory {
       const security_interstitials::UnsafeResource& unsafe_resource,
       scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory,
       history::HistoryService* history_service,
+      ReferrerChainProvider* referrer_chain_provider,
       bool trim_to_ad_tags,
       ThreatDetailsDoneCallback done_callback) override {
     MockThreatDetails* threat_details = new MockThreatDetails();
@@ -60,7 +61,9 @@ class MockTriggerThrottler : public TriggerThrottler {
 
 class TriggerManagerTest : public ::testing::Test {
  public:
-  TriggerManagerTest() : trigger_manager_(/*ui_manager=*/nullptr) {}
+  TriggerManagerTest()
+      : trigger_manager_(/*ui_manager=*/nullptr,
+                         /*referrer_chain_provider=*/nullptr) {}
   ~TriggerManagerTest() override {}
 
   void SetUp() override {