Improve referrer chain by adding transition type

Proto change copied from cl/176384092. This CL expand the
"has_user_gesture" boolean field into a enum to include
more information on how the navigation happens.
Tests updated accordingly.

Bug: 780532
Change-Id: Ic13d8472aa6ea3e42f268f92bf46c6cae64f4066
Reviewed-on: https://chromium-review.googlesource.com/802185
Commit-Queue: Jialiu Lin <jialiul@chromium.org>
Reviewed-by: Luke Z <lpz@chromium.org>
Reviewed-by: Nathan Parker <nparker@chromium.org>
Cr-Commit-Position: refs/heads/master@{#524157}

chrome/browser/safe_browsing/download_protection/download_protection_service.h

@@ -223,16 +223,15 @@ class DownloadProtectionService {
       const net::X509Certificate& issuer,
       std::vector<std::string>* whitelist_strings);
 
-  // If kDownloadAttribution feature is enabled, identify referrer chain info of
-  // a download. This function also records UMA stats of download attribution
-  // result.
+  // Identify referrer chain info of a download. This function also records UMA
+  // stats of download attribution result.
   std::unique_ptr<ReferrerChain> IdentifyReferrerChain(
       const content::DownloadItem& item);
 
-  // If kDownloadAttribution feature is enabled, identify referrer chain of the
-  // PPAPI download based on the frame URL where the download is initiated.
-  // Then add referrer chain info to ClientDownloadRequest proto. This function
-  // also records UMA stats of download attribution result.
+  // Identify referrer chain of the PPAPI download based on the frame URL where
+  // the download is initiated. Then add referrer chain info to
+  // ClientDownloadRequest proto. This function also records UMA stats of
+  // download attribution result.
   void AddReferrerChainToPPAPIClientDownloadRequest(
       const GURL& initiating_frame_url,
       const GURL& initiating_main_frame_url,

chrome/browser/safe_browsing/safe_browsing_navigation_observer.cc

@@ -38,7 +38,7 @@ NavigationEvent::NavigationEvent()
       target_tab_id(-1),
       frame_id(-1),
       last_updated(base::Time::Now()),
-      is_user_initiated(false),
+      navigation_initiation(ReferrerChainEntry::UNDEFINED),
       has_committed(false) {}
 
 NavigationEvent::NavigationEvent(NavigationEvent&& nav_event)
@@ -50,7 +50,7 @@ NavigationEvent::NavigationEvent(NavigationEvent&& nav_event)
       target_tab_id(std::move(nav_event.target_tab_id)),
       frame_id(nav_event.frame_id),
       last_updated(nav_event.last_updated),
-      is_user_initiated(nav_event.is_user_initiated),
+      navigation_initiation(nav_event.navigation_initiation),
       has_committed(nav_event.has_committed) {}
 
 NavigationEvent& NavigationEvent::operator=(NavigationEvent&& nav_event) {
@@ -61,7 +61,7 @@ NavigationEvent& NavigationEvent::operator=(NavigationEvent&& nav_event) {
   target_tab_id = nav_event.target_tab_id;
   frame_id = nav_event.frame_id;
   last_updated = nav_event.last_updated;
-  is_user_initiated = nav_event.is_user_initiated;
+  navigation_initiation = nav_event.navigation_initiation;
   has_committed = nav_event.has_committed;
   server_redirect_urls = std::move(nav_event.server_redirect_urls);
   return *this;
@@ -118,22 +118,28 @@ void SafeBrowsingNavigationObserver::DidStartNavigation(
   auto it = navigation_handle_map_.find(navigation_handle);
   // It is possible to see multiple DidStartNavigation(..) with the same
   // navigation_handle (e.g. cross-process transfer). If that's the case,
-  // we need to copy the is_user_initiated field.
-  if (it != navigation_handle_map_.end()) {
-    nav_event->is_user_initiated = it->second->is_user_initiated;
+  // we need to copy the navigation_initiation field.
+  if (it != navigation_handle_map_.end() &&
+      it->second->navigation_initiation != ReferrerChainEntry::UNDEFINED) {
+    nav_event->navigation_initiation = it->second->navigation_initiation;
   } else {
     // If this is the first time we see this navigation_handle, create a new
     // NavigationEvent, and decide if it is triggered by user.
-    if ((has_user_gesture_ &&
-         !SafeBrowsingNavigationObserverManager::IsUserGestureExpired(
-             last_user_gesture_timestamp_)) ||
-        !navigation_handle->IsRendererInitiated()) {
-      nav_event->is_user_initiated = true;
-      if (has_user_gesture_) {
-        manager_->OnUserGestureConsumed(web_contents(),
-                                        last_user_gesture_timestamp_);
-        has_user_gesture_ = false;
-      }
+    if (!navigation_handle->IsRendererInitiated()) {
+      nav_event->navigation_initiation = ReferrerChainEntry::BROWSER_INITIATED;
+    } else if (has_user_gesture_ &&
+               !SafeBrowsingNavigationObserverManager::IsUserGestureExpired(
+                   last_user_gesture_timestamp_)) {
+      nav_event->navigation_initiation =
+          ReferrerChainEntry::RENDERER_INITIATED_WITH_USER_GESTURE;
+    } else {
+      nav_event->navigation_initiation =
+          ReferrerChainEntry::RENDERER_INITIATED_WITHOUT_USER_GESTURE;
+    }
+    if (has_user_gesture_) {
+      manager_->OnUserGestureConsumed(web_contents(),
+                                      last_user_gesture_timestamp_);
+      has_user_gesture_ = false;
     }
   }
 

chrome/browser/safe_browsing/safe_browsing_navigation_observer.h

@@ -9,6 +9,7 @@
 #include "base/supports_user_data.h"
 #include "components/content_settings/core/browser/content_settings_observer.h"
 #include "components/content_settings/core/common/content_settings.h"
+#include "components/safe_browsing/proto/csd.pb.h"
 #include "content/public/browser/web_contents_observer.h"
 #include "url/gurl.h"
 
@@ -30,27 +31,40 @@ struct NavigationEvent {
   NavigationEvent& operator=(NavigationEvent&& nav_event);
   ~NavigationEvent();
 
-  GURL source_url;  // URL that caused this navigation to occur.
-                    // TODO(jialiul): source_url may be incorrect when
-                    // navigation involves frames targeting each other.
-                    // http://crbug.com/651895.
-  GURL source_main_frame_url;  // Main frame url of the source_url. Could be the
-                               // same as source_url, if source_url was loaded
-                               // in main frame.
-  GURL original_request_url;   // The original request URL of this navigation.
-  std::vector<GURL> server_redirect_urls;  // Server redirect url chain.
-                                           // Empty if there is no server
-                                           // redirect. If set, last url in this
-                                           // vector is the destination url.
-  int source_tab_id;  // Which tab contains the frame with source_url. Tab ID is
-                      // returned by SessionTabHelper::IdForTab. This ID is
-                      // immutable for a given tab and unique across Chrome
-                      // within the current session.
-  int target_tab_id;  // Which tab this request url is targeting to.
-  int frame_id;  // Frame tree node ID of the frame where this navigation takes
-                 // place.
-  base::Time last_updated;  // When this NavigationEvent was last updated.
-  bool is_user_initiated;  // browser_initiated || has_user_gesture.
+  // URL that caused this navigation to occur.
+  GURL source_url;
+
+  // Main frame url of the source_url. Could be the same as source_url, if
+  // source_url was loaded in main frame.
+  GURL source_main_frame_url;
+
+  // The original request URL of this navigation.
+  GURL original_request_url;
+
+  // Server redirect url chain. Empty if there is no server redirect. If set,
+  // last url in this vector is the destination url.
+  std::vector<GURL> server_redirect_urls;
+
+  // Which tab contains the frame with source_url. Tab ID is returned by
+  // SessionTabHelper::IdForTab. This ID is immutable for a given tab and unique
+  // across Chrome within the current session.
+  int source_tab_id;
+
+  // Which tab this request url is targeting to.
+  int target_tab_id;
+
+  // Frame tree node ID of the frame where this navigation takes place.
+  int frame_id;
+
+  // When this NavigationEvent was last updated.
+  base::Time last_updated;
+
+  // If this navigation is triggered by browser or renderer, and if it is
+  // associated with any user gesture.
+  ReferrerChainEntry::NavigationInitiation navigation_initiation;
+
+  // Whether this a committed navigation. Navigation leads to download is not
+  // committed.
   bool has_committed;
 
   const GURL& GetDestinationUrl() const {
@@ -59,6 +73,12 @@ struct NavigationEvent {
     else
       return original_request_url;
   }
+
+  bool IsUserInitiated() const {
+    return navigation_initiation == ReferrerChainEntry::BROWSER_INITIATED ||
+           navigation_initiation ==
+               ReferrerChainEntry::RENDERER_INITIATED_WITH_USER_GESTURE;
+  }
 };
 
 // Structure to keep track of resolved IP address of a host.

chrome/browser/safe_browsing/safe_browsing_navigation_observer_manager.cc

@@ -132,7 +132,7 @@ NavigationEvent* NavigationEventList::FindNavigationEvent(
       // looks for the retargeting navigation event.
       if (nav_event->source_url.is_empty() &&
           nav_event->source_main_frame_url.is_empty() &&
-          !nav_event->is_user_initiated) {
+          !nav_event->IsUserInitiated()) {
         // If there is a server redirection immediately after retargeting, we
         // need to adjust our search url to the original request.
         if (!nav_event->server_redirect_urls.empty()) {
@@ -250,6 +250,7 @@ void SafeBrowsingNavigationObserverManager::SanitizeReferrerChain(
           GetOrigin(entry_copy.referrer_main_frame_url()));
     entry->set_is_retargeting(entry_copy.is_retargeting());
     entry->set_navigation_time_msec(entry_copy.navigation_time_msec());
+    entry->set_navigation_initiation(entry_copy.navigation_initiation());
     for (int j = 0; j < entry_copy.server_redirect_chain_size(); j++) {
       ReferrerChainEntry::ServerRedirect* server_redirect_entry =
           entry->add_server_redirect_chain();
@@ -434,7 +435,7 @@ void SafeBrowsingNavigationObserverManager::RecordNewWebContents(
     int source_render_frame_id,
     GURL target_url,
     content::WebContents* target_web_contents,
-    bool not_yet_in_tabstrip) {
+    bool renderer_initiated) {
   DCHECK(source_web_contents);
   DCHECK(target_web_contents);
 
@@ -458,14 +459,18 @@ void SafeBrowsingNavigationObserverManager::RecordNewWebContents(
   nav_event->original_request_url = cleaned_target_url;
   nav_event->target_tab_id = SessionTabHelper::IdForTab(target_web_contents);
   nav_event->frame_id = rfh ? rfh->GetFrameTreeNodeId() : -1;
+
   auto it = user_gesture_map_.find(source_web_contents);
-  if (it != user_gesture_map_.end() &&
-      !SafeBrowsingNavigationObserverManager::IsUserGestureExpired(
-          it->second)) {
-    nav_event->is_user_initiated = true;
-    OnUserGestureConsumed(it->first, it->second);
+  if (it == user_gesture_map_.end() ||
+      SafeBrowsingNavigationObserverManager::IsUserGestureExpired(it->second)) {
+    nav_event->navigation_initiation =
+        ReferrerChainEntry::RENDERER_INITIATED_WITHOUT_USER_GESTURE;
   } else {
-    nav_event->is_user_initiated = false;
+    OnUserGestureConsumed(it->first, it->second);
+    nav_event->navigation_initiation =
+        renderer_initiated
+            ? ReferrerChainEntry::RENDERER_INITIATED_WITH_USER_GESTURE
+            : ReferrerChainEntry::BROWSER_INITIATED;
   }
 
   navigation_event_list_.RecordNavigationEvent(std::move(nav_event));
@@ -530,6 +535,8 @@ void SafeBrowsingNavigationObserverManager::AddToReferrerChain(
     ReferrerChainEntry::URLType type) {
   std::unique_ptr<ReferrerChainEntry> referrer_chain_entry =
       base::MakeUnique<ReferrerChainEntry>();
+  referrer_chain_entry->set_navigation_initiation(
+      nav_event->navigation_initiation);
   const GURL destination_url = nav_event->GetDestinationUrl();
   referrer_chain_entry->set_url(ShortURLForReporting(destination_url));
   if (destination_main_frame_url.is_valid() &&
@@ -583,7 +590,7 @@ void SafeBrowsingNavigationObserverManager::GetRemainingReferrerChain(
   GURL last_main_frame_url_traced(last_nav_event_traced->source_main_frame_url);
   while (current_user_gesture_count < user_gesture_count_limit) {
     // Back trace to the next nav_event that was initiated by the user.
-    while (!last_nav_event_traced->is_user_initiated) {
+    while (!last_nav_event_traced->IsUserInitiated()) {
       last_nav_event_traced = navigation_event_list_.FindNavigationEvent(
           last_nav_event_traced->source_url,
           last_nav_event_traced->source_main_frame_url,
@@ -602,14 +609,11 @@ void SafeBrowsingNavigationObserverManager::GetRemainingReferrerChain(
 
     current_user_gesture_count++;
 
-
-    // If the source_url and source_main_frame_url of current navigation event
-    // are empty, and is_user_initiated is true, this is a browser initiated
-    // navigation (e.g. trigged by typing in address bar, clicking on bookmark,
-    // etc). We reached the end of the referrer chain.
-    if (last_nav_event_traced->source_url.is_empty() &&
-        last_nav_event_traced->source_main_frame_url.is_empty()) {
-      DCHECK(last_nav_event_traced->is_user_initiated);
+    // If this is a browser initiated navigation (e.g. trigged by typing in
+    // address bar, clicking on bookmark, etc). We reached the end of the
+    // referrer chain.
+    if (last_nav_event_traced->navigation_initiation ==
+        ReferrerChainEntry::BROWSER_INITIATED) {
       return;
     }
 

chrome/browser/safe_browsing/safe_browsing_navigation_observer_manager.h

@@ -195,7 +195,7 @@ class SafeBrowsingNavigationObserverManager
                             int source_render_frame_id,
                             GURL target_url,
                             content::WebContents* target_web_contents,
-                            bool not_yet_in_tabstrip);
+                            bool renderer_initiated);
 
  private:
   friend class base::RefCountedThreadSafe<

chrome/browser/safe_browsing/safe_browsing_navigation_observer_unittest.cc

@@ -42,16 +42,17 @@ class SBNavigationObserverTest : public BrowserWithTestWindowTest {
     delete navigation_observer_;
     BrowserWithTestWindowTest::TearDown();
   }
-  void VerifyNavigationEvent(const GURL& expected_source_url,
-                             const GURL& expected_source_main_frame_url,
-                             const GURL& expected_original_request_url,
-                             const GURL& expected_destination_url,
-                             int expected_source_tab,
-                             int expected_target_tab,
-                             bool expected_is_user_initiated,
-                             bool expected_has_committed,
-                             bool expected_has_server_redirect,
-                             NavigationEvent* actual_nav_event) {
+  void VerifyNavigationEvent(
+      const GURL& expected_source_url,
+      const GURL& expected_source_main_frame_url,
+      const GURL& expected_original_request_url,
+      const GURL& expected_destination_url,
+      int expected_source_tab,
+      int expected_target_tab,
+      ReferrerChainEntry::NavigationInitiation expected_nav_initiation,
+      bool expected_has_committed,
+      bool expected_has_server_redirect,
+      NavigationEvent* actual_nav_event) {
     EXPECT_EQ(expected_source_url, actual_nav_event->source_url);
     EXPECT_EQ(expected_source_main_frame_url,
               actual_nav_event->source_main_frame_url);
@@ -60,7 +61,7 @@ class SBNavigationObserverTest : public BrowserWithTestWindowTest {
     EXPECT_EQ(expected_destination_url, actual_nav_event->GetDestinationUrl());
     EXPECT_EQ(expected_source_tab, actual_nav_event->source_tab_id);
     EXPECT_EQ(expected_target_tab, actual_nav_event->target_tab_id);
-    EXPECT_EQ(expected_is_user_initiated, actual_nav_event->is_user_initiated);
+    EXPECT_EQ(expected_nav_initiation, actual_nav_event->navigation_initiation);
     EXPECT_EQ(expected_has_committed, actual_nav_event->has_committed);
     EXPECT_EQ(expected_has_server_redirect,
               !actual_nav_event->server_redirect_urls.empty());
@@ -170,9 +171,9 @@ TEST_F(SBNavigationObserverTest, BasicNavigationAndCommit) {
                         GURL("http://foo/1"),  // destination_url
                         tab_id,                // source_tab_id
                         tab_id,                // target_tab_id
-                        true,                  // is_user_initiated
-                        true,                  // has_committed
-                        false,                 // has_server_redirect
+                        ReferrerChainEntry::BROWSER_INITIATED,
+                        true,   // has_committed
+                        false,  // has_server_redirect
                         nav_list->Get(0U));
 }
 
@@ -187,16 +188,17 @@ TEST_F(SBNavigationObserverTest, ServerRedirect) {
       browser()->tab_strip_model()->GetWebContentsAt(0));
   auto* nav_list = navigation_event_list();
   ASSERT_EQ(1U, nav_list->Size());
-  VerifyNavigationEvent(GURL("http://foo/0"),       // source_url
-                        GURL("http://foo/0"),       // source_main_frame_url
-                        GURL("http://foo/3"),       // original_request_url
-                        GURL("http://redirect/1"),  // destination_url
-                        tab_id,                     // source_tab_id
-                        tab_id,                     // target_tab_id
-                        false,                      // is_user_initiated
-                        true,                       // has_committed
-                        true,                       // has_server_redirect
-                        nav_list->Get(0U));
+  VerifyNavigationEvent(
+      GURL("http://foo/0"),       // source_url
+      GURL("http://foo/0"),       // source_main_frame_url
+      GURL("http://foo/3"),       // original_request_url
+      GURL("http://redirect/1"),  // destination_url
+      tab_id,                     // source_tab_id
+      tab_id,                     // target_tab_id
+      ReferrerChainEntry::RENDERER_INITIATED_WITHOUT_USER_GESTURE,
+      true,  // has_committed
+      true,  // has_server_redirect
+      nav_list->Get(0U));
 }
 
 TEST_F(SBNavigationObserverTest, TestCleanUpStaleNavigationEvents) {

components/safe_browsing/proto/csd.proto

@@ -188,10 +188,18 @@ message LoginReputationClientRequest {
     // Whether the frame contains password field.
     optional bool has_password_field = 4;
 
-    // URLs transitions in reverse chronological order, i.e. the top level url
-    // or the url of the iframe comes first in the list.
+    // If we can find the complete referrer chain, this field will contains URLs
+    // transitions from landing referrer to event in reverse chronological
+    // order, i.e. event url comes first in this list, and landing referrer
+    // comes last.
+    // For Safe Browsing Extended Reporting or Scout users, if the referrer
+    // chain is empty or partially missing, we will add/append recent navigation
+    // events to this list. The type of these entries will be RECENT_NAVIGATION.
     repeated ReferrerChainEntry referrer_chain = 5;
 
+    // Options and metadata about the above referrer chain.
+    optional ReferrerChainOptions referrer_chain_options = 7;
+
     // The message contains features of a form.
     message Form {
       // Action url of the form.
@@ -202,6 +210,8 @@ message LoginReputationClientRequest {
     }
 
     repeated Form forms = 6;
+
+    // next available tag number: 8.
   }
 
   repeated Frame frames = 3;
@@ -520,11 +530,18 @@ message ClientDownloadRequest {
   // the leading extension separator.
   repeated string alternate_extensions = 35;
 
-  // URLs transitions from landing referrer to download in reverse chronological
-  // order, i.e. download url comes first in this list, and landing referrer
+  // If we can find the complete referrer chain, this field will contains URLs
+  // transitions from landing referrer to event in reverse chronological
+  // order, i.e. event url comes first in this list, and landing referrer
   // comes last.
+  // For Safe Browsing Extended Reporting or Scout users, if the referrer chain
+  // is empty or partially missing, we will add/append recent navigation events
+  // to this list. The type of these entries will be RECENT_NAVIGATION.
   repeated ReferrerChainEntry referrer_chain = 36;
 
+  // Options and metadata about the above referrer chain.
+  optional ReferrerChainOptions referrer_chain_options = 49;
+
   // Deprecated.
   optional bool DEPRECATED_download_attribution_finch_enabled = 39
       [deprecated = true];
@@ -533,6 +550,14 @@ message ClientDownloadRequest {
   // The underlying structure of code signature is defined at
   // https://opensource.apple.com/source/xnu/xnu-2782.1.97/bsd/sys/codesign.h
   optional bytes udif_code_signature = 40;
+
+  // next available tag number: 50;
+}
+
+message ReferrerChainOptions {
+  // The number of recent navigations we'd like to collect. This number is
+  // controlled by Finch parameter and its default value is 0.
+  optional int32 recent_navigations_to_collect = 1 [default = 0];
 }
 
 // Please update SafeBrowsingNavigationObserverManager::SanitizeReferrerChain()
@@ -557,6 +582,32 @@ message ReferrerChainEntry {
     CLIENT_REDIRECT = 4;
 
     DEPRECATED_SERVER_REDIRECT = 5;  // Deprecated
+
+    // For Safe Browsing Extended Reporting or Scout users, if the referrer
+    // chain is empty or partially missing, we will add/append recent navigation
+    // events to this list. RECENT_NAVIGATION type is set for these cases.
+    RECENT_NAVIGATION = 6;
+  }
+
+  enum NavigationInitiation {
+    // Convenient default value, should not be used in real ReferrerChainEntry.
+    UNDEFINED = 0;
+
+    // Navigation is initiated by the browser process. e.g. user enters url into
+    // address bar, or user opens a bookmark.
+    BROWSER_INITIATED = 1;
+
+    // Navigation is initiated by the renderer process and there is no user
+    // gesture associate with this navigation. e.g.
+    // * redirect via the <meta http-equiv="refresh"> tag
+    // * change window.location.href
+    RENDERER_INITIATED_WITHOUT_USER_GESTURE = 2;
+
+    // Navigation is initiated by the renderer process and there is a clear user
+    // gesture associate with this navigation. e.g.
+    //  * <a> link click
+    //  * using window.history.pushState (i.e. back, forward button interaction)
+    RENDERER_INITIATED_WITH_USER_GESTURE = 3;
   }
 
   message ServerRedirect {
@@ -594,6 +645,11 @@ message ReferrerChainEntry {
   // The first entry in |server_redirect_chain| should be the original request
   // url, and the last entry should be the same as |url|.
   repeated ServerRedirect server_redirect_chain = 8;
+
+  // How this navigation is initiated.
+  optional NavigationInitiation navigation_initiation = 10;
+
+  // next available tag number: 11.
 }  // End of ReferrerChainEntry
 
 message ClientDownloadResponse {