Adds the functionality for the config service to ignore the blacklist

When the DRP config service sends a bool to ignore the previews
blacklist for server previews, the long term (host and user) rules will
be ignored for server previews. Client previews will still respect those
rules.

Bug: 849328
Change-Id: I4ef5f6b3e71592cba1eaa9c6e0006780191de6b5
Reviewed-on: https://chromium-review.googlesource.com/1086364Reviewed-by: Peter Williamson <petewil@chromium.org>
Reviewed-by: Tarun Bansal <tbansal@chromium.org>
Commit-Queue: Ryan Sturm <ryansturm@chromium.org>
Cr-Commit-Position: refs/heads/master@{#565690}

chrome/browser/offline_pages/offline_page_request_handler_unittest.cc

@@ -260,8 +260,8 @@ class TestPreviewsDecider : public previews::PreviewsDecider {
       const net::URLRequest& request,
       previews::PreviewsType type,
       net::EffectiveConnectionType effective_connection_type_threshold,
-      const std::vector<std::string>& host_blacklist_from_server)
-      const override {
+      const std::vector<std::string>& host_blacklist_from_server,
+      bool ignore_long_term_black_list_rules) const override {
     return should_allow_preview_;
   }
 

components/data_reduction_proxy/core/browser/data_reduction_proxy_config.cc

@@ -196,6 +196,7 @@ DataReductionProxyConfig::DataReductionProxyConfig(
       configurator_(configurator),
       event_creator_(event_creator),
       connection_type_(net::NetworkChangeNotifier::GetConnectionType()),
+      ignore_long_term_black_list_rules_(false),
       network_properties_manager_(nullptr),
       weak_factory_(this) {
   DCHECK(io_task_runner_);
@@ -766,7 +767,7 @@ bool DataReductionProxyConfig::IsBlackListedOrDisabled(
   // TODO(crbug.com/720102): Consider new method to just check blacklist.
   return !previews_decider.ShouldAllowPreviewAtECT(
       request, previews_type, net::EFFECTIVE_CONNECTION_TYPE_4G,
-      std::vector<std::string>());
+      std::vector<std::string>(), ignore_long_term_black_list_rules_);
 }
 
 bool DataReductionProxyConfig::ShouldAcceptServerPreview(
@@ -852,4 +853,15 @@ void DataReductionProxyConfig::EnableGetNetworkIdAsynchronously() {
 }
 #endif  // defined(OS_CHROMEOS)
 
+void DataReductionProxyConfig::SetIgnoreLongTermBlackListRules(
+    bool ignore_long_term_black_list_rules) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  ignore_long_term_black_list_rules_ = ignore_long_term_black_list_rules;
+}
+
+bool DataReductionProxyConfig::IgnoreBlackListLongTermRulesForTesting() const {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  return ignore_long_term_black_list_rules_;
+}
+
 }  // namespace data_reduction_proxy

components/data_reduction_proxy/core/browser/data_reduction_proxy_config.h

@@ -202,6 +202,12 @@ class DataReductionProxyConfig
   void EnableGetNetworkIdAsynchronously();
 #endif  // defined(OS_CHROMEOS)
 
+  // When triggering previews, prevent long term black list rules.
+  void SetIgnoreLongTermBlackListRules(bool ignore_long_term_black_list_rules);
+
+  // Returns the value set in SetIgnoreLongTermBlackListRules.
+  bool IgnoreBlackListLongTermRulesForTesting() const;
+
  protected:
   virtual base::TimeTicks GetTicksNow() const;
 
@@ -363,6 +369,9 @@ class DataReductionProxyConfig
   bool warmup_url_fetch_in_flight_secure_proxy_;
   bool warmup_url_fetch_in_flight_core_proxy_;
 
+  // When triggerring previews, prevent long term black list rules.
+  bool ignore_long_term_black_list_rules_;
+
   // Should be accessed only on the IO thread. Guaranteed to be non-null during
   // the lifetime of |this| if accessed on the IO thread.
   NetworkPropertiesManager* network_properties_manager_;

components/data_reduction_proxy/core/browser/data_reduction_proxy_config_service_client.cc

@@ -590,6 +590,9 @@ bool DataReductionProxyConfigServiceClient::ParseAndApplyProxyConfig(
   if (!config.has_proxy_config())
     return false;
 
+  config_->SetIgnoreLongTermBlackListRules(
+      config.ignore_long_term_black_list_rules());
+
   // An empty proxy config is OK, and allows the server to effectively turn off
   // DataSaver if needed. See http://crbug.com/840978.
   std::vector<DataReductionProxyServer> proxies =

components/data_reduction_proxy/core/browser/data_reduction_proxy_config_service_client_unittest.cc

@@ -91,7 +91,8 @@ ClientConfig CreateConfig(const std::string& session_key,
                           const std::string& secondary_host,
                           int secondary_port,
                           const ProxyServer_ProxyType& secondary_proxy_type,
-                          float reporting_fraction) {
+                          float reporting_fraction,
+                          bool ignore_long_term_black_list_rules) {
   ClientConfig config;
 
   config.set_session_key(session_key);
@@ -104,6 +105,8 @@ ClientConfig CreateConfig(const std::string& session_key,
     config.mutable_pageload_metrics_config()->set_reporting_fraction(
         reporting_fraction);
   }
+  config.set_ignore_long_term_black_list_rules(
+      ignore_long_term_black_list_rules);
   ProxyServer* primary_proxy =
       config.mutable_proxy_config()->add_http_proxy_servers();
   primary_proxy->set_scheme(primary_scheme);
@@ -172,11 +175,11 @@ class DataReductionProxyConfigServiceClientTest : public testing::Test {
 
     ASSERT_NE(nullptr, context_->network_delegate());
     // Set up the various test ClientConfigs.
-    ClientConfig config =
-        CreateConfig(kSuccessSessionKey, kConfigRefreshDurationSeconds, 0,
-                     ProxyServer_ProxyScheme_HTTPS, "origin.net", 443,
-                     ProxyServer::CORE, ProxyServer_ProxyScheme_HTTP,
-                     "fallback.net", 80, ProxyServer::UNSPECIFIED_TYPE, 0.5f);
+    ClientConfig config = CreateConfig(
+        kSuccessSessionKey, kConfigRefreshDurationSeconds, 0,
+        ProxyServer_ProxyScheme_HTTPS, "origin.net", 443, ProxyServer::CORE,
+        ProxyServer_ProxyScheme_HTTP, "fallback.net", 80,
+        ProxyServer::UNSPECIFIED_TYPE, 0.5f, false);
     config.SerializeToString(&config_);
     encoded_config_ = EncodeConfig(config);
 
@@ -184,21 +187,21 @@ class DataReductionProxyConfigServiceClientTest : public testing::Test {
         kOldSuccessSessionKey, kConfigRefreshDurationSeconds, 0,
         ProxyServer_ProxyScheme_HTTPS, "old.origin.net", 443, ProxyServer::CORE,
         ProxyServer_ProxyScheme_HTTP, "old.fallback.net", 80,
-        ProxyServer::UNSPECIFIED_TYPE, 0.0f);
+        ProxyServer::UNSPECIFIED_TYPE, 0.0f, false);
     previous_config.SerializeToString(&previous_config_);
 
-    ClientConfig persisted =
-        CreateConfig(kPersistedSessionKey, kConfigRefreshDurationSeconds, 0,
-                     ProxyServer_ProxyScheme_HTTPS, "persisted.net", 443,
-                     ProxyServer::CORE, ProxyServer_ProxyScheme_HTTP,
-                     "persisted.net", 80, ProxyServer::UNSPECIFIED_TYPE, 0.0f);
+    ClientConfig persisted = CreateConfig(
+        kPersistedSessionKey, kConfigRefreshDurationSeconds, 0,
+        ProxyServer_ProxyScheme_HTTPS, "persisted.net", 443, ProxyServer::CORE,
+        ProxyServer_ProxyScheme_HTTP, "persisted.net", 80,
+        ProxyServer::UNSPECIFIED_TYPE, 0.0f, false);
     loaded_config_ = EncodeConfig(persisted);
 
-    ClientConfig zero_reporting_fraction_config =
-        CreateConfig(kSuccessSessionKey, kConfigRefreshDurationSeconds, 0,
-                     ProxyServer_ProxyScheme_HTTPS, "origin.net", 443,
-                     ProxyServer::CORE, ProxyServer_ProxyScheme_HTTP,
-                     "origin.net", 0, ProxyServer::UNSPECIFIED_TYPE, 0.0f);
+    ClientConfig zero_reporting_fraction_config = CreateConfig(
+        kSuccessSessionKey, kConfigRefreshDurationSeconds, 0,
+        ProxyServer_ProxyScheme_HTTPS, "origin.net", 443, ProxyServer::CORE,
+        ProxyServer_ProxyScheme_HTTP, "origin.net", 0,
+        ProxyServer::UNSPECIFIED_TYPE, 0.0f, false);
     zero_reporting_fraction_encoded_config_ =
         EncodeConfig(zero_reporting_fraction_config);
 
@@ -206,29 +209,36 @@ class DataReductionProxyConfigServiceClientTest : public testing::Test {
         CreateConfig(kSuccessSessionKey, kConfigRefreshDurationSeconds, 0,
                      ProxyServer_ProxyScheme_HTTPS, "", 443, ProxyServer::CORE,
                      ProxyServer_ProxyScheme_HTTP, "", 0,
-                     ProxyServer::UNSPECIFIED_TYPE, 1.0f);
+                     ProxyServer::UNSPECIFIED_TYPE, 1.0f, false);
     one_reporting_fraction_encoded_config_ =
         EncodeConfig(one_reporting_fraction_config);
 
     // Passing in -1.0f as the reporting fraction causes the
     // |empty_reporting_fraction_config| to have no pageload_metrics_config()
     // set.
-    ClientConfig empty_reporting_fraction_config =
-        CreateConfig(kSuccessSessionKey, kConfigRefreshDurationSeconds, 0,
-                     ProxyServer_ProxyScheme_HTTPS, "origin.net", 443,
-                     ProxyServer::CORE, ProxyServer_ProxyScheme_HTTP,
-                     "origin.net", 0, ProxyServer::UNSPECIFIED_TYPE, -1.0f);
+    ClientConfig empty_reporting_fraction_config = CreateConfig(
+        kSuccessSessionKey, kConfigRefreshDurationSeconds, 0,
+        ProxyServer_ProxyScheme_HTTPS, "origin.net", 443, ProxyServer::CORE,
+        ProxyServer_ProxyScheme_HTTP, "origin.net", 0,
+        ProxyServer::UNSPECIFIED_TYPE, -1.0f, false);
     empty_reporting_fraction_encoded_config_ =
         EncodeConfig(empty_reporting_fraction_config);
 
-    ClientConfig half_reporting_fraction_config =
-        CreateConfig(kSuccessSessionKey, kConfigRefreshDurationSeconds, 0,
-                     ProxyServer_ProxyScheme_HTTPS, "origin.net", 443,
-                     ProxyServer::CORE, ProxyServer_ProxyScheme_HTTP,
-                     "origin.net", 0, ProxyServer::UNSPECIFIED_TYPE, 0.5f);
+    ClientConfig half_reporting_fraction_config = CreateConfig(
+        kSuccessSessionKey, kConfigRefreshDurationSeconds, 0,
+        ProxyServer_ProxyScheme_HTTPS, "origin.net", 443, ProxyServer::CORE,
+        ProxyServer_ProxyScheme_HTTP, "origin.net", 0,
+        ProxyServer::UNSPECIFIED_TYPE, 0.5f, false);
     half_reporting_fraction_encoded_config_ =
         EncodeConfig(half_reporting_fraction_config);
 
+    ClientConfig ignore_black_list_config = CreateConfig(
+        kSuccessSessionKey, kConfigRefreshDurationSeconds, 0,
+        ProxyServer_ProxyScheme_HTTPS, "origin.net", 443, ProxyServer::CORE,
+        ProxyServer_ProxyScheme_HTTP, "origin.net", 0,
+        ProxyServer::UNSPECIFIED_TYPE, 0.5f, true);
+    ignore_black_list_encoded_config_ = EncodeConfig(ignore_black_list_config);
+
     ClientConfig no_proxies_config;
     no_proxies_config.set_session_key(kSuccessSessionKey);
     no_proxies_config.mutable_refresh_duration()->set_seconds(
@@ -454,6 +464,9 @@ class DataReductionProxyConfigServiceClientTest : public testing::Test {
   const std::string& half_reporting_fraction_encoded_config() const {
     return half_reporting_fraction_encoded_config_;
   }
+  const std::string& ignore_black_list_encoded_config() const {
+    return ignore_black_list_encoded_config_;
+  }
   const std::string& no_proxies_config() const { return no_proxies_config_; }
 
   const std::string& loaded_config() const { return loaded_config_; }
@@ -498,6 +511,9 @@ class DataReductionProxyConfigServiceClientTest : public testing::Test {
   // A configuration where the pingback reporting fraction is set to 0.5f.
   std::string half_reporting_fraction_encoded_config_;
 
+  // A configuration where the black list rules are ignored.
+  std::string ignore_black_list_encoded_config_;
+
   // A configuration where no proxies are configured.
   std::string no_proxies_config_;
 
@@ -1249,11 +1265,11 @@ TEST_F(DataReductionProxyConfigServiceClientTest, HTTPRequests) {
 TEST_F(DataReductionProxyConfigServiceClientTest, ApplyClientConfigOverride) {
   const std::string override_key = "OverrideSecureSession";
   std::string encoded_config;
-  ClientConfig config =
-      CreateConfig(override_key, kConfigRefreshDurationSeconds, 0,
-                   ProxyServer_ProxyScheme_HTTPS, "origin.net", 443,
-                   ProxyServer::CORE, ProxyServer_ProxyScheme_HTTP,
-                   "fallback.net", 80, ProxyServer::UNSPECIFIED_TYPE, 0.5f);
+  ClientConfig config = CreateConfig(
+      override_key, kConfigRefreshDurationSeconds, 0,
+      ProxyServer_ProxyScheme_HTTPS, "origin.net", 443, ProxyServer::CORE,
+      ProxyServer_ProxyScheme_HTTP, "fallback.net", 80,
+      ProxyServer::UNSPECIFIED_TYPE, 0.5f, false);
   config.SerializeToString(&encoded_config);
   base::Base64Encode(encoded_config, &encoded_config);
 
@@ -1395,6 +1411,15 @@ TEST_F(DataReductionProxyConfigServiceClientTest,
   config_client()->ApplySerializedConfig(
       half_reporting_fraction_encoded_config());
   EXPECT_EQ(0.5f, pingback_reporting_fraction());
+  EXPECT_FALSE(config()->IgnoreBlackListLongTermRulesForTesting());
+}
+
+TEST_F(DataReductionProxyConfigServiceClientTest,
+       ApplySerializedConfigIgnoreBlackList) {
+  Init(true);
+
+  config_client()->ApplySerializedConfig(ignore_black_list_encoded_config());
+  EXPECT_TRUE(config()->IgnoreBlackListLongTermRulesForTesting());
 }
 
 TEST_F(DataReductionProxyConfigServiceClientTest, EmptyConfigDisablesDRP) {

components/data_reduction_proxy/proto/client_config.proto

@@ -27,6 +27,9 @@ message ClientConfig {
   optional Duration refresh_duration = 4;
   // Configuration information for reporting pageload metrics.
   optional PageloadMetricsConfig pageload_metrics_config = 5;
+  // Prevents the host base and user base blacklisting behaviors for lite pages
+  // and server LoFi.
+  optional bool ignore_long_term_black_list_rules = 7;
 }
 
 // The configuration for reporting pageload metrics.

components/previews/content/previews_content_util.cc

@@ -42,7 +42,8 @@ content::PreviewsState DetermineEnabledClientPreviewsState(
       previews_decider->ShouldAllowPreviewAtECT(
           url_request, previews::PreviewsType::LOFI,
           previews::params::EffectiveConnectionTypeThresholdForClientLoFi(),
-          previews::params::GetBlackListedHostsForClientLoFiFieldTrial())) {
+          previews::params::GetBlackListedHostsForClientLoFiFieldTrial(),
+          false)) {
     previews_state |= content::CLIENT_LOFI_ON;
     return previews_state;
   }

components/previews/content/previews_content_util_unittest.cc

@@ -31,8 +31,8 @@ class PreviewEnabledPreviewsDecider : public PreviewsDecider {
       const net::URLRequest& request,
       PreviewsType type,
       net::EffectiveConnectionType effective_connection_type_threshold,
-      const std::vector<std::string>& host_blacklist_from_server)
-      const override {
+      const std::vector<std::string>& host_blacklist_from_server,
+      bool ignore_long_term_black_list_rules) const override {
     return IsEnabled(type);
   }
 
@@ -40,7 +40,7 @@ class PreviewEnabledPreviewsDecider : public PreviewsDecider {
                           PreviewsType type) const override {
     return ShouldAllowPreviewAtECT(request, type,
                                    params::GetECTThresholdForPreview(type),
-                                   std::vector<std::string>());
+                                   std::vector<std::string>(), false);
   }
 
   bool IsURLAllowedForPreview(const net::URLRequest& request,

components/previews/content/previews_io_data.cc

@@ -206,16 +206,20 @@ void PreviewsIOData::SetIgnorePreviewsBlacklistDecision(bool ignored) {
 
 bool PreviewsIOData::ShouldAllowPreview(const net::URLRequest& request,
                                         PreviewsType type) const {
+  DCHECK(type == PreviewsType::OFFLINE || type == PreviewsType::NOSCRIPT);
+  // Consumers that need to specify a blacklist or ignore flag should use
+  // ShouldAllowPreviewAtECT directly.
   return ShouldAllowPreviewAtECT(request, type,
                                  params::GetECTThresholdForPreview(type),
-                                 std::vector<std::string>());
+                                 std::vector<std::string>(), false);
 }
 
 bool PreviewsIOData::ShouldAllowPreviewAtECT(
     const net::URLRequest& request,
     PreviewsType type,
     net::EffectiveConnectionType effective_connection_type_threshold,
-    const std::vector<std::string>& host_blacklist_from_server) const {
+    const std::vector<std::string>& host_blacklist_from_server,
+    bool ignore_long_term_black_list_rules) const {
   if (!previews::params::ArePreviewsAllowed()) {
     return false;
   }
@@ -243,7 +247,9 @@ bool PreviewsIOData::ShouldAllowPreviewAtECT(
     // The blacklist will disallow certain hosts for periods of time based on
     // user's opting out of the preview.
     PreviewsEligibilityReason status = previews_black_list_->IsLoadedAndAllowed(
-        request.url(), type, &passed_reasons);
+        request.url(), type, ignore_long_term_black_list_rules,
+        &passed_reasons);
+
     if (status != PreviewsEligibilityReason::ALLOWED) {
       if (type == PreviewsType::LITE_PAGE) {
         PreviewsUserData::GetData(request)->set_black_listed_for_lite_page(
@@ -343,12 +349,13 @@ bool PreviewsIOData::ShouldAllowPreviewAtECT(
 
 bool PreviewsIOData::IsURLAllowedForPreview(const net::URLRequest& request,
                                             PreviewsType type) const {
+  DCHECK_EQ(PreviewsType::NOSCRIPT, type);
   if (previews_black_list_ && !blacklist_ignored_) {
     std::vector<PreviewsEligibilityReason> passed_reasons;
     // The blacklist will disallow certain hosts for periods of time based on
     // user's opting out of the preview.
     PreviewsEligibilityReason status = previews_black_list_->IsLoadedAndAllowed(
-        request.url(), type, &passed_reasons);
+        request.url(), type, false, &passed_reasons);
     if (status != PreviewsEligibilityReason::ALLOWED) {
       if (type == PreviewsType::LITE_PAGE) {
         PreviewsUserData::GetData(request)->set_black_listed_for_lite_page(

components/previews/content/previews_io_data.h

@@ -107,8 +107,8 @@ class PreviewsIOData : public PreviewsDecider,
       const net::URLRequest& request,
       PreviewsType type,
       net::EffectiveConnectionType effective_connection_type_threshold,
-      const std::vector<std::string>& host_blacklist_from_server)
-      const override;
+      const std::vector<std::string>& host_blacklist_from_server,
+      bool ignore_long_term_black_list_rules) const override;
   bool IsURLAllowedForPreview(const net::URLRequest& request,
                               PreviewsType type) const override;
 

components/previews/core/previews_black_list.cc

@@ -147,6 +147,7 @@ void PreviewsBlackList::AddPreviewNavigationSync(const GURL& url,
 PreviewsEligibilityReason PreviewsBlackList::IsLoadedAndAllowed(
     const GURL& url,
     PreviewsType type,
+    bool ignore_long_term_black_list_rules,
     std::vector<PreviewsEligibilityReason>* passed_reasons) const {
   DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK(url.has_host());
@@ -161,6 +162,8 @@ PreviewsEligibilityReason PreviewsBlackList::IsLoadedAndAllowed(
     return PreviewsEligibilityReason::USER_RECENTLY_OPTED_OUT;
   }
   passed_reasons->push_back(PreviewsEligibilityReason::USER_RECENTLY_OPTED_OUT);
+  if (ignore_long_term_black_list_rules)
+    return PreviewsEligibilityReason::ALLOWED;
   if (host_indifferent_black_list_item_->IsBlackListed(clock_->Now()))
     return PreviewsEligibilityReason::USER_BLACKLISTED;
   passed_reasons->push_back(PreviewsEligibilityReason::USER_BLACKLISTED);

components/previews/core/previews_black_list.h

@@ -108,6 +108,7 @@ class PreviewsBlackList {
   virtual PreviewsEligibilityReason IsLoadedAndAllowed(
       const GURL& url,
       PreviewsType type,
+      bool ignore_long_term_black_list_rules,
       std::vector<PreviewsEligibilityReason>* passed_reasons) const;
 
   // Asynchronously deletes all entries in the in-memory black list. Informs

components/previews/core/previews_decider.h

@@ -30,7 +30,8 @@ class PreviewsDecider {
       const net::URLRequest& request,
       PreviewsType type,
       net::EffectiveConnectionType effective_connection_type_threshold,
-      const std::vector<std::string>& host_blacklist_from_server) const = 0;
+      const std::vector<std::string>& host_blacklist_from_server,
+      bool ignore_long_term_black_list_rules) const = 0;
 
   // Same as ShouldAllowPreviewAtECT, but uses the previews default
   // EffectiveConnectionType and no blacklisted hosts from the server.

components/previews/core/test_previews_decider.cc

@@ -15,7 +15,8 @@ bool TestPreviewsDecider::ShouldAllowPreviewAtECT(
     const net::URLRequest& request,
     previews::PreviewsType type,
     net::EffectiveConnectionType effective_connection_type_threshold,
-    const std::vector<std::string>& host_blacklist_from_server) const {
+    const std::vector<std::string>& host_blacklist_from_server,
+    bool ignore_long_term_black_list_rules) const {
   return allow_previews_;
 }
 

components/previews/core/test_previews_decider.h

@@ -20,8 +20,8 @@ class TestPreviewsDecider : public previews::PreviewsDecider {
       const net::URLRequest& request,
       previews::PreviewsType type,
       net::EffectiveConnectionType effective_connection_type_threshold,
-      const std::vector<std::string>& host_blacklist_from_server)
-      const override;
+      const std::vector<std::string>& host_blacklist_from_server,
+      bool ignore_long_term_black_list_rules) const override;
   bool ShouldAllowPreview(const net::URLRequest& request,
                           previews::PreviewsType type) const override;
   bool IsURLAllowedForPreview(const net::URLRequest& request,