Add login robots decider agent

LoginRobotsDeciderAgent is added which is the login and robots checking
based decider implementation that overrides PublicResourceDecider.

This new agent class maintains a cache of robots rules for different
origins, for checking robots.txt eligibility.

Change-Id: I58d0cc6afd6131b2f26f26e8aea8757a6c002542
Bug: 1144836
TBR: reillyg@chromium.org
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2574073
Commit-Queue: rajendrant <rajendrant@chromium.org>
Reviewed-by: Michael Crouse <mcrouse@chromium.org>
Cr-Commit-Position: refs/heads/master@{#835874}

chrome/renderer/BUILD.gn

@@ -96,6 +96,8 @@ static_library("renderer") {
     "plugins/plugin_uma.h",
     "previews/resource_loading_hints_agent.cc",
     "previews/resource_loading_hints_agent.h",
+    "subresource_redirect/login_robots_decider_agent.cc",
+    "subresource_redirect/login_robots_decider_agent.h",
     "subresource_redirect/public_image_hints_decider_agent.cc",
     "subresource_redirect/public_image_hints_decider_agent.h",
     "subresource_redirect/public_resource_decider.h",
@@ -104,6 +106,8 @@ static_library("renderer") {
     "subresource_redirect/redirect_result.h",
     "subresource_redirect/robots_rules_parser.cc",
     "subresource_redirect/robots_rules_parser.h",
+    "subresource_redirect/robots_rules_parser_cache.cc",
+    "subresource_redirect/robots_rules_parser_cache.h",
     "subresource_redirect/subresource_redirect_params.cc",
     "subresource_redirect/subresource_redirect_params.h",
     "subresource_redirect/subresource_redirect_url_loader_throttle.cc",

chrome/renderer/chrome_content_renderer_client.cc

@@ -60,6 +60,7 @@
 #include "chrome/renderer/plugins/pdf_plugin_placeholder.h"
 #include "chrome/renderer/plugins/plugin_uma.h"
 #include "chrome/renderer/previews/resource_loading_hints_agent.h"
+#include "chrome/renderer/subresource_redirect/login_robots_decider_agent.h"
 #include "chrome/renderer/subresource_redirect/public_image_hints_decider_agent.h"
 #include "chrome/renderer/subresource_redirect/subresource_redirect_params.h"
 #include "chrome/renderer/sync_encryption_keys_extension.h"
@@ -605,10 +606,13 @@ void ChromeContentRendererClient::RenderFrameCreated(
 
   new previews::ResourceLoadingHintsAgent(associated_interfaces, render_frame);
 
-  if (subresource_redirect::IsPublicImageHintsBasedCompressionEnabled())
+  if (subresource_redirect::IsPublicImageHintsBasedCompressionEnabled()) {
     new subresource_redirect::PublicImageHintsDeciderAgent(
         associated_interfaces, render_frame);
-
+  } else if (subresource_redirect::IsLoginRobotsCheckedCompressionEnabled()) {
+    new subresource_redirect::LoginRobotsDeciderAgent(associated_interfaces,
+                                                      render_frame);
+  }
   if (translate::IsSubFrameTranslationEnabled()) {
     new translate::PerFrameTranslateAgent(
         render_frame, ISOLATED_WORLD_ID_TRANSLATE, associated_interfaces);

chrome/renderer/subresource_redirect/DEPS

@@ -2,4 +2,5 @@ include_rules = [
   "+components/base32",
   "+services/metrics/public/cpp",
   "+components/data_reduction_proxy/proto/robots_rules.pb.h",
+  "+services/network/test/test_utils.h",
 ]

chrome/renderer/subresource_redirect/login_robots_decider_agent.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/renderer/subresource_redirect/login_robots_decider_agent.h"
+
+#include "base/metrics/field_trial_params.h"
+#include "base/metrics/histogram_macros.h"
+#include "base/no_destructor.h"
+#include "base/strings/strcat.h"
+#include "chrome/renderer/subresource_redirect/redirect_result.h"
+#include "chrome/renderer/subresource_redirect/robots_rules_parser.h"
+#include "chrome/renderer/subresource_redirect/robots_rules_parser_cache.h"
+#include "chrome/renderer/subresource_redirect/subresource_redirect_params.h"
+#include "content/public/renderer/render_frame.h"
+
+namespace subresource_redirect {
+
+namespace {
+
+// Returns the robots rules parser cache that is shared across the RenderFrames
+// in the renderer.
+RobotsRulesParserCache& GetRobotsRulesParserCache() {
+  static base::NoDestructor<RobotsRulesParserCache> instance;
+  return *instance;
+}
+
+// Converts the RobotsRulesParser::CheckResult enum to RedirectResult enum.
+RedirectResult ConvertToRedirectResult(
+    RobotsRulesParser::CheckResult check_result) {
+  switch (check_result) {
+    case RobotsRulesParser::CheckResult::kAllowed:
+      return RedirectResult::kRedirectable;
+    case RobotsRulesParser::CheckResult::kDisallowed:
+      return RedirectResult::kIneligibleRobotsDisallowed;
+    case RobotsRulesParser::CheckResult::kTimedout:
+    case RobotsRulesParser::CheckResult::kDisallowedAfterTimeout:
+      return RedirectResult::kIneligibleRobotsTimeout;
+  }
+}
+
+// Converts the robots rules CheckResult to RedirectResult and passes to the
+// callback.
+void SendRedirectResultToCallback(
+    LoginRobotsDeciderAgent::ShouldRedirectDecisionCallback callback,
+    RobotsRulesParser::CheckResult check_result) {
+  std::move(callback).Run(ConvertToRedirectResult(check_result));
+}
+
+}  // namespace
+
+LoginRobotsDeciderAgent::LoginRobotsDeciderAgent(
+    blink::AssociatedInterfaceRegistry* associated_interfaces,
+    content::RenderFrame* render_frame)
+    : PublicResourceDeciderAgent(associated_interfaces, render_frame) {
+  DCHECK(IsLoginRobotsCheckedCompressionEnabled());
+  DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+}
+
+LoginRobotsDeciderAgent::~LoginRobotsDeciderAgent() = default;
+
+void LoginRobotsDeciderAgent::UpdateRobotsRules(const url::Origin& origin,
+                                                const std::string& rules) {
+  DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+  DCHECK(!origin.opaque());
+  if (!render_frame()->IsMainFrame())
+    return;
+  GetRobotsRulesParserCache().UpdateRobotsRules(origin, rules);
+}
+
+base::Optional<RedirectResult>
+LoginRobotsDeciderAgent::ShouldRedirectSubresource(
+    const GURL& url,
+    ShouldRedirectDecisionCallback callback) {
+  DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+  DCHECK(url.is_valid());
+  if (!render_frame()->IsMainFrame())
+    return RedirectResult::kIneligibleSubframeResource;
+
+  base::Optional<RobotsRulesParser::CheckResult> result =
+      GetRobotsRulesParserCache().CheckRobotsRules(
+          url,
+          base::BindOnce(&SendRedirectResultToCallback, std::move(callback)));
+  if (result)
+    return ConvertToRedirectResult(*result);
+
+  return base::nullopt;
+}
+
+void LoginRobotsDeciderAgent::RecordMetricsOnLoadFinished(
+    const GURL& url,
+    int64_t content_length,
+    RedirectResult redirect_result) {
+  LOCAL_HISTOGRAM_ENUMERATION(
+      "SubresourceRedirect.LoginRobotsDeciderAgent.RedirectResult",
+      redirect_result);
+  // TODO(crbug.com/1148980): Record coverage metrics
+}
+
+void LoginRobotsDeciderAgent::SetCompressPublicImagesHints(
+    mojom::CompressPublicImagesHintsPtr images_hints) {
+  // This mojo from browser process should not be called for robots rules based
+  // subresource compression on non logged-in pages.
+  DCHECK(IsLoginRobotsCheckedCompressionEnabled());
+  NOTREACHED();
+}
+
+}  // namespace subresource_redirect

chrome/renderer/subresource_redirect/login_robots_decider_agent.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_RENDERER_SUBRESOURCE_REDIRECT_LOGIN_ROBOTS_DECIDER_AGENT_H_
+#define CHROME_RENDERER_SUBRESOURCE_REDIRECT_LOGIN_ROBOTS_DECIDER_AGENT_H_
+
+#include "base/macros.h"
+#include "base/threading/thread_checker.h"
+#include "base/timer/timer.h"
+#include "chrome/renderer/subresource_redirect/public_resource_decider_agent.h"
+#include "chrome/renderer/subresource_redirect/robots_rules_parser.h"
+#include "url/gurl.h"
+#include "url/origin.h"
+
+namespace subresource_redirect {
+
+// The decider agent implementation that allows subresource redirect compression
+// based on robots rules on non-logged-in pages. Currently only handles
+// mainframes.
+// TODO(crbug.com/1148974): Handle subresources in subframes as well.
+// TODO(crbug.com/1149853): Add the logged-in checks.
+class LoginRobotsDeciderAgent : public PublicResourceDeciderAgent {
+ public:
+  LoginRobotsDeciderAgent(
+      blink::AssociatedInterfaceRegistry* associated_interfaces,
+      content::RenderFrame* render_frame);
+  ~LoginRobotsDeciderAgent() override;
+
+  LoginRobotsDeciderAgent(const LoginRobotsDeciderAgent&) = delete;
+  LoginRobotsDeciderAgent& operator=(const LoginRobotsDeciderAgent&) = delete;
+
+  // Updates the robots rules for the origin.
+  void UpdateRobotsRules(const url::Origin& origin, const std::string& rules);
+
+ private:
+  friend class SubresourceRedirectLoginRobotsDeciderAgentTest;
+
+  // mojom::SubresourceRedirectHintsReceiver:
+  void SetCompressPublicImagesHints(
+      mojom::CompressPublicImagesHintsPtr images_hints) override;
+
+  // PublicResourceDeciderAgent:
+  base::Optional<RedirectResult> ShouldRedirectSubresource(
+      const GURL& url,
+      ShouldRedirectDecisionCallback callback) override;
+  void RecordMetricsOnLoadFinished(const GURL& url,
+                                   int64_t content_length,
+                                   RedirectResult redirect_result) override;
+
+  bool IsMainFrame() const;
+
+  THREAD_CHECKER(thread_checker_);
+};
+
+}  // namespace subresource_redirect
+
+#endif  // CHROME_RENDERER_SUBRESOURCE_REDIRECT_LOGIN_ROBOTS_DECIDER_AGENT_H_

chrome/renderer/subresource_redirect/login_robots_decider_agent_browsertest.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/memory/weak_ptr.h"
+#include "base/test/scoped_feature_list.h"
+#include "chrome/renderer/subresource_redirect/login_robots_decider_agent.h"
+#include "chrome/renderer/subresource_redirect/login_robots_decider_test_util.h"
+#include "chrome/renderer/subresource_redirect/subresource_redirect_url_loader_throttle.h"
+#include "chrome/renderer/subresource_redirect/subresource_redirect_util.h"
+#include "chrome/test/base/chrome_render_view_test.h"
+#include "chrome/test/base/ui_test_utils.h"
+#include "content/public/renderer/render_frame.h"
+#include "content/public/renderer/render_view.h"
+#include "services/network/public/cpp/resource_request.h"
+#include "services/network/public/mojom/fetch_api.mojom-shared.h"
+#include "testing/gtest/include/gtest/gtest.h"
+#include "third_party/blink/public/common/features.h"
+#include "third_party/blink/public/common/loader/previews_state.h"
+#include "third_party/blink/public/platform/web_network_state_notifier.h"
+#include "third_party/blink/public/platform/web_url.h"
+#include "third_party/blink/public/platform/web_url_request.h"
+
+namespace subresource_redirect {
+
+// Helper class that exposes a callback for the decider to call, and maintains
+// the redirect result for verification.
+class RedirectResultReceiver {
+ public:
+  PublicResourceDecider::ShouldRedirectDecisionCallback GetCallback() {
+    return base::BindOnce(
+        &RedirectResultReceiver::OnShouldRedirectDecisionCallback,
+        weak_ptr_factory_.GetWeakPtr());
+  }
+
+  RedirectResult redirect_result() const { return redirect_result_; }
+
+  bool did_receive_result() const { return did_receive_result_; }
+
+ private:
+  void OnShouldRedirectDecisionCallback(RedirectResult redirect_result) {
+    EXPECT_FALSE(did_receive_result_);
+    did_receive_result_ = true;
+    redirect_result_ = redirect_result;
+  }
+
+  RedirectResult redirect_result_;
+  bool did_receive_result_ = false;
+  base::WeakPtrFactory<RedirectResultReceiver> weak_ptr_factory_{this};
+};
+
+class SubresourceRedirectLoginRobotsDeciderAgentTest
+    : public ChromeRenderViewTest {
+ public:
+  void SetUpRobotsRules(const std::string& origin,
+                        const std::vector<Rule>& patterns) {
+    login_robots_decider_agent_->UpdateRobotsRules(
+        url::Origin::Create(GURL(origin)), GetRobotsRulesProtoString(patterns));
+  }
+
+  bool ShouldRedirectSubresource(const std::string& url) {
+    RedirectResultReceiver result_receiver;
+    auto immediate_result =
+        login_robots_decider_agent_->ShouldRedirectSubresource(
+            GURL(url), result_receiver.GetCallback());
+    if (immediate_result) {
+      // When the reult was sent immediately, callback should not be invoked.
+      EXPECT_FALSE(result_receiver.did_receive_result());
+      return *immediate_result == RedirectResult::kRedirectable;
+    }
+    task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(10));
+    return result_receiver.did_receive_result() &&
+           result_receiver.redirect_result() == RedirectResult::kRedirectable;
+  }
+
+ protected:
+  void SetUp() override {
+    ChromeRenderViewTest::SetUp();
+    scoped_feature_list_.InitWithFeaturesAndParameters(
+        {{blink::features::kSubresourceRedirect,
+          {{"enable_login_robots_based_compression", "true"},
+           {"enable_public_image_hints_based_compression", "false"}}}},
+        {});
+    login_robots_decider_agent_ = new LoginRobotsDeciderAgent(
+        &associated_interfaces_, view_->GetMainRenderFrame());
+  }
+
+  LoginRobotsDeciderAgent* login_robots_decider_agent_;
+  base::test::ScopedFeatureList scoped_feature_list_;
+};
+
+TEST_F(SubresourceRedirectLoginRobotsDeciderAgentTest,
+       TestAllowDisallowSingleOrigin) {
+  SetUpRobotsRules("https://foo.com", {{kRuleTypeAllow, "/public"},
+                                       {kRuleTypeDisallow, "/private"}});
+  EXPECT_TRUE(ShouldRedirectSubresource("https://foo.com/public.jpg"));
+  EXPECT_FALSE(ShouldRedirectSubresource("https://foo.com/private.jpg"));
+
+  EXPECT_FALSE(ShouldRedirectSubresource("https://m.foo.com/public.jpg"));
+  EXPECT_FALSE(ShouldRedirectSubresource("https://www.foo.com/public.jpg"));
+  EXPECT_FALSE(ShouldRedirectSubresource("https://bar.com/public.jpg"));
+}
+
+TEST_F(SubresourceRedirectLoginRobotsDeciderAgentTest,
+       TestHTTPRulesAreSeparate) {
+  SetUpRobotsRules("https://foo.com", {{kRuleTypeAllow, "/public"},
+                                       {kRuleTypeDisallow, "/private"}});
+  EXPECT_FALSE(ShouldRedirectSubresource("http://foo.com/public.jpg"));
+
+  SetUpRobotsRules("http://foo.com", {{kRuleTypeAllow, "/public"},
+                                      {kRuleTypeDisallow, "/private"}});
+  EXPECT_TRUE(ShouldRedirectSubresource("http://foo.com/public.jpg"));
+
+  EXPECT_FALSE(ShouldRedirectSubresource("http://m.foo.com/public.jpg"));
+  EXPECT_FALSE(ShouldRedirectSubresource("http://www.foo.com/public.jpg"));
+}
+
+TEST_F(SubresourceRedirectLoginRobotsDeciderAgentTest, TestURLWithArguments) {
+  SetUpRobotsRules("https://foo.com",
+                   {{kRuleTypeAllow, "/*.jpg$"},
+                    {kRuleTypeDisallow, "/*.png?*arg_disallowed"},
+                    {kRuleTypeAllow, "/*.png"},
+                    {kRuleTypeDisallow, "/*.gif?*arg_disallowed"},
+                    {kRuleTypeAllow, "/*.gif"},
+                    {kRuleTypeDisallow, "/"}});
+  EXPECT_TRUE(ShouldRedirectSubresource("https://foo.com/allowed.jpg"));
+  EXPECT_TRUE(ShouldRedirectSubresource("https://foo.com/allowed.png"));
+  EXPECT_TRUE(ShouldRedirectSubresource("https://foo.com/allowed.gif"));
+  EXPECT_FALSE(ShouldRedirectSubresource("https://foo.com/disallowed.jpg?arg"));
+  EXPECT_TRUE(ShouldRedirectSubresource("https://foo.com/allowed.png?arg"));
+  EXPECT_TRUE(
+      ShouldRedirectSubresource("https://foo.com/allowed.png?arg_allowed"));
+  EXPECT_TRUE(ShouldRedirectSubresource(
+      "https://foo.com/allowed.png?arg_allowed&arg_alllowed2"));
+  EXPECT_FALSE(ShouldRedirectSubresource(
+      "https://foo.com/disallowed.png?arg_disallowed"));
+  EXPECT_FALSE(ShouldRedirectSubresource(
+      "https://foo.com/disallowed.png?arg_disallowed&arg_disallowed2"));
+}
+
+TEST_F(SubresourceRedirectLoginRobotsDeciderAgentTest,
+       TestRulesAreCaseSensitive) {
+  SetUpRobotsRules("https://foo.com", {{kRuleTypeAllow, "/allowed"},
+                                       {kRuleTypeAllow, "/CamelCase"},
+                                       {kRuleTypeAllow, "/CAPITALIZE"},
+                                       {kRuleTypeDisallow, "/"}});
+  EXPECT_TRUE(ShouldRedirectSubresource("https://foo.com/allowed.jpg"));
+  EXPECT_TRUE(ShouldRedirectSubresource("https://foo.com/CamelCase.jpg"));
+  EXPECT_TRUE(ShouldRedirectSubresource("https://foo.com/CAPITALIZE.jpg"));
+  EXPECT_FALSE(ShouldRedirectSubresource("https://foo.com/Allowed.jpg"));
+  EXPECT_FALSE(ShouldRedirectSubresource("https://foo.com/camelcase.jpg"));
+  EXPECT_FALSE(ShouldRedirectSubresource("https://foo.com/capitalize.jpg"));
+}
+
+}  // namespace subresource_redirect

chrome/renderer/subresource_redirect/public_image_hints_decider_agent.cc

@@ -84,6 +84,8 @@ base::Optional<RedirectResult>
 PublicImageHintsDeciderAgent::ShouldRedirectSubresource(
     const GURL& url,
     ShouldRedirectDecisionCallback callback) {
+  if (!IsMainFrame())
+    return RedirectResult::kIneligibleSubframeResource;
   if (!public_image_urls_)
     return RedirectResult::kIneligibleImageHintsUnavailable;
 
@@ -152,9 +154,13 @@ void PublicImageHintsDeciderAgent::RecordMetrics(
     case RedirectResult::kUnknown:
     case RedirectResult::kIneligibleRedirectFailed:
     case RedirectResult::kIneligibleBlinkDisallowed:
+    case RedirectResult::kIneligibleSubframeResource:
       public_image_compression_data_use.SetIneligibleOtherImageBytes(
           content_length);
       break;
+    case RedirectResult::kIneligibleRobotsDisallowed:
+    case RedirectResult::kIneligibleRobotsTimeout:
+      NOTREACHED();
   }
   mojo::PendingRemote<ukm::mojom::UkmRecorderInterface> recorder;
   content::RenderThread::Get()->BindHostReceiver(

chrome/renderer/subresource_redirect/redirect_result.h

@@ -22,6 +22,9 @@ enum class RedirectResult {
   // element, CSP/CORS security restrictions, javascript initiated image, etc.
   kIneligibleBlinkDisallowed,
 
+  // Because the resource is fetched for a subframe.
+  kIneligibleSubframeResource,
+
   // Because the compressed subresource fetch failed, and then the original
   // subresource was loaded.
   kIneligibleRedirectFailed,
@@ -45,7 +48,16 @@ enum class RedirectResult {
   // Because the image URL was not found in the image hints.
   kIneligibleMissingInImageHints,
 
-  kMaxValue = RedirectResult::kIneligibleMissingInImageHints
+  // Possible reasons for ineligibility due to login and robots rules
+  // based approach:
+
+  // Because the image was disallowed by robots rules of the image origin.
+  kIneligibleRobotsDisallowed,
+
+  // Because the robots rules fetch timedout.
+  kIneligibleRobotsTimeout,
+
+  kMaxValue = RedirectResult::kIneligibleRobotsTimeout
 };
 
 }  // namespace subresource_redirect

chrome/renderer/subresource_redirect/robots_rules_parser.h

@@ -64,7 +64,7 @@ class RobotsRulesParser {
   RobotsRulesParser& operator=(const RobotsRulesParser&) = delete;
 
   // Update the robots rules. This causes any pending check requests to be
-  // processed immediately and called with th result.
+  // processed immediately and called with the result.
   void UpdateRobotsRules(const std::string& rules);
 
   // Check whether the URL is allowed or disallowed by robots rules. When the

chrome/renderer/subresource_redirect/robots_rules_parser_cache.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/renderer/subresource_redirect/robots_rules_parser_cache.h"
+
+#include "chrome/renderer/subresource_redirect/subresource_redirect_params.h"
+
+namespace subresource_redirect {
+
+RobotsRulesParserCache::RobotsRulesParserCache()
+    : parsers_cache_(MaxRobotsRulesParsersCacheSize()) {}
+
+RobotsRulesParserCache::~RobotsRulesParserCache() = default;
+
+void RobotsRulesParserCache::UpdateRobotsRules(const url::Origin& origin,
+                                               const std::string& rules) {
+  GetRobotsRulesParserForOrigin(origin).UpdateRobotsRules(rules);
+}
+
+base::Optional<RobotsRulesParser::CheckResult>
+RobotsRulesParserCache::CheckRobotsRules(
+    const GURL& url,
+    RobotsRulesParser::CheckResultCallback callback) {
+  return GetRobotsRulesParserForOrigin(url::Origin::Create(url))
+      .CheckRobotsRules(url, std::move(callback));
+}
+
+RobotsRulesParser& RobotsRulesParserCache::GetRobotsRulesParserForOrigin(
+    const url::Origin& origin) {
+  auto it = parsers_cache_.Get(origin);
+  if (it == parsers_cache_.end())
+    it = parsers_cache_.Put(origin, std::make_unique<RobotsRulesParser>());
+  return *it->second;
+}
+
+}  // namespace subresource_redirect

chrome/renderer/subresource_redirect/robots_rules_parser_cache.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_RENDERER_SUBRESOURCE_REDIRECT_ROBOTS_RULES_PARSER_CACHE_H_
+#define CHROME_RENDERER_SUBRESOURCE_REDIRECT_ROBOTS_RULES_PARSER_CACHE_H_
+
+#include "base/containers/mru_cache.h"
+#include "chrome/renderer/subresource_redirect/robots_rules_parser.h"
+#include "url/gurl.h"
+#include "url/origin.h"
+
+namespace subresource_redirect {
+
+// The store of robots rules parsers keyed by origin
+class RobotsRulesParserCache {
+ public:
+  RobotsRulesParserCache();
+  ~RobotsRulesParserCache();
+
+  RobotsRulesParserCache(const RobotsRulesParserCache&) = delete;
+  RobotsRulesParserCache& operator=(const RobotsRulesParserCache&) = delete;
+
+  // Update the robots rules to the parser for the |origin|.
+  void UpdateRobotsRules(const url::Origin& origin, const std::string& rules);
+
+  // Returns the result of checking whether resource |url| is allowed by robots
+  // rules parser for the url origin. When the determination can be made
+  // immediately, the decision should be returned. Otherwise base::nullopt
+  // should be returned and the |callback| will be invoked when the decision was
+  // made.
+  base::Optional<RobotsRulesParser::CheckResult> CheckRobotsRules(
+      const GURL& url,
+      RobotsRulesParser::CheckResultCallback callback);
+
+ private:
+  // Returns a reference to the robots rules parser for the |origin| from the
+  // cache. An entry is created if it does not exist.
+  RobotsRulesParser& GetRobotsRulesParserForOrigin(const url::Origin& origin);
+
+  // The underlying cache of robots rules parsers.
+  base::MRUCache<url::Origin, std::unique_ptr<RobotsRulesParser>>
+      parsers_cache_;
+};
+
+}  // namespace subresource_redirect
+
+#endif  // CHROME_RENDERER_SUBRESOURCE_REDIRECT_ROBOTS_RULES_PARSER_CACHE_H_

chrome/renderer/subresource_redirect/subresource_redirect_params.cc

@@ -31,10 +31,25 @@ url::Origin GetSubresourceRedirectOrigin() {
 }
 
 bool IsPublicImageHintsBasedCompressionEnabled() {
-  return IsSubresourceRedirectEnabled() &&
+  bool is_enabled = IsSubresourceRedirectEnabled() &&
                     base::GetFieldTrialParamByFeatureAsBool(
                         blink::features::kSubresourceRedirect,
                         "enable_public_image_hints_based_compression", true);
+  // Only one of the public image hints or login and robots based image
+  // compression should be active.
+  DCHECK(!is_enabled || !IsLoginRobotsCheckedCompressionEnabled());
+  return is_enabled;
+}
+
+bool IsLoginRobotsCheckedCompressionEnabled() {
+  bool is_enabled = IsSubresourceRedirectEnabled() &&
+                    base::GetFieldTrialParamByFeatureAsBool(
+                        blink::features::kSubresourceRedirect,
+                        "enable_login_robots_based_compression", false);
+  // Only one of the public image hints or login and robots based image
+  // compression should be active.
+  DCHECK(!is_enabled || !IsPublicImageHintsBasedCompressionEnabled());
+  return is_enabled;
 }
 
 bool ShouldCompressionServerRedirectSubresource() {
@@ -63,4 +78,10 @@ base::TimeDelta GetRobotsRulesReceiveTimeout() {
           10));
 }
 
+int MaxRobotsRulesParsersCacheSize() {
+  return base::GetFieldTrialParamByFeatureAsInt(
+      blink::features::kSubresourceRedirect,
+      "max_robots_rules_parsers_cache_size", 20);
+}
+
 }  // namespace subresource_redirect

chrome/renderer/subresource_redirect/subresource_redirect_params.h

@@ -19,6 +19,11 @@ url::Origin GetSubresourceRedirectOrigin();
 // Returns if the public image hints based subresource compression is enabled.
 bool IsPublicImageHintsBasedCompressionEnabled();
 
+// Returns if the login and robots checks based subresource compression is
+// enabled. This compresses non logged-in pages and subresources allowed by
+// robots.txt rules.
+bool IsLoginRobotsCheckedCompressionEnabled();
+
 // Should the subresource be redirected to its compressed version. This returns
 // false if only coverage metrics need to be recorded and actual redirection
 // should not happen.
@@ -35,6 +40,10 @@ int64_t GetHintsReceiveTimeout();
 // the subresource should be fetched directly from the origin.
 base::TimeDelta GetRobotsRulesReceiveTimeout();
 
+// The maximum number of robots rules parsers the renderer should cache locally
+// for reuse by the renderframes in the renderer process.
+int MaxRobotsRulesParsersCacheSize();
+
 }  // namespace subresource_redirect
 
 #endif  // CHROME_RENDERER_SUBRESOURCE_REDIRECT_SUBRESOURCE_REDIRECT_PARAMS_H_

chrome/renderer/subresource_redirect/subresource_redirect_url_loader_throttle.cc

@@ -50,8 +50,11 @@ std::unique_ptr<SubresourceRedirectURLLoaderThrottle>
 SubresourceRedirectURLLoaderThrottle::MaybeCreateThrottle(
     const blink::WebURLRequest& request,
     int render_frame_id) {
-  if (IsPublicImageHintsBasedCompressionEnabled() &&
-      request.GetRequestDestination() ==
+  if (!IsPublicImageHintsBasedCompressionEnabled() &&
+      !IsLoginRobotsCheckedCompressionEnabled()) {
+    return nullptr;
+  }
+  if (request.GetRequestDestination() ==
           network::mojom::RequestDestination::kImage &&
       request.Url().ProtocolIs(url::kHttpsScheme) &&
       blink::WebNetworkStateNotifier::SaveDataEnabled() &&
@@ -69,6 +72,8 @@ SubresourceRedirectURLLoaderThrottle::SubresourceRedirectURLLoaderThrottle(
     int render_frame_id,
     bool allowed_to_redirect)
     : render_frame_id_(render_frame_id) {
+  DCHECK(IsPublicImageHintsBasedCompressionEnabled() ||
+         IsLoginRobotsCheckedCompressionEnabled());
   redirect_result_ = allowed_to_redirect
                          ? RedirectResult::kRedirectable
                          : RedirectResult::kIneligibleBlinkDisallowed;
@@ -80,7 +85,6 @@ SubresourceRedirectURLLoaderThrottle::~SubresourceRedirectURLLoaderThrottle() =
 void SubresourceRedirectURLLoaderThrottle::WillStartRequest(
     network::ResourceRequest* request,
     bool* defer) {
-  DCHECK(IsPublicImageHintsBasedCompressionEnabled());
   DCHECK_EQ(request->destination, network::mojom::RequestDestination::kImage);
   DCHECK(request->url.SchemeIs(url::kHttpsScheme));