Previews Server HTTPS URLLoader pingback support

This CL supports the pingback for HTTPS Server Previews URLLoader
implementation. During committed preview decision time, if we are
serving a lite page, the ServerLitePageInfo is backfilled from the
navigation handle. This does not support reporting on failures.

Bug: 921757,936496
Change-Id: Ic474e0d1bd49f6a181704edcfe24e5f1025d6089
Reviewed-on: https://chromium-review.googlesource.com/c/1487916
Commit-Queue: Ryan Sturm <ryansturm@chromium.org>
Reviewed-by: Robert Ogden <robertogden@chromium.org>
Cr-Commit-Position: refs/heads/master@{#636241}

chrome/browser/chrome_content_browser_client.cc

@@ -5384,7 +5384,8 @@ ChromeContentBrowserClient::DetermineCommittedPreviewsForURL(
     data_reduction_proxy::DataReductionProxyData* drp_data,
     previews::PreviewsUserData* previews_user_data,
     const previews::PreviewsDecider* previews_decider,
-    content::PreviewsState initial_state) {
+    content::PreviewsState initial_state,
+    content::NavigationHandle* navigation_handle) {
   DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
   if (!previews::HasEnabledPreviews(initial_state))
     return content::PREVIEWS_OFF;
@@ -5396,7 +5397,8 @@ ChromeContentBrowserClient::DetermineCommittedPreviewsForURL(
 
   // Check the various other client previews types.
   return previews::DetermineCommittedClientPreviewsState(
-      previews_user_data, url, previews_state, previews_decider);
+      previews_user_data, url, previews_state, previews_decider,
+      navigation_handle);
 }
 
 content::PreviewsState ChromeContentBrowserClient::DetermineCommittedPreviews(
@@ -5478,7 +5480,7 @@ content::PreviewsState ChromeContentBrowserClient::DetermineCommittedPreviews(
   // Determine effective PreviewsState for this committed main frame response.
   content::PreviewsState committed_state = DetermineCommittedPreviewsForURL(
       navigation_handle->GetURL(), drp_data.get(), previews_user_data,
-      previews_decider_impl, initial_state);
+      previews_decider_impl, initial_state, navigation_handle);
 
   // Double check that we never serve a preview when we have a
   // cache-control:no-transform directive.

chrome/browser/chrome_content_browser_client.h

@@ -576,7 +576,8 @@ class ChromeContentBrowserClient : public content::ContentBrowserClient {
       data_reduction_proxy::DataReductionProxyData* drp_data,
       previews::PreviewsUserData* previews_user_data,
       const previews::PreviewsDecider* previews_decider,
-      content::PreviewsState initial_state);
+      content::PreviewsState initial_state,
+      content::NavigationHandle* navigation_handle);
 
  protected:
   static bool HandleWebUI(GURL* url, content::BrowserContext* browser_context);

chrome/browser/previews/previews_content_util.cc

@@ -4,11 +4,14 @@
 
 #include "chrome/browser/previews/previews_content_util.h"
 
+#include <memory>
 #include <string>
 #include <vector>
 
+#include "base/feature_list.h"
 #include "base/metrics/histogram_functions.h"
 #include "base/metrics/histogram_macros.h"
+#include "base/optional.h"
 #include "base/strings/stringprintf.h"
 #include "chrome/browser/content_settings/cookie_settings_factory.h"
 #include "chrome/browser/previews/previews_lite_page_decider.h"
@@ -18,8 +21,10 @@
 #include "chrome/browser/previews/previews_service_factory.h"
 #include "chrome/browser/profiles/profile.h"
 #include "components/content_settings/core/browser/cookie_settings.h"
+#include "components/data_reduction_proxy/core/browser/data_reduction_proxy_request_options.h"
 #include "components/previews/content/previews_user_data.h"
 #include "components/previews/core/previews_experiments.h"
+#include "components/previews/core/previews_features.h"
 #include "components/previews/core/previews_lite_page_redirect.h"
 #include "content/public/browser/browser_context.h"
 #include "content/public/browser/navigation_handle.h"
@@ -27,6 +32,7 @@
 #include "net/base/ip_address.h"
 #include "net/base/url_util.h"
 #include "net/nqe/effective_connection_type.h"
+#include "services/network/public/cpp/features.h"
 
 namespace previews {
 
@@ -150,6 +156,38 @@ bool ShouldAllowRedirectPreview(content::NavigationHandle* navigation_handle) {
   return true;
 }
 
+std::unique_ptr<PreviewsUserData::ServerLitePageInfo>
+CreateServerLitePageInfoFromNavigationHandle(
+    content::NavigationHandle* navigation_handle) {
+  auto server_lite_page_info =
+      std::make_unique<PreviewsUserData::ServerLitePageInfo>();
+
+  server_lite_page_info->original_navigation_start =
+      navigation_handle->NavigationStart();
+
+  const net::HttpRequestHeaders& headers =
+      navigation_handle->GetRequestHeaders();
+
+  base::Optional<uint64_t> page_id = data_reduction_proxy::
+      DataReductionProxyRequestOptions::GetPageIdFromRequestHeaders(headers);
+  if (page_id) {
+    server_lite_page_info->page_id = page_id.value();
+  }
+
+  base::Optional<std::string> session_key =
+      data_reduction_proxy::DataReductionProxyRequestOptions::
+          GetSessionKeyFromRequestHeaders(headers);
+  if (session_key) {
+    server_lite_page_info->drp_session_key = session_key.value();
+  }
+
+  server_lite_page_info->status = ServerLitePageStatus::kSuccess;
+
+  server_lite_page_info->restart_count = 0;
+
+  return server_lite_page_info;
+}
+
 bool HasEnabledPreviews(content::PreviewsState previews_state) {
   return previews_state != content::PREVIEWS_UNSPECIFIED &&
          !(previews_state & content::PREVIEWS_OFF) &&
@@ -257,7 +295,8 @@ content::PreviewsState DetermineCommittedClientPreviewsState(
     previews::PreviewsUserData* previews_data,
     const GURL& url,
     content::PreviewsState previews_state,
-    const previews::PreviewsDecider* previews_decider) {
+    const previews::PreviewsDecider* previews_decider,
+    content::NavigationHandle* navigation_handle) {
   bool is_https = url.SchemeIs(url::kHttpsScheme);
 
   // Record whether the hint cache has a matching entry for this committed URL.
@@ -296,6 +335,13 @@ content::PreviewsState DetermineCommittedClientPreviewsState(
   // Check if a LITE_PAGE_REDIRECT preview was actually served.
   if (previews_state & content::LITE_PAGE_REDIRECT_ON) {
     if (IsLitePageRedirectPreviewURL(url)) {
+      if (navigation_handle &&
+          base::FeatureList::IsEnabled(network::features::kNetworkService) &&
+          base::FeatureList::IsEnabled(
+              previews::features::kHTTPSServerPreviewsUsingURLLoader)) {
+        previews_data->set_server_lite_page_info(
+            CreateServerLitePageInfoFromNavigationHandle(navigation_handle));
+      }
       LogCommittedPreview(previews_data, PreviewsType::LITE_PAGE_REDIRECT);
       return content::LITE_PAGE_REDIRECT_ON;
     }

chrome/browser/previews/previews_content_util.h

@@ -42,7 +42,8 @@ content::PreviewsState DetermineCommittedClientPreviewsState(
     previews::PreviewsUserData* previews_data,
     const GURL& url,
     content::PreviewsState previews_state,
-    const previews::PreviewsDecider* previews_decider);
+    const previews::PreviewsDecider* previews_decider,
+    content::NavigationHandle* navigation_handle);
 
 // Returns the effective PreviewsType known on a main frame basis given the
 // |previews_state| bitmask for the committed main frame. This uses the same

chrome/browser/previews/previews_content_util_unittest.cc

@@ -339,7 +339,7 @@ TEST_F(PreviewsContentUtilTest, DetermineCommittedClientPreviewsState) {
                 &user_data, GURL("https://www.google.com"),
                 content::SERVER_LITE_PAGE_ON | content::SERVER_LOFI_ON |
                     content::CLIENT_LOFI_ON | content::NOSCRIPT_ON,
-                enabled_previews_decider()));
+                enabled_previews_decider(), nullptr));
   histogram_tester.ExpectUniqueSample(
       "Previews.Triggered.EffectiveConnectionType2.LitePage",
       static_cast<int>(net::EFFECTIVE_CONNECTION_TYPE_2G), 1);
@@ -357,7 +357,7 @@ TEST_F(PreviewsContentUtilTest, DetermineCommittedClientPreviewsState) {
       content::LITE_PAGE_REDIRECT_ON,
       previews::DetermineCommittedClientPreviewsState(
           &user_data, GURL("https://litepages.googlezip.net/?u=google.com"),
-          lite_page_redirect_enabled, enabled_previews_decider()));
+          lite_page_redirect_enabled, enabled_previews_decider(), nullptr));
   histogram_tester.ExpectUniqueSample(
       "Previews.Triggered.EffectiveConnectionType2.LitePageRedirect",
       static_cast<int>(net::EFFECTIVE_CONNECTION_TYPE_2G), 1);
@@ -365,10 +365,11 @@ TEST_F(PreviewsContentUtilTest, DetermineCommittedClientPreviewsState) {
       "Previews.Triggered.EffectiveConnectionType2", 2);
 
   // Verify LITE_PAGE_REDIRECT_ON not committed for non-lite-page-sever URL.
-  EXPECT_NE(content::LITE_PAGE_REDIRECT_ON,
-            previews::DetermineCommittedClientPreviewsState(
-                &user_data, GURL("https://www.google.com"),
-                lite_page_redirect_enabled, enabled_previews_decider()));
+  EXPECT_NE(
+      content::LITE_PAGE_REDIRECT_ON,
+      previews::DetermineCommittedClientPreviewsState(
+          &user_data, GURL("https://www.google.com"),
+          lite_page_redirect_enabled, enabled_previews_decider(), nullptr));
   histogram_tester.ExpectUniqueSample(
       "Previews.Triggered.EffectiveConnectionType2.ResourceLoadingHints",
       static_cast<int>(net::EFFECTIVE_CONNECTION_TYPE_2G), 1);
@@ -380,7 +381,7 @@ TEST_F(PreviewsContentUtilTest, DetermineCommittedClientPreviewsState) {
             previews::DetermineCommittedClientPreviewsState(
                 &user_data, GURL("https://www.google.com"),
                 content::CLIENT_LOFI_ON | content::NOSCRIPT_ON,
-                enabled_previews_decider()));
+                enabled_previews_decider(), nullptr));
   histogram_tester.ExpectUniqueSample(
       "Previews.Triggered.EffectiveConnectionType2.NoScript",
       static_cast<int>(net::EFFECTIVE_CONNECTION_TYPE_2G), 1);
@@ -396,7 +397,7 @@ TEST_F(PreviewsContentUtilTest, DetermineCommittedClientPreviewsState) {
                 &user_data, GURL("https://www.google.com"),
                 content::CLIENT_LOFI_ON | content::NOSCRIPT_ON |
                     content::RESOURCE_LOADING_HINTS_ON,
-                enabled_previews_decider()));
+                enabled_previews_decider(), nullptr));
   histogram_tester.ExpectBucketCount(
       "Previews.Triggered.EffectiveConnectionType2.ResourceLoadingHints",
       static_cast<int>(net::EFFECTIVE_CONNECTION_TYPE_SLOW_2G), 1);
@@ -409,7 +410,7 @@ TEST_F(PreviewsContentUtilTest, DetermineCommittedClientPreviewsState) {
                 &user_data, GURL("http://www.google.com"),
                 content::CLIENT_LOFI_ON | content::NOSCRIPT_ON |
                     content::RESOURCE_LOADING_HINTS_ON,
-                enabled_previews_decider()));
+                enabled_previews_decider(), nullptr));
   histogram_tester.ExpectUniqueSample(
       "Previews.Triggered.EffectiveConnectionType2.LoFi",
       static_cast<int>(net::EFFECTIVE_CONNECTION_TYPE_SLOW_2G), 1);
@@ -420,7 +421,7 @@ TEST_F(PreviewsContentUtilTest, DetermineCommittedClientPreviewsState) {
   EXPECT_EQ(content::CLIENT_LOFI_ON,
             previews::DetermineCommittedClientPreviewsState(
                 &user_data, GURL("https://www.google.com"),
-                content::CLIENT_LOFI_ON, enabled_previews_decider()));
+                content::CLIENT_LOFI_ON, enabled_previews_decider(), nullptr));
   histogram_tester.ExpectUniqueSample(
       "Previews.Triggered.EffectiveConnectionType2.LoFi",
       static_cast<int>(net::EFFECTIVE_CONNECTION_TYPE_SLOW_2G), 2);
@@ -431,7 +432,7 @@ TEST_F(PreviewsContentUtilTest, DetermineCommittedClientPreviewsState) {
   EXPECT_EQ(content::NOSCRIPT_ON,
             previews::DetermineCommittedClientPreviewsState(
                 &user_data, GURL("https://www.google.com"),
-                content::NOSCRIPT_ON, enabled_previews_decider()));
+                content::NOSCRIPT_ON, enabled_previews_decider(), nullptr));
 }
 
 TEST_F(PreviewsContentUtilTest,
@@ -445,7 +446,7 @@ TEST_F(PreviewsContentUtilTest,
             previews::DetermineCommittedClientPreviewsState(
                 &user_data, GURL("https://www.google.com"),
                 content::CLIENT_LOFI_ON | content::NOSCRIPT_ON,
-                enabled_previews_decider()));
+                enabled_previews_decider(), nullptr));
 }
 
 TEST_F(PreviewsContentUtilTest, GetMainFramePreviewsType) {

chrome/browser/previews/previews_lite_page_browsertest.cc

@@ -137,6 +137,7 @@ class PreviewsLitePageServerBrowserTest
     cmd->AppendSwitchASCII("force-effective-connection-type", "Slow-2G");
     cmd->AppendSwitchASCII("force-variation-ids", "42");
     cmd->AppendSwitchASCII("host-rules", "MAP * 127.0.0.1");
+    cmd->AppendSwitch("enable-data-reduction-proxy-force-pingback");
   }
 
   void SetUp() override {
@@ -229,6 +230,14 @@ class PreviewsLitePageServerBrowserTest
     slow_http_url_ = slow_http_server_->GetURL(kOriginHost, "/");
     ASSERT_TRUE(slow_http_url_.SchemeIs(url::kHttpScheme));
 
+    pingback_server_ = std::make_unique<net::EmbeddedTestServer>(
+        net::EmbeddedTestServer::TYPE_HTTPS);
+
+    pingback_server_->RegisterRequestHandler(base::BindRepeating(
+        &PreviewsLitePageServerBrowserTest::HandlePingbackRequest,
+        base::Unretained(this)));
+    ASSERT_TRUE(pingback_server_->Start());
+
     std::map<std::string, std::string> feature_parameters = {
         {"previews_host", previews_server_url().spec()},
         {"blacklisted_path_suffixes", ".mp4,.jpg"},
@@ -238,6 +247,10 @@ class PreviewsLitePageServerBrowserTest
          use_timeout ? base::NumberToString(kTimeoutMs) : "60000"},
         {"control_group", is_control ? "true" : "false"}};
 
+    base::CommandLine::ForCurrentProcess()->AppendSwitchASCII(
+        "data-reduction-proxy-pingback-url",
+        pingback_server_->GetURL("pingback.com", "/").spec());
+
     scoped_parameterized_feature_list_.InitAndEnableFeatureWithParameters(
         previews::features::kLitePageServerPreviews, feature_parameters);
 
@@ -519,6 +532,12 @@ class PreviewsLitePageServerBrowserTest
   const GURL& subframe_url() const { return subframe_url_; }
   int subresources_requested() const { return subresources_requested_; }
 
+  void WaitForPingback() {
+    base::RunLoop run_loop;
+    waiting_for_pingback_closure_ = run_loop.QuitClosure();
+    run_loop.Run();
+  }
+
  private:
   std::unique_ptr<net::test_server::HttpResponse> HandleRedirectRequest(
       const net::test_server::HttpRequest& request) {
@@ -527,6 +546,7 @@ class PreviewsLitePageServerBrowserTest
       std::unique_ptr<net::test_server::BasicHttpResponse> response =
           std::make_unique<net::test_server::BasicHttpResponse>();
       response->set_code(net::HTTP_TEMPORARY_REDIRECT);
+      response->set_content_type("text/html");
       response->AddCustomHeader("Location", https_url().spec());
       return std::move(response);
     }
@@ -546,6 +566,7 @@ class PreviewsLitePageServerBrowserTest
       std::unique_ptr<net::test_server::BasicHttpResponse> response =
           std::make_unique<net::test_server::BasicHttpResponse>();
       response->set_code(net::HTTP_TEMPORARY_REDIRECT);
+      response->set_content_type("text/html");
 
       if (request.GetURL().SchemeIsCryptographic()) {
         response->AddCustomHeader("Location", http_redirect_loop_url().spec());
@@ -576,6 +597,19 @@ class PreviewsLitePageServerBrowserTest
     return std::move(response);
   }
 
+  std::unique_ptr<net::test_server::HttpResponse> HandlePingbackRequest(
+      const net::test_server::HttpRequest& request) {
+    std::unique_ptr<net::test_server::BasicHttpResponse> response =
+        std::make_unique<net::test_server::BasicHttpResponse>();
+    response->set_code(net::HTTP_OK);
+
+    if (!waiting_for_pingback_closure_.is_null()) {
+      std::move(waiting_for_pingback_closure_).Run();
+    }
+
+    return response;
+  }
+
   std::unique_ptr<net::test_server::HttpResponse> HandleResourceRequest(
       const net::test_server::HttpRequest& request) {
     std::unique_ptr<net::test_server::BasicHttpResponse> response =
@@ -591,6 +625,8 @@ class PreviewsLitePageServerBrowserTest
       return response;
     }
 
+    response->set_content_type("text/html");
+
     std::string original_url_str;
 
     // EmbeddedTestServer's request URL is 127.0.0.1 which causes
@@ -651,6 +687,7 @@ class PreviewsLitePageServerBrowserTest
     if (delay_ms > 0) {
       response = std::make_unique<net::test_server::DelayedHttpResponse>(
           base::TimeDelta::FromMilliseconds(delay_ms));
+      response->set_content_type("text/html");
     }
 
     std::string code_query_param;
@@ -705,7 +742,6 @@ class PreviewsLitePageServerBrowserTest
         response->set_code(net::HTTP_SERVICE_UNAVAILABLE);
         break;
       case kSubresources:
-        response->set_content_type("text/html");
         response->set_content(subresource_body);
         break;
       default:
@@ -732,6 +768,7 @@ class PreviewsLitePageServerBrowserTest
   std::unique_ptr<net::EmbeddedTestServer> https_server_;
   std::unique_ptr<net::EmbeddedTestServer> http_server_;
   std::unique_ptr<net::EmbeddedTestServer> slow_http_server_;
+  std::unique_ptr<net::EmbeddedTestServer> pingback_server_;
   GURL https_url_;
   GURL base_https_lite_page_url_;
   GURL https_media_url_;
@@ -746,6 +783,7 @@ class PreviewsLitePageServerBrowserTest
   GURL previews_server_url_;
   GURL slow_http_url_;
   int subresources_requested_ = 0;
+  base::OnceClosure waiting_for_pingback_closure_;
 };
 
 // True if testing using the URLLoader Interceptor implementation.
@@ -1318,6 +1356,16 @@ IN_PROC_BROWSER_TEST_P(PreviewsLitePageServerBrowserTest,
   VerifyPreviewLoaded();
 }
 
+IN_PROC_BROWSER_TEST_P(PreviewsLitePageServerBrowserTest,
+                       DISABLE_ON_WIN_MAC(LitePageCreatesPingback)) {
+  ui_test_utils::NavigateToURL(browser(), HttpsLitePageURL(kSuccess));
+  VerifyPreviewLoaded();
+
+  // Starting a new page load will send a pingback for the previous page load.
+  GetWebContents()->GetController().Reload(content::ReloadType::NORMAL, false);
+  WaitForPingback();
+}
+
 class PreviewsLitePageServerTimeoutBrowserTest
     : public PreviewsLitePageServerBrowserTest {
  public:

components/data_reduction_proxy/core/common/data_reduction_proxy_headers.cc

@@ -474,7 +474,8 @@ int64_t GetDataReductionProxyOFCL(const net::HttpResponseHeaders* headers) {
 double EstimateCompressionRatioFromHeaders(
     const network::ResourceResponseHead* response_head) {
   if (!response_head->network_accessed || !response_head->headers ||
-      response_head->headers->GetContentLength() <= 0) {
+      response_head->headers->GetContentLength() <= 0 ||
+      response_head->proxy_server.is_direct()) {
     return 1.0;  // No compression
   }