Add a new test, testServerReturnsBypassWithHostBlacklisted

I also added a redirect_chain field to HTTPResponse. This is the ordered
list of URLs in the redirect chain. If a request did not have any
redirects, this list is empty. This is used to verify that a server
bypass happend.

For example, testServerReturnsBypass checks that we show the original
page, and also that we got there via a redirect (bypass) from the
server. OTOH, testServerReturnsBypassWithHostBlacklisted loads the page
twice, where the second load should _not_ go through the server.

R=robertogden@chromium.org

Bug: 908491
Change-Id: I0e4c66917098013f2137d0cf2306567ddf8f4a60
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1584436Reviewed-by: Robert Ogden <robertogden@chromium.org>
Commit-Queue: Tom Bergan <tombergan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#654594}

tools/chrome_proxy/webdriver/common.py

@@ -25,6 +25,10 @@ sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
 from selenium import webdriver
 from selenium.webdriver.chrome.options import Options
 
+# Pretty printer for debug output.
+def PrettyPrintJSON(obj):
+  return json.dumps(obj, indent=2)
+
 # These network condition values are used in SetNetworkConnection()
 NETWORKS = {
     '4G': {
@@ -322,7 +326,7 @@ class TestDriver:
       self._logger.info('Using the Chrome binary at this path: %s',
         self._flags.chrome_exec)
     self._logger.debug('ChromeOptions will be parsed into these capabilities: '
-      '%s', json.dumps(chrome_options.to_capabilities()))
+      '%s', PrettyPrintJSON(chrome_options.to_capabilities()))
     driver = webdriver.Chrome(executable_path=self._flags.chrome_driver,
       desired_capabilities=capabilities, chrome_options=chrome_options)
     driver.command_executor._commands.update({
@@ -598,7 +602,8 @@ class TestDriver:
       json_file_content = json_file_content[:end] + ']}'
       return json.loads(json_file_content)
 
-  def GetPerformanceLogs(self, method_filter=r'Network\.responseReceived'):
+  def GetPerformanceLogs(self, method_filter=r'Network\.(requestWillBeSent|' +
+                                                        'responseReceived)'):
     """Returns all logged Performance events from Chrome. Raises an Exception if
     no pages have been loaded since the last time this function was called.
 
@@ -614,7 +619,7 @@ class TestDriver:
     all_messages = []
     for log in self._driver.execute('getLog', {'type': 'performance'})['value']:
       message = json.loads(log['message'])['message']
-      self._logger.debug('Got Performance log: %s', log['message'])
+      self._logger.debug('Got Performance log:\n%s', PrettyPrintJSON(message))
       if re.match(method_filter, message['method']):
         all_messages.append(message)
     self._logger.info('Got %d performance logs with filter method=%s',
@@ -664,8 +669,12 @@ class TestDriver:
     """
     if override_has_logs:
       self._has_logs = True
-    def MakeHTTPResponse(log_dict):
-      params = log_dict['params']
+
+    all_requests = {}  # map from requestId to params
+    all_responses = [] # list of HTTPResponse
+
+    def MakeHTTPResponse(message):
+      params = message['params']
       response_dict = params['response']
       http_response_dict = {
         'response_headers': response_dict['headers'] if 'headers' in
@@ -678,11 +687,21 @@ class TestDriver:
         'port': response_dict['remotePort'] if 'remotePort' in response_dict
           else -1,
         'status': response_dict['status'] if 'status' in response_dict else -1,
-        'request_type': params['type'] if 'type' in params else ''
+        'request_type': params['type'] if 'type' in params else '',
+        'redirect_chain': [],
       }
+      for request in all_requests[params['requestId']][:-1]:
+        http_response_dict['redirect_chain'].append(request['request']['url'])
       return HTTPResponse(**http_response_dict)
-    all_responses = []
+
     for message in self.GetPerformanceLogs():
+      if message['method'] == 'Network.requestWillBeSent':
+        requestId = message['params']['requestId']
+        if requestId not in all_requests:
+          all_requests[requestId] = [message['params']]
+        else:
+          all_requests[requestId].append(message['params'])
+        continue
       response = MakeHTTPResponse(message)
       self._logger.debug('New HTTPResponse: %s', str(response))
       is_favicon = response.url.endswith('favicon.ico')
@@ -717,7 +736,7 @@ class HTTPResponse:
   """
 
   def __init__(self, response_headers, request_headers, url, protocol, port,
-      status, request_type):
+      status, request_type, redirect_chain):
     self._response_headers = {}
     self._request_headers = {}
     self._url = url
@@ -725,6 +744,7 @@ class HTTPResponse:
     self._port = port
     self._status = status
     self._request_type = request_type
+    self._redirect_chain = redirect_chain # empty if no redirects
     self._flags = ParseFlags()
     # Make all header names lower case.
     for name in response_headers:
@@ -733,16 +753,16 @@ class HTTPResponse:
       self._request_headers[name.lower()] = request_headers[name]
 
   def __str__(self):
-    self_dict = {
+    return PrettyPrintJSON({
       'response_headers': self._response_headers,
       'request_headers': self._request_headers,
       'url': self._url,
       'protocol': self._protocol,
       'port': self._port,
       'status': self._status,
-      'request_type': self._request_type
-    }
-    return json.dumps(self_dict, indent=2)
+      'request_type': self._request_type,
+      'redirect_chain': self._redirect_chain,
+    })
 
   @property
   def response_headers(self):
@@ -772,6 +792,10 @@ class HTTPResponse:
   def request_type(self):
     return self._request_type
 
+  @property
+  def redirect_chain(self):
+    return self._redirect_chain
+
   def ResponseHasViaHeader(self):
     return 'via' in self._response_headers and (self._response_headers['via'] ==
       self._flags.via_header_value)

tools/chrome_proxy/webdriver/https_previews.py

@@ -16,6 +16,7 @@ from selenium.common.exceptions import TimeoutException
 
 NAV_THROTTLE_VERSION = "v1_NavThrottle"
 URL_LOADER_VERSION = "v2_URLLoader"
+LITEPAGES_REGEXP = r'https://\w+\.litepages\.googlezip\.net/.*'
 
 # These are integration tests for server provided previews and the
 # protocol that supports them. This class is intended as an abstract base class
@@ -67,6 +68,8 @@ class HttpsPreviewsBaseClass():
 
     Args:
       t: the TestDriver object.
+      expectedText: text that should appear in the HTML response body.
+      expectedImages: the number of images that should be fetched.
     """
     lite_page_responses = 0
     image_responses = 0
@@ -75,15 +78,12 @@ class HttpsPreviewsBaseClass():
       content_type = ''
       if 'content-type' in response.response_headers:
         content_type = response.response_headers['content-type']
-
       if 'text/html' in content_type:
-        self.assertRegexpMatches(response.url,
-                                 r"https://\w+\.litepages\.googlezip\.net/")
+        self.assertRegexpMatches(response.url, LITEPAGES_REGEXP)
         self.assertEqual(200, response.status)
         lite_page_responses += 1
       if 'image/' in content_type:
-        self.assertRegexpMatches(response.url,
-                                 r"https://\w+\.litepages\.googlezip\.net/")
+        self.assertRegexpMatches(response.url, LITEPAGES_REGEXP)
         self.assertEqual(200, response.status)
         image_responses += 1
 
@@ -95,11 +95,15 @@ class HttpsPreviewsBaseClass():
 
     self.assertPreviewShownViaHistogram(t, 'LitePageRedirect')
 
-  def _AssertShowingOriginalPage(self, t, expectedURL, expectedStatus):
+  def _AssertShowingOriginalPage(self, t, expectedURL, expectedStatus,
+                                 expectedBypassCount = 1):
     """Asserts that Chrome has not loaded a Lite Page from the litepages server.
 
     Args:
       t: the TestDriver object.
+      expectedURL: the URL of the mainframe HTML response.
+      expectedStatus: the HTTP response status for the mainframe HTML response.
+      expectBypass: true if we expect a bypass from the litepages server.
     """
     html_responses = 0
 
@@ -108,8 +112,9 @@ class HttpsPreviewsBaseClass():
         self.assertEqual(expectedStatus, response.status)
         html_responses += 1
 
+    bypass_count = t.GetHistogram('Previews.ServerLitePage.ServerResponse', 2)
     self.assertEqual(1, html_responses)
-
+    self.assertEqual(expectedBypassCount, bypass_count['count'])
     self.assertPreviewNotShownViaHistogram(t, 'LitePageRedirect')
 
   # Verifies that a Lite Page is not served when the server returns a bypass.
@@ -121,6 +126,23 @@ class HttpsPreviewsBaseClass():
       t.LoadURL(url)
       self._AssertShowingOriginalPage(t, url, 200)
 
+  # Verifies that a Lite Page is not served when the server returns a bypass.
+  # Additionally, verifies that after receiving the host-blacklisted directive,
+  # previews will not be attempted for future navigations on the same host.
+  @ChromeVersionEqualOrAfterM(74)
+  def testServerReturnsBypassWithHostBlacklisted(self):
+    with TestDriver() as t:
+      self.EnableLitePageServerPreviewsAndInit(t)
+      url = 'https://mobilespeed-test2.appspot.com/static/litepagetests/bypass.html'
+      t.LoadURL(url)
+      self._AssertShowingOriginalPage(t, url, 200)
+      # Ensure the reload doesn't use a cached page.
+      t.LoadURL('chrome://settings/clearBrowserData')
+      # This second navigation should not attempt a preview, so the bypass count
+      # should not have been incremented a second time.
+      t.LoadURL(url)
+      self._AssertShowingOriginalPage(t, url, 200, expectedBypassCount = 1)
+
   # Verifies that a Lite Page is not served when the server returns a 404.
   @ChromeVersionEqualOrAfterM(74)
   def testServerReturns404(self):
@@ -215,9 +237,8 @@ class HttpsPreviewsBaseClass():
 
       # Verify that the request is served by a Lite Page.
       lite_page_responses = 0
-      lite_page_regexp = re.compile('https://\w+\.litepages\.googlezip\.net/p')
       for response in t.GetHTTPResponses():
-        if lite_page_regexp.search(response.url) and response.status == 200:
+        if re.match(LITEPAGES_REGEXP, response.url) and response.status == 200:
           lite_page_responses += 1
       self.assertEqual(1, lite_page_responses)