tools/android/loading: Add support for multiple redirects.

This properly identify redirect chains. Before this CL, each link in the
redirect chain would seem to be initiated by the eventual request,
instead of the previous redirect.

Also logs the number of cases where the redirected request doesn't have
the same initiator as the initial request.

Review URL: https://codereview.chromium.org/1633813005

Cr-Commit-Position: refs/heads/master@{#371522}

tools/android/loading/loading_model.py

@@ -247,7 +247,9 @@ class ResourceGraph(object):
       for s in n.Successors():
         style = 'color = orange'
         annotations = self._EdgeAnnotation(n, s)
-        if 'parser' in annotations:
+        if 'redirect' in annotations:
+          style = 'color = black'
+        elif 'parser' in annotations:
           style = 'color = red'
         elif 'stack' in annotations:
           style = 'color = blue'

tools/android/loading/loading_model_unittest.py

@@ -72,7 +72,9 @@ class LoadingModelTestCase(unittest.TestCase):
                        self.MakeParserRequest(1, 0, 102, 103).ToJsonDict(),
                        self.MakeParserRequest(2, 0, 102, 103).ToJsonDict(),
                        self.MakeParserRequest(3, 2, 104, 105).ToJsonDict()],
-            'metadata': { 'duplicates_count' : 0 }},
+            'metadata': {
+                request_track.RequestTrack._DUPLICATES_KEY: 0,
+                request_track.RequestTrack._INCONSISTENT_INITIATORS_KEY: 0}},
          'url': 'foo.com',
          'tracing_track': {'events': []},
          'page_track': {'events': []},

tools/android/loading/request_dependencies_lens.py

@@ -64,8 +64,7 @@ class RequestDependencyLens(object):
     """
     reason = request.initiator['type']
     assert reason in request_track.Request.INITIATORS
-    # Redirect suffixes are added in RequestTrack.
-    if request.request_id.endswith(request_track.RequestTrack.REDIRECT_SUFFIX):
+    if reason == 'redirect':
       return self._GetInitiatingRequestRedirect(request)
     elif reason == 'parser':
       return self._GetInitiatingRequestParser(request)
@@ -76,14 +75,11 @@ class RequestDependencyLens(object):
       return self._GetInitiatingRequestOther(request)
 
   def _GetInitiatingRequestRedirect(self, request):
-    request_id = request.request_id[:request.request_id.index(
-        request_track.RequestTrack.REDIRECT_SUFFIX)]
-    assert request_id in self._requests_by_id
-    dependent_request = self._requests_by_id[request_id]
-    assert request.timing.request_time < \
-        dependent_request.timing.request_time, '.\n'.join(
-            [str(request), str(dependent_request)])
-    return (request, dependent_request, 'redirect')
+    assert request_track.Request.INITIATING_REQUEST in request.initiator
+    initiating_request_id = request.initiator[
+        request_track.Request.INITIATING_REQUEST]
+    assert initiating_request_id in self._requests_by_id
+    return (self._requests_by_id[initiating_request_id], request, 'redirect')
 
   def _GetInitiatingRequestParser(self, request):
     url = request.initiator['url']

tools/android/loading/request_dependencies_lens_unittest.py

@@ -13,9 +13,17 @@ import test_utils
 
 class RequestDependencyLensTestCase(unittest.TestCase):
   _REDIRECT_REQUEST = Request.FromJsonDict(
-      {'url': 'http://bla.com', 'request_id': '1234.1.redirect',
+      {'url': 'http://bla.com', 'request_id': '1234.redirect.1',
        'initiator': {'type': 'other'},
        'timestamp': 1, 'timing': TimingFromDict({})})
+  _REDIRECTED_REQUEST = Request.FromJsonDict({
+      'url': 'http://bla.com',
+      'request_id': '1234.1',
+      'frame_id': '123.1',
+      'initiator': {'type': 'redirect',
+                    'initiating_request': '1234.redirect.1'},
+      'timestamp': 2,
+      'timing': TimingFromDict({})})
   _REQUEST = Request.FromJsonDict({'url': 'http://bla.com',
                                    'request_id': '1234.1',
                                    'frame_id': '123.1',
@@ -55,7 +63,7 @@ class RequestDependencyLensTestCase(unittest.TestCase):
 
   def testRedirectDependency(self):
     loading_trace = test_utils.LoadingTraceFromEvents(
-        [self._REDIRECT_REQUEST, self._REQUEST])
+        [self._REDIRECT_REQUEST, self._REDIRECTED_REQUEST])
     request_dependencies_lens = RequestDependencyLens(loading_trace)
     deps = request_dependencies_lens.GetRequestDependencies()
     self.assertEquals(1, len(deps))
@@ -86,7 +94,7 @@ class RequestDependencyLensTestCase(unittest.TestCase):
 
   def testSeveralDependencies(self):
     loading_trace = test_utils.LoadingTraceFromEvents(
-        [self._REDIRECT_REQUEST, self._REQUEST, self._JS_REQUEST,
+        [self._REDIRECT_REQUEST, self._REDIRECTED_REQUEST, self._JS_REQUEST,
          self._JS_REQUEST_2])
     request_dependencies_lens = RequestDependencyLens(loading_trace)
     deps = request_dependencies_lens.GetRequestDependencies()

tools/android/loading/request_track.py

@@ -46,7 +46,7 @@ class Request(object):
   third_party/WebKit/Source/devtools/protocol.json.
 
   Fields:
-    request_id: (str) unique request ID. Postfixed with REDIRECT_SUFFIX for
+    request_id: (str) unique request ID. Postfixed with _REDIRECT_SUFFIX for
                 redirects.
     frame_id: (str) unique frame identifier.
     loader_id: (str) unique frame identifier.
@@ -77,7 +77,9 @@ class Request(object):
   RESOURCE_TYPES = ('Document', 'Stylesheet', 'Image', 'Media', 'Font',
                     'Script', 'TextTrack', 'XHR', 'Fetch', 'EventSource',
                     'WebSocket', 'Manifest', 'Other')
-  INITIATORS = ('parser', 'script', 'other')
+  INITIATORS = ('parser', 'script', 'other', 'redirect')
+  INITIATING_REQUEST = 'initiating_request'
+  ORIGINAL_INITIATOR = 'original_initiator'
   def __init__(self):
     self.request_id = None
     self.frame_id = None
@@ -172,7 +174,7 @@ class Request(object):
 
 class RequestTrack(devtools_monitor.Track):
   """Aggregates request data."""
-  REDIRECT_SUFFIX = '.redirect'
+  _REDIRECT_SUFFIX = '.redirect'
   # Request status
   _STATUS_SENT = 0
   _STATUS_RESPONSE = 1
@@ -183,12 +185,14 @@ class RequestTrack(devtools_monitor.Track):
   _EVENTS_KEY = 'events'
   _METADATA_KEY = 'metadata'
   _DUPLICATES_KEY = 'duplicates_count'
+  _INCONSISTENT_INITIATORS_KEY = 'inconsistent_initiators'
   def __init__(self, connection):
     super(RequestTrack, self).__init__(connection)
     self._connection = connection
     self._requests = []
     self._requests_in_flight = {}  # requestId -> (request, status)
     self._completed_requests_by_id = {}
+    self._redirects_count_by_id = collections.defaultdict(int)
     if connection:  # Optional for testing.
       for method in RequestTrack._METHOD_TO_HANDLER:
         self._connection.RegisterListener(method, self)
@@ -196,6 +200,7 @@ class RequestTrack(devtools_monitor.Track):
     # detect this.
     self._request_id_to_response_received = {}
     self.duplicates_count = 0
+    self.inconsistent_initiators_count = 0
 
   def Handle(self, method, msg):
     assert method in RequestTrack._METHOD_TO_HANDLER
@@ -214,7 +219,10 @@ class RequestTrack(devtools_monitor.Track):
       logging.warning('Requests in flight, will be ignored in the dump')
     return {self._EVENTS_KEY: [
         request.ToJsonDict() for request in self._requests],
-            self._METADATA_KEY: {self._DUPLICATES_KEY: self.duplicates_count}}
+            self._METADATA_KEY: {
+                self._DUPLICATES_KEY: self.duplicates_count,
+                self._INCONSISTENT_INITIATORS_KEY:
+                self.inconsistent_initiators_count}}
 
   @classmethod
   def FromJsonDict(cls, json_dict):
@@ -224,14 +232,18 @@ class RequestTrack(devtools_monitor.Track):
     requests = [Request.FromJsonDict(request)
                 for request in json_dict[cls._EVENTS_KEY]]
     result._requests = requests
-    result.duplicates_count = json_dict[cls._METADATA_KEY][cls._DUPLICATES_KEY]
+    metadata = json_dict[cls._METADATA_KEY]
+    result.duplicates_count = metadata.get(cls._DUPLICATES_KEY, 0)
+    result.inconsistent_initiators_count = metadata.get(
+        cls._INCONSISTENT_INITIATORS_KEY, 0)
     return result
 
   def _RequestWillBeSent(self, request_id, params):
     # Several "requestWillBeSent" events can be dispatched in a row in the case
     # of redirects.
+    redirect_initiator = None
     if request_id in self._requests_in_flight:
-      self._HandleRedirect(request_id, params)
+      redirect_initiator = self._HandleRedirect(request_id, params)
     assert (request_id not in self._requests_in_flight
             and request_id not in self._completed_requests_by_id)
     r = Request()
@@ -247,6 +259,16 @@ class RequestTrack(devtools_monitor.Track):
                      ('headers', 'headers'),
                      ('initialPriority', 'initial_priority')))
     r.resource_type = params.get('type', 'Other')
+    if redirect_initiator:
+      original_initiator = r.initiator
+      r.initiator = redirect_initiator
+      r.initiator[Request.ORIGINAL_INITIATOR] = original_initiator
+      initiating_request = self._completed_requests_by_id[
+          redirect_initiator[Request.INITIATING_REQUEST]]
+      initiating_initiator = initiating_request.initiator.get(
+          Request.ORIGINAL_INITIATOR, initiating_request.initiator)
+      if initiating_initiator != original_initiator:
+        self.inconsistent_initiators_count += 1
     self._requests_in_flight[request_id] = (r, RequestTrack._STATUS_SENT)
 
   def _HandleRedirect(self, request_id, params):
@@ -256,15 +278,23 @@ class RequestTrack(devtools_monitor.Track):
     # one. Finalize the first request.
     assert 'redirectResponse' in params
     redirect_response = params['redirectResponse']
+
     _CopyFromDictToObject(redirect_response, r,
                           (('headers', 'response_headers'),
                            ('encodedDataLength', 'encoded_data_length'),
                            ('fromDiskCache', 'from_disk_cache')))
     r.timing = TimingFromDict(redirect_response['timing'])
-    r.request_id = request_id + self.REDIRECT_SUFFIX
+
+    redirect_index = self._redirects_count_by_id[request_id]
+    self._redirects_count_by_id[request_id] += 1
+    r.request_id = '%s%s.%d' % (request_id, self._REDIRECT_SUFFIX,
+                                 redirect_index + 1)
+    initiator = {
+        'type': 'redirect', Request.INITIATING_REQUEST: r.request_id}
     self._requests_in_flight[r.request_id] = (r, RequestTrack._STATUS_FINISHED)
     del self._requests_in_flight[request_id]
     self._FinalizeRequest(r.request_id)
+    return initiator
 
   def _RequestServedFromCache(self, request_id, _):
     assert request_id in self._requests_in_flight

tools/android/loading/request_track_unittest.py

@@ -195,7 +195,47 @@ class RequestTrackTestCase(unittest.TestCase):
     self.assertEquals(1, len(self.request_track.GetEvents()))
     redirect_request = self.request_track.GetEvents()[0]
     self.assertTrue(redirect_request.request_id.endswith(
-        RequestTrack.REDIRECT_SUFFIX))
+        RequestTrack._REDIRECT_SUFFIX + '.1'))
+    request = self.request_track._requests_in_flight.values()[0][0]
+    self.assertEquals('redirect', request.initiator['type'])
+    self.assertEquals(
+        redirect_request.request_id,
+        request.initiator[Request.INITIATING_REQUEST])
+    self.assertEquals(0, self.request_track.inconsistent_initiators_count)
+
+  def testMultipleRedirects(self):
+    self.request_track.Handle('Network.requestWillBeSent',
+                              RequestTrackTestCase._REQUEST_WILL_BE_SENT)
+    self.request_track.Handle('Network.requestWillBeSent',
+                              RequestTrackTestCase._REDIRECT)
+    self.request_track.Handle('Network.requestWillBeSent',
+                              RequestTrackTestCase._REDIRECT)
+    self.assertEquals(1, len(self.request_track._requests_in_flight))
+    self.assertEquals(2, len(self.request_track.GetEvents()))
+    first_redirect_request = self.request_track.GetEvents()[0]
+    self.assertTrue(first_redirect_request.request_id.endswith(
+        RequestTrack._REDIRECT_SUFFIX + '.1'))
+    second_redirect_request = self.request_track.GetEvents()[1]
+    self.assertTrue(second_redirect_request.request_id.endswith(
+        RequestTrack._REDIRECT_SUFFIX + '.2'))
+    self.assertEquals('redirect', second_redirect_request.initiator['type'])
+    self.assertEquals(
+        first_redirect_request.request_id,
+        second_redirect_request.initiator[Request.INITIATING_REQUEST])
+    request = self.request_track._requests_in_flight.values()[0][0]
+    self.assertEquals('redirect', request.initiator['type'])
+    self.assertEquals(
+        second_redirect_request.request_id,
+        request.initiator[Request.INITIATING_REQUEST])
+    self.assertEquals(0, self.request_track.inconsistent_initiators_count)
+
+  def testInconsistentInitiators(self):
+    self.request_track.Handle('Network.requestWillBeSent',
+                              RequestTrackTestCase._REQUEST_WILL_BE_SENT)
+    request = copy.deepcopy(RequestTrackTestCase._REDIRECT)
+    request['params']['initiator']['type'] = 'script'
+    self.request_track.Handle('Network.requestWillBeSent', request)
+    self.assertEquals(1, self.request_track.inconsistent_initiators_count)
 
   def testRejectDuplicates(self):
     msg = RequestTrackTestCase._REQUEST_WILL_BE_SENT
@@ -281,6 +321,7 @@ class RequestTrackTestCase(unittest.TestCase):
   def testCanDeserialize(self):
     self._ValidSequence(self.request_track)
     self.request_track.duplicates_count = 142
+    self.request_track.inconsistent_initiators_count = 123
     json_dict = self.request_track.ToJsonDict()
     request_track = RequestTrack.FromJsonDict(json_dict)
     self.assertEquals(self.request_track, request_track)