Added support to listen on Network.responseReceived and expose that through tab.py

BUG=337526

Review URL: https://codereview.chromium.org/145923002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@255200 0039d316-1c4b-4281-b951-d872f2087c98

tools/telemetry/telemetry/core/backends/chrome/inspector_backend.py

@@ -21,6 +21,8 @@ from telemetry.core.backends.chrome import inspector_runtime
 from telemetry.core.backends.chrome import inspector_timeline
 from telemetry.core.backends.chrome import websocket
 from telemetry.core.heap import model
+from telemetry.core.timeline import model as timeline_model
+from telemetry.core.timeline import recording_options
 
 
 class InspectorException(Exception):
@@ -44,6 +46,7 @@ class InspectorBackend(object):
     self._runtime = inspector_runtime.InspectorRuntime(self)
     self._timeline = inspector_timeline.InspectorTimeline(self)
     self._network = inspector_network.InspectorNetwork(self)
+    self._timeline_model = None
 
   def __del__(self):
     self._Disconnect()
@@ -191,13 +194,29 @@ class InspectorBackend(object):
 
   @property
   def timeline_model(self):
-    return self._timeline.timeline_model
+    return self._timeline_model
 
-  def StartTimelineRecording(self):
-    self._timeline.Start()
+  def StartTimelineRecording(self, options=None):
+    if not options:
+      options = recording_options.TimelineRecordingOptions()
+    if options.record_timeline:
+      self._timeline.Start()
+    if options.record_network:
+      self._network.timeline_recorder.Start()
 
   def StopTimelineRecording(self):
-    self._timeline.Stop()
+    data = []
+    timeline_data = self._timeline.Stop()
+    if timeline_data:
+      data.append(timeline_data)
+    network_data = self._network.timeline_recorder.Stop()
+    if network_data:
+      data.append(network_data)
+    if data:
+      self._timeline_model = timeline_model.TimelineModel(
+          timeline_data=data, shift_world_to_zero=False)
+    else:
+      self._timeline_model = None
 
   # Network public methods.
 

tools/telemetry/telemetry/core/backends/chrome/inspector_network.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+import logging
+
+from telemetry.core import util
+from telemetry.core.backends.chrome import inspector_timeline_data
+from telemetry.core.backends.chrome import timeline_recorder
+
+
+class InspectorNetworkException(Exception):
+  pass
+
+
+class InspectorNetworkResponseData(object):
+  def __init__(self, inspector_network, params):
+    self._inspector_network = inspector_network
+    self._request_id = params['requestId']
+    self._timestamp = params['timestamp']
+
+    self._response = params['response']
+    if not self._response:
+      raise InspectorNetworkException('response must exist')
+
+    # Response headers.
+    headers = self._response['headers']
+    self._header_map = {}
+    for k, v in headers.iteritems():
+      # Camel-case header keys.
+      self._header_map[k.title()] = v
+
+    # Request headers.
+    self._request_header_map = {}
+    if 'requestHeaders' in self._response:
+      # Camel-case header keys.
+      for k, v in self._response['requestHeaders'].iteritems():
+        self._request_header_map[k.title()] = v
+
+    self._body = None
+    self._base64_encoded = False
+    if self._inspector_network:
+      self._served_from_cache = (
+          self._inspector_network.HTTPResponseServedFromCache(self._request_id))
+    else:
+      self._served_from_cache = False
+
+    # Whether constructed from a timeline event.
+    self._from_event = False
+
+  @property
+  def status(self):
+    return self._response['status']
+
+  def status_text(self):
+    return self._response['status_text']
+
+  @property
+  def headers(self):
+    return self._header_map
+
+  @property
+  def request_headers(self):
+    return self._request_header_map
+
+  @property
+  def timestamp(self):
+    return self._timestamp
+
+  @property
+  def timing(self):
+    if 'timing' in self._response:
+      return self._response['timing']
+    return None
+
+  @property
+  def url(self):
+    return self._response['url']
+
+  @property
+  def request_id(self):
+    return self._request_id
+
+  @property
+  def served_from_cache(self):
+    self._served_from_cache = False
+
+  def GetHeader(self, name):
+    if name in self.headers:
+      return self.headers[name]
+    return None
+
+  def GetBody(self, timeout=60):
+    if not self._body and not self._from_event:
+      self._body, self._base64_encoded = (
+        self._inspector_network.GetHTTPResponseBody(self._request_id, timeout))
+    return self._body, self._base64_encoded
+
+  def AsTimelineEvent(self):
+    event = {}
+    event['type'] = 'HTTPResponse'
+    event['startTime'] = self.timestamp
+    # There is no end time. Just return the timestamp instead.
+    event['endTime'] = self.timestamp
+    event['requestId'] = self.request_id
+    event['response'] = self._response
+    event['body'], event['base64_encoded_body'] = self.GetBody()
+    event['served_from_cache'] = self.served_from_cache
+    return event
+
+  @staticmethod
+  def FromTimelineEvent(event):
+    assert event.name == 'HTTPResponse'
+    params = {}
+    params['timestamp'] = event.start
+    params['requestId'] = event.args['requestId']
+    params['response'] = event.args['response']
+    recorded = InspectorNetworkResponseData(None, params)
+    recorded._body = event.args['body']
+    recorded._base64_encoded = event.args['base64_encoded_body']
+    recorded._served_from_cache = event.args['served_from_cache']
+    recorded._from_event = True
+    return recorded
+
 
 class InspectorNetwork(object):
   def __init__(self, inspector_backend):
     self._inspector_backend = inspector_backend
+    self._http_responses = []
+    self._served_from_cache = set()
+    self._timeline_recorder = None
 
   def ClearCache(self, timeout=60):
     """Clears the browser's disk and memory cache."""
     res = self._inspector_backend.SyncRequest({
         'method': 'Network.canClearBrowserCache'
         }, timeout)
-    assert res['result'], 'Cache clearing is not supported by this browser'
+    assert res['result'], 'Cache clearing is not supported by this browser.'
     self._inspector_backend.SyncRequest({
         'method': 'Network.clearBrowserCache'
         }, timeout)
+
+  def StartMonitoringNetwork(self):
+    """Starts monitoring network notifications and recording HTTP responses."""
+    self.ClearResponseData()
+    self._inspector_backend.RegisterDomain(
+        'Network',
+        self._OnNetworkNotification,
+        self._OnClose)
+    request = {
+        'method': 'Network.enable'
+        }
+    self._inspector_backend.SyncRequest(request)
+
+  def StopMonitoringNetwork(self):
+    """Stops monitoring network notifications and recording HTTP responses."""
+    self._inspector_backend.UnregisterDomain('Network')
+    request = {
+        'method': 'Network.disable'
+        }
+    self._inspector_backend.SyncRequest(request)
+
+  def GetResponseData(self):
+    """Returns all recorded HTTP responses."""
+    return self._http_responses
+
+  def ClearResponseData(self):
+    """Clears recorded HTTP responses."""
+    self._http_responses = []
+    self._served_from_cache.clear()
+
+  def _OnNetworkNotification(self, msg):
+    if msg['method'] == 'Network.responseReceived':
+      self._RecordHTTPResponse(msg['params'])
+    elif msg['method'] == 'Network.requestServedFromCache':
+      self._served_from_cache.add(msg['params']['requestId'])
+
+  def _RecordHTTPResponse(self, params):
+    required_fields = ['requestId', 'timestamp', 'response']
+    for field in required_fields:
+      if field not in params:
+        logging.waring('HTTP Response missing required field: %s', field)
+        return
+    self._http_responses.append(InspectorNetworkResponseData(self, params))
+
+  def GetHTTPResponseBody(self, request_id, timeout=60):
+    try:
+      res =  self._inspector_backend.SyncRequest({
+          'method': 'Network.getResponseBody',
+          'params': {
+              'requestId': request_id,
+              }
+          }, timeout)
+    except util.TimeoutException:
+      logging.warning('Timeout during fetching body for %s' % request_id)
+      return None, False
+    if 'error' in res:
+      return None, False
+    return res['result']['body'], res['result']['base64Encoded']
+
+  def HTTPResponseServedFromCache(self, request_id):
+    return request_id and request_id in self._served_from_cache
+
+  def _OnClose(self):
+    pass
+
+  @property
+  def timeline_recorder (self):
+    if not self._timeline_recorder:
+      self._timeline_recorder = TimelineRecorder(self)
+    return self._timeline_recorder
+
+
+class TimelineRecorder(timeline_recorder.TimelineRecorder):
+  def __init__(self, inspector_network):
+    super(TimelineRecorder, self).__init__()
+    self._inspector_network = inspector_network
+    self._is_recording = False
+
+  def Start(self):
+    assert not self._is_recording, 'Start should only be called once.'
+    self._is_recording = True
+    self._inspector_network.StartMonitoringNetwork()
+
+  def Stop(self):
+    if not self._is_recording:
+      return None
+    responses = self._inspector_network.GetResponseData()
+    events = [r.AsTimelineEvent() for r in list(responses)]
+    self._inspector_network.StopMonitoringNetwork()
+    self._is_recording = False
+    if len(events) == 0:
+      return None
+    return inspector_timeline_data.InspectorTimelineData(events)

tools/telemetry/telemetry/core/backends/chrome/inspector_network_unittest.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+from telemetry.core import util
+from telemetry.core.backends.chrome import inspector_network
+from telemetry.core.timeline import recording_options
+from telemetry.unittest import tab_test_case
+
+
+class InspectorNetworkTabTest(tab_test_case.TabTestCase):
+  class TestCase(object):
+    def __init__(self, monitoring = False, responses_count = 0,
+                 subresources = None):
+      # Whether to monitor network for this case.
+      self.monitoring = monitoring
+      # Number of responses expected for this case.
+      self.responses_count = responses_count
+      # List of subresource links for this case.
+      self.subresources = subresources
+
+  def __init__(self, *args):
+    super(InspectorNetworkTabTest, self).__init__(*args)
+
+  def testHTTPResponseTimelineRecorder(self):
+    self._browser.SetHTTPServerDirectories(util.GetUnittestDataDir())
+    tests = {
+        'blank.html': InspectorNetworkTabTest.TestCase(),
+        'green_rect.html': InspectorNetworkTabTest.TestCase(
+            monitoring=True, responses_count=1),
+        'image_decoding.html': InspectorNetworkTabTest.TestCase(
+            monitoring=True, responses_count=2, subresources=['image.png']),
+        }
+    for page, test in tests.iteritems():
+      opts = recording_options.TimelineRecordingOptions()
+      if test.monitoring:
+        opts.record_network = True
+      self._tab.StartTimelineRecording(opts)
+      self._tab.Navigate(self._browser.http_server.UrlOf(page))
+      self._tab.WaitForDocumentReadyStateToBeComplete()
+      self._tab.StopTimelineRecording()
+
+      self.assertTrue(self._tab.timeline_model)
+      events = self._tab.timeline_model.GetAllEventsOfName('HTTPResponse')
+      self.assertEqual(test.responses_count, len(events))
+      if not test.monitoring:
+        continue
+
+      # Verify required event fields
+      for event in events:
+        self.assertEqual('HTTPResponse', event.name)
+        resp = inspector_network.InspectorNetworkResponseData.FromTimelineEvent(
+            event)
+        self.assertLess(0.0, resp.timestamp)
+
+        self.assertTrue(resp.headers)
+        self.assertTrue(resp.headers['Content-Length'])
+        body, base64_encoded = resp.GetBody()
+
+        link = resp.url[resp.url.rfind('/') + 1 :]
+        self.assertTrue(link == page or link in test.subresources)
+        if link == page:
+          self.assertEqual(resp.GetHeader('Content-Type'), 'text/html')
+          self.assertTrue('<!DOCTYPE HTML>' in body)
+          self.assertEqual(False, base64_encoded)
+        else:
+          # We know this is the only subresource type in our setup.
+          self.assertEqual(resp.GetHeader('Content-Type'), 'image/png')
+          self.assertFalse('<!DOCTYPE HTML>' in body)
+          self.assertEqual(True, base64_encoded)

tools/telemetry/telemetry/core/backends/chrome/inspector_timeline.py

@@ -3,7 +3,7 @@
 # found in the LICENSE file.
 
 from telemetry.core.backends.chrome import inspector_timeline_data
-from telemetry.core.timeline import model
+from telemetry.core.backends.chrome import timeline_recorder
 
 
 class TabBackendException(Exception):
@@ -11,7 +11,7 @@ class TabBackendException(Exception):
   pass
 
 
-class InspectorTimeline(object):
+class InspectorTimeline(timeline_recorder.TimelineRecorder):
   """Implementation of dev tools timeline."""
 
   class Recorder(object):
@@ -34,13 +34,9 @@ class InspectorTimeline(object):
       self._tab.StopTimelineRecording()
 
   def __init__(self, inspector_backend):
+    super(InspectorTimeline, self).__init__()
     self._inspector_backend = inspector_backend
     self._is_recording = False
-    self._timeline_model = None
-
-  @property
-  def timeline_model(self):
-    return self._timeline_model
 
   def Start(self):
     """Starts recording."""
@@ -58,17 +54,17 @@ class InspectorTimeline(object):
     self._SendSyncRequest(request)
 
   def Stop(self):
-    """Stops recording and makes a TimelineModel with the event data."""
-    assert self._is_recording, 'Stop should be called after Start.'
+    """Stops recording and returns timeline event data."""
+    if not self._is_recording:
+      return None
     request = {'method': 'Timeline.stop'}
     result = self._SendSyncRequest(request)
-    raw_events = result['events']
-    timeline_data = inspector_timeline_data.InspectorTimelineData(raw_events)
-    self._timeline_model = model.TimelineModel(
-        timeline_data=timeline_data, shift_world_to_zero=False)
     self._inspector_backend.UnregisterDomain('Timeline')
     self._is_recording = False
 
+    raw_events = result['events']
+    return inspector_timeline_data.InspectorTimelineData(raw_events)
+
   def _SendSyncRequest(self, request, timeout=60):
     """Sends a devtools remote debugging protocol request.
 

tools/telemetry/telemetry/core/backends/chrome/timeline_recorder.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+class TimelineRecorder(object):
+  """Interface for classes that can record timeline raw events."""
+  def Start(self):
+    """Starts recording."""
+    raise NotImplementedError
+
+  def Stop(self):
+    """Stops recording and returns timeline event data."""
+    raise NotImplementedError

tools/telemetry/telemetry/core/timeline/recording_options.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+class TimelineRecordingOptions(object):
+  def __init__(self):
+    self.record_timeline = True
+    self.record_network = False

tools/telemetry/telemetry/core/web_contents.py

@@ -79,8 +79,8 @@ class WebContents(object):
   def timeline_model(self):
     return self._inspector_backend.timeline_model
 
-  def StartTimelineRecording(self):
-    self._inspector_backend.StartTimelineRecording()
+  def StartTimelineRecording(self, options=None):
+    self._inspector_backend.StartTimelineRecording(options)
 
   def StopTimelineRecording(self):
     self._inspector_backend.StopTimelineRecording()