Add JS heap snapshotting functionality to Telemetry (part 2)

This CL adds code which parses the JS heap snapshot retrieved from the
Dev tools.

BUG=274456

Review URL: https://chromiumcodereview.appspot.com/24076016

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@223392 0039d316-1c4b-4281-b951-d872f2087c98

tools/telemetry/telemetry/core/backends/chrome/inspector_backend.py

@@ -16,7 +16,7 @@ from telemetry.core.backends.chrome import inspector_page
 from telemetry.core.backends.chrome import inspector_runtime
 from telemetry.core.backends.chrome import inspector_timeline
 from telemetry.core.backends.chrome import websocket
-from telemetry.core.jsheap import model
+from telemetry.core.heap import model
 
 class InspectorException(Exception):
   pass
@@ -354,5 +354,4 @@ class InspectorBackend(object):
                       'params': {'uid': snapshot_uid[0]}}, timeout)
 
     self.UnregisterDomain('HeapProfiler')
-
-    return model.JsHeapSnapshotModel(snapshot[0])
+    return model.Model(snapshot[0])

tools/telemetry/telemetry/core/heap/chrome_js_heap_snapshot_parser.py

+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+
+from telemetry.core.heap import live_heap_object
+from telemetry.core.heap import retaining_edge
+
+class ChromeJsHeapSnapshotParser(object):
+  """ Parser for the heap snapshot.
+
+  The heap snapshot JSON format is defined by HeapSnapshotJSONSerializer in V8.
+
+  The snapshot contains a list of integers describing nodes (types, names, etc.)
+  and a list of integers describing edges (types, the node the edge points to,
+  etc.) and a string table. All strings are expressed as indices to the string
+  table.
+
+  In addition, the snapshot contains meta information describing the data fields
+  for nodes and the data fields for edges.
+
+  Attributes:
+    _node_dict: {int -> LiveHeapObject}, maps integer ids to LiveHeapObject
+        objects.
+    _node_list: [int], the raw node data of the heap snapshot.
+    _edge_list: [int], the raw edge data of the heap snapshot.
+    _node_types: [str], the possible node types in the heap snapshot.
+    _edge_types: [str], the possible edge types in the heap snapshot.
+    _node_fields: [str], the fields present in the heap snapshot for each node.
+    _edge_fields: [str], the fields present in the heap snapshot for each node.
+    _node_type_ix: int, index of the node type field.
+    _node_name_ix: int, index of the node name field.
+    _node_id_ix: int, index of the node id field.
+    _node_edge_count_ix: int, index of the node edge count field.
+    _node_field_count: int, number of node fields.
+    _edge_type_ix: int, index of the edge type field.
+    _edge_name_or_ix_ix: int, index of the "edge name or index" field.
+    _edge_to_node_ix: int, index of the "to node for an edge" field.
+    _edge_field_count: int, number of edge fields.
+  """
+
+  def __init__(self, raw_data):
+    heap = json.loads(raw_data)
+    self._node_dict = {}
+
+    # Read the snapshot components (nodes, edges, strings, metadata).
+    self._node_list = heap['nodes']
+    self._edge_list = heap['edges']
+    self._strings = heap['strings']
+
+    self._node_types = heap['snapshot']['meta']['node_types'][0]
+    self._edge_types = heap['snapshot']['meta']['edge_types'][0]
+    node_fields = heap['snapshot']['meta']['node_fields']
+    edge_fields = heap['snapshot']['meta']['edge_fields']
+
+    # Find the indices of the required node and edge fields based on the
+    # metadata.
+    self._node_type_ix = node_fields.index('type')
+    self._node_name_ix = node_fields.index('name')
+    self._node_id_ix = node_fields.index('id')
+    self._node_edge_count_ix = node_fields.index('edge_count')
+    self._node_field_count = len(node_fields)
+
+    self._edge_type_ix = edge_fields.index('type')
+    self._edge_name_or_ix_ix = edge_fields.index('name_or_index')
+    self._edge_to_node_ix = edge_fields.index('to_node')
+    self._edge_field_count = len(edge_fields)
+
+    self._ParseSnapshot()
+
+  @staticmethod
+  def CanImport(raw_data):
+    heap = json.loads(raw_data)
+    if ('nodes' not in heap or 'edges' not in heap or 'strings' not in heap or
+        'snapshot' not in heap or 'meta' not in heap['snapshot']):
+      return False
+    meta = heap['snapshot']['meta']
+    if ('node_types' not in meta or 'edge_types' not in meta or
+        'node_fields' not in meta or 'edge_fields' not in meta):
+      return False
+    node_fields = meta['node_fields']
+    edge_fields = meta['edge_fields']
+    if ('type' not in node_fields or 'name' not in node_fields or
+        'id' not in node_fields or 'edge_count' not in node_fields):
+      return False
+    if ('type' not in edge_fields or 'name_or_index' not in edge_fields or
+        'to_node' not in edge_fields):
+      return False
+    return True
+
+  def GetAllLiveHeapObjects(self):
+    return self._node_dict.values()
+
+  @staticmethod
+  def LiveHeapObjectToJavaScript(heap_object):
+    return heap_object.name or str(heap_object)
+
+  @staticmethod
+  def RetainingEdgeToJavaScript(edge):
+    if edge.type_string == 'property':
+      return '.' + edge.name_string
+    if edge.type_string == 'element':
+      return '[' + edge.name_string + ']'
+    return str(edge)
+
+  def _ParseSnapshot(self):
+    """Parses the stored JSON snapshot data.
+
+    Fills in self._node_dict with LiveHeapObject objects constructed based on
+    the heap snapshot. The LiveHeapObject objects contain the associated
+    RetainingEdge objects.
+    """
+    edge_start_ix = 0
+    for ix in xrange(0, len(self._node_list), self._node_field_count):
+      edge_start_ix = self._ReadNodeFromIndex(ix, edge_start_ix)
+
+    # Add pointers to the endpoints to the edges, and associate the edges with
+    # the "to" nodes.
+    for node_id in self._node_dict:
+      n = self._node_dict[node_id]
+      for e in n.edges_from:
+        self._node_dict[e.to_object_id].AddEdgeTo(e)
+        e.SetFromObject(n)
+        e.SetToObject(self._node_dict[e.to_object_id])
+
+  def _ReadNodeFromIndex(self, ix, edges_start):
+    """Reads the data for a node from the heap snapshot.
+
+    If the index contains an interesting node, constructs a Node object and adds
+    it to self._node_dict.
+
+    Args:
+      ix: int, index into the self._node_list array.
+      edges_start: int, the index of the edge array where the edges for the node
+          start.
+    Returns:
+      int, the edge start index for the next node.
+
+    Raises:
+      Exception: The node list of the snapshot is malformed.
+    """
+    if ix + self._node_field_count > len(self._node_list):
+      raise Exception('Snapshot node list too short')
+
+    type_ix = self._node_list[ix + self._node_type_ix]
+    type_string = self._node_types[int(type_ix)]
+
+    # edges_end is noninclusive (the index of the first edge that is not part of
+    # this node).
+    edge_count = self._node_list[ix + self._node_edge_count_ix]
+    edges_end = edges_start + edge_count * self._edge_field_count
+
+    if ChromeJsHeapSnapshotParser._IsNodeTypeUninteresting(type_string):
+      return edges_end
+
+    name_ix = self._node_list[ix + self._node_name_ix]
+    node_id = self._node_list[ix + self._node_id_ix]
+
+    def ConstructorName(type_string, node_name_ix):
+      if type_string == 'object':
+        return self._strings[int(node_name_ix)]
+      return '(%s)' % type_string
+
+    ctor_name = ConstructorName(type_string, name_ix)
+    n = live_heap_object.LiveHeapObject(node_id, type_string, ctor_name)
+    if type_string == 'string':
+      n.string = self._strings[int(name_ix)]
+
+    for edge_ix in xrange(edges_start, edges_end, self._edge_field_count):
+      edge = self._ReadEdgeFromIndex(node_id, edge_ix)
+      if edge:
+        # The edge will be associated with the other endpoint when all the data
+        # has been read.
+        n.AddEdgeFrom(edge)
+
+    self._node_dict[node_id] = n
+    return edges_end
+
+  @staticmethod
+  def _IsNodeTypeUninteresting(type_string):
+    """Helper function for filtering out nodes from the heap snapshot.
+
+    Args:
+      type_string: str, type of the node.
+    Returns:
+      bool, True if the node is of an uninteresting type and shouldn't be
+          included in the heap snapshot analysis.
+    """
+    uninteresting_types = ('hidden', 'code', 'number', 'native', 'synthetic')
+    return type_string in uninteresting_types
+
+  @staticmethod
+  def _IsEdgeTypeUninteresting(edge_type_string):
+    """Helper function for filtering out edges from the heap snapshot.
+
+    Args:
+      edge_type_string: str, type of the edge.
+    Returns:
+      bool, True if the edge is of an uninteresting type and shouldn't be
+          included in the heap snapshot analysis.
+    """
+    uninteresting_types = ('weak', 'hidden', 'internal')
+    return edge_type_string in uninteresting_types
+
+  def _ReadEdgeFromIndex(self, node_id, edge_ix):
+    """Reads the data for an edge from the heap snapshot.
+
+    Args:
+      node_id: int, id of the node which is the starting point of the edge.
+      edge_ix: int, index into the self._edge_list array.
+    Returns:
+      Edge, if the index contains an interesting edge, otherwise None.
+    Raises:
+      Exception: The node list of the snapshot is malformed.
+    """
+    if edge_ix + self._edge_field_count > len(self._edge_list):
+      raise Exception('Snapshot edge list too short')
+
+    edge_type_ix = self._edge_list[edge_ix + self._edge_type_ix]
+    edge_type_string = self._edge_types[int(edge_type_ix)]
+
+    if ChromeJsHeapSnapshotParser._IsEdgeTypeUninteresting(edge_type_string):
+      return None
+
+    child_name_or_ix = self._edge_list[edge_ix + self._edge_name_or_ix_ix]
+    child_node_ix = self._edge_list[edge_ix + self._edge_to_node_ix]
+
+    # The child_node_ix is an index into the node list. Read the actual
+    # node information.
+    child_node_type_ix = self._node_list[child_node_ix + self._node_type_ix]
+    child_node_type_string = self._node_types[int(child_node_type_ix)]
+    child_node_id = self._node_list[child_node_ix + self._node_id_ix]
+
+    if ChromeJsHeapSnapshotParser._IsNodeTypeUninteresting(
+        child_node_type_string):
+      return None
+
+    child_name_string = ''
+    # For element nodes, the child has no name (only an index).
+    if (edge_type_string == 'element' or
+        int(child_name_or_ix) >= len(self._strings)):
+      child_name_string = str(child_name_or_ix)
+    else:
+      child_name_string = self._strings[int(child_name_or_ix)]
+    return retaining_edge.RetainingEdge(node_id, child_node_id,
+                                        edge_type_string, child_name_string)

tools/telemetry/telemetry/core/heap/chrome_js_heap_snapshot_parser_unittest.py

+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import unittest
+
+from telemetry.core.heap import chrome_js_heap_snapshot_parser
+
+class ChromeJsHeapSnapshotParserUnittest(unittest.TestCase):
+  def _HeapSnapshotData(self, node_types, edge_types, node_fields, edge_fields,
+                        node_list, edge_list, strings):
+    """Helper for creating heap snapshot data."""
+    return {'snapshot': {'meta': {'node_types': [node_types],
+                                  'edge_types': [edge_types],
+                                  'node_fields': node_fields,
+                                  'edge_fields': edge_fields}},
+            'nodes': node_list,
+            'edges': edge_list,
+            'strings': strings}
+
+  def testParseSimpleSnapshot(self):
+    # Create a snapshot containing 2 nodes and an edge between them.
+    node_types = ['object']
+    edge_types = ['property']
+    node_fields = ['type', 'name', 'id', 'edge_count']
+    edge_fields = ['type', 'name_or_index', 'to_node']
+    node_list = [0, 0, 0, 1,
+                 0, 1, 1, 0]
+    edge_list = [0, 2, 4]
+    strings = ['node1', 'node2', 'edge1']
+    heap = self._HeapSnapshotData(node_types, edge_types, node_fields,
+                                  edge_fields, node_list, edge_list, strings)
+    objects = list(chrome_js_heap_snapshot_parser.ChromeJsHeapSnapshotParser(
+        json.dumps(heap)).GetAllLiveHeapObjects())
+    self.assertEqual(2, len(objects))
+    if objects[0].edges_from:
+      from_ix = 0
+      to_ix = 1
+    else:
+      from_ix = 1
+      to_ix = 0
+    self.assertEqual('node1', objects[from_ix].class_name)
+    self.assertEqual('node2', objects[to_ix].class_name)
+    self.assertEqual(1, len(objects[from_ix].edges_from))
+    self.assertEqual(0, len(objects[from_ix].edges_to))
+    self.assertEqual(0, len(objects[to_ix].edges_from))
+    self.assertEqual(1, len(objects[to_ix].edges_to))
+    self.assertEqual('node1',
+                     objects[from_ix].edges_from[0].from_object.class_name)
+    self.assertEqual('node2',
+                     objects[from_ix].edges_from[0].to_object.class_name)
+    self.assertEqual('edge1', objects[from_ix].edges_from[0].name_string)
+    self.assertEqual('node1', objects[to_ix].edges_to[0].from_object.class_name)
+    self.assertEqual('node2', objects[to_ix].edges_to[0].to_object.class_name)
+    self.assertEqual('edge1', objects[to_ix].edges_to[0].name_string)

tools/telemetry/telemetry/core/heap/live_heap_object.py

+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+class LiveHeapObject(object):
+  """Data structure for representing an object in the heap snapshot.
+
+  Attributes:
+    object_id: int, identifier for the object.
+    type_string: str, describes the type of the node.
+    class_name: str, describes the class of the JavaScript object
+        represented by this LiveHeapObject.
+    edges_to: [RetainingEdge], edges whose end point this LiveHeapObject is.
+    edges_from: [RetainingEdge], edges whose start point this LiveHeapObject is.
+    string: str, for string LiveHeapObjects, contains the string the
+        LiveHeapObject represents. Empty string for LiveHeapObjects which are
+        not strings.
+    name: str, how to refer to this LiveHeapObject.
+  """
+
+  def __init__(self, object_id, type_string, class_name):
+    """Initializes the LiveHeapObject object.
+
+    Args:
+      object_id: int, identifier for the LiveHeapObject.
+      type_string: str, the type of the node.
+      class_name: str, the class of the object this LiveHeapObject represents.
+    """
+    self.object_id = object_id
+    self.type_string = type_string
+    self.class_name = class_name
+    self.edges_to = []
+    self.edges_from = []
+    self.string = ''
+    self.name = ''
+
+  def AddEdgeTo(self, edge):
+    """Associates an Edge with the LiveHeapObject (the end point).
+
+    Args:
+      edge: Edge, an edge whose end point this LiveHeapObject is.
+    """
+    self.edges_to.append(edge)
+
+  def AddEdgeFrom(self, edge):
+    """Associates an Edge with the LiveHeapObject (the start point).
+
+    Args:
+      edge: Edge, an edge whose start point this LiveHeapObject is.
+    """
+    self.edges_from.append(edge)
+
+  def __str__(self):
+    prefix = 'LiveHeapObject(' + str(self.object_id) + ' '
+    if self.type_string == 'object':
+      return prefix + self.class_name + ')'
+    return prefix + self.type_string + ')'

tools/telemetry/telemetry/core/heap/model.py

+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from telemetry.core.heap import chrome_js_heap_snapshot_parser
+
+class Model(object):
+  """ The heap snapshot model is a set of LiveHeapObjects. The LiveHeapObjects
+  contain the RetainingEdge objects describing the relationships between the
+  LiveHeapObjects."""
+
+  def __init__(self, raw_data):
+    if not chrome_js_heap_snapshot_parser.ChromeJsHeapSnapshotParser.CanImport(
+        raw_data):
+      raise ValueError("Cannot import snapshot data")
+    parser = chrome_js_heap_snapshot_parser.ChromeJsHeapSnapshotParser(raw_data)
+    self._all_live_heap_objects = parser.GetAllLiveHeapObjects()
+
+  @property
+  def all_live_heap_objects(self):
+    return self._all_live_heap_objects

tools/telemetry/telemetry/core/heap/retaining_edge.py

+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+class RetainingEdge(object):
+  """Data structure for representing a retainer relationship between objects.
+
+  Attributes:
+    from_object_id: int, id of the object which is the start point of this
+        RetainingEdge.  Used when the corresponding LiveHeapObject object is not
+        yet contstructed.
+    to_object_id: int, id of the object which is the end point of this
+        RetainingEdge. Used when the corresponding LiveHeapObject object is not
+        yet contstructed.
+    from_object: LiveHeapObject, the start point of this RetainingEdge.
+    to_object: LiveHeapObject, the end point of this RetainingEdge.
+    type_string: str, the type of the RetainingEdge.
+    name_string: str, the JavaScript attribute name this RetainingEdge
+        represents.
+  """
+
+  def __init__(self, from_object_id, to_object_id, type_string, name_string):
+    """Initializes the RetainingEdge object.
+
+    Args:
+      from_object_id: int, id of the object which is the start point of this
+          RetainingEdge. Used when the corresponding LiveHeapObject object is
+          not yet contstructed.
+      to_object_id: int, id of the object which is the end point of this
+          RetainingEdge. Used when the corresponding LiveHeapObject object is
+          not yet contstructed.
+      type_string: str, the type of the RetainingEdge.
+      name_string: str, the JavaScript attribute name this RetainingEdge
+          represents.
+    """
+    self.from_object_id = from_object_id
+    self.to_object_id = to_object_id
+    self.from_object = {}
+    self.to_object = {}
+    self.type_string = type_string
+    self.name_string = name_string
+
+  def SetFromObject(self, obj):
+    self.from_object = obj
+    return self
+
+  def SetToObject(self, obj):
+    self.to_object = obj
+    return self
+
+  def __str__(self):
+    return 'RetainingEdge(' + self.type_string + ' ' + self.name_string + ')'

tools/telemetry/telemetry/core/jsheap/model.py

-# Copyright 2013 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-class JsHeapSnapshotModel(object):
-  def __init__(self, raw_data):
-    # FIXME(marja): Parse the raw data (copy the parsing code from jsleakcheck).
-    pass