Add mmap and native heap classifiers to memory_inspector.

    
This introduces the final piece to the classifier which are
able to do basic parsing for mmap and native heap traces.
    
BUG=340294
NOTRY=true

Review URL: https://codereview.chromium.org/184043006

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@255933 0039d316-1c4b-4281-b951-d872f2087c98

tools/memory_inspector/memory_inspector/classification/mmap_classifier.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""This module classifies MemoryMap objects filtering their mmap entries.
+
+Two filters are currently available:
+  - 'mmap_file': 'foo.*\.so': matches any entry which mmap file is foo*.so.
+  - 'mmap_prot': 'rw.-': matches any entry which prot. flags is rw*-.
+"""
+
+import re
+
+from memory_inspector.classification import results
+from memory_inspector.classification import rules
+from memory_inspector.core import exceptions
+from memory_inspector.core import memory_map
+
+
+_RESULT_KEYS = ['total_rss', 'priv_clean_bytes', 'priv_dirty_bytes',
+                'shared_clean_bytes', 'shared_dirty_bytes']
+
+
+def LoadRules(content):
+  """Loads and parses a mmap rule tree from a content (string).
+
+  Returns:
+    An instance of |rules.Rule|, which nodes are |_MmapRule| instances.
+  """
+  return rules.Load(content, _MmapRule)
+
+
+def Classify(mmap, rule_tree):
+  """Create aggregated results of memory maps using the provided rules.
+
+  Args:
+    mmap: the memory map dump being processed (a |memory_map.Map| instance).
+    rule_tree: the user-defined rules that define the filtering categories.
+
+  Returns:
+    An instance of |AggreatedResults|.
+  """
+  assert(isinstance(mmap, memory_map.Map))
+  assert(isinstance(rule_tree, rules.Rule))
+
+  res = results.AggreatedResults(rule_tree, _RESULT_KEYS)
+  for map_entry in mmap.entries:
+    values = [0, map_entry.priv_dirty_bytes, map_entry.priv_clean_bytes,
+              map_entry.shared_dirty_bytes, map_entry.shared_clean_bytes]
+    values[0] = values[1] + values[2] + values[3] + values[4]
+    res.AddToMatchingNodes(map_entry, values)
+  return res
+
+
+class _MmapRule(rules.Rule):
+  def __init__(self, name, filters):
+    super(_MmapRule, self).__init__(name)
+    try:
+      self._file_re = (
+          re.compile(filters['mmap_file']) if 'mmap_file' in filters else None)
+      self._prot_re = (
+          re.compile(filters['mmap_prot']) if 'mmap_prot' in filters else None)
+    except re.error, descr:
+      raise exceptions.MemoryInspectorException(
+        'Regex parse error "%s" : %s' % (filters, descr))
+
+  def Match(self, map_entry):
+    if self._file_re and not self._file_re.search(map_entry.mapped_file):
+      return False
+    if self._prot_re and not self._prot_re.search(map_entry.prot_flags):
+      return False
+    return True
\ No newline at end of file

tools/memory_inspector/memory_inspector/classification/mmap_classifier_unittest.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import unittest
+
+from memory_inspector.classification import mmap_classifier
+from memory_inspector.core import memory_map
+
+
+_TEST_RULES = """
+[
+{
+  'name': 'anon',
+  'mmap_file': r'^\[anon',
+  'children': [
+    {
+      'name': 'jit',
+      'mmap_prot': 'r-x',
+    },
+  ],
+},
+{
+  'name': 'dev',
+  'mmap_file': r'^/dev',
+  'children': [
+    {
+      'name': 'gpu',
+      'mmap_file': r'/gpu',
+    },
+  ],
+},
+{
+  'name': 'lib',
+  'mmap_file': r'.so$',
+  'children': [
+    {
+      'name': 'data',
+      'mmap_prot': 'rw',
+    },
+    {
+      'name': 'text',
+      'mmap_prot': 'r-x',
+    },
+  ],
+},
+]
+"""
+
+_TEST_MMAPS = [
+#    START    END      PROT    FILE          P.Dirt  P.Clean  S.Dirt  S.Clean
+    (0x00000, 0x03fff, 'rw--', '[anon]',     4096,   0,       4096,   0),
+    (0x04000, 0x07fff, 'rw--', '/lib/1.so',  8192,   0,       0,      0),
+    (0x08000, 0x0bfff, 'r-x-', '/lib/1.so',  4096,   8192,    0,      0),
+    (0x0c000, 0x0ffff, 'rw--', '/lib/2.so',  0,      0,       4096,   8192),
+    (0x10000, 0x13fff, 'r-x-', '/lib/2.so',  0,      12288,   0,      4096),
+    (0x14000, 0x17fff, 'rw--', '/dev/gpu/1', 4096,   0,       0,      0),
+    (0x18000, 0x1bfff, 'rw--', '/dev/gpu/2', 8192,   0,       4096,   0),
+    (0x1c000, 0x1ffff, 'rw--', '/dev/foo',   0,      4096,    0,      8192),
+    (0x20000, 0x23fff, 'r-x-', '[anon:jit]', 8192,   0,       4096,   0),
+    (0x24000, 0x27fff, 'r---', 'OTHER',      0,      0,       8192,   0),
+]
+
+_EXPECTED_RESULTS = {
+    'Total':                                [36864,  24576,  24576,  20480],
+    'Total::anon':                          [12288,  0,      8192,   0],
+    'Total::anon::jit':                     [8192,   0,      4096,   0],
+    'Total::anon::anon-other':              [4096,   0,      4096,   0],
+    'Total::dev':                           [12288,  4096,   4096,   8192],
+    'Total::dev::gpu':                      [12288,  0,      4096,   0],
+    'Total::dev::dev-other':                [0,      4096,   0,      8192],
+    'Total::lib':                           [12288,  20480,  4096,   12288],
+    'Total::lib::data':                     [8192,   0,      4096,   8192],
+    'Total::lib::text':                     [4096,   20480,  0,      4096],
+    'Total::lib::lib-other':                [0,      0,      0,      0],
+    'Total::Total-other':                   [0,      0,      8192,   0],
+}
+
+
+
+class MmapClassifierTest(unittest.TestCase):
+  def runTest(self):
+    rule_tree = mmap_classifier.LoadRules(_TEST_RULES)
+    mmap = memory_map.Map()
+    for m in _TEST_MMAPS:
+      mmap.Add(memory_map.MapEntry(
+          m[0], m[1], m[2], m[3], 0, m[4], m[5], m[6], m[7]))
+
+    res = mmap_classifier.Classify(mmap, rule_tree)
+
+    def CheckResult(node, prefix):
+      node_name = prefix + node.name
+      self.assertIn(node_name, _EXPECTED_RESULTS)
+      subtotal = node.values[0]
+      values = node.values[1:]
+
+      # First check that the subtotal matches clean + dirty + shared + priv.
+      self.assertEqual(subtotal, values[0] + values[1] + values[2] + values[3])
+
+      # Then check that the single values match the expectations.
+      self.assertEqual(values, _EXPECTED_RESULTS[node_name])
+
+      for child in node.children:
+        CheckResult(child, node_name + '::')
+
+    CheckResult(res.total, '')
\ No newline at end of file

tools/memory_inspector/memory_inspector/classification/native_heap_classifier.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""This module classifies NativeHeap objects filtering their allocations.
+
+The only filter currently available is 'stacktrace', which works as follows:
+
+{'name': 'rule-1', 'stacktrace': 'foo' }
+{'name': 'rule-2', 'stacktrace': ['foo', r'bar\s+baz']}
+
+rule-1 will match any allocation that has 'foo' in one of its  stack frames.
+rule-2 will match any allocation that has a stack frame matching 'foo' AND a
+followed by a stack frame matching 'bar baz'. Note that order matters, so rule-2
+will not match a stacktrace like ['bar baz', 'foo'].
+
+TODO(primiano): introduce more filters after the first prototype with UI, for
+instance, filter by source file path, library file name or by allocation size.
+"""
+
+import re
+
+from memory_inspector.classification import results
+from memory_inspector.classification import rules
+from memory_inspector.core import exceptions
+from memory_inspector.core import native_heap
+
+
+_RESULT_KEYS = ['bytes_allocated']
+
+
+def LoadRules(content):
+  """Loads and parses a native-heap rule tree from a content (string).
+
+  Returns:
+    An instance of |rules.Rule|, which nodes are |_NHeapRule| instances.
+  """
+  return rules.Load(content, _NHeapRule)
+
+
+def Classify(nativeheap, rule_tree):
+  """Create aggregated results of native heaps using the provided rules.
+
+  Args:
+    nativeheap: the heap dump being processed (a |NativeHeap| instance).
+    rule_tree: the user-defined rules that define the filtering categories.
+
+  Returns:
+    An instance of |AggreatedResults|.
+  """
+  assert(isinstance(nativeheap, native_heap.NativeHeap))
+  assert(isinstance(rule_tree, rules.Rule))
+
+  res = results.AggreatedResults(rule_tree, _RESULT_KEYS)
+  for allocation in nativeheap.allocations:
+    res.AddToMatchingNodes(allocation, [allocation.total_size])
+  return res
+
+
+class _NHeapRule(rules.Rule):
+  def __init__(self, name, filters):
+    super(_NHeapRule, self).__init__(name)
+    stacktrace_regexs = filters.get('stacktrace', [])
+    # The 'stacktrace' filter can be either a string (simple case, one regex) or
+    # a list of strings (complex case, see doc in the header of this file).
+    if isinstance(stacktrace_regexs, basestring):
+      stacktrace_regexs = [stacktrace_regexs]
+    self._stacktrace_regexs = []
+    for regex in stacktrace_regexs:
+      try:
+        self._stacktrace_regexs.append(re.compile(regex))
+      except re.error, descr:
+        raise exceptions.MemoryInspectorException(
+            'Regex parse error "%s" : %s' % (regex, descr))
+
+  def Match(self, allocation):
+    if not self._stacktrace_regexs:
+      return True
+    cur_regex_idx = 0
+    cur_regex = self._stacktrace_regexs[0]
+    for frame in allocation.stack_trace.frames:
+      if frame.symbol and cur_regex.search(frame.symbol.name):
+        # The current regex has been matched.
+        if cur_regex_idx == len(self._stacktrace_regexs) - 1:
+          return True  # All the provided regexs have been matched, we're happy.
+        cur_regex_idx += 1
+        cur_regex = self._stacktrace_regexs[cur_regex_idx]
+
+    return False  # Not all the provided regexs have been matched.
\ No newline at end of file

tools/memory_inspector/memory_inspector/classification/native_heap_classifier_unittest.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import unittest
+
+from memory_inspector.classification import native_heap_classifier
+from memory_inspector.core import native_heap
+from memory_inspector.core import stacktrace
+from memory_inspector.core import symbol
+
+
+_TEST_RULES = """
+[
+{
+  'name': 'content',
+  'stacktrace': r'content::',
+  'children': [
+    {
+      'name': 'browser',
+      'stacktrace': r'content::browser',
+    },
+    {
+      'name': 'renderer',
+      'stacktrace': r'content::renderer',
+    },
+  ],
+},
+{
+  'name': 'ashmem_in_skia',
+  'stacktrace': [r'sk::', r'ashmem::'],
+},
+]
+"""
+
+_TEST_STACK_TRACES = [
+    (3, ['stack_frame_0::foo()', 'this_goes_under_totals_other']),
+    (5, ['foo', 'content::browser::something()', 'bar']),
+    (7, ['content::browser::something_else()']),
+    (11, ['content::browser::something_else_more()', 'foo']),
+    (13, ['foo', 'content::renderer::something()', 'bar']),
+    (17, ['content::renderer::something_else()']),
+    (19, ['content::renderer::something_else_more()', 'foo']),
+    (23, ['content::something_different']),
+    (29, ['foo', 'sk::something', 'not_ashsmem_goes_into_totals_other']),
+    (31, ['foo', 'sk::something', 'foo::bar', 'sk::foo::ashmem::alloc()']),
+    (37, ['foo', 'sk::something', 'sk::foo::ashmem::alloc()']),
+    (43, ['foo::ashmem::alloc()', 'sk::foo', 'wrong_order_goes_into_totals'])
+]
+
+_EXPECTED_RESULTS = {
+    'Total':                         [238],
+    'Total::content':                [95],
+    'Total::content::browser':       [23],  # 5 + 7 + 11.
+    'Total::content::renderer':      [49],  # 13 + 17 + 19.
+    'Total::content::content-other': [23],
+    'Total::ashmem_in_skia':         [68],  # 31 + 37.
+    'Total::Total-other':            [75],  # 3 + 29 + 43.
+}
+
+
+class NativeHeapClassifierTest(unittest.TestCase):
+  def runTest(self):
+    rule_tree = native_heap_classifier.LoadRules(_TEST_RULES)
+    nheap = native_heap.NativeHeap()
+    mock_addr = 0
+    for test_entry in _TEST_STACK_TRACES:
+      mock_strace = stacktrace.Stacktrace()
+      for mock_btstr in test_entry[1]:
+        mock_addr += 4  # Addr is irrelevant, just keep it distinct.
+        mock_frame = stacktrace.Frame(mock_addr)
+        mock_frame.SetSymbolInfo(symbol.Symbol(mock_btstr))
+        mock_strace.Add(mock_frame)
+      nheap.Add(native_heap.Allocation(
+          size=test_entry[0], count=1, stack_trace=mock_strace))
+
+    res = native_heap_classifier.Classify(nheap, rule_tree)
+
+    def CheckResult(node, prefix):
+      node_name = prefix + node.name
+      self.assertIn(node_name, _EXPECTED_RESULTS)
+      self.assertEqual(node.values, _EXPECTED_RESULTS[node_name])
+      for child in node.children:
+        CheckResult(child, node_name + '::')
+
+    CheckResult(res.total, '')
\ No newline at end of file

tools/memory_inspector/memory_inspector/core/exceptions.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+class MemoryInspectorException(Exception):
+  """Base exception class used for all memory inspector related exceptions."""
+  def __init__(self, value):
+    super(MemoryInspectorException, self).__init__(value)