Breakdown memory usage by source file names in dmprof.

dmprof has classified memory usage by pattern-matching with function names.
It adds another classification by pattern-matching with source file names.

BUG=225343
NOTRY=true

Review URL: https://chromiumcodereview.appspot.com/13514003

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@193022 0039d316-1c4b-4281-b951-d872f2087c98

tools/deep_memory_profiler/policies.json

 {
+  "sourcefile": {
+    "file": "policy.sourcefile.json",
+    "format": "json"
+  },
   "l0": {
     "file": "policy.l0.json",
     "format": "json"

tools/deep_memory_profiler/policy.sourcefile.json

+{
+  "components": [
+    "second",
+    "mmap-profiler",
+    "mmap-type-profiler",
+    "mmap-tcmalloc",
+    "FROM_HERE_FOR_TOTAL",
+    "mustbezero",
+    "unhooked-absent",
+    "unhooked-anonymous",
+    "unhooked-file-exec",
+    "unhooked-file-nonexec",
+    "unhooked-stack",
+    "unhooked-other",
+    "no-bucket",
+    "mmap-v8",
+    "mmap-catch-all",
+    "tc-v8",
+    "tc-skia",
+    "tc-webcore",
+    "tc-webkit",
+    "tc-catch-all",
+    "tc-unused",
+    "UNTIL_HERE_FOR_TOTAL",
+    "total-exclude-profiler",
+    "total",
+    "absent",
+    "anonymous",
+    "file-exec",
+    "file-nonexec",
+    "stack",
+    "other",
+    "mmap-total-log",
+    "mmap-no-log",
+    "mmap-total-record",
+    "other-total-log",
+    "tc-total-log",
+    "tc-no-log",
+    "tc-total-record",
+    "tc-total"
+  ],
+  "rules": [
+    {
+      "name": "second",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "mmap-profiler",
+      "stacktrace": ".*(ProfilerMalloc|MemoryRegionMap::).*",
+      "allocator": "mmap"
+    },
+    {
+      "name": "mmap-type-profiler",
+      "stacktrace": ".*(TypeProfilerMalloc).*",
+      "allocator": "mmap"
+    },
+    {
+      "name": "mmap-tcmalloc",
+      "stacktrace": ".*(DoAllocWithArena|SbrkSysAllocator::Alloc|MmapSysAllocator::Alloc|LowLevelAlloc::Alloc|LowLevelAlloc::AllocWithArena).*",
+      "allocator": "mmap"
+    },
+    {
+      "name": "FROM_HERE_FOR_TOTAL",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "mustbezero",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "unhooked-absent",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "unhooked-anonymous",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "unhooked-file-exec",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "unhooked-file-nonexec",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "unhooked-stack",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "unhooked-other",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "mmap-v8",
+      "stacksourcefile": ".*\\.\\./\\.\\./v8/src/.*",
+      "allocator": "mmap"
+    },
+    {
+      "name": "mmap-catch-all",
+      "stacksourcefile": ".*",
+      "allocator": "mmap"
+    },
+    {
+      "name": "tc-v8",
+      "stacksourcefile": ".*\\.\\./\\.\\./v8/src/.*",
+      "allocator": "malloc"
+    },
+    {
+      "name": "tc-skia",
+      "stacksourcefile": ".*\\.\\./\\.\\./third_party/skia/src/.*",
+      "allocator": "malloc"
+    },
+    {
+      "name": "tc-webcore",
+      "stacksourcefile": ".*\\.\\./\\.\\./third_party/WebKit/Source/WebCore/.*",
+      "allocator": "malloc"
+    },
+    {
+      "name": "tc-webkit",
+      "stacksourcefile": ".*\\.\\./\\.\\./third_party/WebKit/Source/.*",
+      "allocator": "malloc"
+    },
+    {
+      "name": "tc-catch-all",
+      "stacksourcefile": ".*",
+      "allocator": "malloc"
+    },
+    {
+      "name": "UNTIL_HERE_FOR_TOTAL",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "total-exclude-profiler",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "total",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "absent",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "anonymous",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "file-exec",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "file-nonexec",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "stack",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "other",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    }
+  ],
+  "version": "POLICY_DEEP_3"
+}
\ No newline at end of file

tools/deep_memory_profiler/tests/dmprof_test.py

@@ -13,8 +13,17 @@ import unittest
 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, ROOT_DIR)
 
+try:
+  from collections import OrderedDict  # pylint: disable=E0611
+except ImportError:
+  SIMPLEJSON_PATH = os.path.join(ROOT_DIR, os.pardir, os.pardir, 'third_party')
+  sys.path.insert(0, SIMPLEJSON_PATH)
+  from simplejson import OrderedDict
+
 import dmprof
-from dmprof import FUNCTION_ADDRESS, TYPEINFO_ADDRESS
+from find_runtime_symbols import FUNCTION_SYMBOLS
+from find_runtime_symbols import SOURCEFILE_SYMBOLS
+from find_runtime_symbols import TYPEINFO_SYMBOLS
 
 
 class SymbolMappingCacheTest(unittest.TestCase):
@@ -22,7 +31,7 @@ class SymbolMappingCacheTest(unittest.TestCase):
     def __init__(self, addresses):
       self._addresses = addresses
 
-    def iter_addresses(self, address_type):  # pylint: disable=W0613
+    def iter_addresses(self, symbol_type):  # pylint: disable=W0613
       for address in self._addresses:
         yield address
 
@@ -31,7 +40,10 @@ class SymbolMappingCacheTest(unittest.TestCase):
       self._mapping = mapping
 
     def find(self, address_list):
-      return [self._mapping[address] for address in address_list]
+      result = OrderedDict()
+      for address in address_list:
+        result[address] = self._mapping[address]
+      return result
 
   _TEST_FUNCTION_CACHE = textwrap.dedent("""\
       1 0x0000000000000001
@@ -70,35 +82,39 @@ class SymbolMappingCacheTest(unittest.TestCase):
 
     # No update from self._TEST_FUNCTION_CACHE
     symbol_mapping_cache.update(
-        FUNCTION_ADDRESS,
+        FUNCTION_SYMBOLS,
         self.MockBucketSet(self._TEST_FUNCTION_ADDRESS_LIST1),
         self.MockSymbolFinder(self._TEST_FUNCTION_DICT), cache_f)
     for address in self._TEST_FUNCTION_ADDRESS_LIST1:
       self.assertEqual(self._TEST_FUNCTION_DICT[address],
-                       symbol_mapping_cache.lookup(FUNCTION_ADDRESS, address))
+                       symbol_mapping_cache.lookup(FUNCTION_SYMBOLS, address))
     self.assertEqual(self._TEST_FUNCTION_CACHE, cache_f.getvalue())
 
     # Update to self._TEST_FUNCTION_ADDRESS_LIST2
     symbol_mapping_cache.update(
-        FUNCTION_ADDRESS,
+        FUNCTION_SYMBOLS,
         self.MockBucketSet(self._TEST_FUNCTION_ADDRESS_LIST2),
         self.MockSymbolFinder(self._TEST_FUNCTION_DICT), cache_f)
     for address in self._TEST_FUNCTION_ADDRESS_LIST2:
       self.assertEqual(self._TEST_FUNCTION_DICT[address],
-                       symbol_mapping_cache.lookup(FUNCTION_ADDRESS, address))
+                       symbol_mapping_cache.lookup(FUNCTION_SYMBOLS, address))
     self.assertEqual(self._EXPECTED_TEST_FUNCTION_CACHE, cache_f.getvalue())
 
 
 class PolicyTest(unittest.TestCase):
   class MockSymbolMappingCache(object):
     def __init__(self):
-      self._symbol_caches = {FUNCTION_ADDRESS: {}, TYPEINFO_ADDRESS: {}}
+      self._symbol_caches = {
+          FUNCTION_SYMBOLS: {},
+          SOURCEFILE_SYMBOLS: {},
+          TYPEINFO_SYMBOLS: {},
+          }
 
-    def add(self, address_type, address, symbol):
-      self._symbol_caches[address_type][address] = symbol
+    def add(self, symbol_type, address, symbol):
+      self._symbol_caches[symbol_type][address] = symbol
 
-    def lookup(self, address_type, address):
-      symbol = self._symbol_caches[address_type].get(address)
+    def lookup(self, symbol_type, address):
+      symbol = self._symbol_caches[symbol_type].get(address)
       return symbol if symbol else '0x%016x' % address
 
   _TEST_POLICY = textwrap.dedent("""\
@@ -157,8 +173,8 @@ class PolicyTest(unittest.TestCase):
     self.assertTrue(policy)
 
     symbol_mapping_cache = self.MockSymbolMappingCache()
-    symbol_mapping_cache.add(FUNCTION_ADDRESS, 0x1212, 'v8::create')
-    symbol_mapping_cache.add(FUNCTION_ADDRESS, 0x1381, 'WebKit::create')
+    symbol_mapping_cache.add(FUNCTION_SYMBOLS, 0x1212, 'v8::create')
+    symbol_mapping_cache.add(FUNCTION_SYMBOLS, 0x1381, 'WebKit::create')
 
     bucket1 = dmprof.Bucket([0x1212, 0x013], False, 0x29492, '_Z')
     bucket1.symbolize(symbol_mapping_cache)

tools/find_runtime_symbols/find_runtime_symbols.py

@@ -2,6 +2,11 @@
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+"""Find symbols in a binary corresponding to given runtime virtual addresses.
+
+Note that source file names are treated as symbols in this script while they
+are actually not.
+"""
 
 import json
 import logging
@@ -11,37 +16,21 @@ import sys
 from static_symbols import StaticSymbolsInFile
 from proc_maps import ProcMaps
 
+try:
+  from collections import OrderedDict  # pylint: disable=E0611
+except ImportError:
+  BASE_PATH = os.path.dirname(os.path.abspath(__file__))
+  SIMPLEJSON_PATH = os.path.join(BASE_PATH, os.pardir, os.pardir, 'third_party')
+  sys.path.insert(0, SIMPLEJSON_PATH)
+  from simplejson import OrderedDict
 
-_MAPS_FILENAME = 'maps'
-_FILES_FILENAME = 'files.json'
-
-
-class _ListOutput(object):
-  def __init__(self, result):
-    self.result = result
-
-  def output(self, address, symbol):  # pylint: disable=W0613
-    self.result.append(symbol)
 
+FUNCTION_SYMBOLS = 0
+SOURCEFILE_SYMBOLS = 1
+TYPEINFO_SYMBOLS = 2
 
-class _DictOutput(object):
-  def __init__(self, result):
-    self.result = result
-
-  def output(self, address, symbol):
-    self.result[address] = symbol
-
-
-class _FileOutput(object):
-  def __init__(self, result, with_address):
-    self.result = result
-    self.with_address = with_address
-
-  def output(self, address, symbol):
-    if self.with_address:
-      self.result.write('%016x %s\n' % (address, symbol))
-    else:
-      self.result.write('%s\n' % symbol)
+_MAPS_FILENAME = 'maps'
+_FILES_FILENAME = 'files.json'
 
 
 class RuntimeSymbolsInProcess(object):
@@ -60,6 +49,17 @@ class RuntimeSymbolsInProcess(object):
           return None
     return None
 
+  def find_sourcefile(self, runtime_address):
+    for vma in self._maps.iter(ProcMaps.executable):
+      if vma.begin <= runtime_address < vma.end:
+        static_symbols = self._static_symbols_in_filse.get(vma.name)
+        if static_symbols:
+          return static_symbols.find_sourcefile_by_runtime_address(
+              runtime_address, vma)
+        else:
+          return None
+    return None
+
   def find_typeinfo(self, runtime_address):
     for vma in self._maps.iter(ProcMaps.constants):
       if vma.begin <= runtime_address < vma.end:
@@ -99,73 +99,71 @@ class RuntimeSymbolsInProcess(object):
                   'r') as f:
           static_symbols.load_readelf_ew(f)
 
+      decodedline_file_entry = file_entry.get('readelf-debug-decodedline-file')
+      if decodedline_file_entry:
+        with open(os.path.join(prepared_data_dir,
+                               decodedline_file_entry['file']), 'r') as f:
+          static_symbols.load_readelf_debug_decodedline_file(f)
+
       symbols_in_process._static_symbols_in_filse[vma.name] = static_symbols
 
     return symbols_in_process
 
 
-def _find_runtime_symbols(symbols_in_process, addresses, outputter):
+def _find_runtime_function_symbols(symbols_in_process, addresses):
+  result = OrderedDict()
   for address in addresses:
     if isinstance(address, basestring):
       address = int(address, 16)
     found = symbols_in_process.find_procedure(address)
     if found:
-      outputter.output(address, found.name)
+      result[address] = found.name
     else:
-      outputter.output(address, '0x%016x' % address)
+      result[address] = '0x%016x' % address
+  return result
 
 
-def _find_runtime_typeinfo_symbols(symbols_in_process, addresses, outputter):
+def _find_runtime_sourcefile_symbols(symbols_in_process, addresses):
+  result = OrderedDict()
+  for address in addresses:
+    if isinstance(address, basestring):
+      address = int(address, 16)
+    found = symbols_in_process.find_sourcefile(address)
+    if found:
+      result[address] = found
+    else:
+      result[address] = ''
+  return result
+
+
+def _find_runtime_typeinfo_symbols(symbols_in_process, addresses):
+  result = OrderedDict()
   for address in addresses:
     if isinstance(address, basestring):
       address = int(address, 16)
     if address == 0:
-      outputter.output(address, 'no typeinfo')
+      result[address] = 'no typeinfo'
     else:
       found = symbols_in_process.find_typeinfo(address)
       if found:
         if found.startswith('typeinfo for '):
-          outputter.output(address, found[13:])
+          result[address] = found[13:]
         else:
-          outputter.output(address, found)
+          result[address] = found
       else:
-        outputter.output(address, '0x%016x' % address)
-
-
-def find_runtime_typeinfo_symbols_list(symbols_in_process, addresses):
-  result = []
-  _find_runtime_typeinfo_symbols(
-      symbols_in_process, addresses, _ListOutput(result))
+        result[address] = '0x%016x' % address
   return result
 
 
-def find_runtime_typeinfo_symbols_dict(symbols_in_process, addresses):
-  result = {}
-  _find_runtime_typeinfo_symbols(
-      symbols_in_process, addresses, _DictOutput(result))
-  return result
-
-
-def find_runtime_typeinfo_symbols_file(symbols_in_process, addresses, f):
-  _find_runtime_typeinfo_symbols(
-      symbols_in_process, addresses, _FileOutput(f, False))
+_INTERNAL_FINDERS = {
+    FUNCTION_SYMBOLS: _find_runtime_function_symbols,
+    SOURCEFILE_SYMBOLS: _find_runtime_sourcefile_symbols,
+    TYPEINFO_SYMBOLS: _find_runtime_typeinfo_symbols,
+    }
 
 
-def find_runtime_symbols_list(symbols_in_process, addresses):
-  result = []
-  _find_runtime_symbols(symbols_in_process, addresses, _ListOutput(result))
-  return result
-
-
-def find_runtime_symbols_dict(symbols_in_process, addresses):
-  result = {}
-  _find_runtime_symbols(symbols_in_process, addresses, _DictOutput(result))
-  return result
-
-
-def find_runtime_symbols_file(symbols_in_process, addresses, f):
-  _find_runtime_symbols(
-      symbols_in_process, addresses, _FileOutput(f, False))
+def find_runtime_symbols(symbol_type, symbols_in_process, addresses):
+  return _INTERNAL_FINDERS[symbol_type](symbols_in_process, addresses)
 
 
 def main():
@@ -193,7 +191,16 @@ def main():
     return 1
 
   symbols_in_process = RuntimeSymbolsInProcess.load(prepared_data_dir)
-  return find_runtime_symbols_file(symbols_in_process, sys.stdin, sys.stdout)
+  symbols_dict = find_runtime_symbols(FUNCTION_SYMBOLS,
+                                      symbols_in_process,
+                                      sys.stdin)
+  for address, symbol in symbols_dict:
+    if symbol:
+      print '%016x %s' % (address, symbol)
+    else:
+      print '%016x' % address
+
+  return 0
 
 
 if __name__ == '__main__':

tools/find_runtime_symbols/static_symbols.py

@@ -33,7 +33,7 @@ class AddressMapping(object):
 
 class RangeAddressMapping(AddressMapping):
   def __init__(self):
-    AddressMapping.__init__(self)
+    super(RangeAddressMapping, self).__init__()
     self._sorted_start_list = []
     self._is_sorted = True
 
@@ -47,6 +47,8 @@ class RangeAddressMapping(AddressMapping):
     self._symbol_map[start] = entry
 
   def find(self, address):
+    if not self._sorted_start_list:
+      return None
     if not self._is_sorted:
       self._sorted_start_list.sort()
       self._is_sorted = True
@@ -119,6 +121,7 @@ class StaticSymbolsInFile(object):
     self.my_name = my_name
     self._elf_sections = []
     self._procedures = RangeAddressMapping()
+    self._sourcefiles = RangeAddressMapping()
     self._typeinfos = AddressMapping()
 
   def _append_elf_section(self, elf_section):
@@ -127,6 +130,9 @@ class StaticSymbolsInFile(object):
   def _append_procedure(self, start, procedure):
     self._procedures.append(start, procedure)
 
+  def _append_sourcefile(self, start, sourcefile):
+    self._sourcefiles.append(start, sourcefile)
+
   def _append_typeinfo(self, start, typeinfo):
     self._typeinfos.append(start, typeinfo)
 
@@ -150,6 +156,9 @@ class StaticSymbolsInFile(object):
   def find_procedure_by_runtime_address(self, address, vma):
     return self._find_symbol_by_runtime_address(address, vma, self._procedures)
 
+  def find_sourcefile_by_runtime_address(self, address, vma):
+    return self._find_symbol_by_runtime_address(address, vma, self._sourcefiles)
+
   def find_typeinfo_by_runtime_address(self, address, vma):
     return self._find_symbol_by_runtime_address(address, vma, self._typeinfos)
 
@@ -183,6 +192,11 @@ class StaticSymbolsInFile(object):
         if line in ('Key to Flags:', 'Program Headers:'):
           break
 
+  def load_readelf_debug_decodedline_file(self, input_file):
+    for line in input_file:
+      splitted = line.rstrip().split(None, 2)
+      self._append_sourcefile(int(splitted[0], 16), splitted[1])
+
   @staticmethod
   def _parse_nm_bsd_line(line):
     if line[8] == ' ':