Breakdown memory usage by source file names in dmprof.

dmprof has classified memory usage by pattern-matching with function names.
It adds another classification by pattern-matching with source file names.

BUG=225343
NOTRY=true

Review URL: https://chromiumcodereview.appspot.com/13514003

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@193022 0039d316-1c4b-4281-b951-d872f2087c98

tools/deep_memory_profiler/dmprof.py

@@ -4,7 +4,7 @@
 
 """The deep heap profiler script for Chrome."""
 
-from datetime import datetime
+import datetime
 import json
 import logging
 import optparse
@@ -20,10 +20,12 @@ FIND_RUNTIME_SYMBOLS_PATH = os.path.join(
     BASE_PATH, os.pardir, 'find_runtime_symbols')
 sys.path.append(FIND_RUNTIME_SYMBOLS_PATH)
 
-from find_runtime_symbols import find_runtime_symbols_list
-from find_runtime_symbols import find_runtime_typeinfo_symbols_list
-from find_runtime_symbols import RuntimeSymbolsInProcess
-from prepare_symbol_info import prepare_symbol_info
+import find_runtime_symbols
+import prepare_symbol_info
+
+from find_runtime_symbols import FUNCTION_SYMBOLS
+from find_runtime_symbols import SOURCEFILE_SYMBOLS
+from find_runtime_symbols import TYPEINFO_SYMBOLS
 
 BUCKET_ID = 5
 VIRTUAL = 0
@@ -34,8 +36,6 @@ NULL_REGEX = re.compile('')
 
 LOGGER = logging.getLogger('dmprof')
 POLICIES_JSON_PATH = os.path.join(BASE_PATH, 'policies.json')
-FUNCTION_ADDRESS = 'function'
-TYPEINFO_ADDRESS = 'typeinfo'
 
 
 # Heap Profile Dump versions
@@ -155,8 +155,12 @@ class SymbolDataSources(object):
         True if succeeded.
     """
     LOGGER.info('Preparing symbol mapping...')
-    self._prepared_symbol_data_sources_path, used_tempdir = prepare_symbol_info(
-        self._prefix + '.maps', self._prefix + '.symmap', True)
+    self._prepared_symbol_data_sources_path, used_tempdir = (
+        prepare_symbol_info.prepare_symbol_info(
+            self._prefix + '.maps',
+            output_dir_path=self._prefix + '.symmap',
+            use_tempdir=True,
+            use_source_file_name=True))
     if self._prepared_symbol_data_sources_path:
       LOGGER.info('  Prepared symbol mapping.')
       if used_tempdir:
@@ -178,8 +182,9 @@ class SymbolDataSources(object):
       return None
     if not self._loaded_symbol_data_sources:
       LOGGER.info('Loading symbol mapping...')
-      self._loaded_symbol_data_sources = RuntimeSymbolsInProcess.load(
-          self._prepared_symbol_data_sources_path)
+      self._loaded_symbol_data_sources = (
+          find_runtime_symbols.RuntimeSymbolsInProcess.load(
+              self._prepared_symbol_data_sources_path))
     return self._loaded_symbol_data_sources
 
   def path(self):
@@ -195,17 +200,13 @@ class SymbolFinder(object):
   This class does only 'find()' symbols from a specified |address_list|.
   It is introduced to make a finder mockable.
   """
-  _FIND_RUNTIME_SYMBOLS_FUNCTIONS = {
-      FUNCTION_ADDRESS: find_runtime_symbols_list,
-      TYPEINFO_ADDRESS: find_runtime_typeinfo_symbols_list,
-      }
-
-  def __init__(self, address_type, symbol_data_sources):
-    self._finder_function = self._FIND_RUNTIME_SYMBOLS_FUNCTIONS[address_type]
+  def __init__(self, symbol_type, symbol_data_sources):
+    self._symbol_type = symbol_type
     self._symbol_data_sources = symbol_data_sources
 
   def find(self, address_list):
-    return self._finder_function(self._symbol_data_sources.get(), address_list)
+    return find_runtime_symbols.find_runtime_symbols(
+        self._symbol_type, self._symbol_data_sources.get(), address_list)
 
 
 class SymbolMappingCache(object):
@@ -216,11 +217,12 @@ class SymbolMappingCache(object):
   """
   def __init__(self):
     self._symbol_mapping_caches = {
-        FUNCTION_ADDRESS: {},
-        TYPEINFO_ADDRESS: {},
+        FUNCTION_SYMBOLS: {},
+        SOURCEFILE_SYMBOLS: {},
+        TYPEINFO_SYMBOLS: {},
         }
 
-  def update(self, address_type, bucket_set, symbol_finder, cache_f):
+  def update(self, symbol_type, bucket_set, symbol_finder, cache_f):
     """Updates symbol mapping cache on memory and in a symbol cache file.
 
     It reads cached symbol mapping from a symbol cache file |cache_f| if it
@@ -234,18 +236,18 @@ class SymbolMappingCache(object):
       ...
 
     Args:
-        address_type: A type of addresses to update.
-            It should be one of FUNCTION_ADDRESS or TYPEINFO_ADDRESS.
+        symbol_type: A type of symbols to update.  It should be one of
+            FUNCTION_SYMBOLS, SOURCEFILE_SYMBOLS and TYPEINFO_SYMBOLS.
         bucket_set: A BucketSet object.
         symbol_finder: A SymbolFinder object to find symbols.
         cache_f: A readable and writable IO object of the symbol cache file.
     """
     cache_f.seek(0, os.SEEK_SET)
-    self._load(cache_f, address_type)
+    self._load(cache_f, symbol_type)
 
     unresolved_addresses = sorted(
-        address for address in bucket_set.iter_addresses(address_type)
-        if address not in self._symbol_mapping_caches[address_type])
+        address for address in bucket_set.iter_addresses(symbol_type)
+        if address not in self._symbol_mapping_caches[symbol_type])
 
     if not unresolved_addresses:
       LOGGER.info('No need to resolve any more addresses.')
@@ -253,36 +255,36 @@ class SymbolMappingCache(object):
 
     cache_f.seek(0, os.SEEK_END)
     LOGGER.info('Loading %d unresolved addresses.' %
-                   len(unresolved_addresses))
-    symbol_list = symbol_finder.find(unresolved_addresses)
+                len(unresolved_addresses))
+    symbol_dict = symbol_finder.find(unresolved_addresses)
 
-    for address, symbol in zip(unresolved_addresses, symbol_list):
-      stripped_symbol = symbol.strip() or '??'
-      self._symbol_mapping_caches[address_type][address] = stripped_symbol
+    for address, symbol in symbol_dict.iteritems():
+      stripped_symbol = symbol.strip() or '?'
+      self._symbol_mapping_caches[symbol_type][address] = stripped_symbol
       cache_f.write('%x %s\n' % (address, stripped_symbol))
 
-  def lookup(self, address_type, address):
+  def lookup(self, symbol_type, address):
     """Looks up a symbol for a given |address|.
 
     Args:
-        address_type: A type of addresses to lookup.
-            It should be one of FUNCTION_ADDRESS or TYPEINFO_ADDRESS.
+        symbol_type: A type of symbols to update.  It should be one of
+            FUNCTION_SYMBOLS, SOURCEFILE_SYMBOLS and TYPEINFO_SYMBOLS.
         address: An integer that represents an address.
 
     Returns:
         A string that represents a symbol.
     """
-    return self._symbol_mapping_caches[address_type].get(address)
+    return self._symbol_mapping_caches[symbol_type].get(address)
 
-  def _load(self, cache_f, address_type):
+  def _load(self, cache_f, symbol_type):
     try:
       for line in cache_f:
         items = line.rstrip().split(None, 1)
         if len(items) == 1:
           items.append('??')
-        self._symbol_mapping_caches[address_type][int(items[0], 16)] = items[1]
+        self._symbol_mapping_caches[symbol_type][int(items[0], 16)] = items[1]
       LOGGER.info('Loaded %d entries from symbol cache.' %
-                     len(self._symbol_mapping_caches[address_type]))
+                     len(self._symbol_mapping_caches[symbol_type]))
     except IOError as e:
       LOGGER.info('The symbol cache file is invalid: %s' % e)
 
@@ -290,14 +292,28 @@ class SymbolMappingCache(object):
 class Rule(object):
   """Represents one matching rule in a policy file."""
 
-  def __init__(self, name, mmap, stacktrace_pattern, typeinfo_pattern=None):
+  def __init__(self,
+               name,
+               mmap,
+               stackfunction_pattern=None,
+               stacksourcefile_pattern=None,
+               typeinfo_pattern=None):
     self._name = name
     self._mmap = mmap
-    self._stacktrace_pattern = re.compile(stacktrace_pattern + r'\Z')
+
+    self._stackfunction_pattern = None
+    if stackfunction_pattern:
+      self._stackfunction_pattern = re.compile(
+          stackfunction_pattern + r'\Z')
+
+    self._stacksourcefile_pattern = None
+    if stacksourcefile_pattern:
+      self._stacksourcefile_pattern = re.compile(
+          stacksourcefile_pattern + r'\Z')
+
+    self._typeinfo_pattern = None
     if typeinfo_pattern:
       self._typeinfo_pattern = re.compile(typeinfo_pattern + r'\Z')
-    else:
-      self._typeinfo_pattern = None
 
   @property
   def name(self):
@@ -308,8 +324,12 @@ class Rule(object):
     return self._mmap
 
   @property
-  def stacktrace_pattern(self):
-    return self._stacktrace_pattern
+  def stackfunction_pattern(self):
+    return self._stackfunction_pattern
+
+  @property
+  def stacksourcefile_pattern(self):
+    return self._stacksourcefile_pattern
 
   @property
   def typeinfo_pattern(self):
@@ -350,14 +370,18 @@ class Policy(object):
     if bucket.component_cache:
       return bucket.component_cache
 
-    stacktrace = bucket.symbolized_joined_stacktrace
+    stackfunction = bucket.symbolized_joined_stackfunction
+    stacksourcefile = bucket.symbolized_joined_stacksourcefile
     typeinfo = bucket.symbolized_typeinfo
     if typeinfo.startswith('0x'):
       typeinfo = bucket.typeinfo_name
 
     for rule in self._rules:
       if (bucket.mmap == rule.mmap and
-          rule.stacktrace_pattern.match(stacktrace) and
+          (not rule.stackfunction_pattern or
+           rule.stackfunction_pattern.match(stackfunction)) and
+          (not rule.stacksourcefile_pattern or
+           rule.stacksourcefile_pattern.match(stacksourcefile)) and
           (not rule.typeinfo_pattern or rule.typeinfo_pattern.match(typeinfo))):
         bucket.component_cache = rule.name
         return rule.name
@@ -414,11 +438,15 @@ class Policy(object):
 
     rules = []
     for rule in policy['rules']:
+      stackfunction = rule.get('stackfunction') or rule.get('stacktrace')
+      stacksourcefile = rule.get('stacksourcefile')
       rules.append(Rule(
           rule['name'],
           rule['allocator'] == 'mmap',
-          rule['stacktrace'],
+          stackfunction,
+          stacksourcefile,
           rule['typeinfo'] if 'typeinfo' in rule else None))
+
     return Policy(rules, policy['version'], policy['components'])
 
 
@@ -493,8 +521,10 @@ class Bucket(object):
     self._typeinfo = typeinfo
     self._typeinfo_name = typeinfo_name
 
-    self._symbolized_stacktrace = stacktrace
-    self._symbolized_joined_stacktrace = ''
+    self._symbolized_stackfunction = stacktrace
+    self._symbolized_joined_stackfunction = ''
+    self._symbolized_stacksourcefile = stacktrace
+    self._symbolized_joined_stacksourcefile = ''
     self._symbolized_typeinfo = typeinfo_name
 
     self.component_cache = ''
@@ -506,15 +536,21 @@ class Bucket(object):
         symbol_mapping_cache: A SymbolMappingCache object.
     """
     # TODO(dmikurube): Fill explicitly with numbers if symbol not found.
-    self._symbolized_stacktrace = [
-        symbol_mapping_cache.lookup(FUNCTION_ADDRESS, address)
+    self._symbolized_stackfunction = [
+        symbol_mapping_cache.lookup(FUNCTION_SYMBOLS, address)
+        for address in self._stacktrace]
+    self._symbolized_joined_stackfunction = ' '.join(
+        self._symbolized_stackfunction)
+    self._symbolized_stacksourcefile = [
+        symbol_mapping_cache.lookup(SOURCEFILE_SYMBOLS, address)
         for address in self._stacktrace]
-    self._symbolized_joined_stacktrace = ' '.join(self._symbolized_stacktrace)
+    self._symbolized_joined_stacksourcefile = ' '.join(
+        self._symbolized_stacksourcefile)
     if not self._typeinfo:
       self._symbolized_typeinfo = 'no typeinfo'
     else:
       self._symbolized_typeinfo = symbol_mapping_cache.lookup(
-          TYPEINFO_ADDRESS, self._typeinfo)
+          TYPEINFO_SYMBOLS, self._typeinfo)
       if not self._symbolized_typeinfo:
         self._symbolized_typeinfo = 'no typeinfo'
 
@@ -538,12 +574,20 @@ class Bucket(object):
     return self._typeinfo_name
 
   @property
-  def symbolized_stacktrace(self):
-    return self._symbolized_stacktrace
+  def symbolized_stackfunction(self):
+    return self._symbolized_stackfunction
 
   @property
-  def symbolized_joined_stacktrace(self):
-    return self._symbolized_joined_stacktrace
+  def symbolized_joined_stackfunction(self):
+    return self._symbolized_joined_stackfunction
+
+  @property
+  def symbolized_stacksourcefile(self):
+    return self._symbolized_stacksourcefile
+
+  @property
+  def symbolized_joined_stacksourcefile(self):
+    return self._symbolized_joined_stacksourcefile
 
   @property
   def symbolized_typeinfo(self):
@@ -554,10 +598,8 @@ class BucketSet(object):
   """Represents a set of bucket."""
   def __init__(self):
     self._buckets = {}
-    self._addresses = {
-        FUNCTION_ADDRESS: set(),
-        TYPEINFO_ADDRESS: set(),
-        }
+    self._code_addresses = set()
+    self._typeinfo_addresses = set()
 
   def load(self, prefix):
     """Loads all related bucket files.
@@ -591,7 +633,7 @@ class BucketSet(object):
           continue
         if word[0] == 't':
           typeinfo = int(word[1:], 16)
-          self._addresses[TYPEINFO_ADDRESS].add(typeinfo)
+          self._typeinfo_addresses.add(typeinfo)
         elif word[0] == 'n':
           typeinfo_name = word[1:]
         else:
@@ -599,7 +641,7 @@ class BucketSet(object):
           break
       stacktrace = [int(address, 16) for address in words[stacktrace_begin:]]
       for frame in stacktrace:
-        self._addresses[FUNCTION_ADDRESS].add(frame)
+        self._code_addresses.add(frame)
       self._buckets[int(words[0])] = Bucket(
           stacktrace, words[1] == 'mmap', typeinfo, typeinfo_name)
 
@@ -621,9 +663,13 @@ class BucketSet(object):
     for bucket_content in self._buckets.itervalues():
       bucket_content.clear_component_cache()
 
-  def iter_addresses(self, address_type):
-    for function in self._addresses[address_type]:
-      yield function
+  def iter_addresses(self, symbol_type):
+    if symbol_type in [FUNCTION_SYMBOLS, SOURCEFILE_SYMBOLS]:
+      for function in self._code_addresses:
+        yield function
+    else:
+      for function in self._typeinfo_addresses:
+        yield function
 
 
 class Dump(object):
@@ -840,14 +886,18 @@ class Command(object):
     else:
       dump = Dump.load(dump_path)
     symbol_mapping_cache = SymbolMappingCache()
-    with open(prefix + '.funcsym', 'a+') as cache_f:
+    with open(prefix + '.cache.function', 'a+') as cache_f:
+      symbol_mapping_cache.update(
+          FUNCTION_SYMBOLS, bucket_set,
+          SymbolFinder(FUNCTION_SYMBOLS, symbol_data_sources), cache_f)
+    with open(prefix + '.cache.typeinfo', 'a+') as cache_f:
       symbol_mapping_cache.update(
-          FUNCTION_ADDRESS, bucket_set,
-          SymbolFinder(FUNCTION_ADDRESS, symbol_data_sources), cache_f)
-    with open(prefix + '.typesym', 'a+') as cache_f:
+          TYPEINFO_SYMBOLS, bucket_set,
+          SymbolFinder(TYPEINFO_SYMBOLS, symbol_data_sources), cache_f)
+    with open(prefix + '.cache.sourcefile', 'a+') as cache_f:
       symbol_mapping_cache.update(
-          TYPEINFO_ADDRESS, bucket_set,
-          SymbolFinder(TYPEINFO_ADDRESS, symbol_data_sources), cache_f)
+          SOURCEFILE_SYMBOLS, bucket_set,
+          SymbolFinder(SOURCEFILE_SYMBOLS, symbol_data_sources), cache_f)
     bucket_set.symbolize(symbol_mapping_cache)
     if multiple:
       return (bucket_set, dump_list)
@@ -936,7 +986,7 @@ class StacktraceCommand(Command):
         continue
       for i in range(0, BUCKET_ID - 1):
         out.write(words[i] + ' ')
-      for frame in bucket.symbolized_stacktrace:
+      for frame in bucket.symbolized_stackfunction:
         out.write(frame + ' ')
       out.write('\n')
 
@@ -1121,7 +1171,7 @@ class JSONCommand(PolicyCommands):
         component_sizes = PolicyCommands._apply_policy(
             dump, policy_set[label], bucket_set, dumps[0].time)
         component_sizes['dump_path'] = dump.path
-        component_sizes['dump_time'] = datetime.fromtimestamp(
+        component_sizes['dump_time'] = datetime.datetime.fromtimestamp(
             dump.time).strftime('%Y-%m-%d %H:%M:%S')
         json_base['policies'][label]['snapshots'].append(component_sizes)
 
@@ -1197,6 +1247,7 @@ class ExpandCommand(Command):
     sorted_sizes_list = sorted(
         sizes.iteritems(), key=(lambda x: x[1]), reverse=True)
     total = 0
+    # TODO(dmikurube): Better formatting.
     for size_pair in sorted_sizes_list:
       out.write('%10d %s\n' % (size_pair[1], size_pair[0]))
       total += size_pair[1]
@@ -1213,9 +1264,12 @@ class ExpandCommand(Command):
         if bucket.typeinfo:
           stacktrace_sequence += '(type=%s)' % bucket.symbolized_typeinfo
           stacktrace_sequence += ' (type.name=%s) ' % bucket.typeinfo_name
-        for stack in bucket.symbolized_stacktrace[
-            0 : min(len(bucket.symbolized_stacktrace), 1 + depth)]:
-          stacktrace_sequence += stack + ' '
+        for function, sourcefile in zip(
+            bucket.symbolized_stackfunction[
+                0 : min(len(bucket.symbolized_stackfunction), 1 + depth)],
+            bucket.symbolized_stacksourcefile[
+                0 : min(len(bucket.symbolized_stacksourcefile), 1 + depth)]):
+          stacktrace_sequence += '%s(@%s) ' % (function, sourcefile)
         if not stacktrace_sequence in sizes:
           sizes[stacktrace_sequence] = 0
         sizes[stacktrace_sequence] += int(words[COMMITTED])

tools/deep_memory_profiler/policies.json

 {
+  "sourcefile": {
+    "file": "policy.sourcefile.json",
+    "format": "json"
+  },
   "l0": {
     "file": "policy.l0.json",
     "format": "json"

tools/deep_memory_profiler/policy.sourcefile.json

+{
+  "components": [
+    "second",
+    "mmap-profiler",
+    "mmap-type-profiler",
+    "mmap-tcmalloc",
+    "FROM_HERE_FOR_TOTAL",
+    "mustbezero",
+    "unhooked-absent",
+    "unhooked-anonymous",
+    "unhooked-file-exec",
+    "unhooked-file-nonexec",
+    "unhooked-stack",
+    "unhooked-other",
+    "no-bucket",
+    "mmap-v8",
+    "mmap-catch-all",
+    "tc-v8",
+    "tc-skia",
+    "tc-webcore",
+    "tc-webkit",
+    "tc-catch-all",
+    "tc-unused",
+    "UNTIL_HERE_FOR_TOTAL",
+    "total-exclude-profiler",
+    "total",
+    "absent",
+    "anonymous",
+    "file-exec",
+    "file-nonexec",
+    "stack",
+    "other",
+    "mmap-total-log",
+    "mmap-no-log",
+    "mmap-total-record",
+    "other-total-log",
+    "tc-total-log",
+    "tc-no-log",
+    "tc-total-record",
+    "tc-total"
+  ],
+  "rules": [
+    {
+      "name": "second",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "mmap-profiler",
+      "stacktrace": ".*(ProfilerMalloc|MemoryRegionMap::).*",
+      "allocator": "mmap"
+    },
+    {
+      "name": "mmap-type-profiler",
+      "stacktrace": ".*(TypeProfilerMalloc).*",
+      "allocator": "mmap"
+    },
+    {
+      "name": "mmap-tcmalloc",
+      "stacktrace": ".*(DoAllocWithArena|SbrkSysAllocator::Alloc|MmapSysAllocator::Alloc|LowLevelAlloc::Alloc|LowLevelAlloc::AllocWithArena).*",
+      "allocator": "mmap"
+    },
+    {
+      "name": "FROM_HERE_FOR_TOTAL",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "mustbezero",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "unhooked-absent",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "unhooked-anonymous",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "unhooked-file-exec",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "unhooked-file-nonexec",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "unhooked-stack",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "unhooked-other",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "mmap-v8",
+      "stacksourcefile": ".*\\.\\./\\.\\./v8/src/.*",
+      "allocator": "mmap"
+    },
+    {
+      "name": "mmap-catch-all",
+      "stacksourcefile": ".*",
+      "allocator": "mmap"
+    },
+    {
+      "name": "tc-v8",
+      "stacksourcefile": ".*\\.\\./\\.\\./v8/src/.*",
+      "allocator": "malloc"
+    },
+    {
+      "name": "tc-skia",
+      "stacksourcefile": ".*\\.\\./\\.\\./third_party/skia/src/.*",
+      "allocator": "malloc"
+    },
+    {
+      "name": "tc-webcore",
+      "stacksourcefile": ".*\\.\\./\\.\\./third_party/WebKit/Source/WebCore/.*",
+      "allocator": "malloc"
+    },
+    {
+      "name": "tc-webkit",
+      "stacksourcefile": ".*\\.\\./\\.\\./third_party/WebKit/Source/.*",
+      "allocator": "malloc"
+    },
+    {
+      "name": "tc-catch-all",
+      "stacksourcefile": ".*",
+      "allocator": "malloc"
+    },
+    {
+      "name": "UNTIL_HERE_FOR_TOTAL",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "total-exclude-profiler",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "total",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "absent",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "anonymous",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "file-exec",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "file-nonexec",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "stack",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
+    {
+      "name": "other",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    }
+  ],
+  "version": "POLICY_DEEP_3"
+}
\ No newline at end of file

tools/deep_memory_profiler/tests/dmprof_test.py

@@ -13,8 +13,17 @@ import unittest
 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, ROOT_DIR)
 
+try:
+  from collections import OrderedDict  # pylint: disable=E0611
+except ImportError:
+  SIMPLEJSON_PATH = os.path.join(ROOT_DIR, os.pardir, os.pardir, 'third_party')
+  sys.path.insert(0, SIMPLEJSON_PATH)
+  from simplejson import OrderedDict
+
 import dmprof
-from dmprof import FUNCTION_ADDRESS, TYPEINFO_ADDRESS
+from find_runtime_symbols import FUNCTION_SYMBOLS
+from find_runtime_symbols import SOURCEFILE_SYMBOLS
+from find_runtime_symbols import TYPEINFO_SYMBOLS
 
 
 class SymbolMappingCacheTest(unittest.TestCase):
@@ -22,7 +31,7 @@ class SymbolMappingCacheTest(unittest.TestCase):
     def __init__(self, addresses):
       self._addresses = addresses
 
-    def iter_addresses(self, address_type):  # pylint: disable=W0613
+    def iter_addresses(self, symbol_type):  # pylint: disable=W0613
       for address in self._addresses:
         yield address
 
@@ -31,7 +40,10 @@ class SymbolMappingCacheTest(unittest.TestCase):
       self._mapping = mapping
 
     def find(self, address_list):
-      return [self._mapping[address] for address in address_list]
+      result = OrderedDict()
+      for address in address_list:
+        result[address] = self._mapping[address]
+      return result
 
   _TEST_FUNCTION_CACHE = textwrap.dedent("""\
       1 0x0000000000000001
@@ -70,35 +82,39 @@ class SymbolMappingCacheTest(unittest.TestCase):
 
     # No update from self._TEST_FUNCTION_CACHE
     symbol_mapping_cache.update(
-        FUNCTION_ADDRESS,
+        FUNCTION_SYMBOLS,
         self.MockBucketSet(self._TEST_FUNCTION_ADDRESS_LIST1),
         self.MockSymbolFinder(self._TEST_FUNCTION_DICT), cache_f)
     for address in self._TEST_FUNCTION_ADDRESS_LIST1:
       self.assertEqual(self._TEST_FUNCTION_DICT[address],
-                       symbol_mapping_cache.lookup(FUNCTION_ADDRESS, address))
+                       symbol_mapping_cache.lookup(FUNCTION_SYMBOLS, address))
     self.assertEqual(self._TEST_FUNCTION_CACHE, cache_f.getvalue())
 
     # Update to self._TEST_FUNCTION_ADDRESS_LIST2
     symbol_mapping_cache.update(
-        FUNCTION_ADDRESS,
+        FUNCTION_SYMBOLS,
         self.MockBucketSet(self._TEST_FUNCTION_ADDRESS_LIST2),
         self.MockSymbolFinder(self._TEST_FUNCTION_DICT), cache_f)
     for address in self._TEST_FUNCTION_ADDRESS_LIST2:
       self.assertEqual(self._TEST_FUNCTION_DICT[address],
-                       symbol_mapping_cache.lookup(FUNCTION_ADDRESS, address))
+                       symbol_mapping_cache.lookup(FUNCTION_SYMBOLS, address))
     self.assertEqual(self._EXPECTED_TEST_FUNCTION_CACHE, cache_f.getvalue())
 
 
 class PolicyTest(unittest.TestCase):
   class MockSymbolMappingCache(object):
     def __init__(self):
-      self._symbol_caches = {FUNCTION_ADDRESS: {}, TYPEINFO_ADDRESS: {}}
+      self._symbol_caches = {
+          FUNCTION_SYMBOLS: {},
+          SOURCEFILE_SYMBOLS: {},
+          TYPEINFO_SYMBOLS: {},
+          }
 
-    def add(self, address_type, address, symbol):
-      self._symbol_caches[address_type][address] = symbol
+    def add(self, symbol_type, address, symbol):
+      self._symbol_caches[symbol_type][address] = symbol
 
-    def lookup(self, address_type, address):
-      symbol = self._symbol_caches[address_type].get(address)
+    def lookup(self, symbol_type, address):
+      symbol = self._symbol_caches[symbol_type].get(address)
       return symbol if symbol else '0x%016x' % address
 
   _TEST_POLICY = textwrap.dedent("""\
@@ -157,8 +173,8 @@ class PolicyTest(unittest.TestCase):
     self.assertTrue(policy)
 
     symbol_mapping_cache = self.MockSymbolMappingCache()
-    symbol_mapping_cache.add(FUNCTION_ADDRESS, 0x1212, 'v8::create')
-    symbol_mapping_cache.add(FUNCTION_ADDRESS, 0x1381, 'WebKit::create')
+    symbol_mapping_cache.add(FUNCTION_SYMBOLS, 0x1212, 'v8::create')
+    symbol_mapping_cache.add(FUNCTION_SYMBOLS, 0x1381, 'WebKit::create')
 
     bucket1 = dmprof.Bucket([0x1212, 0x013], False, 0x29492, '_Z')
     bucket1.symbolize(symbol_mapping_cache)

tools/find_runtime_symbols/find_runtime_symbols.py

@@ -2,6 +2,11 @@
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+"""Find symbols in a binary corresponding to given runtime virtual addresses.
+
+Note that source file names are treated as symbols in this script while they
+are actually not.
+"""
 
 import json
 import logging
@@ -11,37 +16,21 @@ import sys
 from static_symbols import StaticSymbolsInFile
 from proc_maps import ProcMaps
 
+try:
+  from collections import OrderedDict  # pylint: disable=E0611
+except ImportError:
+  BASE_PATH = os.path.dirname(os.path.abspath(__file__))
+  SIMPLEJSON_PATH = os.path.join(BASE_PATH, os.pardir, os.pardir, 'third_party')
+  sys.path.insert(0, SIMPLEJSON_PATH)
+  from simplejson import OrderedDict
 
-_MAPS_FILENAME = 'maps'
-_FILES_FILENAME = 'files.json'
-
-
-class _ListOutput(object):
-  def __init__(self, result):
-    self.result = result
-
-  def output(self, address, symbol):  # pylint: disable=W0613
-    self.result.append(symbol)
 
+FUNCTION_SYMBOLS = 0
+SOURCEFILE_SYMBOLS = 1
+TYPEINFO_SYMBOLS = 2
 
-class _DictOutput(object):
-  def __init__(self, result):
-    self.result = result
-
-  def output(self, address, symbol):
-    self.result[address] = symbol
-
-
-class _FileOutput(object):
-  def __init__(self, result, with_address):
-    self.result = result
-    self.with_address = with_address
-
-  def output(self, address, symbol):
-    if self.with_address:
-      self.result.write('%016x %s\n' % (address, symbol))
-    else:
-      self.result.write('%s\n' % symbol)
+_MAPS_FILENAME = 'maps'
+_FILES_FILENAME = 'files.json'
 
 
 class RuntimeSymbolsInProcess(object):
@@ -60,6 +49,17 @@ class RuntimeSymbolsInProcess(object):
           return None
     return None
 
+  def find_sourcefile(self, runtime_address):
+    for vma in self._maps.iter(ProcMaps.executable):
+      if vma.begin <= runtime_address < vma.end:
+        static_symbols = self._static_symbols_in_filse.get(vma.name)
+        if static_symbols:
+          return static_symbols.find_sourcefile_by_runtime_address(
+              runtime_address, vma)
+        else:
+          return None
+    return None
+
   def find_typeinfo(self, runtime_address):
     for vma in self._maps.iter(ProcMaps.constants):
       if vma.begin <= runtime_address < vma.end:
@@ -99,73 +99,71 @@ class RuntimeSymbolsInProcess(object):
                   'r') as f:
           static_symbols.load_readelf_ew(f)
 
+      decodedline_file_entry = file_entry.get('readelf-debug-decodedline-file')
+      if decodedline_file_entry:
+        with open(os.path.join(prepared_data_dir,
+                               decodedline_file_entry['file']), 'r') as f:
+          static_symbols.load_readelf_debug_decodedline_file(f)
+
       symbols_in_process._static_symbols_in_filse[vma.name] = static_symbols
 
     return symbols_in_process
 
 
-def _find_runtime_symbols(symbols_in_process, addresses, outputter):
+def _find_runtime_function_symbols(symbols_in_process, addresses):
+  result = OrderedDict()
   for address in addresses:
     if isinstance(address, basestring):
       address = int(address, 16)
     found = symbols_in_process.find_procedure(address)
     if found:
-      outputter.output(address, found.name)
+      result[address] = found.name
     else:
-      outputter.output(address, '0x%016x' % address)
+      result[address] = '0x%016x' % address
+  return result
 
 
-def _find_runtime_typeinfo_symbols(symbols_in_process, addresses, outputter):
+def _find_runtime_sourcefile_symbols(symbols_in_process, addresses):
+  result = OrderedDict()
+  for address in addresses:
+    if isinstance(address, basestring):
+      address = int(address, 16)
+    found = symbols_in_process.find_sourcefile(address)
+    if found:
+      result[address] = found
+    else:
+      result[address] = ''
+  return result
+
+
+def _find_runtime_typeinfo_symbols(symbols_in_process, addresses):
+  result = OrderedDict()
   for address in addresses:
     if isinstance(address, basestring):
       address = int(address, 16)
     if address == 0:
-      outputter.output(address, 'no typeinfo')
+      result[address] = 'no typeinfo'
     else:
       found = symbols_in_process.find_typeinfo(address)
       if found:
         if found.startswith('typeinfo for '):
-          outputter.output(address, found[13:])
+          result[address] = found[13:]
         else:
-          outputter.output(address, found)
+          result[address] = found
       else:
-        outputter.output(address, '0x%016x' % address)
-
-
-def find_runtime_typeinfo_symbols_list(symbols_in_process, addresses):
-  result = []
-  _find_runtime_typeinfo_symbols(
-      symbols_in_process, addresses, _ListOutput(result))
+        result[address] = '0x%016x' % address
   return result
 
 
-def find_runtime_typeinfo_symbols_dict(symbols_in_process, addresses):
-  result = {}
-  _find_runtime_typeinfo_symbols(
-      symbols_in_process, addresses, _DictOutput(result))
-  return result
-
-
-def find_runtime_typeinfo_symbols_file(symbols_in_process, addresses, f):
-  _find_runtime_typeinfo_symbols(
-      symbols_in_process, addresses, _FileOutput(f, False))
+_INTERNAL_FINDERS = {
+    FUNCTION_SYMBOLS: _find_runtime_function_symbols,
+    SOURCEFILE_SYMBOLS: _find_runtime_sourcefile_symbols,
+    TYPEINFO_SYMBOLS: _find_runtime_typeinfo_symbols,
+    }
 
 
-def find_runtime_symbols_list(symbols_in_process, addresses):
-  result = []
-  _find_runtime_symbols(symbols_in_process, addresses, _ListOutput(result))
-  return result
-
-
-def find_runtime_symbols_dict(symbols_in_process, addresses):
-  result = {}
-  _find_runtime_symbols(symbols_in_process, addresses, _DictOutput(result))
-  return result
-
-
-def find_runtime_symbols_file(symbols_in_process, addresses, f):
-  _find_runtime_symbols(
-      symbols_in_process, addresses, _FileOutput(f, False))
+def find_runtime_symbols(symbol_type, symbols_in_process, addresses):
+  return _INTERNAL_FINDERS[symbol_type](symbols_in_process, addresses)
 
 
 def main():
@@ -193,7 +191,16 @@ def main():
     return 1
 
   symbols_in_process = RuntimeSymbolsInProcess.load(prepared_data_dir)
-  return find_runtime_symbols_file(symbols_in_process, sys.stdin, sys.stdout)
+  symbols_dict = find_runtime_symbols(FUNCTION_SYMBOLS,
+                                      symbols_in_process,
+                                      sys.stdin)
+  for address, symbol in symbols_dict:
+    if symbol:
+      print '%016x %s' % (address, symbol)
+    else:
+      print '%016x' % address
+
+  return 0
 
 
 if __name__ == '__main__':

tools/find_runtime_symbols/static_symbols.py

@@ -33,7 +33,7 @@ class AddressMapping(object):
 
 class RangeAddressMapping(AddressMapping):
   def __init__(self):
-    AddressMapping.__init__(self)
+    super(RangeAddressMapping, self).__init__()
     self._sorted_start_list = []
     self._is_sorted = True
 
@@ -47,6 +47,8 @@ class RangeAddressMapping(AddressMapping):
     self._symbol_map[start] = entry
 
   def find(self, address):
+    if not self._sorted_start_list:
+      return None
     if not self._is_sorted:
       self._sorted_start_list.sort()
       self._is_sorted = True
@@ -119,6 +121,7 @@ class StaticSymbolsInFile(object):
     self.my_name = my_name
     self._elf_sections = []
     self._procedures = RangeAddressMapping()
+    self._sourcefiles = RangeAddressMapping()
     self._typeinfos = AddressMapping()
 
   def _append_elf_section(self, elf_section):
@@ -127,6 +130,9 @@ class StaticSymbolsInFile(object):
   def _append_procedure(self, start, procedure):
     self._procedures.append(start, procedure)
 
+  def _append_sourcefile(self, start, sourcefile):
+    self._sourcefiles.append(start, sourcefile)
+
   def _append_typeinfo(self, start, typeinfo):
     self._typeinfos.append(start, typeinfo)
 
@@ -150,6 +156,9 @@ class StaticSymbolsInFile(object):
   def find_procedure_by_runtime_address(self, address, vma):
     return self._find_symbol_by_runtime_address(address, vma, self._procedures)
 
+  def find_sourcefile_by_runtime_address(self, address, vma):
+    return self._find_symbol_by_runtime_address(address, vma, self._sourcefiles)
+
   def find_typeinfo_by_runtime_address(self, address, vma):
     return self._find_symbol_by_runtime_address(address, vma, self._typeinfos)
 
@@ -183,6 +192,11 @@ class StaticSymbolsInFile(object):
         if line in ('Key to Flags:', 'Program Headers:'):
           break
 
+  def load_readelf_debug_decodedline_file(self, input_file):
+    for line in input_file:
+      splitted = line.rstrip().split(None, 2)
+      self._append_sourcefile(int(splitted[0], 16), splitted[1])
+
   @staticmethod
   def _parse_nm_bsd_line(line):
     if line[8] == ' ':