[Android] memory_inspector: move to libheap_profiler.

This change moves towards the new libheap_profiler + heap_dump for
profiling native allocations, instead of lib.debug.malloc.
It introduces the code to install the library in the system (wrapping
the Android zygote) and the parsers.
Compared to the lib.debug.malloc, libheap_profiler is much much faster
(less overhead) and able to hook also mmaps. Its output is very
detailed and allows to get stack traces for each VM region. This will
make it possible to intersect, in the near future, mmaps and native
stack traces.
This change also introduces the support to multiple ABIs to support
both arm and arm64.

BUG=340294
NOTRY=true

Review URL: https://codereview.chromium.org/549313006

Cr-Commit-Position: refs/heads/master@{#295031}

tools/memory_inspector/PRESUBMIT.py

@@ -9,7 +9,7 @@ details on the presubmit API built into gcl.
 """
 
 
-def CommonChecks(input_api, output_api):
+def _CommonChecks(input_api, output_api):
   output = []
   blacklist = [r'classification_rules.*']
   output.extend(input_api.canned_checks.RunPylint(
@@ -30,9 +30,46 @@ def CommonChecks(input_api, output_api):
   return output
 
 
+def _CheckPrebuiltsAreUploaded(input_api, output_api):
+  import sys
+  import urllib2
+  old_sys_path = sys.path
+  try:
+    sys.path.append(input_api.os_path.join(input_api.PresubmitLocalPath()))
+    from memory_inspector import constants
+  finally:
+    sys.path = old_sys_path
+  missing_files = []
+  for f in input_api.os_listdir(constants.PREBUILTS_PATH):
+    if not f.endswith('.sha1'):
+      continue
+    prebuilt_sha_path = input_api.os_path.join(constants.PREBUILTS_PATH, f)
+    with open(prebuilt_sha_path) as sha_file:
+      sha = sha_file.read().strip()
+    url = constants.PREBUILTS_BASE_URL + sha
+    request = urllib2.Request(url)
+    request.get_method = lambda : 'HEAD'
+    try:
+      urllib2.urlopen(request)
+    except Exception, e:
+      if isinstance(e, urllib2.HTTPError) and e.code == 404:
+        missing_files += [prebuilt_sha_path]
+      else:
+        return [output_api.PresubmitError('HTTP Error while checking %s' % url,
+                                          long_text=str(e))]
+  if missing_files:
+    return [output_api.PresubmitError(
+        'Some prebuilts have not been uploaded. Perhaps you forgot to '
+        'upload_to_google_storage.py?', missing_files)]
+  return []
+
+
 def CheckChangeOnUpload(input_api, output_api):
-  return CommonChecks(input_api, output_api)
+  results = []
+  results.extend(_CommonChecks(input_api, output_api))
+  results.extend(_CheckPrebuiltsAreUploaded(input_api, output_api))
+  return results
 
 
 def CheckChangeOnCommit(input_api, output_api):
-  return CommonChecks(input_api, output_api)
+  return _CommonChecks(input_api, output_api)

tools/memory_inspector/memory_inspector/backends/android/android_backend.py

@@ -16,7 +16,7 @@ import posixpath
 
 from memory_inspector import constants
 from memory_inspector.backends import prebuilts_fetcher
-from memory_inspector.backends.android import dumpheap_native_parser
+from memory_inspector.backends.android import native_heap_dump_parser
 from memory_inspector.backends.android import memdump_parser
 from memory_inspector.core import backends
 from memory_inspector.core import exceptions
@@ -31,14 +31,20 @@ from pylib.device import device_utils
 from pylib.symbols import elf_symbolizer
 
 
+_SUPPORTED_32BIT_ABIS = {'armeabi': 'arm', 'armeabi-v7a': 'arm'}
+_SUPPORTED_64BIT_ABIS = {'arm64-v8a': 'arm64'}
 _MEMDUMP_PREBUILT_PATH = os.path.join(constants.PREBUILTS_PATH,
-                                      'memdump-android-arm')
+                                      'memdump-android-%(arch)s')
 _MEMDUMP_PATH_ON_DEVICE = '/data/local/tmp/memdump'
 _PSEXT_PREBUILT_PATH = os.path.join(constants.PREBUILTS_PATH,
-                                    'ps_ext-android-arm')
+                                    'ps_ext-android-%(arch)s')
 _PSEXT_PATH_ON_DEVICE = '/data/local/tmp/ps_ext'
-_DLMALLOC_DEBUG_SYSPROP = 'libc.debug.malloc'
-_DUMPHEAP_OUT_FILE_PATH = '/data/local/tmp/heap-%d-native.dump'
+_HEAP_DUMP_PREBUILT_PATH = os.path.join(constants.PREBUILTS_PATH,
+                                        'heap_dump-android-%(arch)s')
+_HEAP_DUMP_PATH_ON_DEVICE = '/data/local/tmp/heap_dump'
+_LIBHEAPPROF_PREBUILT_PATH = os.path.join(constants.PREBUILTS_PATH,
+                                          'libheap_profiler-android-%(arch)s')
+_LIBHEAPPROF_FILE_NAME = 'libheap_profiler.so'
 
 
 class AndroidBackend(backends.Backend):
@@ -179,6 +185,22 @@ class AndroidDevice(backends.Device):
     self._processes = {}  # pid (int) -> |Process|
     self._initialized = False
 
+    # Determine the available ABIs, |_arch| will contain the primary ABI.
+    # TODO(primiano): For the moment we support only one ABI per device (i.e. we
+    # assume that all processes are 64 bit on 64 bit device, failing to profile
+    # 32 bit ones). Dealing properly with multi-ABIs requires work on ps_ext and
+    # at the moment is not an interesting use case.
+    self._arch = None
+    self._arch32 = None
+    self._arch64 = None
+    abi = adb.GetProp('ro.product.cpu.abi')
+    if abi in _SUPPORTED_64BIT_ABIS:
+      self._arch = self._arch64 = _SUPPORTED_64BIT_ABIS[abi]
+    elif abi in _SUPPORTED_32BIT_ABIS:
+      self._arch = self._arch32 = _SUPPORTED_32BIT_ABIS[abi]
+    else:
+      raise exceptions.MemoryInspectorException('ABI %s not supported' % abi)
+
   def Initialize(self):
     """Starts adb root and deploys the prebuilt binaries on initialization."""
     try:
@@ -190,26 +212,89 @@ class AndroidDevice(backends.Device):
           'The device must be adb root-able in order to use memory_inspector')
 
     # Download (from GCS) and deploy prebuilt helper binaries on the device.
-    self._DeployPrebuiltOnDeviceIfNeeded(_MEMDUMP_PREBUILT_PATH,
-                                         _MEMDUMP_PATH_ON_DEVICE)
-    self._DeployPrebuiltOnDeviceIfNeeded(_PSEXT_PREBUILT_PATH,
-                                         _PSEXT_PATH_ON_DEVICE)
+    self._DeployPrebuiltOnDeviceIfNeeded(
+        _MEMDUMP_PREBUILT_PATH % {'arch': self._arch}, _MEMDUMP_PATH_ON_DEVICE)
+    self._DeployPrebuiltOnDeviceIfNeeded(
+        _PSEXT_PREBUILT_PATH % {'arch': self._arch}, _PSEXT_PATH_ON_DEVICE)
+    self._DeployPrebuiltOnDeviceIfNeeded(
+        _HEAP_DUMP_PREBUILT_PATH % {'arch': self._arch},
+        _HEAP_DUMP_PATH_ON_DEVICE)
+
     self._initialized = True
 
   def IsNativeTracingEnabled(self):
-    """Checks for the libc.debug.malloc system property."""
-    return bool(self.adb.GetProp(_DLMALLOC_DEBUG_SYSPROP))
+    """Checks whether the libheap_profiler is preloaded in the zygote."""
+    zygote_name = 'zygote64' if self._arch64 else 'zygote'
+    zygote_process = [p for p in self.ListProcesses() if p.name == zygote_name]
+    if not zygote_process:
+      raise exceptions.MemoryInspectorException('Zygote process not found')
+    zygote_pid = zygote_process[0].pid
+    zygote_maps = self.adb.RunShellCommand('cat /proc/%d/maps' % zygote_pid)
+    return any(('libheap_profiler' in line for line in zygote_maps))
 
   def EnableNativeTracing(self, enabled):
-    """Enables libc.debug.malloc and restarts the shell."""
+    """Installs libheap_profiler in and injects it in the Zygote."""
+
+    def WrapZygote(app_process):
+      WRAPPER_SCRIPT = ('#!/system/bin/sh\n'
+                        'LD_PRELOAD="libheap_profiler.so:$LD_PRELOAD" '
+                        'exec %s.real "$@"\n' % app_process)
+      self.adb.RunShellCommand('mv %(0)s %(0)s.real' % {'0': app_process})
+      self.adb.WriteFile(app_process, WRAPPER_SCRIPT)
+      self.adb.RunShellCommand('chown root.shell ' + app_process)
+      self.adb.RunShellCommand('chmod 755 ' + app_process)
+
+    def UnwrapZygote():
+      for suffix in ('', '32', '64'):
+        # We don't really care if app_processX.real doesn't exists and mv fails.
+        # If app_processX.real doesn't exists, either app_processX is already
+        # unwrapped or it doesn't exists for the current arch.
+        app_process = '/system/bin/app_process' + suffix
+        self.adb.RunShellCommand('mv %(0)s.real %(0)s' % {'0': app_process})
+
     assert(self._initialized)
-    prop_value = '1' if enabled else ''
-    self.adb.SetProp(_DLMALLOC_DEBUG_SYSPROP, prop_value)
-    assert(self.IsNativeTracingEnabled())
-    # The libc.debug property takes effect only after restarting the Zygote.
+    self.adb.old_interface.MakeSystemFolderWritable()
+
+    # Start restoring the original state in any case.
+    UnwrapZygote()
+
+    if enabled:
+      # Temporarily disable SELinux (until next reboot).
+      self.adb.RunShellCommand('setenforce 0')
+
+      # Wrap the Zygote startup binary (app_process) with a script which
+      # LD_PRELOADs libheap_profiler and invokes the original Zygote process.
+      if self._arch64:
+        app_process = '/system/bin/app_process64'
+        assert(self.adb.FileExists(app_process))
+        self._DeployPrebuiltOnDeviceIfNeeded(
+            _LIBHEAPPROF_PREBUILT_PATH % {'arch': self._arch64},
+            '/system/lib64/' + _LIBHEAPPROF_FILE_NAME)
+        WrapZygote(app_process)
+
+      if self._arch32:
+        # Path is app_process32 for Android >= L, app_process when < L.
+        app_process = '/system/bin/app_process32'
+        if not self.adb.FileExists(app_process):
+          app_process = '/system/bin/app_process'
+          assert(self.adb.FileExists(app_process))
+        self._DeployPrebuiltOnDeviceIfNeeded(
+            _LIBHEAPPROF_PREBUILT_PATH % {'arch': self._arch32},
+            '/system/lib/' + _LIBHEAPPROF_FILE_NAME)
+        WrapZygote(app_process)
+
+    # Respawn the zygote (the device will kind of reboot at this point).
     self.adb.old_interface.RestartShell()
     self.adb.old_interface.Adb().WaitForDevicePm(wait_time=30)
 
+    # Remove the wrapper. This won't have effect until the next reboot, when
+    # the profiler will be automatically disarmed.
+    UnwrapZygote()
+
+    # We can also unlink the lib files at this point. Once the Zygote has
+    # started it will keep the inodes refcounted anyways through its lifetime.
+    self.adb.RunShellCommand('rm /system/lib*/' + _LIBHEAPPROF_FILE_NAME)
+
   def ListProcesses(self):
     """Returns a sequence of |AndroidProcess|."""
     self._RefreshProcessesList()
@@ -324,17 +409,16 @@ class AndroidProcess(backends.Process):
     return memdump_parser.Parse(dump_out)
 
   def DumpNativeHeap(self):
-    """Grabs and parses malloc traces through am dumpheap -n."""
-    # TODO(primiano): grab also mmap bt (depends on pending framework change).
-    dump_file_path = _DUMPHEAP_OUT_FILE_PATH % self.pid
-    cmd = 'am dumpheap -n %d %s' % (self.pid, dump_file_path)
-    self.device.adb.RunShellCommand(cmd)
-    # TODO(primiano): Some pre-KK versions of Android might need a sleep here
-    # as, IIRC, 'am dumpheap' did not wait for the dump to be completed before
-    # returning. Double check this and either add a sleep or remove this TODO.
-    dump_out = self.device.adb.ReadFile(dump_file_path)
-    self.device.adb.RunShellCommand('rm %s' % dump_file_path)
-    return dumpheap_native_parser.Parse(dump_out)
+    """Grabs and parses native heap traces using heap_dump."""
+    cmd = '%s -n -x %d' % (_HEAP_DUMP_PATH_ON_DEVICE, self.pid)
+    out_lines = self.device.adb.RunShellCommand(cmd)
+    return native_heap_dump_parser.Parse('\n'.join(out_lines))
+
+  def Freeze(self):
+    self.device.adb.RunShellCommand('kill -STOP %d' % self.pid)
+
+  def Unfreeze(self):
+    self.device.adb.RunShellCommand('kill -CONT %d' % self.pid)
 
   def GetStats(self):
     """Calculate process CPU/VM stats (CPU stats are relative to last call)."""

tools/memory_inspector/memory_inspector/backends/android/android_backend_unittest.py

@@ -40,22 +40,20 @@ ffff00ffff0000-ffff00ffff1000 r-xp 0 private_unevictable=0 private=0 """
     """shared_app=[] shared_other_unevictable=0 shared_other=0 "[vectors]" []"""
 )
 
-_MOCK_DUMPHEAP_OUT = """Android Native Heap Dump v1.0
-
-Total memory: 1608601
-Allocation records: 2
-
-z 1  sz    100    num    2  bt 9dcd1000 9dcd2000 b570e100 b570e000
-z 0  sz    1000   num    3  bt b570e100 00001234 b570e200
-MAPS
-9dcd0000-9dcd6000 r-xp 00000000 103:00 815       /system/lib/libnbaio.so
-9dcd6000-9dcd7000 r--p 00005000 103:00 815       /system/lib/libnbaio.so
-9dcd7000-9dcd8000 rw-p 00006000 103:00 815       /system/lib/libnbaio.so
-b570e000-b598c000 r-xp 00000000 103:00 680       /system/lib/libart.so
-b598c000-b598d000 ---p 00000000 00:00 0
-b598d000-b5993000 r--p 0027e000 103:00 680       /system/lib/libart.so
-b5993000-b5994000 rw-p 00284000 103:00 680       /system/lib/libart.so
-END"""
+_MOCK_HEAP_DUMP_OUT = """{
+  "total_allocated": 8192,
+  "num_allocs":      2,
+  "num_stacks":      2,
+  "allocs": {
+   "a1000": {"l": 100, "f": 0, "s": "a"},
+   "a2000": {"l": 100, "f": 0, "s": "a"},
+   "b1000": {"l": 1000, "f": 0, "s": "b"},
+   "b2000": {"l": 1000, "f": 0, "s": "b"},
+   "b3000": {"l": 1000, "f": 0, "s": "b"}},
+  "stacks": {
+    "a": {"l": 200,  "f": [10, 20, 30, 40]},
+    "b": {"l": 3000, "f": [50, 60, 70]}}
+  }"""
 
 _MOCK_PS_EXT_OUT = """
 {
@@ -83,10 +81,11 @@ class AndroidBackendTest(unittest.TestCase):
       'devices': _MOCK_DEVICES_OUT,
       'shell getprop ro.product.model': 'Mock device',
       'shell getprop ro.build.type': 'userdebug',
+      'shell getprop ro.product.cpu.abi': 'armeabi',
       'root': 'adbd is already running as root',
       'shell /data/local/tmp/ps_ext': _MOCK_PS_EXT_OUT,
       'shell /data/local/tmp/memdump': _MOCK_MEMDUMP_OUT,
-      'shell cat "/data/local/tmp/heap': _MOCK_DUMPHEAP_OUT,
+      'shell /data/local/tmp/heap_dump': _MOCK_HEAP_DUMP_OUT,
       'shell test -e "/data/local/tmp/heap': '0',
     }
     for (cmd, response) in planned_adb_responses.iteritems():
@@ -156,35 +155,22 @@ class AndroidBackendTest(unittest.TestCase):
     for i in xrange(16, 33):
       self.assertFalse(entry.IsPageResident(i))
 
-    # Test dumpheap parsing.
+    # Test heap_dump parsing.
     heap = processes[0].DumpNativeHeap()
-    self.assertEqual(len(heap.allocations), 2)
-
-    alloc_1 = heap.allocations[0]
-    self.assertEqual(alloc_1.size, 100)
-    self.assertEqual(alloc_1.count, 2)
-    self.assertEqual(alloc_1.total_size, 200)
-    self.assertEqual(alloc_1.stack_trace.depth, 4)
-    self.assertEqual(alloc_1.stack_trace[0].exec_file_rel_path,
-                     '/system/lib/libnbaio.so')
-    self.assertEqual(alloc_1.stack_trace[0].address, 0x9dcd1000)
-    self.assertEqual(alloc_1.stack_trace[0].offset, 0x1000)
-    self.assertEqual(alloc_1.stack_trace[1].offset, 0x2000)
-    self.assertEqual(alloc_1.stack_trace[2].exec_file_rel_path,
-                     '/system/lib/libart.so')
-    self.assertEqual(alloc_1.stack_trace[2].offset, 0x100)
-    self.assertEqual(alloc_1.stack_trace[3].offset, 0)
-
-    alloc_2 = heap.allocations[1]
-    self.assertEqual(alloc_2.size, 1000)
-    self.assertEqual(alloc_2.count, 3)
-    self.assertEqual(alloc_2.total_size, 3000)
-    self.assertEqual(alloc_2.stack_trace.depth, 3)
-    # 0x00001234 is not present in the maps. It should be parsed anyways but
-    # no executable info is expected.
-    self.assertEqual(alloc_2.stack_trace[1].address, 0x00001234)
-    self.assertIsNone(alloc_2.stack_trace[1].exec_file_rel_path)
-    self.assertIsNone(alloc_2.stack_trace[1].offset)
+    self.assertEqual(len(heap.allocations), 5)
+
+    for alloc in heap.allocations:
+      self.assertTrue(alloc.size == 100 or alloc.size == 1000)
+      if alloc.size == 100:
+        self.assertEqual(alloc.size, 100)
+        self.assertEqual(alloc.stack_trace.depth, 4)
+        self.assertEqual([x.address for x in alloc.stack_trace.frames],
+                         [10, 20, 30, 40])
+      elif alloc.size == 1000:
+        self.assertEqual(alloc.size, 1000)
+        self.assertEqual(alloc.stack_trace.depth, 3)
+        self.assertEqual([x.address for x in alloc.stack_trace.frames],
+                         [50, 60, 70])
 
   def tearDown(self):
     self._mock_adb.Stop()

tools/memory_inspector/memory_inspector/backends/android/dumpheap_native_parser.py

-# Copyright 2014 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""This parser turns the am dumpheap -n output into a |NativeHeap| object."""
-
-import logging
-import re
-
-from memory_inspector.core import native_heap
-from memory_inspector.core import memory_map
-from memory_inspector.core import stacktrace
-
-
-def Parse(lines):
-  """Parses the output of Android's am dumpheap -n.
-
-  am dumpheap dumps the oustanding malloc information (when the system property
-  libc.debug.malloc == 1).
-
-  The expected dumpheap output looks like this:
-  ------------------------------------------------------------------------------
-  ... Some irrelevant banner lines ...
-  z 0  sz    1000   num    3  bt 1234 5678 9abc ...
-  ...
-  MAPS
-  9dcd0000-9dcd6000 r-xp 00000000 103:00 815       /system/lib/libnbaio.so
-  ...
-  ------------------------------------------------------------------------------
-  The lines before MAPS list the allocations grouped by {size, backtrace}. In
-  the example above, "1000" is the size of each alloc, "3" is their cardinality
-  and "1234 5678 9abc" are the first N stack frames (absolute addresses in the
-  process virtual address space).  The lines after MAPS provide essentially the
-  same information of /proc/PID/smaps.
-  See tests/android_backend_test.py for a more complete example.
-
-  Args:
-      lines: array of strings containing the am dumpheap -n output.
-
-  Returns:
-      An instance of |native_heap.NativeHeap|.
-  """
-  (STATE_PARSING_BACKTRACES, STATE_PARSING_MAPS, STATE_ENDED) = range(3)
-  BT_RE = re.compile(
-      r'^\w+\s+\d+\s+sz\s+(\d+)\s+num\s+(\d+)\s+bt\s+((?:[0-9a-f]+\s?)+)$')
-  MAP_RE = re.compile(
-      r'^([0-9a-f]+)-([0-9a-f]+)\s+....\s*([0-9a-f]+)\s+\w+:\w+\s+\d+\s*(.*)$')
-
-  state = STATE_PARSING_BACKTRACES
-  skip_first_n_lines = 5
-  mmap = memory_map.Map()
-  nativeheap = native_heap.NativeHeap()
-
-  for line in lines:
-    line = line.rstrip('\r\n')
-    if skip_first_n_lines > 0:
-      skip_first_n_lines -= 1
-      continue
-
-    if state == STATE_PARSING_BACKTRACES:
-      if line == 'MAPS':
-        state = STATE_PARSING_MAPS
-        continue
-      m = BT_RE.match(line)
-      if not m:
-        logging.warning('Skipping unrecognized dumpheap alloc: "%s"' % line)
-        continue
-      alloc_size = int(m.group(1))
-      alloc_count = int(m.group(2))
-      alloc_bt_str = m.group(3)
-      strace = stacktrace.Stacktrace()
-      # Keep only one |stacktrace.Frame| per distinct |absolute_addr|, in order
-      # to ease the complexity of the final de-offset pass.
-      for absolute_addr in alloc_bt_str.split():
-        absolute_addr = int(absolute_addr, 16)
-        stack_frame = nativeheap.GetStackFrame(absolute_addr)
-        strace.Add(stack_frame)
-      nativeheap.Add(native_heap.Allocation(alloc_size, alloc_count, strace))
-
-    # The am dumpheap output contains also a list of mmaps. This information is
-    # used in this module for the only purpose of normalizing addresses (i.e.
-    # translating an absolute addr into its offset inside the mmap-ed library).
-    # The mmap information is not further retained. A more complete mmap dump is
-    # performed (and retained) using the memdump tool (see memdump_parser.py).
-    elif state == STATE_PARSING_MAPS:
-      if line == 'END':
-        state = STATE_ENDED
-        continue
-      m = MAP_RE.match(line)
-      if not m:
-        logging.warning('Skipping unrecognized dumpheap mmap: "%s"' % line)
-        continue
-      mmap.Add(memory_map.MapEntry(
-          start=int(m.group(1), 16),
-          end=int(m.group(2), 16),
-          prot_flags='----', # Not really needed for lookup
-          mapped_file=m.group(4),
-          mapped_offset=int(m.group(3), 16)))
-
-    elif state == STATE_ENDED:
-      pass
-
-    else:
-      assert(False)
-
-  # Final pass: translate all the stack frames' absolute addresses into
-  # relative offsets (exec_file + offset) using the memory maps just processed.
-  for abs_addr, stack_frame in nativeheap.stack_frames.iteritems():
-    assert(abs_addr == stack_frame.address)
-    map_entry = mmap.Lookup(abs_addr)
-    if not map_entry:
-      continue
-    stack_frame.SetExecFileInfo(map_entry.mapped_file,
-                                map_entry.GetRelativeFileOffset(abs_addr))
-
-  return nativeheap

tools/memory_inspector/memory_inspector/backends/android/native_heap_dump_parser.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""This parser turns the heap_dump output into a |NativeHeap| object."""
+
+import json
+
+from memory_inspector.core import native_heap
+from memory_inspector.core import stacktrace
+
+
+# These are defined in heap_profiler/heap_profiler.h
+FLAGS_MALLOC = 1
+FLAGS_MMAP = 2
+FLAGS_MMAP_FILE = 4
+FLAGS_IN_ZYGOTE = 8
+
+
+def Parse(content):
+  """Parses the output of the heap_dump binary (part of libheap_profiler).
+
+  heap_dump provides a conveniente JSON output.
+  See the header of tools/android/heap_profiler/heap_dump.c for more details.
+
+  Args:
+      content: string containing the command output.
+
+  Returns:
+      An instance of |native_heap.NativeHeap|.
+  """
+  data = json.loads(content)
+  assert('allocs' in data), 'Need to run heap_dump with the -x (extended) arg.'
+  nativeheap = native_heap.NativeHeap()
+  strace_by_index = {}   # index (str) -> |stacktrace.Stacktrace|
+
+  for index, entry in data['stacks'].iteritems():
+    strace = stacktrace.Stacktrace()
+    for absolute_addr in entry['f']:
+      strace.Add(nativeheap.GetStackFrame(absolute_addr))
+    strace_by_index[index] = strace
+
+  for start_addr, entry in data['allocs'].iteritems():
+    flags = int(entry['f'])
+    # TODO(primiano): For the moment we just skip completely the allocations
+    # made in the Zygote (pre-fork) because this is usually reasonable. In the
+    # near future we will expose them with some UI to selectively filter them.
+    if flags & FLAGS_IN_ZYGOTE:
+      continue
+    nativeheap.Add(native_heap.Allocation(
+      size=entry['l'],
+      stack_trace=strace_by_index[entry['s']],
+      start=int(start_addr, 16),
+      flags=flags))
+
+  return nativeheap

tools/memory_inspector/memory_inspector/classification/native_heap_classifier.py

@@ -61,7 +61,7 @@ def Classify(nativeheap, rule_tree):
 
   res = results.AggreatedResults(rule_tree, _RESULT_KEYS)
   for allocation in nativeheap.allocations:
-    res.AddToMatchingNodes(allocation, [allocation.total_size])
+    res.AddToMatchingNodes(allocation, [allocation.size])
   return res
 
 
@@ -125,8 +125,8 @@ def InferHeuristicRulesFromHeap(nheap, max_depth=3, threshold=0.02):
       continue
     # Add the blamed dir to the leaderboard.
     blamed_dir = dir_histogram.most_common()[0][0]
-    blamed_dirs.update({blamed_dir : alloc.total_size})
-    total_allocated += alloc.total_size
+    blamed_dirs.update({blamed_dir : alloc.size})
+    total_allocated += alloc.size
 
   # Select only the top paths from the leaderboard which contribute for more
   # than |threshold| and make a radix tree out of them.

tools/memory_inspector/memory_inspector/classification/native_heap_classifier_unittest.py

@@ -113,8 +113,8 @@ class NativeHeapClassifierTest(unittest.TestCase):
         mock_frame = stacktrace.Frame(mock_addr)
         mock_frame.SetSymbolInfo(symbol.Symbol(mock_btstr, mock_source_path))
         mock_strace.Add(mock_frame)
-      nheap.Add(native_heap.Allocation(
-          size=test_entry[0], count=1, stack_trace=mock_strace))
+      nheap.Add(native_heap.Allocation(size=test_entry[0],
+                                       stack_trace=mock_strace))
 
     res = native_heap_classifier.Classify(nheap, rule_tree)
     self._CheckResult(res.total, '', _EXPECTED_RESULTS)
@@ -129,8 +129,8 @@ class NativeHeapClassifierTest(unittest.TestCase):
       mock_frame.SetSymbolInfo(symbol.Symbol(str(mock_addr), mock_source_path))
       for _ in xrange(10):  # Just repeat the same stack frame 10 times
         mock_strace.Add(mock_frame)
-      nheap.Add(native_heap.Allocation(
-          size=mock_alloc_size, count=1, stack_trace=mock_strace))
+      nheap.Add(native_heap.Allocation(size=mock_alloc_size,
+                                       stack_trace=mock_strace))
 
     rule_tree = native_heap_classifier.InferHeuristicRulesFromHeap(
         nheap, threshold=0.05)

tools/memory_inspector/memory_inspector/core/backends.py

@@ -134,6 +134,14 @@ class Process(object):
     """Returns an instance of |native_heap.NativeHeap|."""
     raise NotImplementedError()
 
+  def Freeze(self):
+    """Stops the process and all its threads."""
+    raise NotImplementedError()
+
+  def Unfreeze(self):
+    """Resumes the process."""
+    raise NotImplementedError()
+
   def GetStats(self):
     """Returns an instance of |ProcessStats|."""
     raise NotImplementedError()

tools/memory_inspector/memory_inspector/core/native_heap.py

@@ -40,21 +40,27 @@ class NativeHeap(object):
 
 
 class Allocation(object):
-  """A Native allocation, modeled in a size*count fashion.
+  """Records profiling information aobut a native heap allocation.
 
-  |count| is the number of identical stack_traces which performed the allocation
-  of |size| bytes.
+  Args:
+      size: size of the allocation, in bytes.
+      stack_trace: the allocation call-site. See |stacktrace.Stacktrace|.
+      start: (Optional) Absolute start address in the process VMA. Optional.
+          It is required only for |CalculateResidentSize|.
+      flags: (Optional) More details about the call site (e.g., mmap vs malloc).
   """
 
-  def __init__(self, size, count, stack_trace):
+  def __init__(self, size, stack_trace, start=0, flags=0):
+    assert(size > 0)
     assert(isinstance(stack_trace, stacktrace.Stacktrace))
     self.size = size  # in bytes.
-    self.count = count
     self.stack_trace = stack_trace
+    self.start = start  # Optional, for using the resident size logic.
+    self.flags = flags
 
   @property
-  def total_size(self):
-    return self.size * self.count
+  def end(self):
+    return self.start + self.size - 1
 
   def __str__(self):
-    return '%d x %d : %s' % (self.count, self.size, self.stack_trace)
+    return '%d : %s' % (self.size, self.stack_trace)

tools/memory_inspector/memory_inspector/data/file_storage_unittest.py

@@ -87,7 +87,10 @@ class FileStorageTest(unittest.TestCase):
       frame = nh.GetStackFrame(i * 10 + 2)
       frame.SetExecFileInfo('bar.so', 2)
       stack_trace.Add(frame)
-      nh.Add(native_heap.Allocation(i * 2, i * 3, stack_trace))
+      nh.Add(native_heap.Allocation(size=i * 10,
+                                    stack_trace=stack_trace,
+                                    start=i * 20,
+                                    flags=i * 30))
     archive.StoreNativeHeap(nh)
     nh_deser = archive.LoadNativeHeap(timestamp)
     self._DeepCompare(nh, nh_deser)

tools/memory_inspector/memory_inspector/data/serialization.py

@@ -105,8 +105,8 @@ class NativeHeapDecoder(json.JSONDecoder):
       stack_trace = stacktrace.Stacktrace()
       for absolute_addr in alloc_dict['stack_trace']:
         stack_trace.Add(nh.GetStackFrame(absolute_addr))
-      allocation = native_heap.Allocation(alloc_dict['size'],
-                                          alloc_dict['count'],
-                                          stack_trace)
-      nh.Add(allocation)
+      nh.Add(native_heap.Allocation(start=alloc_dict['start'],
+                                    size=alloc_dict['size'],
+                                    stack_trace=stack_trace,
+                                    flags=alloc_dict['flags']))
     return nh
\ No newline at end of file

tools/memory_inspector/memory_inspector/frontends/background_tasks.py

@@ -90,14 +90,25 @@ def TracerMain_(log, storage_path, backend_name, device_id, pid, interval,
     completion = 80 * i / count
     log.put((completion, 'Dumping trace %d of %d' % (i, count)))
     archive.StartNewSnapshot()
-    mmaps = process.DumpMemoryMaps()
-    log.put((completion, 'Dumped %d memory maps' % len(mmaps)))
-    archive.StoreMemMaps(mmaps)
-    if trace_native_heap:
-      nheap = process.DumpNativeHeap()
-      log.put((completion, 'Dumped %d native allocs' % len(nheap.allocations)))
-      archive.StoreNativeHeap(nheap)
-      heaps_to_symbolize += [nheap]
+    # Freeze the process, so that the mmaps and the heap dump are consistent.
+    process.Freeze()
+    try:
+      if trace_native_heap:
+        nheap = process.DumpNativeHeap()
+        log.put((completion,
+                 'Dumped %d native allocations' % len(nheap.allocations)))
+
+      # TODO(primiano): memdump has the bad habit of sending SIGCONT to the
+      # process. Fix that, so we are the only one in charge of controlling it.
+      mmaps = process.DumpMemoryMaps()
+      log.put((completion, 'Dumped %d memory maps' % len(mmaps)))
+      archive.StoreMemMaps(mmaps)
+
+      if trace_native_heap:
+        archive.StoreNativeHeap(nheap)
+        heaps_to_symbolize += [nheap]
+    finally:
+      process.Unfreeze()
 
     if i < count:
       time.sleep(interval)

tools/memory_inspector/memory_inspector/frontends/www_content/css/processes.css

@@ -20,4 +20,13 @@
 #ps-tracer-dialog > div > :last-child {
   float: right;
   width: 60%;
+}
+
+#android_provision_dialog {
+  display: none;
+}
+
+#android_provision_dialog code {
+  display: block;
+  white-space: pre-line;
 }
\ No newline at end of file

tools/memory_inspector/memory_inspector/frontends/www_content/index.html

@@ -204,6 +204,27 @@
     <div id="progress_bar"><div id="progress_bar-label">Progress...</div></div>
   </div>
 
+  <div id="android_provision_dialog" title="Warning: this might brick your device">
+    <p>
+      <span class="ui-icon ui-icon-alert" style="float:left;"></span>
+      Full heap profiling requires some modifications to the Android system
+      (preload a library into the Android Zygote).
+      <br>
+      This feature has currently been tested on Nexus devices running Android K
+      and L. On some other untested devices / Android releases this might end up
+      bricking the device.
+      <br>
+      <b>Use this feature only on development rooted phones.</b>
+      <br><br>
+      If something goes wrong these are the steps to restore your device:
+      <code>
+        $ adb root && adb wait-for-device && adb remount
+        $ adb shell mv /system/bin/app_process.real /system/bin/app_process
+        $ adb reboot
+      </code>
+    </p>
+  </div>
+
   <div id="js_loading_banner">
   Loading JavaScript content. If you see this message something has probably gone wrong. Check JS console.
   </div>

tools/memory_inspector/memory_inspector/frontends/www_content/js/processes.js

@@ -77,6 +77,26 @@ this.dumpSelectedProcessMmaps_ = function() {
   rootUi.showTab('mm');
 };
 
+this.showAndroidProvisionDialog_ = function() {
+  $("#android_provision_dialog").dialog({
+      modal: true,
+      width: '50em',
+       buttons: {
+        Continue: function() {
+          devices.initializeSelectedDevice(true);
+          $(this).dialog('close');
+          rootUi.showDialog(
+              'Wait device to complete reboot (~30 s) then retry.',
+              'Device rebooting');
+          processes.clear();
+        },
+        Cancel: function() {
+          $(this).dialog('close');
+        }
+      }
+  });
+};
+
 this.showTracingDialog_ = function() {
   if (!this.selProcUri_)
     return alert('Must select a process!');
@@ -92,13 +112,8 @@ this.startTracingSelectedProcess_ = function() {
   $('#ps-tracer-dialog').dialog('close');
 
   if (traceNativeHeap && !devices.getSelectedDevice().isNativeTracingEnabled) {
-    var shouldProvision = confirm('Native heap tracing is not enabled.\n' +
-        'Do you want to enable it (will cause a reboot on Android)?');
-    if (shouldProvision) {
-      devices.initializeSelectedDevice(true);
-      alert('Wait device to complete reboot and then retry.');
+      this.showAndroidProvisionDialog_();
       return;
-    }
   }
 
   var postArgs = {interval: $('#ps-tracer-period').val(),

tools/memory_inspector/memory_inspector/frontends/www_server.py

@@ -582,9 +582,9 @@ def _LoadNheapFromStorage(args, req_vars):
 
   resp = {
       'cols': [
-          {'label': 'Total size [KB]', 'type':'number'},
-          {'label': 'Alloc size [B]', 'type':'number'},
-          {'label': 'Count', 'type':'number'},
+          {'label': 'Allocated', 'type':'number'},
+          {'label': 'Resident', 'type':'number'},
+          {'label': 'Flags', 'type':'number'},
           {'label': 'Stack Trace', 'type':'string'},
         ],
       'rows': []}
@@ -602,9 +602,9 @@ def _LoadNheapFromStorage(args, req_vars):
     strace += '</dl>'
 
     resp['rows'] += [{'c': [
-        {'v': alloc.total_size, 'f': alloc.total_size / 1024},
-        {'v': alloc.size, 'f': None},
-        {'v': alloc.count, 'f': None},
+        {'v': alloc.size, 'f': _StrMem(alloc.size)},
+        {'v': 0, 'f': 0},  # TODO(primiano): support resident_size (next CLs).
+        {'v': alloc.flags, 'f': None},
         {'v': strace, 'f': None},
     ]}]
   return _HTTP_OK, [], resp

tools/memory_inspector/prebuilts/heap_dump-android-arm.sha1

+d179b777f4664dd42311f2f51cdf79481e79158b
\ No newline at end of file

tools/memory_inspector/prebuilts/heap_dump-android-arm64.sha1

+ad962c1504ca0ea613926de25913fe52524e05d8
\ No newline at end of file

tools/memory_inspector/prebuilts/libheap_profiler-android-arm.sha1

+de4429c4fe458488a3875e40650d5ced5ead3929
\ No newline at end of file

tools/memory_inspector/prebuilts/libheap_profiler-android-arm64.sha1

+22bba39a6540575b413d890535050aacdc888d40
\ No newline at end of file

tools/memory_inspector/prebuilts/memdump-android-arm.sha1

-e9576bce3acbb3475a99ffa4624bc7eecbf2988d
+168ba1cf32442931e93a3636e14a613dec13315b
\ No newline at end of file

tools/memory_inspector/prebuilts/memdump-android-arm64.sha1

+2a60fe2928e8b66d4c821406adebc90e5007c1d0
\ No newline at end of file

tools/memory_inspector/prebuilts/ps_ext-android-arm.sha1

-f76a501a09bf31606b4e0561ffea92e75f1993ad
+23682576a76415e84750357806afbdad01433dce
\ No newline at end of file

tools/memory_inspector/prebuilts/ps_ext-android-arm64.sha1

+dd1be941c4a5b060dfbfc78e038e8031d8b8cb16
\ No newline at end of file