Commit 78695003 authored by primiano's avatar primiano Committed by Commit bot

[Android] memory_inspector: add resident memory accounting.

Introduce the logic which is able to intersect mmap stats (dirty/clean
priv/shared) with allocations. Essentially it contains the math to
calculate the overlap (even partial) between allocations and mmaps
and attribute stats counters to allocations. For the moment only
resident memory is accounted. Finer grained accounting (dirty/clean)
requires some changes to memdump (to have more than one bit per page in
its output bitmap) and will come soon.

BUG=340294
NOTRY=true

Review URL: https://codereview.chromium.org/563183003

Cr-Commit-Position: refs/heads/master@{#295032}
parent c8cd4b80
......@@ -124,6 +124,8 @@ class AndroidBackend(backends.Backend):
exec_file_name = posixpath.basename(exec_file_rel_path)
if exec_file_rel_path.startswith('/'):
exec_file_rel_path = exec_file_rel_path[1:]
if not exec_file_rel_path:
continue
exec_file_abs_path = ''
for sym_path in sym_paths:
# First try to locate the symbol file following the full relative path
......@@ -145,7 +147,7 @@ class AndroidBackend(backends.Backend):
if os.path.exists(exec_file_abs_path):
break
if not os.path.exists(exec_file_abs_path):
if not os.path.isfile(exec_file_abs_path):
continue
symbolizer = elf_symbolizer.ELFSymbolizer(
......@@ -177,8 +179,9 @@ class AndroidDevice(backends.Device):
backend=backend,
settings=backends.Settings(AndroidDevice._SETTINGS_KEYS))
self.adb = adb
self._name = '%s %s' % (adb.GetProp('ro.product.model'),
adb.GetProp('ro.build.id'))
self._id = str(adb)
self._name = adb.GetProp('ro.product.model')
self._sys_stats = None
self._last_device_stats = None
self._sys_stats_last_update = None
......
......@@ -81,6 +81,7 @@ class AndroidBackendTest(unittest.TestCase):
'devices': _MOCK_DEVICES_OUT,
'shell getprop ro.product.model': 'Mock device',
'shell getprop ro.build.type': 'userdebug',
'shell getprop ro.build.id': 'ZZ007',
'shell getprop ro.product.cpu.abi': 'armeabi',
'root': 'adbd is already running as root',
'shell /data/local/tmp/ps_ext': _MOCK_PS_EXT_OUT,
......@@ -103,9 +104,9 @@ class AndroidBackendTest(unittest.TestCase):
# Test device enumeration.
devices = list(ab.EnumerateDevices())
self.assertEqual(len(devices), 2)
self.assertEqual(devices[0].name, 'Mock device')
self.assertEqual(devices[0].name, 'Mock device ZZ007')
self.assertEqual(devices[0].id, '0000000000000001')
self.assertEqual(devices[1].name, 'Mock device')
self.assertEqual(devices[1].name, 'Mock device ZZ007')
self.assertEqual(devices[1].id, '0000000000000002')
# Initialize device (checks that sha1 are checked in).
......
......@@ -34,7 +34,7 @@ from memory_inspector.core import exceptions
from memory_inspector.core import native_heap
_RESULT_KEYS = ['bytes_allocated']
_RESULT_KEYS = ['bytes_allocated', 'bytes_resident']
def LoadRules(content):
......@@ -61,7 +61,8 @@ def Classify(nativeheap, rule_tree):
res = results.AggreatedResults(rule_tree, _RESULT_KEYS)
for allocation in nativeheap.allocations:
res.AddToMatchingNodes(allocation, [allocation.size])
res.AddToMatchingNodes(allocation,
[allocation.size, allocation.resident_size])
return res
......
......@@ -66,13 +66,13 @@ _TEST_STACK_TRACES = [
]
_EXPECTED_RESULTS = {
'Total': [238],
'Total::content': [95],
'Total::content::browser': [12], # 5 + 7.
'Total::content::renderer': [49], # 13 + 17 + 19.
'Total::content::content-other': [34],
'Total::ashmem_in_skia': [68], # 31 + 37.
'Total::Total-other': [75], # 3 + 29 + 43.
'Total': [238, 0],
'Total::content': [95, 0],
'Total::content::browser': [12, 0], # 5 + 7.
'Total::content::renderer': [49, 0], # 13 + 17 + 19.
'Total::content::content-other': [34, 0],
'Total::ashmem_in_skia': [68, 0], # 31 + 37.
'Total::Total-other': [75, 0], # 3 + 29 + 43.
}
_HEURISTIC_TEST_STACK_TRACES = [
......@@ -86,18 +86,18 @@ _HEURISTIC_TEST_STACK_TRACES = [
]
_HEURISTIC_EXPECTED_RESULTS = {
'Total': [76],
'Total::/root/': [76],
'Total::/root/::base1/foo/': [31], # 10 + 20 +1
'Total::/root/::base1/foo/::bar/': [10],
'Total::/root/::base1/foo/::baz/': [20],
'Total::/root/::base1/foo/::base1/foo/-other': [1],
'Total::/root/::base2/': [43], # 3 + 22 + 18
'Total::/root/::base2/::subpath/': [22],
'Total::/root/::base2/::subpath2/': [18],
'Total::/root/::base2/::base2/-other': [3],
'Total::/root/::/root/-other': [2],
'Total::Total-other': [0],
'Total': [76, 0],
'Total::/root/': [76, 0],
'Total::/root/::base1/foo/': [31, 0], # 10 + 20 +1
'Total::/root/::base1/foo/::bar/': [10, 0],
'Total::/root/::base1/foo/::baz/': [20, 0],
'Total::/root/::base1/foo/::base1/foo/-other': [1, 0],
'Total::/root/::base2/': [43, 0], # 3 + 22 + 18
'Total::/root/::base2/::subpath/': [22, 0],
'Total::/root/::base2/::subpath2/': [18, 0],
'Total::/root/::base2/::base2/-other': [3, 0],
'Total::/root/::/root/-other': [2, 0],
'Total::Total-other': [0, 0],
}
......@@ -113,8 +113,8 @@ class NativeHeapClassifierTest(unittest.TestCase):
mock_frame = stacktrace.Frame(mock_addr)
mock_frame.SetSymbolInfo(symbol.Symbol(mock_btstr, mock_source_path))
mock_strace.Add(mock_frame)
nheap.Add(native_heap.Allocation(size=test_entry[0],
stack_trace=mock_strace))
nheap.Add(native_heap.Allocation(
size=test_entry[0], stack_trace=mock_strace))
res = native_heap_classifier.Classify(nheap, rule_tree)
self._CheckResult(res.total, '', _EXPECTED_RESULTS)
......@@ -129,8 +129,8 @@ class NativeHeapClassifierTest(unittest.TestCase):
mock_frame.SetSymbolInfo(symbol.Symbol(str(mock_addr), mock_source_path))
for _ in xrange(10): # Just repeat the same stack frame 10 times
mock_strace.Add(mock_frame)
nheap.Add(native_heap.Allocation(size=mock_alloc_size,
stack_trace=mock_strace))
nheap.Add(native_heap.Allocation(
size=mock_alloc_size, stack_trace=mock_strace))
rule_tree = native_heap_classifier.InferHeuristicRulesFromHeap(
nheap, threshold=0.05)
......
......@@ -2,9 +2,12 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from memory_inspector.core import memory_map
from memory_inspector.core import stacktrace
from memory_inspector.core import symbol
from memory_inspector.core.memory_map import PAGE_SIZE
class NativeHeap(object):
"""A snapshot of outstanding (i.e. not freed) native allocations.
......@@ -32,31 +35,88 @@ class NativeHeap(object):
def SymbolizeUsingSymbolDB(self, symbols):
assert(isinstance(symbols, symbol.Symbols))
for stack_frame in self.stack_frames.itervalues():
if stack_frame.exec_file_rel_path is None:
if not stack_frame.exec_file_rel_path:
continue
sym = symbols.Lookup(stack_frame.exec_file_rel_path, stack_frame.offset)
if sym:
stack_frame.SetSymbolInfo(sym)
def RelativizeStackFrames(self, mmap):
"""Turns stack frames' absolute addresses into mmap relative addresses.
For each absolute address, the containing mmap is looked up and the frame
is decorated with the mapped file + relative address in the file."""
assert(isinstance(mmap, memory_map.Map))
for abs_addr, stack_frame in self.stack_frames.iteritems():
assert(abs_addr == stack_frame.address)
map_entry = mmap.Lookup(abs_addr)
if not map_entry:
continue
stack_frame.SetExecFileInfo(map_entry.mapped_file,
map_entry.GetRelativeFileOffset(abs_addr))
def CalculateResidentSize(self, mmap):
"""Updates the |Allocation|.|resident_size|s by looking at mmap stats.
Not all the allocated memory is always used (read: resident). This function
estimates the resident size of an allocation intersecting the mmaps dump.
"""
assert(isinstance(mmap, memory_map.Map))
for alloc in self.allocations:
# This function loops over all the memory pages that intersect, partially
# or fully, with each allocation. For each of them, the allocation is
# attributed a resident size equal to the size of intersecting range iff
# the page is resident.
# The tricky part is that, in the general case, an allocation can span
# over multiple (contiguous) mmaps. See the chart below for a reference:
#
# VA space: |0 |4k |8k |12k |16k |20k |24k |28k |32k |
# Mmaps: [ mm 1 ][ mm2 ] [ map 3 ]
# Allocs: <a1> < a2 > < a3 >
#
# Note: this accounting technique is not fully correct but is generally a
# good tradeoff between accuracy and speed of profiling. The OS provides
# resident information with the page granularity (typ. 4k). Finer values
# would require more fancy techniques based, for instance, on run-time
# instrumentation tools like Valgrind or *sanitizer.
cur_start = alloc.start
mm = None
while cur_start < alloc.end:
if not mm or not mm.Contains(cur_start):
mm = mmap.Lookup(cur_start)
if mm:
page, page_off = mm.GetRelativeMMOffset(cur_start)
if mm.IsPageResident(page):
page_end = mm.start + page * PAGE_SIZE + PAGE_SIZE - 1
alloc_memory_in_current_page = PAGE_SIZE - page_off
if alloc.end < page_end:
alloc_memory_in_current_page -= page_end - alloc.end
alloc.resident_size += alloc_memory_in_current_page
# Move to the next page boundary.
cur_start = (cur_start + PAGE_SIZE) & ~(PAGE_SIZE - 1)
class Allocation(object):
"""Records profiling information aobut a native heap allocation.
"""Records profiling information about a native heap allocation.
Args:
size: size of the allocation, in bytes.
stack_trace: the allocation call-site. See |stacktrace.Stacktrace|.
start: (Optional) Absolute start address in the process VMA. Optional.
It is required only for |CalculateResidentSize|.
start: (Optional) Absolute start address in the process VMA. It is
required only for |CalculateResidentSize|.
flags: (Optional) More details about the call site (e.g., mmap vs malloc).
resident_size: this is normally obtained through |CalculateResidentSize|
and is part of the initializer just for deserialization purposes.
"""
def __init__(self, size, stack_trace, start=0, flags=0):
def __init__(self, size, stack_trace, start=0, flags=0, resident_size=0):
assert(size > 0)
assert(isinstance(stack_trace, stacktrace.Stacktrace))
self.size = size # in bytes.
self.stack_trace = stack_trace
self.start = start # Optional, for using the resident size logic.
self.flags = flags
self.resident_size = resident_size # see |CalculateResidentSize|.
@property
def end(self):
......
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
The test scenario is as follows:
VA space: |0 |4k |8k |12k |16k |20k ... |64k |65k |66k
Mmaps: [ anon 1 ][anon 2] [anon 3] ... [ exe1 ][ exe2 ]
Resident: *******-------*******-------******* (*:resident, -:not resident)
Allocs: <1> <2> < 3 >
| | |
S.Traces: | | +-----------> st1[exe1 + 0, exe1 + 4]
| +--------------------------> st1[exe1 + 0, exe1 + 4]
+------------------------------> st2[exe1 + 0, exe2 + 4, post-exe2]
Furthermore, the exe2 is a file mapping with non-zero (8k) offset.
"""
import unittest
from memory_inspector.core import memory_map
from memory_inspector.core import native_heap
from memory_inspector.core import stacktrace
from memory_inspector.core import symbol
from memory_inspector.core.memory_map import PAGE_SIZE
class NativeHeapTest(unittest.TestCase):
def runTest(self):
nheap = native_heap.NativeHeap()
EXE_1_MM_BASE = 64 * PAGE_SIZE
EXE_2_MM_BASE = 65 * PAGE_SIZE
EXE_2_FILE_OFF = 8192
st1 = stacktrace.Stacktrace()
st1.Add(nheap.GetStackFrame(EXE_1_MM_BASE))
st1.Add(nheap.GetStackFrame(EXE_1_MM_BASE + 4))
st2 = stacktrace.Stacktrace()
st2.Add(nheap.GetStackFrame(EXE_1_MM_BASE))
st2.Add(nheap.GetStackFrame(EXE_2_MM_BASE + 4))
st2.Add(nheap.GetStackFrame(EXE_2_MM_BASE + PAGE_SIZE + 4))
# Check that GetStackFrames keeps one unique object instance per address.
# This is to guarantee that the symbolization logic (SymbolizeUsingSymbolDB)
# can cheaply iterate on distinct stack frames rather than re-processing
# every stack frame for each allocation (and save memory as well).
self.assertIs(st1[0], st2[0])
self.assertIsNot(st1[0], st1[1])
self.assertIsNot(st2[0], st2[1])
alloc1 = native_heap.Allocation(start=4, size=4, stack_trace=st1)
alloc2 = native_heap.Allocation(start=4090, size=8, stack_trace=st1)
alloc3 = native_heap.Allocation(start=8190, size=10000, stack_trace=st2)
nheap.Add(alloc1)
nheap.Add(alloc2)
nheap.Add(alloc3)
self.assertEqual(len(nheap.allocations), 3)
self.assertIn(alloc1, nheap.allocations)
self.assertIn(alloc2, nheap.allocations)
self.assertIn(alloc3, nheap.allocations)
############################################################################
# Test the relativization (absolute address -> mmap + offset) logic.
############################################################################
mmap = memory_map
mmap = memory_map.Map()
mmap.Add(memory_map.MapEntry(EXE_1_MM_BASE, EXE_1_MM_BASE + PAGE_SIZE - 1,
'rw--', '/d/exe1', 0))
mmap.Add(memory_map.MapEntry(EXE_2_MM_BASE, EXE_2_MM_BASE + PAGE_SIZE - 1,
'rw--', 'exe2',EXE_2_FILE_OFF))
# Entry for EXE_3 is deliberately missing to check the fallback behavior.
nheap.RelativizeStackFrames(mmap)
self.assertEqual(st1[0].exec_file_rel_path, '/d/exe1')
self.assertEqual(st1[0].exec_file_name, 'exe1')
self.assertEqual(st1[0].offset, 0)
self.assertEqual(st1[1].exec_file_rel_path, '/d/exe1')
self.assertEqual(st1[1].exec_file_name, 'exe1')
self.assertEqual(st1[1].offset, 4)
self.assertEqual(st2[0].exec_file_rel_path, '/d/exe1')
self.assertEqual(st2[0].exec_file_name, 'exe1')
self.assertEqual(st2[0].offset, 0)
self.assertEqual(st2[1].exec_file_rel_path, 'exe2')
self.assertEqual(st2[1].exec_file_name, 'exe2')
self.assertEqual(st2[1].offset, 4 + EXE_2_FILE_OFF)
self.assertIsNone(st2[2].exec_file_rel_path)
self.assertIsNone(st2[2].exec_file_name)
self.assertIsNone(st2[2].offset)
############################################################################
# Test the symbolization logic.
############################################################################
syms = symbol.Symbols()
syms.Add('/d/exe1', 0, symbol.Symbol('sym1', 'src1.c', 1)) # st1[0]
syms.Add('exe2', 4 + EXE_2_FILE_OFF, symbol.Symbol('sym3')) # st2[1]
nheap.SymbolizeUsingSymbolDB(syms)
self.assertEqual(st1[0].symbol.name, 'sym1')
self.assertEqual(st1[0].symbol.source_info[0].source_file_path, 'src1.c')
self.assertEqual(st1[0].symbol.source_info[0].line_number, 1)
# st1[1] should have no symbol info, because we didn't provide any above.
self.assertIsNone(st1[1].symbol)
# st2[0] and st1[0] were the same Frame. Expect identical symbols instances.
self.assertIs(st2[0].symbol, st1[0].symbol)
# st2[1] should have a symbols name, but no source line info.
self.assertEqual(st2[1].symbol.name, 'sym3')
self.assertEqual(len(st2[1].symbol.source_info), 0)
# st2[2] should have no sym because we didn't even provide a mmap for exe3.
self.assertIsNone(st2[2].symbol)
############################################################################
# Test the resident size calculation logic (intersects mmaps and allocs).
############################################################################
mmap.Add(
memory_map.MapEntry(0, 8191, 'rw--', '', 0, resident_pages=[1]))
mmap.Add(
memory_map.MapEntry(8192, 12287, 'rw--', '', 0, resident_pages=[1]))
# [12k, 16k] is deliberately missing to check the fallback behavior.
mmap.Add(
memory_map.MapEntry(16384, 20479, 'rw--', '', 0, resident_pages=[1]))
nheap.CalculateResidentSize(mmap)
# alloc1 [4, 8] is fully resident because it lays in the first resident 4k.
self.assertEqual(alloc1.resident_size, 4)
# alloc2 [4090, 4098] should have only 6 resident bytes ([4090,4096]), but
# not the last two, which lay on the second page which is noijt resident.
self.assertEqual(alloc2.resident_size, 6)
# alloc3 [8190, 18190] is split as follows (* = resident):
# [8190, 8192]: these 2 bytes are NOT resident, they lay in the 2nd page.
# *[8192, 12288]: the 3rd page is resident and is fully covered by alloc3.
# [12288, 16384]: the 4th page is fully covered as well, but not resident.
# *[16384, 18190]: the 5th page is partially covered and resident.
self.assertEqual(alloc3.resident_size, (12288 - 8192) + (18190 - 16384))
......@@ -67,7 +67,7 @@ class Frame(object):
@property
def exec_file_name(self):
"""Returns the file name (stripped of the path) of the executable."""
if self.exec_file_rel_path is None:
if not self.exec_file_rel_path:
return None
return posixpath.basename(self.exec_file_rel_path.replace('\\', '/'))
......
......@@ -108,5 +108,6 @@ class NativeHeapDecoder(json.JSONDecoder):
nh.Add(native_heap.Allocation(start=alloc_dict['start'],
size=alloc_dict['size'],
stack_trace=stack_trace,
flags=alloc_dict['flags']))
flags=alloc_dict['flags'],
resident_size=alloc_dict['resident_size']))
return nh
\ No newline at end of file
......@@ -105,6 +105,8 @@ def TracerMain_(log, storage_path, backend_name, device_id, pid, interval,
archive.StoreMemMaps(mmaps)
if trace_native_heap:
nheap.RelativizeStackFrames(mmaps)
nheap.CalculateResidentSize(mmaps)
archive.StoreNativeHeap(nheap)
heaps_to_symbolize += [nheap]
finally:
......
......@@ -169,10 +169,14 @@
<div id="tabs-nheap">
<div id="nheap-toolbar" class="ui-widget-header ui-corner-all">
<label>Totals: </label>
<input type="text" id="nheap-totals" values="0 KB" readonly>
<label>Selected: </label>
<input type="text" id="nheap-selected" values="0 KB" readonly>
<label>Total (allocated): </label>
<input type="text" id="nheap-total-allocated" values="0 KB" readonly>
<label>Total (resident): </label>
<input type="text" id="nheap-total-resident" values="0 KB" readonly>
<label>Selected (allocated): </label>
<input type="text" id="nheap-selected-allocated" values="0 KB" readonly>
<label>Selected (resident): </label>
<input type="text" id="nheap-selected-resident" values="0 KB" readonly>
<label>Filter: </label>
<input type="text" id="nheap-filter">
</div>
......
......@@ -4,8 +4,10 @@
nheap = new (function() {
this.COL_STACKTRACE = 3;
this.COL_TOTAL = 0;
this.COL_RESIDENT = 1;
this.COL_STACKTRACE = 3;
this.nheapData_ = null;
this.nheapTable_ = null;
......@@ -44,17 +46,20 @@ this.applyTableFilters_ = function() {
var rx = $('#nheap-filter').val();
var rows = [];
var total = 0;
var total_allocated = 0;
var total_resident = 0;
for (var row = 0; row < this.nheapData_.getNumberOfRows(); ++row) {
stackTrace = this.nheapData_.getValue(row, this.COL_STACKTRACE);
if (!stackTrace.match(rx))
continue;
rows.push(row);
total += this.nheapData_.getValue(row, this.COL_TOTAL);
total_allocated += this.nheapData_.getValue(row, this.COL_TOTAL);
total_resident += this.nheapData_.getValue(row, this.COL_RESIDENT);
}
$('#nheap-totals').val(Math.floor(total / 1024) + ' KB');
$('#nheap-total-allocated').val(Math.floor(total_allocated / 1024) + ' KB');
$('#nheap-total-resident').val(Math.floor(total_resident / 1024) + ' KB');
this.nheapFilter_.setRows(rows);
this.redraw();
};
......@@ -63,14 +68,18 @@ this.onNheapTableRowSelect_ = function() {
if (!this.nheapFilter_)
return;
var total = 0;
var total_allocated = 0;
var total_resident = 0;
this.nheapTable_.getSelection().forEach(function(sel) {
var row = sel.row;
total += this.nheapFilter_.getValue(row, this.COL_TOTAL);
total_allocated += this.nheapFilter_.getValue(row, this.COL_TOTAL);
total_resident += this.nheapFilter_.getValue(row, this.COL_RESIDENT);
}, this);
$('#nheap-selected').val(Math.floor(total / 1024) + ' KB');
$('#nheap-selected-allocated').val(Math.floor(total_allocated / 1024) +
' KB');
$('#nheap-selected-resident').val(Math.floor(total_resident / 1024) + ' KB');
};
this.redraw = function() {
......
......@@ -603,7 +603,7 @@ def _LoadNheapFromStorage(args, req_vars):
resp['rows'] += [{'c': [
{'v': alloc.size, 'f': _StrMem(alloc.size)},
{'v': 0, 'f': 0}, # TODO(primiano): support resident_size (next CLs).
{'v': alloc.resident_size, 'f': _StrMem(alloc.resident_size)},
{'v': alloc.flags, 'f': None},
{'v': strace, 'f': None},
]}]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment