[Android] Add 64-bit support to memdump (and memory_inspector parser)

This change makes memdump able to parse and print out 64 bit addresses.
No change is intended for the 32-bit output.
This also updates the memdump parser in memory_inspector to match the
memdump change and updates its prebuilts.

BUG=340294

Review URL: https://codereview.chromium.org/550093003

Cr-Commit-Position: refs/heads/master@{#293927}

tools/android/memdump/memdump.cc

@@ -23,6 +23,7 @@
 #include "base/containers/hash_tables.h"
 #include "base/file_util.h"
 #include "base/files/scoped_file.h"
+#include "base/format_macros.h"
 #include "base/logging.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/string_split.h"
@@ -45,7 +46,16 @@ class BitSet {
   }
 
   std::string AsB64String() const {
-    std::string bits(&data_[0], data_.size());
+    // Simple optimization: strip trailing zero bytes from the bitmap.
+    // For instance, if a region has 32 pages but only the first 9 are resident,
+    // The full bitmap would be 0xff 0x01 0x00 0x00, the stripped one 0xff 0x01.
+    // It can save up to some seconds when printing large mmaps, in particular
+    // in presence of large virtual address space reservations (where none of
+    // the pages are resident).
+    size_t end = data_.size();
+    while (end > 0 && data_[end - 1] == '\0')
+      --end;
+    std::string bits(&data_[0], end);
     std::string b64_string;
     base::Base64Encode(bits, &b64_string);
     return b64_string;
@@ -79,9 +89,9 @@ struct PageCount {
 struct MemoryMap {
   std::string name;
   std::string flags;
-  uint start_address;
-  uint end_address;
-  uint offset;
+  uint64 start_address;
+  uint64 end_address;
+  uint64 offset;
   PageCount private_pages;
   // app_shared_pages[i] contains the number of pages mapped in i+2 processes
   // (only among the processes that are being analyzed).
@@ -127,23 +137,20 @@ bool ParseMemoryMapLine(const std::string& line,
   const std::string& addr_range = tokens->at(0);
   std::vector<std::string> range_tokens;
   base::SplitString(addr_range, '-', &range_tokens);
-  uint64 tmp = 0;
   const std::string& start_address_token = range_tokens.at(0);
-  if (!base::HexStringToUInt64(start_address_token, &tmp)) {
+  if (!base::HexStringToUInt64(start_address_token,
+                               &memory_map->start_address)) {
     return false;
   }
-  memory_map->start_address = static_cast<uint>(tmp);
   const std::string& end_address_token = range_tokens.at(1);
-  if (!base::HexStringToUInt64(end_address_token, &tmp)) {
+  if (!base::HexStringToUInt64(end_address_token, &memory_map->end_address)) {
     return false;
   }
-  memory_map->end_address = static_cast<uint>(tmp);
   if (tokens->at(1).size() != strlen("rwxp"))
     return false;
   memory_map->flags.swap(tokens->at(1));
-  if (!base::HexStringToUInt64(tokens->at(2), &tmp))
+  if (!base::HexStringToUInt64(tokens->at(2), &memory_map->offset))
     return false;
-  memory_map->offset = static_cast<uint>(tmp);
   memory_map->committed_pages_bits.resize(
       (memory_map->end_address - memory_map->start_address) / kPageSize);
   const int map_name_index = 5;
@@ -206,7 +213,7 @@ bool GetPagesForMemoryMap(int pagemap_fd,
     PLOG(ERROR) << "lseek";
     return false;
   }
-  for (uint addr = memory_map.start_address, page_index = 0;
+  for (uint64 addr = memory_map.start_address, page_index = 0;
        addr < memory_map.end_address;
        addr += kPageSize, ++page_index) {
     DCHECK_EQ(0, addr % kPageSize);
@@ -415,8 +422,9 @@ void DumpProcessesMemoryMapsInExtendedFormat(
       AppendAppSharedField(memory_map.app_shared_pages, &app_shared_buf);
       base::SStringPrintf(
           &buf,
-          "%x-%x %s %x private_unevictable=%d private=%d shared_app=%s "
-          "shared_other_unevictable=%d shared_other=%d \"%s\" [%s]\n",
+          "%"PRIx64"-%"PRIx64" %s %"PRIx64" private_unevictable=%d private=%d "
+          "shared_app=%s shared_other_unevictable=%d shared_other=%d "
+          "\"%s\" [%s]\n",
           memory_map.start_address,
           memory_map.end_address,
           memory_map.flags.c_str(),

tools/memory_inspector/memory_inspector/backends/android/android_backend_unittest.py

@@ -34,10 +34,11 @@ _MOCK_MEMDUMP_OUT = ("""[ PID=1]
     """shared_other_unevictable=0 shared_other=0 "" [AQ==]
 40127000-40147000 rw-s 0 private_unevictable=4096 private=4096 shared_app=[] """
     """shared_other_unevictable=32768 shared_other=32768 "/X" [/////w==]
-be8b7000-be8d8000 rw-p 0 private_unevictable=8192 private=8192 shared_app=[] """
-    """shared_other_unevictable=4096 shared_other=12288 "[stack]" [AAAA4AE=]
-ffff0000-ffff1000 r-xp 0 private_unevictable=0 private=0 shared_app=[] """
-    """shared_other_unevictable=0 shared_other=0 "[vectors]" [AA==]""")
+be8b7000-be8d8000 rw-p 8 private_unevictable=8192 private=8192 shared_app=[] """
+    """shared_other_unevictable=4096 shared_other=12288 "[stack]" [//8=]
+ffff00ffff0000-ffff00ffff1000 r-xp 0 private_unevictable=0 private=0 """
+    """shared_app=[] shared_other_unevictable=0 shared_other=0 "[vectors]" []"""
+)
 
 _MOCK_DUMPHEAP_OUT = """Android Native Heap Dump v1.0
 
@@ -137,21 +138,23 @@ class AndroidBackendTest(unittest.TestCase):
     self.assertIsNotNone(mmaps.Lookup(0x32FFF))
     self.assertIsNotNone(mmaps.Lookup(0x33000))
     self.assertIsNone(mmaps.Lookup(0x6d000))
-    self.assertIsNotNone(mmaps.Lookup(0xffff0000))
+    self.assertIsNotNone(mmaps.Lookup(0xffff00ffff0000))
+    self.assertIsNone(mmaps.Lookup(0xffff0000))
 
     entry = mmaps.Lookup(0xbe8b7000)
     self.assertIsNotNone(entry)
     self.assertEqual(entry.start, 0xbe8b7000)
     self.assertEqual(entry.end, 0xbe8d7fff)
+    self.assertEqual(entry.mapped_offset, 0x8)
     self.assertEqual(entry.prot_flags, 'rw-p')
     self.assertEqual(entry.priv_dirty_bytes, 8192)
     self.assertEqual(entry.priv_clean_bytes, 0)
     self.assertEqual(entry.shared_dirty_bytes, 4096)
     self.assertEqual(entry.shared_clean_bytes, 8192)
-    for i in xrange(29):
-      self.assertFalse(entry.IsPageResident(i))
-    for i in xrange(29, 33):
+    for i in xrange(16):
       self.assertTrue(entry.IsPageResident(i))
+    for i in xrange(16, 33):
+      self.assertFalse(entry.IsPageResident(i))
 
     # Test dumpheap parsing.
     heap = processes[0].DumpNativeHeap()

tools/memory_inspector/memory_inspector/backends/android/memdump_parser.py

@@ -42,7 +42,8 @@ def Parse(lines):
       An instance of |memory_map.Map|.
   """
   RE = (r'^([0-9a-f]+)-([0-9a-f]+)\s+'
-        r'([rwxps-]{4})\s*.*?'
+        r'([rwxps-]{4})\s+'
+        r'([0-9a-f]+)\s+'
         r'private_unevictable=(\d+) private=(\d+) '
         r'shared_app=(.*?) '
         r'shared_other_unevictable=(\d+) shared_other=(\d+) '
@@ -64,20 +65,23 @@ def Parse(lines):
       logging.warning('Skipping unrecognized memdump line "%s"' % line)
       continue
 
-    # TODO(primiano): proper offset handling requires a change in memdump. In
-    # the meanwhile, it should pretty safe assuming zero-offset for libs (for
-    # symbolization). Also, offsets for other mappings don't really matter.
+    start = int(m.group(1), 16)
+    end = int(m.group(2), 16) - 1 # end addr is inclusive in memdump output.
+    if (start > end):
+      # Sadly, this actually happened. Probably a kernel bug, see b/17402069.
+      logging.warning('Skipping unfeasible mmap "%s"' % line)
+      continue
     entry = memory_map.MapEntry(
-        start=int(m.group(1), 16),
-        end=int(m.group(2), 16) - 1, # end addr is inclusive in memdump output.
+        start=start,
+        end=end,
         prot_flags=m.group(3),
-        mapped_file=m.group(9),
-        mapped_offset=0)
-    entry.priv_dirty_bytes = int(m.group(4))
-    entry.priv_clean_bytes = int(m.group(5)) - entry.priv_dirty_bytes
-    entry.shared_dirty_bytes = int(m.group(7))
-    entry.shared_clean_bytes = int(m.group(8)) - entry.shared_dirty_bytes
-    entry.resident_pages = [ord(c) for c in base64.b64decode(m.group(10))]
+        mapped_file=m.group(10),
+        mapped_offset=int(m.group(4), 16))
+    entry.priv_dirty_bytes = int(m.group(5))
+    entry.priv_clean_bytes = int(m.group(6)) - entry.priv_dirty_bytes
+    entry.shared_dirty_bytes = int(m.group(8))
+    entry.shared_clean_bytes = int(m.group(9)) - entry.shared_dirty_bytes
+    entry.resident_pages = [ord(c) for c in base64.b64decode(m.group(11))]
     maps.Add(entry)
 
   return maps

tools/memory_inspector/memory_inspector/core/memory_map.py

@@ -5,13 +5,16 @@
 import bisect
 
 
+PAGE_SIZE = 4096
+
+
 class Map(object):
   """Models the memory map of a given |backends.Process|.
 
   This is typically obtained by calling backends.Process.DumpMemoryMaps()."""
 
   def __init__(self):
-    self.entries = []
+    self.entries = []  # List of |MapEntry|, sorted by start address.
 
   def Add(self, entry):
     assert(isinstance(entry, MapEntry))
@@ -39,7 +42,6 @@ class Map(object):
 
 class MapEntry(object):
   """An entry (address range + stats) in a memory |Map|."""
-  PAGE_SIZE = 4096
 
   def __init__(self, start, end, prot_flags, mapped_file, mapped_offset,
       priv_dirty_bytes=0, priv_clean_bytes=0, shared_dirty_bytes=0,
@@ -67,17 +69,19 @@ class MapEntry(object):
   def IsPageResident(self, relative_page_index):
     """Checks whether a given memory page is resident in memory."""
     assert(relative_page_index >= 0 and
-           relative_page_index < self.len / MapEntry.PAGE_SIZE)
-    assert(len(self.resident_pages) * MapEntry.PAGE_SIZE * 8 >= self.len)
+           relative_page_index < self.len / PAGE_SIZE)
     arr_idx = relative_page_index / 8
     arr_bit = relative_page_index % 8
+    # Trailing zeros are trimmed by memdump (to optimize dump time).
+    if arr_idx >= len(self.resident_pages):
+      return False
     return (self.resident_pages[arr_idx] & (1 << arr_bit)) != 0
 
   def __cmp__(self, other):
     """Comparison operator required for bisect."""
     if isinstance(other, MapEntry):
       return self.start - other.start
-    elif isinstance(other, int):
+    elif isinstance(other, (long, int)):
       return self.start - other
     else:
       raise Exception('Cannot compare with %s' % other.__class__)
@@ -85,3 +89,8 @@ class MapEntry(object):
   @property
   def len(self):
     return self.end - self.start + 1
+
+  @property
+  def rss_bytes(self):
+    return (self.priv_dirty_bytes + self.priv_clean_bytes +
+            self.shared_dirty_bytes + self.shared_clean_bytes)

tools/memory_inspector/memory_inspector/core/stacktrace.py

@@ -37,7 +37,7 @@ class Frame(object):
         address: the absolute (virtual) address of the stack frame in the
                  original process virtual address space.
     """
-    assert(isinstance(address, int))
+    assert(isinstance(address, (long, int)))
     self.address = address
     self.symbol = None
     self.exec_file_rel_path = None
@@ -54,7 +54,7 @@ class Frame(object):
             relative to the target device (e.g., /system/lib/libc.so).
         offset: the offset in the executable.
     """
-    assert(isinstance(offset, int))
+    assert(isinstance(offset, (long, int)))
     self.exec_file_rel_path = exec_file_rel_path
     self.offset = offset
 

tools/memory_inspector/memory_inspector/data/file_storage_unittest.py

@@ -114,7 +114,7 @@ class FileStorageTest(unittest.TestCase):
     if a is None:
       return
 
-    _BASICTYPES = (int, basestring, float)
+    _BASICTYPES = (long, int, basestring, float)
     if isinstance(a, _BASICTYPES) and isinstance(b, _BASICTYPES):
       return self.assertEqual(a, b, prefix)
 

tools/memory_inspector/prebuilts/memdump-android-arm.sha1

-43e7d83da69a418d6fe6a72dbed168ddd9748aef
\ No newline at end of file
+e9576bce3acbb3475a99ffa4624bc7eecbf2988d
\ No newline at end of file

tools/memory_inspector/prebuilts/memdump-linux-x86_64.sha1

+a91c8177cef1ff9c9064dbe4f9cf2eb8fb40e174
\ No newline at end of file