Supersize: Add .dex symbols

Use apk analyzer to parse dex files in apks. Make use of .jar.info files
to attribute size to source files.

Bug: 723820
Change-Id: I5cdbff6000d585d7903a40bccaaf95ead0370b34
Reviewed-on: https://chromium-review.googlesource.com/949452
Commit-Queue: Peter Wen <wnwen@chromium.org>
Reviewed-by: agrieve <agrieve@chromium.org>
Cr-Commit-Position: refs/heads/master@{#544792}

tools/binary_size/libsupersize/apkanalyzer.py

+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Runs apkanalyzer to parse dex files in an apk.
+
+Assumes that apk_path.mapping and apk_path.jar.info is available.
+"""
+
+import os
+import subprocess
+import zipfile
+
+import models
+import path_util
+
+
+_TOTAL_NODE_NAME = '<TOTAL>'
+_DEX_PATH_COMPONENT = 'prebuilt'
+
+
+def _ParseJarInfoFile(file_name):
+  with open(file_name, 'r') as info:
+    source_map = dict()
+    for line in info:
+      package_path, file_path = line.strip().split(',', 1)
+      source_map[package_path] = file_path
+  return source_map
+
+
+def _LoadSourceMap(apk_name, output_directory):
+  apk_jar_info_name = apk_name + '.jar.info'
+  jar_info_path = os.path.join(
+      output_directory, 'size-info', apk_jar_info_name)
+  return _ParseJarInfoFile(jar_info_path)
+
+
+def _RunApkAnalyzer(apk_path, output_directory):
+  args = [path_util.GetApkAnalyzerPath(output_directory), 'dex', 'packages',
+          '--defined-only', apk_path]
+  mapping_path = apk_path + '.mapping'
+  if os.path.exists(mapping_path):
+    args.extend(['--proguard-mappings', mapping_path])
+  output = subprocess.check_output(args)
+  data = []
+  for line in output.splitlines():
+    vals = line.split()
+    # We want to name these columns so we know exactly which is which.
+    # pylint: disable=unused-variable
+    node_type, state, defined_methods, referenced_methods, size, name = (
+        vals[0], vals[1], vals[2], vals[3], vals[4], vals[5:])
+    data.append((' '.join(name), int(size)))
+  return data
+
+
+def _ExpectedDexTotalSize(apk_path):
+  dex_total = 0
+  with zipfile.ZipFile(apk_path) as z:
+    for zip_info in z.infolist():
+      if not zip_info.filename.endswith('.dex'):
+        continue
+      dex_total += zip_info.file_size
+  return dex_total
+
+
+# VisibleForTesting
+def UndoHierarchicalSizing(data):
+  """Subtracts child node sizes from parent nodes.
+
+  Note that inner classes
+  should be considered as siblings rather than child nodes.
+
+  Example nodes:
+    [
+      ('<TOTAL>', 37),
+      ('org', 30),
+      ('org.chromium', 25),
+      ('org.chromium.ClassA', 14),
+      ('org.chromium.ClassA void methodA()', 10),
+      ('org.chromium.ClassA$Proxy', 8),
+    ]
+
+  Processed nodes:
+    [
+      ('<TOTAL>', 7),
+      ('org', 5),
+      ('org.chromium', 3),
+      ('org.chromium.ClassA', 4),
+      ('org.chromium.ClassA void methodA()', 10),
+      ('org.chromium.ClassA$Proxy', 8),
+    ]
+  """
+  num_nodes = len(data)
+  nodes = []
+
+  def process_node(start_idx):
+    assert start_idx < num_nodes, 'Attempting to parse beyond data array.'
+    name, size = data[start_idx]
+    total_child_size = 0
+    next_idx = start_idx + 1
+    name_len = len(name)
+    while next_idx < num_nodes:
+      next_name = data[next_idx][0]
+      if name == _TOTAL_NODE_NAME or (
+          next_name.startswith(name) and next_name[name_len] in '. '):
+        # Child node
+        child_next_idx, child_node_size = process_node(next_idx)
+        next_idx = child_next_idx
+        total_child_size += child_node_size
+      else:
+        # Sibling or higher nodes
+        break
+    node_size = size - total_child_size
+    nodes.append((name, node_size))
+    return next_idx, size
+
+  idx = 0
+  while idx < num_nodes:
+    idx = process_node(idx)[0]
+  return nodes
+
+
+def CreateDexSymbols(apk_path, output_directory):
+  apk_name = os.path.basename(apk_path)
+  source_map = _LoadSourceMap(apk_name, output_directory)
+  nodes = UndoHierarchicalSizing(_RunApkAnalyzer(apk_path, output_directory))
+  dex_expected_size = _ExpectedDexTotalSize(apk_path)
+  total_node_size = sum(map(lambda x: x[1], nodes))
+  assert dex_expected_size >= total_node_size, (
+      'Node size too large, check for node processing errors.')
+  # We have 1+MB of just ids for methods, strings
+  id_metadata_overhead_size = dex_expected_size - total_node_size
+  symbols = []
+  for name, node_size in nodes:
+    package = name.split(' ')[0]
+    class_path = package.split('$')[0]
+    source_path = source_map.get(class_path, '')
+    if source_path:
+      object_path = package
+    elif package == _TOTAL_NODE_NAME:
+      name = '* Unattributed Dex'
+      object_path = os.path.join(apk_name, _DEX_PATH_COMPONENT)
+      node_size += id_metadata_overhead_size
+    else:
+      object_path = os.path.join(
+          apk_name, _DEX_PATH_COMPONENT, *package.split('.'))
+    # TODO(wnwen): Split into .dex.methods.
+    symbols.append(models.Symbol(models.SECTION_DEX, node_size,
+        full_name=name, object_path=object_path, source_path=source_path))
+  return symbols

tools/binary_size/libsupersize/apkanalyzer_test.py

+#!/usr/bin/env python
+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import unittest
+
+import apkanalyzer
+
+
+class ApkAnalyzerTest(unittest.TestCase):
+  def assertEqualLists(self, list1, list2):
+    self.assertEqual(set(list1), set(list2))
+
+  def testUndoHierarchicalSizing_Empty(self):
+    data = []
+    nodes = apkanalyzer.UndoHierarchicalSizing(data)
+    self.assertEqual(0, len(nodes))
+
+  def testUndoHierarchicalSizing_TotalSingleRootNode(self):
+    data = [
+      ('<TOTAL>', 5),
+    ]
+    nodes = apkanalyzer.UndoHierarchicalSizing(data)
+    # No changes expected since there are no child nodes.
+    self.assertEqualLists(data, nodes)
+
+  def testUndoHierarchicalSizing_TotalSizeMinusChildNode(self):
+    data = [
+      ('<TOTAL>', 10),
+      ('child1', 7),
+    ]
+    nodes = apkanalyzer.UndoHierarchicalSizing(data)
+    self.assertEqualLists([
+      ('<TOTAL>', 3),
+      ('child1', 7),
+    ], nodes)
+
+  def testUndoHierarchicalSizing_SiblingAnonymousClass(self):
+    data = [
+      ('class1', 10),
+      ('class1$inner', 8),
+    ]
+    nodes = apkanalyzer.UndoHierarchicalSizing(data)
+    # No change in size expected since these should be siblings.
+    self.assertEqualLists(data, nodes)
+
+  def testUndoHierarchicalSizing_MethodsShouldBeChildNodes(self):
+    data = [
+      ('class1', 10),
+      ('class1 method', 8),
+    ]
+    nodes = apkanalyzer.UndoHierarchicalSizing(data)
+    self.assertEqualLists([
+      ('class1', 2),
+      ('class1 method', 8),
+    ], nodes)
+
+  def testUndoHierarchicalSizing_ClassIsChildNodeOfPackage(self):
+    data = [
+      ('package1', 10),
+      ('package1.class1', 3),
+    ]
+    nodes = apkanalyzer.UndoHierarchicalSizing(data)
+    self.assertEqualLists([
+      ('package1', 7),
+      ('package1.class1', 3),
+    ], nodes)
+
+  def testUndoHierarchicalSizing_TotalIncludesAllPackages(self):
+    data = [
+      ('<TOTAL>', 10),
+      ('package1', 3),
+      ('package2', 4),
+      ('package3', 2),
+    ]
+    nodes = apkanalyzer.UndoHierarchicalSizing(data)
+    self.assertEqualLists([
+      ('<TOTAL>', 1),
+      ('package1', 3),
+      ('package2', 4),
+      ('package3', 2),
+    ], nodes)
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/binary_size/libsupersize/archive.py

@@ -19,6 +19,7 @@ import sys
 import tempfile
 import zipfile
 
+import apkanalyzer
 import concurrent
 import demangle
 import describe
@@ -99,9 +100,10 @@ def _NormalizeNames(raw_symbols):
   found_prefixes = set()
   for symbol in raw_symbols:
     full_name = symbol.full_name
-    if full_name.startswith('*'):
-      # See comment in _CalculatePadding() about when this
-      # can happen.
+
+    # See comment in _CalculatePadding() about when this can happen. Don't
+    # process names for non-native sections.
+    if full_name.startswith('*') or not symbol.IsNative():
       symbol.template_name = full_name
       symbol.name = full_name
       continue
@@ -149,6 +151,7 @@ def _NormalizeNames(raw_symbols):
 
     # Strip out return type, and split out name, template_name.
     # Function parsing also applies to non-text symbols. E.g. Function statics.
+    # TODO(wnwen): Dex methods might want this processing.
     symbol.full_name, symbol.template_name, symbol.name = (
         function_signature.Parse(full_name))
 
@@ -199,7 +202,13 @@ def _ExtractSourcePathsAndNormalizeObjectPaths(raw_symbols, source_mapper):
     logging.info('Looking up source paths from ninja files')
     for symbol in raw_symbols:
       object_path = symbol.object_path
-      if object_path:
+      if symbol.IsDex():
+        symbol.generated_source, symbol.source_path = _NormalizeSourcePath(
+            symbol.source_path)
+      elif symbol.IsOther():
+        # TODO(wnwen): Add source mapping for other symbols.
+        pass
+      elif object_path:
         # We don't have source info for prebuilt .a files.
         if not os.path.isabs(object_path) and not object_path.startswith('..'):
           source_path = source_mapper.FindSourceForPath(object_path)
@@ -457,7 +466,7 @@ def _CalculatePadding(raw_symbols):
       symbol.padding = symbol.size
       continue
     if (symbol.address <= 0 or prev_symbol.address <= 0 or
-        symbol.IsPak() or prev_symbol.IsPak()):
+        not symbol.IsNative() or not prev_symbol.IsNative()):
       continue
 
     if symbol.address == prev_symbol.address:
@@ -793,19 +802,29 @@ def _ParseApkElfSectionSize(section_sizes, metadata, apk_elf_result):
   return section_sizes
 
 
+def _ParseDexSymbols(section_sizes, apk_path, output_directory):
+  symbols = apkanalyzer.CreateDexSymbols(apk_path, output_directory)
+  prev = section_sizes.setdefault(models.SECTION_DEX, 0)
+  section_sizes[models.SECTION_DEX] = prev + sum(s.size for s in symbols)
+  return symbols
+
+
 def _ParseApkOtherSymbols(section_sizes, apk_path):
+  apk_name = os.path.basename(apk_path)
   apk_symbols = []
   zip_info_total = 0
   with zipfile.ZipFile(apk_path) as z:
     for zip_info in z.infolist():
       zip_info_total += zip_info.compress_size
-      # Skip shared library and pak files as they are already accounted for.
+      # Skip shared library, pak, and dex files as they are accounted for.
       if (zip_info.filename.endswith('.so')
+          or zip_info.filename.endswith('.dex')
           or zip_info.filename.endswith('.pak')):
         continue
+      path = os.path.join(apk_name, 'other', zip_info.filename)
       apk_symbols.append(models.Symbol(
             models.SECTION_OTHER, zip_info.compress_size,
-            full_name=zip_info.filename))
+            object_path=path, full_name=os.path.basename(zip_info.filename)))
   overhead_size = os.path.getsize(apk_path) - zip_info_total
   zip_overhead_symbol = models.Symbol(
       models.SECTION_OTHER, overhead_size, full_name='Overhead: APK file')
@@ -910,6 +929,8 @@ def CreateSectionSizesAndSymbols(
                                                knobs)
     section_sizes, elf_overhead_size = _ParseApkElfSectionSize(
         section_sizes, metadata, apk_elf_result)
+    raw_symbols.extend(
+        _ParseDexSymbols(section_sizes, apk_path, output_directory))
     raw_symbols.extend(_ParseApkOtherSymbols(section_sizes, apk_path))
   elif pak_files and pak_info_file:
     pak_symbols_by_id = _FindPakSymbolsFromFiles(

tools/binary_size/libsupersize/models.py

@@ -49,6 +49,7 @@ SECTION_BSS = '.bss'
 SECTION_DATA = '.data'
 SECTION_DATA_REL_RO = '.data.rel.ro'
 SECTION_DATA_REL_RO_LOCAL = '.data.rel.ro.local'
+SECTION_DEX = '.dex'
 SECTION_OTHER = '.other'
 SECTION_PAK_NONTRANSLATED = '.pak.nontranslated'
 SECTION_PAK_TRANSLATIONS = '.pak.translations'
@@ -75,6 +76,7 @@ SECTION_NAME_TO_SECTION = {
     SECTION_DATA: 'd',
     SECTION_DATA_REL_RO_LOCAL: 'R',
     SECTION_DATA_REL_RO: 'R',
+    SECTION_DEX: 'x',
     SECTION_OTHER: 'o',
     SECTION_PAK_NONTRANSLATED: 'P',
     SECTION_PAK_TRANSLATIONS: 'p',
@@ -89,6 +91,7 @@ SECTION_TO_SECTION_NAME = collections.OrderedDict((
     ('R', SECTION_DATA_REL_RO),
     ('d', SECTION_DATA),
     ('b', SECTION_BSS),
+    ('x', SECTION_DEX),
     ('p', SECTION_PAK_TRANSLATIONS),
     ('P', SECTION_PAK_NONTRANSLATED),
     ('o', SECTION_OTHER),
@@ -276,9 +279,17 @@ class BaseSymbol(object):
   def IsBss(self):
     return self.section_name == SECTION_BSS
 
+  def IsDex(self):
+    return self.section_name == SECTION_DEX
+
+  def IsOther(self):
+    return self.section_name == SECTION_OTHER
+
   def IsPak(self):
-    return (self.section_name == SECTION_PAK_TRANSLATIONS or
-        self.section_name == SECTION_PAK_NONTRANSLATED)
+    return self.section_name in PAK_SECTIONS
+
+  def IsNative(self):
+    return self.section_name in NATIVE_SECTIONS
 
   def IsGroup(self):
     return False

tools/binary_size/libsupersize/path_util.py

@@ -94,11 +94,9 @@ class ToolPrefixFinder(_PathFinder):
                            'Release+Asserts', 'bin', 'llvm-')
       else:
         # Auto-detect from build_vars.txt
-        build_vars_path = os.path.join(output_directory, 'build_vars.txt')
-        if os.path.exists(build_vars_path):
-          with open(build_vars_path) as f:
-            build_vars = dict(l.rstrip().split('=', 1) for l in f if '=' in l)
-          tool_prefix = build_vars['android_tool_prefix']
+        build_vars = _LoadBuildVars(output_directory)
+        tool_prefix = build_vars.get('android_tool_prefix')
+        if tool_prefix:
           ret = os.path.normpath(os.path.join(output_directory, tool_prefix))
           # Maintain a trailing '/' if needed.
           if tool_prefix.endswith(os.path.sep):
@@ -128,6 +126,14 @@ class ToolPrefixFinder(_PathFinder):
       raise Exception('Bad --%s. Path not found: %s' % (self._name, full_path))
 
 
+def _LoadBuildVars(output_directory):
+  build_vars_path = os.path.join(output_directory, 'build_vars.txt')
+  if os.path.exists(build_vars_path):
+    with open(build_vars_path) as f:
+      return dict(l.rstrip().split('=', 1) for l in f if '=' in l)
+  return dict()
+
+
 def FromSrcRootRelative(path):
   ret = os.path.relpath(os.path.join(SRC_ROOT, path))
   # Need to maintain a trailing /.
@@ -154,6 +160,13 @@ def GetNmPath(tool_prefix):
   return tool_prefix + 'nm'
 
 
+def GetApkAnalyzerPath(output_directory):
+  build_vars = _LoadBuildVars(output_directory)
+  return os.path.normpath(os.path.join(
+      output_directory, build_vars['android_sdk_root'], 'tools', 'bin',
+      'apkanalyzer'))
+
+
 def GetObjDumpPath(tool_prefix):
   return tool_prefix + 'objdump'
 

tools/binary_size/libsupersize/template/D3SymbolTreeMap.js

@@ -76,8 +76,9 @@ D3SymbolTreeMap._NM_SYMBOL_TYPE_DESCRIPTIONS = {
   'd': '.data and .data.*',
   'r': '.rodata',
   't': '.text',
-  'v': 'Vtable entry',
+  'v': 'Vtable Entry',
   '!': 'Generated Symbols (typeinfo, thunks, etc)',
+  'x': 'Dex Entries',
   'p': 'Locale Pak Entries',
   'P': 'Non-Locale Pak Entries',
   'o': 'Other Entries',
@@ -108,9 +109,10 @@ D3SymbolTreeMap._colorArray = [
   'rgb(128,177,211)',
   'rgb(255,237,111)',
   'rgb(204,235,197)',
+  'rgb(255,111,111)',
   'rgb(93,156,110)',
   'rgb(61,109,55)',
-  'rgb(255,111,111)',
+  'rgb(150,100,111)',
 ]
 
 D3SymbolTreeMap._initColorMap = function() {

tools/binary_size/libsupersize/template/index.html

@@ -24,7 +24,7 @@ body {
 var treemap;
 var filterChanging = false;
 var savedSettings = {};
-var NUM_SYMBOL_TYPES = 9;
+var NUM_SYMBOL_TYPES = 10;
 
 function init() {
     if (window.metadata !== undefined && window.metadata.subtitle) {
@@ -443,9 +443,10 @@ function escape(str) {
                 <br><span class='swatch' id='swatch_3'>&nbsp;&nbsp;&nbsp;</span><input checked type='checkbox' id='check_3' value='t'>Code (.text)
                 <br><span class='swatch' id='swatch_4'>&nbsp;&nbsp;&nbsp;</span><input checked type='checkbox' id='check_4' value='v'>Vtable entries
                 <br><span class='swatch' id='swatch_5'>&nbsp;&nbsp;&nbsp;</span><input checked type='checkbox' id='check_5' value='!'>Generated Symbols (typeinfo, thunks, etc)
-                <br><span class='swatch' id='swatch_6'>&nbsp;&nbsp;&nbsp;</span><input checked type='checkbox' id='check_6' value='p'>Locale Pak Entries
-                <br><span class='swatch' id='swatch_7'>&nbsp;&nbsp;&nbsp;</span><input checked type='checkbox' id='check_7' value='P'>Non-Locale Pak Entries
-                <br><span class='swatch' id='swatch_8'>&nbsp;&nbsp;&nbsp;</span><input type='checkbox' id='check_8' value='o'>Other Entries
+                <br><span class='swatch' id='swatch_6'>&nbsp;&nbsp;&nbsp;</span><input checked type='checkbox' id='check_6' value='x'>Dex entries
+                <br><span class='swatch' id='swatch_7'>&nbsp;&nbsp;&nbsp;</span><input checked type='checkbox' id='check_7' value='p'>Locale Pak Entries
+                <br><span class='swatch' id='swatch_8'>&nbsp;&nbsp;&nbsp;</span><input checked type='checkbox' id='check_8' value='P'>Non-Locale Pak Entries
+                <br><span class='swatch' id='swatch_9'>&nbsp;&nbsp;&nbsp;</span><input type='checkbox' id='check_9' value='o'>Other Entries
             </td>
         </tr>
         <tr><td style='text-align: center; white-space: nowrap; padding-top: 1em;'>

tools/binary_size/libsupersize/testdata/Console.golden

@@ -2,7 +2,7 @@
 Entering interactive Python shell. Quick reference:
 
 SizeInfo: metadata, native_symbols, pak_symbols, raw_symbols, section_sizes, size_path, symbols
-Symbol: FlagsString, IsBss, IsDelta, IsGeneratedByToolchain, IsGroup, IsPak, IsStringLiteral, IterLeafSymbols, address, aliases, end_address, flags, full_name, generated_source, is_anonymous, name, num_aliases, object_path, padding, padding_pss, pss, pss_without_padding, section, section_name, size, size_without_padding, source_path, template_name
+Symbol: FlagsString, IsBss, IsDelta, IsDex, IsGeneratedByToolchain, IsGroup, IsNative, IsOther, IsPak, IsStringLiteral, IterLeafSymbols, address, aliases, end_address, flags, full_name, generated_source, is_anonymous, name, num_aliases, object_path, padding, padding_pss, pss, pss_without_padding, section, section_name, size, size_without_padding, source_path, template_name
 
 SymbolGroup (extends Symbol): CountUniqueSymbols, Filter, GroupedBy, GroupedByAliases, GroupedByFullName, GroupedByName, GroupedByPath, GroupedBySectionName, Inverted, IterUniqueSymbols, SetName, Sorted, SortedByAddress, SortedByCount, SortedByName, WhereAddressInRange, WhereFullNameMatches, WhereGeneratedByToolchain, WhereHasAnyAttribution, WhereHasPath, WhereInSection, WhereIsGroup, WhereIsNative, WhereIsPak, WhereIsTemplate, WhereMatches, WhereNameMatches, WhereObjectPathMatches, WherePathMatches, WherePssBiggerThan, WhereSizeBiggerThan, WhereSourceIsGenerated, WhereSourcePathMatches, WhereTemplateNameMatches, index, is_sorted