Reland of Add dex info and static initializers to resource_sizes.py

First attempt:
https://codereview.chromium.org/2243343004/

Reason for reland:
Now ignores placeholder .so files

The goal is to have a single script that reports all .apk
size related metrics.

* Uses method_count.py to get dex info.
* Already had static initializer logic, but now shows it even without
--so_with_symbols_path.
* Tweaks the static initializer count logic to sum up rather for all
.so files rather than report the same metric repeatedly.

TBR=rnephew@chromium.org,amineer@chromium.org
BUG=609365

Review-Url: https://codereview.chromium.org/2294093006
Cr-Commit-Position: refs/heads/master@{#415725}

build/android/method_count.py

@@ -48,10 +48,10 @@ import perf_tests_results_helper # pylint: disable=import-error
 # https://source.android.com/devices/tech/dalvik/dex-format.html
 
 
-_CONTRIBUTORS_TO_DEX_CACHE = {'type_ids_size': 'types',
-                              'string_ids_size': 'strings',
-                              'method_ids_size': 'methods',
-                              'field_ids_size': 'fields'}
+CONTRIBUTORS_TO_DEX_CACHE = {'type_ids_size': 'types',
+                             'string_ids_size': 'strings',
+                             'method_ids_size': 'methods',
+                             'field_ids_size': 'fields'}
 
 
 def _ExtractSizesFromDexFile(dex_path):
@@ -61,7 +61,7 @@ def _ExtractSizesFromDexFile(dex_path):
       # Each method, type, field, and string contributes 4 bytes (1 reference)
       # to our DexCache size.
       counts['dex_cache_size'] = (
-          sum(counts[x] for x in _CONTRIBUTORS_TO_DEX_CACHE)) * 4
+          sum(counts[x] for x in CONTRIBUTORS_TO_DEX_CACHE)) * 4
       return counts
     m = re.match(r'([a-z_]+_size) *: (\d+)', line)
     if m:
@@ -69,7 +69,7 @@ def _ExtractSizesFromDexFile(dex_path):
   raise Exception('Unexpected end of output.')
 
 
-def _ExtractSizesFromZip(path):
+def ExtractSizesFromZip(path):
   tmpdir = tempfile.mkdtemp(suffix='_dex_extract')
   try:
     counts = collections.defaultdict(int)
@@ -109,7 +109,7 @@ def main():
           'and --apk-name was not provided.' % args.dexfile)
 
   if os.path.splitext(args.dexfile)[1] in ('.zip', '.apk', '.jar'):
-    sizes = _ExtractSizesFromZip(args.dexfile)
+    sizes = ExtractSizesFromZip(args.dexfile)
   else:
     sizes = _ExtractSizesFromDexFile(args.dexfile)
 
@@ -118,7 +118,7 @@ def main():
         '%s_%s' % (args.apk_name, name), 'total', [sizes[value_key]],
         description or name)
 
-  for dex_header_name, readable_name in _CONTRIBUTORS_TO_DEX_CACHE.iteritems():
+  for dex_header_name, readable_name in CONTRIBUTORS_TO_DEX_CACHE.iteritems():
     print_result(readable_name, dex_header_name)
   print_result(
       'DexCache_size', 'dex_cache_size', 'bytes of permanent dirty memory')

build/android/resource_sizes.py

@@ -24,6 +24,7 @@ import zlib
 
 import devil_chromium
 from devil.utils import cmd_helper
+import method_count
 from pylib import constants
 from pylib.constants import host_paths
 
@@ -37,6 +38,7 @@ with host_paths.SysPath(_GRIT_PATH, 1):
 with host_paths.SysPath(host_paths.BUILD_COMMON_PATH):
   import perf_tests_results_helper # pylint: disable=import-error
 
+
 # Python had a bug in zipinfo parsing that triggers on ChromeModern.apk
 # https://bugs.python.org/issue14315
 def _PatchedDecodeExtra(self):
@@ -354,29 +356,53 @@ def _GetResourceIdNameMap():
   return id_name_map
 
 
-def PrintStaticInitializersCount(so_with_symbols_path, chartjson=None):
-  """Emits the performance result for static initializers found in the provided
-     shared library. Additionally, files for which static initializers were
-     found are printed on the standard output.
+def _PrintStaticInitializersCountFromApk(apk_filename, chartjson=None):
+  print 'Finding static initializers (can take a minute)'
+  with zipfile.ZipFile(apk_filename) as z:
+    infolist = z.infolist()
+  out_dir = constants.GetOutDirectory()
+  si_count = 0
+  for zip_info in infolist:
+    # Check file size to account for placeholder libraries.
+    if zip_info.filename.endswith('.so') and zip_info.file_size > 0:
+      unstripped_path = os.path.join(out_dir, 'lib.unstripped',
+                                     os.path.basename(zip_info.filename))
+      if os.path.exists(unstripped_path):
+        si_count += _PrintStaticInitializersCount(unstripped_path)
+      else:
+        raise Exception('Unstripped .so not found. Looked here: %s',
+                        unstripped_path)
+  ReportPerfResult(chartjson, 'StaticInitializersCount', 'count', si_count,
+                   'count')
+
+
+def _PrintStaticInitializersCount(so_with_symbols_path):
+  """Counts the number of static initializers in the given shared library.
+     Additionally, files for which static initializers were found are printed
+     on the standard output.
 
      Args:
        so_with_symbols_path: Path to the unstripped libchrome.so file.
+
+     Returns:
+       The number of static initializers found.
   """
   # GetStaticInitializers uses get-static-initializers.py to get a list of all
   # static initializers. This does not work on all archs (particularly arm).
   # TODO(rnephew): Get rid of warning when crbug.com/585588 is fixed.
   si_count = CountStaticInitializers(so_with_symbols_path)
   static_initializers = GetStaticInitializers(so_with_symbols_path)
-  if si_count != len(static_initializers):
+  static_initializers_count = len(static_initializers) - 1  # Minus summary.
+  if si_count != static_initializers_count:
     print ('There are %d files with static initializers, but '
            'dump-static-initializers found %d:' %
-           (si_count, len(static_initializers)))
+           (si_count, static_initializers_count))
   else:
-    print 'Found %d files with static initializers:' % si_count
+    print '%s - Found %d files with static initializers:' % (
+        os.path.basename(so_with_symbols_path), si_count)
   print '\n'.join(static_initializers)
 
-  ReportPerfResult(chartjson, 'StaticInitializersCount', 'count',
-                   si_count, 'count')
+  return si_count
 
 def _FormatBytes(byts):
   """Pretty-print a number of bytes."""
@@ -400,20 +426,34 @@ def _CalculateCompressedSize(file_path):
   return total_size
 
 
+def _PrintDexAnalysis(apk_filename, chartjson=None):
+  sizes = method_count.ExtractSizesFromZip(apk_filename)
+
+  graph_title = os.path.basename(apk_filename) + '_Dex'
+  dex_metrics = method_count.CONTRIBUTORS_TO_DEX_CACHE
+  for key, label in dex_metrics.iteritems():
+    ReportPerfResult(chartjson, graph_title, label, sizes[key], 'entries')
+
+  graph_title = '%sCache' % graph_title
+  ReportPerfResult(chartjson, graph_title, 'DexCache', sizes['dex_cache_size'],
+                   'bytes')
+
+
 def main(argv):
   usage = """Usage: %prog [options] file1 file2 ...
 
 Pass any number of files to graph their sizes. Any files with the extension
 '.apk' will be broken down into their components on a separate graph."""
   option_parser = optparse.OptionParser(usage=usage)
-  option_parser.add_option('--so-path', help='Path to libchrome.so.')
+  option_parser.add_option('--so-path',
+                           help='Obsolete. Pass .so as positional arg instead.')
   option_parser.add_option('--so-with-symbols-path',
-                           help='Path to libchrome.so with symbols.')
+                           help='Mostly obsolete. Use .so within .apk instead.')
   option_parser.add_option('--min-pak-resource-size', type='int',
                            default=20*1024,
                            help='Minimum byte size of displayed pak resources.')
   option_parser.add_option('--build_type', dest='build_type', default='Debug',
-                           help='Sets the build type, default is Debug.')
+                           help='Obsoleted by --chromium-output-directory.')
   option_parser.add_option('--chromium-output-directory',
                            help='Location of the build artifacts. '
                                 'Takes precidence over --build_type.')
@@ -444,8 +484,9 @@ Pass any number of files to graph their sizes. Any files with the extension
   devil_chromium.Initialize()
 
   if options.so_with_symbols_path:
-    PrintStaticInitializersCount(
-        options.so_with_symbols_path, chartjson=chartjson)
+    si_count = _PrintStaticInitializersCount(options.so_with_symbols_path)
+    ReportPerfResult(chartjson, 'StaticInitializersCount', 'count', si_count,
+                     'count')
 
   PrintResourceSizes(files, chartjson=chartjson)
 
@@ -453,6 +494,9 @@ Pass any number of files to graph their sizes. Any files with the extension
     if f.endswith('.apk'):
       PrintApkAnalysis(f, chartjson=chartjson)
       PrintPakAnalysis(f, options.min_pak_resource_size)
+      _PrintDexAnalysis(f, chartjson=chartjson)
+      if not options.so_with_symbols_path:
+        _PrintStaticInitializersCountFromApk(f, chartjson=chartjson)
 
   if chartjson:
     results_path = os.path.join(options.output_dir, 'results-chart.json')