Add support for split APKs to supersize

This allows supersize to analyze split APKs. These show up as child
containers to the base APK since they are named with
"<base name>/<split name>".

Example report: https://chrome-supersize.firebaseapp.com/viewer.html?load_url=https%3A%2F%2Fstorage.googleapis.com%2Fchrome-supersize%2Foneoffs%2Fcduvall7.size&group_by=container&type=mx
Example diff: https://chrome-supersize.firebaseapp.com/viewer.html?load_url=https%3A%2F%2Fstorage.googleapis.com%2Fchrome-supersize%2Foneoffs%2Fcduvall4.sizediff&group_by=container&type=mx&diff_mode=on

Bug: 1143690
Change-Id: I6ed7d516607e56431cf4c3fb67989d6783520a78
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2614123
Commit-Queue: Clark DuVall <cduvall@chromium.org>
Reviewed-by: Andrew Grieve <agrieve@chromium.org>
Cr-Commit-Position: refs/heads/master@{#841627}

tools/binary_size/libsupersize/archive.py

@@ -8,6 +8,7 @@ import argparse
 import bisect
 import calendar
 import collections
+import copy
 import datetime
 import gzip
 import itertools
@@ -52,7 +53,6 @@ _OWNERS_COMPONENT_REGEX = re.compile(r'^\s*#\s*COMPONENT:\s*(\S+)',
 _OWNERS_FILE_PATH_REGEX = re.compile(r'^\s*file://(\S+)', re.MULTILINE)
 
 _UNCOMPRESSED_COMPRESSION_RATIO_THRESHOLD = 0.9
-_APKS_MAIN_APK = 'splits/base-master.apk'
 
 # Holds computation state that is live only when an output directory exists.
 _OutputDirectoryContext = collections.namedtuple('_OutputDirectoryContext', [
@@ -748,8 +748,8 @@ def LoadAndPostProcessDeltaSizeInfo(path, file_obj=None):
   return before_size_info, after_size_info
 
 
-def _CollectModuleSizes(minimal_apks_path):
-  sizes_by_module = collections.defaultdict(int)
+def _GetModuleInfoList(minimal_apks_path):
+  module_info_list = []
   with zipfile.ZipFile(minimal_apks_path) as z:
     for info in z.infolist():
       # E.g.:
@@ -760,7 +760,14 @@ def _CollectModuleSizes(minimal_apks_path):
       # TODO(agrieve): Might be worth measuring a non-en locale as well.
       m = re.match(r'splits/(.*)-master\.apk', info.filename)
       if m:
-        sizes_by_module[m.group(1)] += info.file_size
+        module_info_list.append((m.group(1), info.file_size))
+  return sorted(module_info_list)
+
+
+def _CollectModuleSizes(minimal_apks_path):
+  sizes_by_module = collections.defaultdict(int)
+  for module_name, file_size in _GetModuleInfoList(minimal_apks_path):
+    sizes_by_module[module_name] += file_size
   return sizes_by_module
 
 
@@ -834,9 +841,13 @@ def CreateMetadata(args, linker_name, build_config):
     metadata[models.METADATA_MAP_FILENAME] = shorten_path(args.map_file)
 
   if args.minimal_apks_file:
-    sizes_by_module = _CollectModuleSizes(args.minimal_apks_file)
     metadata[models.METADATA_APK_FILENAME] = shorten_path(
         args.minimal_apks_file)
+    if args.split_name and args.split_name != 'base':
+      metadata[models.METADATA_APK_SPLIT_NAME] = args.split_name
+      metadata[models.METADATA_APK_SIZE] = os.path.getsize(args.apk_file)
+    else:
+      sizes_by_module = _CollectModuleSizes(args.minimal_apks_file)
       for name, size in sizes_by_module.items():
         key = models.METADATA_APK_SIZE
         if name != 'base':
@@ -1882,6 +1893,9 @@ def _AddContainerArguments(parser):
       help='Include a padding field for each symbol, instead of rederiving '
       'from consecutive symbols on file load.')
 
+  # The split_name arg is used for bundles to identify DFMs.
+  parser.set_defaults(split_name=None)
+
 
 def AddArguments(parser):
   parser.add_argument('size_file', help='Path to output .size file.')
@@ -2076,14 +2090,7 @@ def _ReadMultipleArgsFromFile(ssargs_file, on_config_error):
                                      on_config_error)
 
 
-def _ProcessContainerArgs(top_args, sub_args, main_file, on_config_error):
-  if hasattr(sub_args, 'name'):
-    container_name = sub_args.name
-  else:
-    container_name = os.path.basename(main_file)
-  if set(container_name) & set('<>'):
-    parser.error('Container name cannot have characters in "<>"')
-
+def _ProcessContainerArgs(top_args, sub_args, container_name, on_config_error):
   # Copy output_directory, tool_prefix, etc. into sub_args.
   for k, v in top_args.__dict__.items():
     sub_args.__dict__.setdefault(k, v)
@@ -2132,6 +2139,28 @@ def _ProcessContainerArgs(top_args, sub_args, main_file, on_config_error):
           linker_name, size_info_prefix)
 
 
+def _IsOnDemand(apk_path):
+  # Check if the manifest specifies whether or not to extract native libs.
+  output = subprocess.check_output([
+      path_util.GetAapt2Path(), 'dump', 'xmltree', '--file',
+      'AndroidManifest.xml', apk_path
+  ]).decode('ascii')
+
+  def parse_attr(name):
+    # http://schemas.android.com/apk/res/android:isFeatureSplit(0x0101055b)=true
+    # http://schemas.android.com/apk/distribution:onDemand=true
+    m = re.search(name + r'(?:\(.*?\))?=(\w+)', output)
+    return m and m.group(1) == 'true'
+
+  is_feature_split = parse_attr('android:isFeatureSplit')
+  # Can use <dist:on-demand>, or <module dist:onDemand="true">.
+  on_demand = parse_attr(
+      'distribution:onDemand') or 'distribution:on-demand' in output
+  on_demand = bool(on_demand and is_feature_split)
+
+  return on_demand
+
+
 def _IterSubArgs(top_args, on_config_error):
   """Generates main paths (may be deduced) for each containers given by input.
 
@@ -2167,16 +2196,33 @@ def _IterSubArgs(top_args, on_config_error):
   # Each element in |sub_args_list| specifies a container.
   for sub_args in sub_args_list:
     main_file = _IdentifyInputFile(sub_args, on_config_error)
+    if hasattr(sub_args, 'name'):
+      container_name = sub_args.name
+    else:
+      container_name = os.path.basename(main_file)
+    if set(container_name) & set('<>'):
+      parser.error('Container name cannot have characters in "<>"')
+
 
     # If needed, extract .apk file to a temp file and process that instead.
     if sub_args.minimal_apks_file:
-      with zip_util.UnzipToTemp(sub_args.minimal_apks_file,
-                                _APKS_MAIN_APK) as temp:
-        sub_args.apk_file = temp
-        yield _ProcessContainerArgs(top_args, sub_args, main_file,
-                                    on_config_error)
+      for module_name, _ in _GetModuleInfoList(sub_args.minimal_apks_file):
+        with zip_util.UnzipToTemp(
+            sub_args.minimal_apks_file,
+            'splits/{}-master.apk'.format(module_name)) as temp:
+          if _IsOnDemand(temp):
+            continue
+          module_sub_args = copy.copy(sub_args)
+          module_sub_args.apk_file = temp
+          module_sub_args.split_name = module_name
+          module_sub_args.name = '{}/{}.apk'.format(container_name, module_name)
+          if module_name != 'base':
+            # TODO(crbug.com/1143690): Fix native analysis for split APKs.
+            module_sub_args.map_file = None
+          yield _ProcessContainerArgs(top_args, module_sub_args,
+                                      module_sub_args.name, on_config_error)
     else:
-      yield _ProcessContainerArgs(top_args, sub_args, main_file,
+      yield _ProcessContainerArgs(top_args, sub_args, container_name,
                                   on_config_error)
 
 

tools/binary_size/libsupersize/integration_test.py

@@ -40,10 +40,15 @@ _TEST_PAK_INFO_PATH = os.path.join(
 _TEST_ELF_FILE_BEGIN = os.path.join(_TEST_OUTPUT_DIR, 'elf.begin')
 _TEST_APK_LOCALE_PAK_PATH = os.path.join(_TEST_APK_ROOT_DIR, 'assets/en-US.pak')
 _TEST_APK_PAK_PATH = os.path.join(_TEST_APK_ROOT_DIR, 'assets/resources.pak')
+_TEST_ON_DEMAND_MANIFEST_PATH = os.path.join(_TEST_DATA_DIR,
+                                             'AndroidManifest_OnDemand.xml')
+_TEST_ALWAYS_INSTALLED_MANIFEST_PATH = os.path.join(
+    _TEST_DATA_DIR, 'AndroidManifest_AlwaysInstalled.xml')
 
 # The following files are dynamically created.
 _TEST_ELF_PATH = os.path.join(_TEST_OUTPUT_DIR, 'elf')
 _TEST_APK_PATH = os.path.join(_TEST_OUTPUT_DIR, 'test.apk')
+_TEST_OTHER_SPLIT_APK_PATH = os.path.join(_TEST_OUTPUT_DIR, 'other.apk')
 _TEST_MINIMAL_APKS_PATH = os.path.join(_TEST_OUTPUT_DIR, 'Bundle.minimal.apks')
 _TEST_SSARGS_PATH = os.path.join(_TEST_OUTPUT_DIR, 'test.ssargs')
 
@@ -80,11 +85,13 @@ def _AddMocksToPath():
   os.environ['PATH'] = _TEST_TOOL_PREFIX[:-1] + os.path.pathsep + prev_path
   os.environ['APK_ANALYZER'] = os.path.join(_TEST_SDK_DIR, 'tools', 'bin',
                                             'apkanalyzer')
+  os.environ['AAPT2'] = os.path.join(_TEST_SDK_DIR, 'tools', 'bin', 'aapt2')
   try:
     yield
   finally:
     os.environ['PATH'] = prev_path
     del os.environ['APK_ANALYZER']
+    del os.environ['AAPT2']
 
 
 def _RunApp(name, args, debug_measures=False):
@@ -145,11 +152,21 @@ class IntegrationTest(unittest.TestCase):
       apk_file.writestr(
           _TEST_APK_DEX_PATH, IntegrationTest._CreateBlankData(23))
 
+    with zipfile.ZipFile(_TEST_OTHER_SPLIT_APK_PATH, 'w') as other_apk_zip:
+      other_apk_zip.write(_TEST_ALWAYS_INSTALLED_MANIFEST_PATH,
+                          'AndroidManifest.xml')
+
     with zipfile.ZipFile(_TEST_MINIMAL_APKS_PATH, 'w') as apk_file:
       apk_file.write(_TEST_APK_PATH, 'splits/base-master.apk')
       apk_file.writestr('splits/base-en.apk', 'x' * 10)
-      apk_file.writestr('splits/vr-master.apk', 'x' * 20)
+
+      with tempfile.NamedTemporaryFile(suffix='.apk') as vr_apk_file:
+        with zipfile.ZipFile(vr_apk_file.name, 'w') as vr_apk_zip:
+          vr_apk_zip.write(_TEST_ON_DEMAND_MANIFEST_PATH, 'AndroidManifest.xml')
+        apk_file.write(vr_apk_file.name, 'splits/vr-master.apk')
       apk_file.writestr('splits/vr-en.apk', 'x' * 40)
+
+      apk_file.write(_TEST_OTHER_SPLIT_APK_PATH, 'splits/other-master.apk')
       apk_file.writestr('toc.pb', 'x' * 80)
 
   @classmethod
@@ -157,6 +174,7 @@ class IntegrationTest(unittest.TestCase):
     IntegrationTest._SafeRemoveFiles([
         _TEST_ELF_PATH,
         _TEST_APK_PATH,
+        _TEST_OTHER_SPLIT_APK_PATH,
         _TEST_MINIMAL_APKS_PATH,
     ])
 
@@ -192,11 +210,13 @@ class IntegrationTest(unittest.TestCase):
       apk_so_path = None
       size_info_prefix = None
       extracted_minimal_apk_path = None
+      container_name = ''
       if use_apk:
         args.apk_file = _TEST_APK_PATH
       elif use_minimal_apks:
         args.minimal_apks_file = _TEST_MINIMAL_APKS_PATH
         extracted_minimal_apk_path = _TEST_APK_PATH
+        container_name = 'Bundle.minimal.apks'
       if use_apk or use_minimal_apks:
         apk_so_path = _TEST_APK_SO_PATH
         if args.output_directory:
@@ -219,10 +239,13 @@ class IntegrationTest(unittest.TestCase):
       with _AddMocksToPath():
         build_config = {}
         metadata = archive.CreateMetadata(args, linker_name, build_config)
+        container_list = []
+        raw_symbols_list = []
         container, raw_symbols = archive.CreateContainerAndSymbols(
             knobs=knobs,
             opts=opts,
-            container_name='',
+            container_name='{}/base.apk'.format(container_name)
+            if container_name else '',
             metadata=metadata,
             map_path=args.map_file,
             tool_prefix=args.tool_prefix,
@@ -235,8 +258,29 @@ class IntegrationTest(unittest.TestCase):
             pak_info_file=pak_info_file,
             linker_name=linker_name,
             size_info_prefix=size_info_prefix)
+        container_list.append(container)
+        raw_symbols_list.append(raw_symbols)
+        if use_minimal_apks:
+          opts.analyze_native = False
+          args.split_name = 'other'
+          args.apk_file = _TEST_OTHER_SPLIT_APK_PATH
+          args.elf_file = None
+          args.map_file = None
+          metadata = archive.CreateMetadata(args, None, build_config)
+          container, raw_symbols = archive.CreateContainerAndSymbols(
+              knobs=knobs,
+              opts=opts,
+              container_name='{}/other.apk'.format(container_name),
+              metadata=metadata,
+              tool_prefix=args.tool_prefix,
+              output_directory=args.output_directory,
+              source_directory=args.source_directory,
+              apk_path=_TEST_OTHER_SPLIT_APK_PATH,
+              size_info_prefix=size_info_prefix)
+          container_list.append(container)
+          raw_symbols_list.append(raw_symbols)
         IntegrationTest.cached_size_info[cache_key] = archive.CreateSizeInfo(
-            build_config, [container], [raw_symbols])
+            build_config, container_list, raw_symbols_list)
     return copy.deepcopy(IntegrationTest.cached_size_info[cache_key])
 
   def _DoArchive(self,
@@ -321,9 +365,13 @@ class IntegrationTest(unittest.TestCase):
 
     sym_strs = (repr(sym) for sym in size_info.symbols)
     stats = describe.DescribeSizeInfoCoverage(size_info)
-    assert len(size_info.containers) == 1
+    if len(size_info.containers) == 1:
       # If there's only one container, merge the its metadata into build_config.
       merged_data_desc = describe.DescribeDict(size_info.metadata_legacy)
+    else:
+      merged_data_desc = describe.DescribeDict(size_info.build_config)
+      for m in size_info.metadata:
+        merged_data_desc.extend(describe.DescribeDict(m))
     return itertools.chain(merged_data_desc, stats, sym_strs)
 
   @_CompareWithGolden()

tools/binary_size/libsupersize/models.py

@@ -54,6 +54,7 @@ BUILD_CONFIG_KEYS = (
 
 METADATA_APK_FILENAME = 'apk_file_name'  # Path relative to output_directory.
 METADATA_APK_SIZE = 'apk_size'  # File size of apk in bytes.
+METADATA_APK_SPLIT_NAME = 'apk_split_name'  # Name of the split if applicable.
 METADATA_ZIPALIGN_OVERHEAD = 'zipalign_padding'  # Overhead from zipalign.
 METADATA_SIGNING_BLOCK_SIZE = 'apk_signature_block_size'  # Size in bytes.
 METADATA_MAP_FILENAME = 'map_file_name'  # Path relative to output_directory.

tools/binary_size/libsupersize/path_util.py

@@ -217,6 +217,12 @@ def GetApkAnalyzerPath():
   return os.environ.get('APK_ANALYZER', default_path)
 
 
+def GetAapt2Path():
+  default_path = FromToolsSrcRootRelative(
+      os.path.join('third_party', 'android_build_tools', 'aapt2', 'aapt2'))
+  return os.environ.get('AAPT2', default_path)
+
+
 def GetJavaHome():
   return FromToolsSrcRootRelative(os.path.join('third_party', 'jdk', 'current'))
 

tools/binary_size/libsupersize/testdata/AndroidManifest_AlwaysInstalled.xml

+N: android=http://schemas.android.com/apk/res/android (line=2)
+  N: dist=http://schemas.android.com/apk/distribution (line=2)
+    E: manifest (line=2)
+      A: http://schemas.android.com/apk/res/android:isFeatureSplit(0x0101055b)=true
+        E: http://schemas.android.com/apk/distribution:module (line=8)
+          A: http://schemas.android.com/apk/distribution:onDemand=false
+            E: http://schemas.android.com/apk/distribution:fusing (line=9)
+              A: http://schemas.android.com/apk/distribution:include=true
+        E: application (line=12)

tools/binary_size/libsupersize/testdata/AndroidManifest_OnDemand.xml

+N: android=http://schemas.android.com/apk/res/android (line=2)
+  N: dist=http://schemas.android.com/apk/distribution (line=2)
+    E: manifest (line=2)
+      A: http://schemas.android.com/apk/res/android:isFeatureSplit(0x0101055b)=true
+        E: http://schemas.android.com/apk/distribution:module (line=8)
+          A: http://schemas.android.com/apk/distribution:onDemand=true
+            E: http://schemas.android.com/apk/distribution:fusing (line=9)
+              A: http://schemas.android.com/apk/distribution:include=true
+        E: application (line=12)

tools/binary_size/libsupersize/testdata/mock_sdk/tools/bin/aapt2

+#!/usr/bin/env bash
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+base_dir=$(dirname "$0")
+exec python "$base_dir/mock_aapt2.py" "$@"

tools/binary_size/libsupersize/testdata/mock_sdk/tools/bin/mock_aapt2.py

+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import sys
+import zipfile
+
+
+def main():
+  apk_path = sys.argv[-1]
+  assert os.path.exists(apk_path), 'Apk does not exist: {}'.format(apk_path)
+  with zipfile.ZipFile(apk_path) as z:
+    try:
+      # The AndroidManifest.xml file will have the aapt2 output, and not XML.
+      with z.open('AndroidManifest.xml') as f:
+        sys.stdout.write(f.read())
+    except KeyError:
+      pass
+
+
+if __name__ == '__main__':
+  main()

tools/binary_size/libsupersize/testdata/mock_sdk/tools/bin/mock_apkanalyzer.py

@@ -5,6 +5,7 @@
 
 import os
 import sys
+import zipfile
 
 
 _SCRIPT_DIR = os.path.dirname(__file__)
@@ -15,8 +16,12 @@ def main():
   # Without a proguard mapping file, the last argument is the apk_path.
   apk_path = sys.argv[-1]
   assert os.path.exists(apk_path), 'Apk does not exist: {}'.format(apk_path)
+  with zipfile.ZipFile(apk_path) as z:
+    if any(n.endswith('.dex') for n in z.namelist()):
       with open(_OUTPUT_FILE, 'r') as f:
         sys.stdout.write(f.read())
+    else:
+      sys.stdout.write('P r 0   0       0       <TOTAL>')
 
 
 if __name__ == '__main__':