Android: Add build targets for bundle .apks file

These will be used by bots & diagnose_bloat.py for measuring binary
size.

Also updates related code in apk_operations.py:
 * Makes --output-apks required for build-bundle-apks
 * Adds --minimal for build-bundle-apks
 * Logs bundletool commands when in verbose mode

Bug: 873714
Change-Id: I6fb4bcc366110cacfd8e1b772db4ea1e00f81cc9
Reviewed-on: https://chromium-review.googlesource.com/c/1456223Reviewed-by: David Turner <digit@chromium.org>
Commit-Queue: Andrew Grieve <agrieve@chromium.org>
Cr-Commit-Position: refs/heads/master@{#630017}

PRESUBMIT.py

@@ -658,6 +658,7 @@ _ANDROID_SPECIFIC_PYDEPS_FILES = [
     'build/android/gyp/assert_static_initializers.pydeps',
     'build/android/gyp/bytecode_processor.pydeps',
     'build/android/gyp/compile_resources.pydeps',
+    'build/android/gyp/create_app_bundle_minimal_apks.pydeps',
     'build/android/gyp/create_bundle_wrapper_script.pydeps',
     'build/android/gyp/copy_ex.pydeps',
     'build/android/gyp/create_app_bundle.pydeps',

build/android/apk_operations.py

@@ -96,25 +96,25 @@ BundleGenerationInfo = collections.namedtuple(
     'keystore_alias')
 
 
-def _GenerateBundleApks(info, universal=False):
+def _GenerateBundleApks(info, output_path, minimal=False, universal=False):
   """Generate an .apks archive from a bundle on demand.
 
   Args:
     info: A BundleGenerationInfo instance.
+    output_path: Path of output .apks archive.
+    minimal: Create the minimal set of apks possible (english-only).
     universal: Whether to create a single APK that contains the contents of all
-      modules.
-  Returns:
-    Path of output .apks archive.
+        modules.
   """
   app_bundle_utils.GenerateBundleApks(
       info.bundle_path,
-      info.bundle_apks_path,
+      output_path,
       info.aapt2_path,
       info.keystore_path,
       info.keystore_password,
       info.keystore_alias,
-      universal)
-  return info.bundle_apks_path
+      universal=universal,
+      minimal=minimal)
 
 
 def _InstallBundle(devices, bundle_apks, package_name, command_line_flags_file,
@@ -1102,8 +1102,10 @@ class _InstallCommand(_Command):
 
   def Run(self):
     if self.is_bundle:
-      bundle_apks_path = _GenerateBundleApks(self.bundle_generation_info)
-      _InstallBundle(self.devices, bundle_apks_path, self.args.package_name,
+      # Store .apks file beside the .aab file so that it gets cached.
+      output_path = self.bundle_generation_info.bundle_apks_path
+      _GenerateBundleApks(self.bundle_generation_info, output_path)
+      _InstallBundle(self.devices, output_path, self.args.package_name,
                      self.args.command_line_flags_file, self.args.module,
                      self.args.fake)
     else:
@@ -1416,21 +1418,24 @@ class _BuildBundleApks(_Command):
   need_device_args = False
 
   def _RegisterExtraArgs(self, group):
-    group.add_argument('--output-apks',
-                       help='Destination path for .apks archive copy.')
+    group.add_argument(
+        '--output-apks', required=True, help='Destination path for .apks file.')
+    group.add_argument(
+        '--minimal',
+        action='store_true',
+        help='Build .apks archive that targets the bundle\'s minSdkVersion and '
+        'contains only english splits. It still contains optional splits.')
     group.add_argument('--universal', action='store_true',
                        help='Build .apks archive containing single APK with '
                             'contents of all splits. NOTE: Won\'t add modules '
                             'with <dist:fusing dist:include="false"/> flag.')
 
   def Run(self):
-    bundle_apks_path = _GenerateBundleApks(self.bundle_generation_info,
-                                           self.args.universal)
-    if self.args.output_apks:
-      try:
-        shutil.copyfile(bundle_apks_path, self.args.output_apks)
-      except shutil.Error as e:
-        logging.exception('Failed to copy .apks archive: %s', e)
+    _GenerateBundleApks(
+        self.bundle_generation_info,
+        self.args.output_apks,
+        minimal=self.args.minimal,
+        universal=self.args.universal)
 
 
 # Shared commands for regular APKs and app bundles.

build/android/gyp/bundletool.py

@@ -8,6 +8,7 @@
 Bundletool is distributed as a versioned jar file. This script abstracts the
 location and version of this jar file, as well as the JVM invokation."""
 
+import logging
 import os
 import subprocess
 import sys
@@ -24,6 +25,7 @@ BUNDLETOOL_JAR_PATH = os.path.join(
 
 def RunBundleTool(args):
   args = ['java', '-jar', BUNDLETOOL_JAR_PATH] + args
+  logging.debug(' '.join(args))
   subprocess.check_call(args)
 
 if __name__ == '__main__':

build/android/gyp/create_app_bundle_minimal_apks.py

+#!/usr/bin/env python
+#
+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Creates an .apks from an .aab with only English strings."""
+
+import argparse
+import os
+import sys
+
+sys.path.append(
+    os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)))
+from pylib.utils import app_bundle_utils
+
+
+def main():
+  parser = argparse.ArgumentParser(description=__doc__)
+  parser.add_argument(
+      '--bundle', required=True, help='Path to input .aab file.')
+  parser.add_argument(
+      '--output', required=True, help='Path to output .apks file.')
+  parser.add_argument('--aapt2-path', required=True, help='Path to aapt2.')
+  parser.add_argument(
+      '--keystore-path', required=True, help='Path to keystore.')
+  parser.add_argument(
+      '--keystore-password', required=True, help='Keystore password.')
+  parser.add_argument(
+      '--keystore-name', required=True, help='Key name within keystore')
+
+  args = parser.parse_args()
+
+  app_bundle_utils.GenerateBundleApks(
+      args.bundle,
+      args.output,
+      args.aapt2_path,
+      args.keystore_path,
+      args.keystore_password,
+      args.keystore_name,
+      minimal=True,
+      check_for_noop=False)
+
+
+if __name__ == '__main__':
+  main()

build/android/gyp/create_app_bundle_minimal_apks.pydeps

+# Generated by running:
+#   build/print_python_deps.py --root build/android/gyp --output build/android/gyp/create_app_bundle_minimal_apks.pydeps build/android/gyp/create_app_bundle_minimal_apks.py
+../../gn_helpers.py
+../pylib/__init__.py
+../pylib/utils/__init__.py
+../pylib/utils/app_bundle_utils.py
+bundletool.py
+create_app_bundle_minimal_apks.py
+util/__init__.py
+util/build_utils.py
+util/md5_check.py

build/android/pylib/utils/app_bundle_utils.py

@@ -2,9 +2,13 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import json
 import logging
 import os
+import re
 import sys
+import tempfile
+import zipfile
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'gyp'))
 
@@ -12,9 +16,36 @@ from util import build_utils
 from util import md5_check
 import bundletool
 
-def GenerateBundleApks(bundle_path, bundle_apks_path, aapt2_path,
-                       keystore_path, keystore_password, keystore_alias,
-                       universal):
+
+_ALL_ABIS = ['armeabi-v7a', 'arm64-v8a', 'x86', 'x86_64']
+
+
+def _CreateMinimalDeviceSpec(bundle_path):
+  # Could also use "bundletool dump resources", but reading directly is faster.
+  with zipfile.ZipFile(bundle_path) as f:
+    manifest_data = f.read('base/manifest/AndroidManifest.xml')
+  min_sdk_version = int(
+      re.search(r'minSdkVersion.*?(\d+)', manifest_data).group(1))
+
+  # Setting sdkVersion=minSdkVersion prevents multiple per-minSdkVersion .apk
+  # files from being created within the .apks file.
+  return {
+      'screenDensity': 1000,  # Ignored since we don't split on density.
+      'sdkVersion': min_sdk_version,
+      'supportedAbis': _ALL_ABIS,  # Our .aab files are already split on abi.
+      'supportedLocales': ['en'],
+  }
+
+
+def GenerateBundleApks(bundle_path,
+                       bundle_apks_path,
+                       aapt2_path,
+                       keystore_path,
+                       keystore_password,
+                       keystore_alias,
+                       universal=False,
+                       minimal=False,
+                       check_for_noop=True):
   """Generate an .apks archive from a an app bundle if needed.
 
   Args:
@@ -27,46 +58,54 @@ def GenerateBundleApks(bundle_path, bundle_apks_path, aapt2_path,
     keystore_alias: Keystore signing key alias.
     universal: Whether to create a single APK that contains the contents of all
       modules.
+    minimal: Create the minimal set of apks possible (english-only).
+    check_for_noop: Use md5_check to short-circuit when inputs have not changed.
   """
-  # NOTE: BUNDLETOOL_JAR_PATH is added to input_strings, rather than
-  # input_paths, to speed up MD5 computations by about 400ms (the .jar file
-  # contains thousands of class files which are checked independently,
-  # resulting in an .md5.stamp of more than 60000 lines!).
-  input_paths = [
-      bundle_path,
-      aapt2_path,
-      keystore_path
-  ]
-  input_strings = [
-      keystore_password,
-      keystore_alias,
-      bundletool.BUNDLETOOL_JAR_PATH,
-      # NOTE: BUNDLETOOL_VERSION is already part of BUNDLETOOL_JAR_PATH, but
-      # it's simpler to assume that this may not be the case in the future.
-      bundletool.BUNDLETOOL_VERSION
-  ]
-  output_paths = [bundle_apks_path]
+  device_spec = None
+  if minimal:
+    device_spec = _CreateMinimalDeviceSpec(bundle_path)
 
   def rebuild():
-    logging.info('Building %s', os.path.basename(bundle_apks_path))
-    with build_utils.AtomicOutput(bundle_apks_path) as tmp_apks:
+    logging.info('Building %s', bundle_apks_path)
+    with tempfile.NamedTemporaryFile(suffix='.json') as spec_file, \
+        build_utils.AtomicOutput(bundle_apks_path, only_if_changed=False) as f:
       cmd_args = [
-          'java', '-jar', bundletool.BUNDLETOOL_JAR_PATH, 'build-apks',
+          'build-apks',
           '--aapt2=%s' % aapt2_path,
-          '--output=%s' % tmp_apks.name,
+          '--output=%s' % f.name,
           '--bundle=%s' % bundle_path,
           '--ks=%s' % keystore_path,
           '--ks-pass=pass:%s' % keystore_password,
           '--ks-key-alias=%s' % keystore_alias,
           '--overwrite',
       ]
+      if device_spec:
+        json.dump(device_spec, spec_file)
+        spec_file.flush()
+        cmd_args += ['--device-spec=' + spec_file.name]
       if universal:
         cmd_args += ['--mode=universal']
+      bundletool.RunBundleTool(cmd_args)
 
-      build_utils.CheckOutput(cmd_args)
-
-  md5_check.CallAndRecordIfStale(
-    rebuild,
-    input_paths=input_paths,
-    input_strings=input_strings,
-    output_paths=output_paths)
+  if check_for_noop:
+    # NOTE: BUNDLETOOL_JAR_PATH is added to input_strings, rather than
+    # input_paths, to speed up MD5 computations by about 400ms (the .jar file
+    # contains thousands of class files which are checked independently,
+    # resulting in an .md5.stamp of more than 60000 lines!).
+    input_paths = [bundle_path, aapt2_path, keystore_path]
+    input_strings = [
+        keystore_password,
+        keystore_alias,
+        bundletool.BUNDLETOOL_JAR_PATH,
+        # NOTE: BUNDLETOOL_VERSION is already part of BUNDLETOOL_JAR_PATH, but
+        # it's simpler to assume that this may not be the case in the future.
+        bundletool.BUNDLETOOL_VERSION,
+        device_spec,
+    ]
+    md5_check.CallAndRecordIfStale(
+        rebuild,
+        input_paths=input_paths,
+        input_strings=input_strings,
+        output_paths=[bundle_apks_path])
+  else:
+    rebuild()

build/config/android/rules.gni

@@ -4118,7 +4118,6 @@ if (enable_java_templates) {
     action_with_pydeps("${target_name}__wrapper_script") {
       script = "//build/android/gyp/create_bundle_wrapper_script.py"
       inputs = [
-        "//build/android/gyp/bundletool.py",
         _base_module_build_config,
       ]
       outputs = [
@@ -4161,10 +4160,58 @@ if (enable_java_templates) {
     }
 
     group(target_name) {
-      deps = [
+      public_deps = [
         ":${target_name}__bundle",
         ":${target_name}__wrapper_script",
       ]
     }
   }
+
+  # Create an .apks file from an .aab file. The .apks file will contain the
+  # minimal set of .apk files needed for tracking binary size.
+  # The file will be created at "$bundle_path_without_extension.minimal.apks".
+  #
+  # Variables:
+  #   bundle_path: Path to the input .aab file.
+  #
+  # Example:
+  #   create_app_bundle_minimal_apks("minimal_apks") {
+  #     deps = [
+  #       ":bundle_target",
+  #     ]
+  #     bundle_path = "$root_build_dir/apks/Bundle.aab"
+  #   }
+  template("create_app_bundle_minimal_apks") {
+    action_with_pydeps(target_name) {
+      forward_variables_from(invoker,
+                             [
+                               "deps",
+                               "testonly",
+                             ])
+      script = "//build/android/gyp/create_app_bundle_minimal_apks.py"
+      _dir = get_path_info(invoker.bundle_path, "dir")
+      _name = get_path_info(invoker.bundle_path, "name")
+      _output_path = "$_dir/$_name.minimal.apks"
+      outputs = [
+        _output_path,
+      ]
+      inputs = [
+        invoker.bundle_path,
+      ]
+      args = [
+        "--bundle",
+        rebase_path(invoker.bundle_path, root_build_dir),
+        "--output",
+        rebase_path(_output_path, root_build_dir),
+        "--aapt2-path",
+        rebase_path(android_sdk_tools_bundle_aapt2, root_build_dir),
+        "--keystore-path",
+        rebase_path(android_keystore_path, root_build_dir),
+        "--keystore-name",
+        android_keystore_name,
+        "--keystore-password",
+        android_keystore_password,
+      ]
+    }
+  }
 }

chrome/android/BUILD.gn

@@ -1884,6 +1884,16 @@ android_app_bundle("chrome_modern_public_bundle") {
   }
 }
 
+if (is_official_build) {
+  # Used for binary size monitoring.
+  create_app_bundle_minimal_apks("chrome_modern_public_minimal_apks") {
+    deps = [
+      ":chrome_modern_public_bundle",
+    ]
+    bundle_path = "$root_build_dir/apks/ChromeModernPublic.aab"
+  }
+}
+
 template("monochrome_public_bundle_tmpl") {
   _bundle_name = "MonochromePublic${invoker.bundle_suffix}"
   _base_module_target_name = "${invoker.target_name}__base_bundle_module"
@@ -1955,6 +1965,16 @@ monochrome_public_bundle_tmpl("monochrome_public_bundle") {
   bundle_suffix = ""
 }
 
+if (is_official_build) {
+  # Used for binary size monitoring.
+  create_app_bundle_minimal_apks("monochrome_public_minimal_apks") {
+    deps = [
+      ":monochrome_public_bundle",
+    ]
+    bundle_path = "$root_build_dir/apks/MonochromePublic.aab"
+  }
+}
+
 if (android_64bit_target_cpu && enable_64_bit_browser) {
   monochrome_public_bundle_tmpl("monochrome_64_public_bundle") {
     bundle_suffix = "64"