[aab] Add synchronized proguarding

This means that the compiled Java of each DFM and the base module are optimized
together in one proguarding step. The advantage is that symbols and symbol
usages get obfuscated in the same way and, thus, usages still work even in the
optimized state. To allow modularization, we split the optimized Jar into the
modules after.

Design doc: go/gn-aab-proguarding

Bug: 862696
Change-Id: I0f6032b09b31a572838721e0b90b025ff879891d
Reviewed-on: https://chromium-review.googlesource.com/1149040
Commit-Queue: Tibor Goldschwendt <tiborg@chromium.org>
Reviewed-by: David Turner <digit@chromium.org>
Reviewed-by: Bernhard Bauer <bauerb@chromium.org>
Cr-Commit-Position: refs/heads/master@{#579210}

build/android/gyp/generate_proguarded_module_jar.py

+#!/usr/bin/env python
+#
+# Copyright (c) 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Extracts a bundle module's classes from jar created in the synchronized
+proguarding step and packages them into a new jar.
+
+Synchronized proguarding means that, when several app modules are combined into
+an app bundle, all un-optimized jars for all modules are grouped and sent to a
+single proguard command, which generates a single, common, intermediate
+optimized jar, and its mapping file.
+
+This script is used to extract, from this synchronized proguard jar, all the
+optimized classes corresponding to a single module, into a new .jar file. The
+latter will be compiled later into the module's dex file.
+
+For this, the script reads the module's un-obfuscated class names from the
+module's unoptimized jars. Then, it maps those to obfuscated class names using
+the proguard mapping file. Finally, it extracts the module's class files from
+the proguarded jar and zips them into a new module jar. """
+
+import argparse
+import os
+import sys
+import zipfile
+
+from util import build_utils
+
+MANIFEST = """Manifest-Version: 1.0
+Created-By: generate_proguarded_module_jar.py
+"""
+
+
+# TODO(tiborg): Share with merge_jar_info_files.py.
+def _FullJavaNameFromClassFilePath(path):
+  if not path.endswith('.class'):
+    return ''
+  path = os.path.splitext(path)[0]
+  parts = []
+  while path:
+    # Use split to be platform independent.
+    head, tail = os.path.split(path)
+    path = head
+    parts.append(tail)
+  parts.reverse()  # Package comes first
+  return '.'.join(parts)
+
+
+def main(args):
+  args = build_utils.ExpandFileArgs(args)
+  parser = argparse.ArgumentParser()
+  build_utils.AddDepfileOption(parser)
+  parser.add_argument(
+      '--proguarded-jar',
+      required=True,
+      help='Path to input jar produced by synchronized proguarding')
+  parser.add_argument(
+      '--proguard-mapping',
+      required=True,
+      help='Path to input proguard mapping produced by synchronized '
+      'proguarding')
+  parser.add_argument(
+      '--module-input-jars',
+      required=True,
+      help='GN-list of input paths to un-optimized jar files for the current '
+      'module. The optimized versions of their .class files will go into '
+      'the output jar.')
+  parser.add_argument(
+      '--output-jar',
+      required=True,
+      help='Path to output jar file containing the module\'s optimized class '
+      'files')
+  parser.add_argument(
+      '--is-base-module',
+      action='store_true',
+      help='Inidcates to extract class files for a base module')
+  options = parser.parse_args(args)
+  options.module_input_jars = build_utils.ParseGnList(options.module_input_jars)
+
+  # Read class names of the currently processed module.
+  classes = set()
+  for module_jar in options.module_input_jars:
+    with zipfile.ZipFile(module_jar) as zip_info:
+      for path in zip_info.namelist():
+        fully_qualified_name = _FullJavaNameFromClassFilePath(path)
+        if fully_qualified_name:
+          classes.add(fully_qualified_name)
+
+  # Parse the proguarding mapping to be able to map un-obfuscated to obfuscated
+  # names.
+  # Proguard mapping files have the following format:
+  #
+  # {un-obfuscated class name 1} -> {obfuscated class name 1}:
+  #     {un-obfuscated member name 1} -> {obfuscated member name 1}
+  #     ...
+  # {un-obfuscated class name 2} -> {obfuscated class name 2}:
+  #     ...
+  # ...
+  obfuscation_map = {}
+  with open(options.proguard_mapping, 'r') as proguard_mapping_file:
+    for line in proguard_mapping_file:
+      # Skip indented lines since they map member names and not class names.
+      if line.startswith(' '):
+        continue
+      line = line.strip()
+      # Skip empty lines.
+      if not line:
+        continue
+      assert line.endswith(':')
+      full, obfuscated = line.strip(':').split(' -> ')
+      assert full
+      assert obfuscated
+      obfuscation_map[full] = obfuscated
+
+  # Collect the obfuscated names of classes, which should go into the currently
+  # processed module.
+  obfuscated_module_classes = set(
+      obfuscation_map[c] for c in classes if c in obfuscation_map)
+
+  # Collect horizontally merged classes to later make sure that those only go
+  # into the base module. Merging classes horizontally means that proguard took
+  # two classes that don't inherit from each other and merged them into one.
+  horiz_merged_classes = set()
+  obfuscated_classes = sorted(obfuscation_map.values())
+  prev_obfuscated_class = None
+  for obfuscated_class in obfuscated_classes:
+    if prev_obfuscated_class and obfuscated_class == prev_obfuscated_class:
+      horiz_merged_classes.add(obfuscated_class)
+    prev_obfuscated_class = obfuscated_class
+
+  # Move horizontally merged classes into the base module.
+  if options.is_base_module:
+    obfuscated_module_classes |= horiz_merged_classes
+  else:
+    obfuscated_module_classes -= horiz_merged_classes
+
+  # Extract module class files from proguarded jar and store them in a module
+  # split jar.
+  with zipfile.ZipFile(
+      os.path.abspath(options.output_jar), 'w',
+      zipfile.ZIP_DEFLATED) as output_jar:
+    with zipfile.ZipFile(os.path.abspath(options.proguarded_jar),
+                         'r') as proguarded_jar:
+      for obfuscated_class in obfuscated_module_classes:
+        class_path = obfuscated_class.replace('.', '/') + '.class'
+        class_file_content = proguarded_jar.read(class_path)
+        output_jar.writestr(class_path, class_file_content)
+    output_jar.writestr('META-INF/MANIFEST.MF', MANIFEST)
+
+  if options.depfile:
+    build_utils.WriteDepfile(
+        options.depfile, options.output_jar, options.module_input_jars +
+        [options.proguard_mapping, options.proguarded_jar], add_pydeps=False)
+
+
+if __name__ == '__main__':
+  main(sys.argv[1:])

build/android/gyp/write_build_config.py

@@ -495,6 +495,16 @@ The classpath listing the jars used for annotation processors. I.e. sent as
 The list of annotation processor main classes. I.e. sent as `-processor' when
 invoking `javac`.
 
+## <a name="android_app_bundle">Target type `android_app_bundle`</a>:
+
+This type corresponds to an Android app bundle (`.aab` file).
+
+* `deps_info['synchronized_proguard_enabled"]`:
+True to indicate that the app modules of this bundle should be proguarded in a
+single synchronized proguarding step. Synchronized proguarding means that all
+un-optimized jars for all modules are sent to a single proguard command. The
+resulting optimized jar is then split into optimized app module jars after.
+
 --------------- END_MARKDOWN ---------------------------------------------------
 """
 
@@ -851,7 +861,9 @@ def main(argv):
       help='Path to the build config of the tested apk (for an instrumentation '
       'test apk).')
   parser.add_option('--proguard-enabled', action='store_true',
-      help='Whether proguard is enabled for this apk.')
+      help='Whether proguard is enabled for this apk or bundle module.')
+  parser.add_option('--synchronized-proguard-enabled', action='store_true',
+      help='Whether synchronized proguarding is enabled for this bundle.')
   parser.add_option('--proguard-configs',
       help='GN-list of proguard flag files to use in final apk.')
   parser.add_option('--proguard-output-jar-path',
@@ -1187,6 +1199,11 @@ def main(argv):
     if is_java_target and options.jar_path:
       java_full_classpath.append(options.jar_path)
     java_full_classpath.extend(c['jar_path'] for c in all_library_deps)
+    if options.type == 'android_app_bundle':
+      for d in deps.Direct('android_app_bundle_module'):
+        java_full_classpath.extend(
+            c for c in d.get('java_runtime_classpath', [])
+            if c not in java_full_classpath)
 
   system_jars = [c['jar_path'] for c in system_library_deps]
   system_interface_jars = [c['interface_jar_path'] for c in system_library_deps]
@@ -1209,7 +1226,16 @@ def main(argv):
           p for p in c.get('proguard_configs', []) if p not in all_configs)
       extra_jars.extend(
           p for p in c.get('extra_classpath_jars', []) if p not in extra_jars)
+    if options.type == 'android_app_bundle':
+      for c in deps.Direct('android_app_bundle_module'):
+        all_configs.extend(
+            p for p in c.get('proguard_configs', []) if p not in all_configs)
     deps_info['proguard_all_configs'] = all_configs
+    if options.type == 'android_app_bundle':
+      for d in deps.Direct('android_app_bundle_module'):
+        extra_jars.extend(
+            c for c in d.get('proguard_classpath_jars', [])
+            if c not in extra_jars)
     deps_info['proguard_classpath_jars'] = extra_jars
 
     if options.type == 'android_app_bundle':
@@ -1227,7 +1253,16 @@ def main(argv):
         raise Exception('Deps %s have proguard enabled while deps %s have '
                         'proguard disabled' % (deps_proguard_enabled,
                                                deps_proguard_disabled))
-      deps_info['sync_proguard_enabled'] = bool(deps_proguard_enabled)
+      if (bool(deps_proguard_enabled) !=
+              bool(options.synchronized_proguard_enabled) or
+          bool(deps_proguard_disabled) ==
+              bool(options.synchronized_proguard_enabled)):
+        raise Exception('Modules %s of bundle %s have opposite proguard '
+                        'enabling flags than bundle' % (deps_proguard_enabled +
+                            deps_proguard_disabled, config['deps_info']['name'])
+                        )
+      deps_info['synchronized_proguard_enabled'] = bool(
+          options.synchronized_proguard_enabled)
     else:
       deps_info['proguard_enabled'] = bool(options.proguard_enabled)
       if options.proguard_output_jar_path:

build/config/android/internal_rules.gni

@@ -376,6 +376,10 @@ template("write_build_config") {
     if (defined(invoker.proguard_enabled) && invoker.proguard_enabled) {
       args += [ "--proguard-enabled" ]
     }
+    if (defined(invoker.synchronized_proguard_enabled) &&
+        invoker.synchronized_proguard_enabled) {
+      args += [ "--synchronized-proguard-enabled" ]
+    }
     if (defined(invoker.proguard_output_jar_path)) {
       _rebased_proguard_output_jar_path =
           rebase_path(invoker.proguard_output_jar_path, root_build_dir)
@@ -3267,6 +3271,9 @@ if (enable_java_templates) {
 #
 #   build_config: Path to build_config of the android_apk_or_module() target.
 #
+#   dex_path_file_arg: Path to the module's dex file passed directly to the
+#     creation script. Must be a rebased path or @FileArg expression.
+#
 template("create_android_app_bundle_module") {
   _build_config = invoker.build_config
   _rebased_build_config = rebase_path(_build_config, root_build_dir)
@@ -3298,7 +3305,7 @@ template("create_android_app_bundle_module") {
       "--format=bundle-module",
       "--output-apk",
       rebase_path(invoker.module_zip_path, root_build_dir),
-      "--dex-file=@FileArg($_rebased_build_config:final_dex:path)",
+      "--dex-file=${invoker.dex_path_file_arg}",
       "--resource-apk=@FileArg(" +
           "$_rebased_build_config:deps_info:proto_resources_path)",
       "--assets=@FileArg($_rebased_build_config:assets)",
@@ -3319,3 +3326,42 @@ template("create_android_app_bundle_module") {
     }
   }
 }
+
+# Extracts a bundle module's classes from jar created in the synchronized
+# proguarding step and packages them into a new jar.
+#
+# Variables:
+#   proguarded_jar: Path to input jar produced by synchronized proguarding.
+#   proguard_mapping: Path to input proguard mapping produced by synchronized
+#     proguarding
+#   output_jar: Path to output jar file containing the module's optimized class
+#     files.
+#   build_config: Path to build config of module.
+#   is_base_module: True if this is processing a base module.
+template("generate_proguarded_module_jar") {
+  _rebased_build_config = rebase_path(invoker.build_config, root_build_dir)
+
+  action(target_name) {
+    forward_variables_from(invoker, [ "deps" ])
+    script = "//build/android/gyp/generate_proguarded_module_jar.py"
+    inputs = build_utils_py
+    outputs = [
+      invoker.output_jar,
+    ]
+    depfile = "${target_gen_dir}/${target_name}.d"
+    args = [
+      "--depfile",
+      rebase_path(depfile, root_build_dir),
+      "--proguarded-jar",
+      rebase_path(invoker.proguarded_jar, root_build_dir),
+      "--proguard-mapping",
+      rebase_path(invoker.proguard_mapping, root_build_dir),
+      "--module-input-jars=@FileArg(${_rebased_build_config}:deps_info:java_runtime_classpath)",
+      "--output-jar",
+      rebase_path(invoker.output_jar, root_build_dir),
+    ]
+    if (defined(invoker.is_base_module) && invoker.is_base_module) {
+      args += [ "--is-base-module" ]
+    }
+  }
+}

chrome/android/BUILD.gn

@@ -1609,12 +1609,15 @@ android_app_bundle("chrome_public_bundle") {
   sign_bundle = is_official_build
 
   enable_language_splits = true
+  synchronized_proguard_enabled = !is_java_debug
 }
 
 android_app_bundle("chrome_modern_public_bundle") {
   base_module_target = ":chrome_modern_public_base_module"
+  synchronized_proguard_enabled = !is_java_debug
 }
 
 android_app_bundle("monochrome_public_bundle") {
   base_module_target = ":monochrome_public_base_module"
+  synchronized_proguard_enabled = !is_java_debug
 }