[Android] Dexlayout with obfuscated profiles

Extends the dexlayout build path with profile obfuscation. This
hooks build/android/gyp/dex.py into the recent convert_dex_profile.py,
and makes the necessary changes to GN to pass the proguard mapping
to the dex pass.

Bug: 875276
Change-Id: Id595b17fdba69c7e2e2dcd252c467954a558c56f
Reviewed-on: https://chromium-review.googlesource.com/c/1366315
Commit-Queue: Matthew Cary <mattcary@chromium.org>
Reviewed-by: agrieve <agrieve@chromium.org>
Cr-Commit-Position: refs/heads/master@{#615863}

build/android/convert_dex_profile.py

@@ -477,6 +477,26 @@ def ProcessProfile(input_profile, proguard_mapping):
   return profile
 
 
+def ObfuscateProfile(nonobfuscated_profile, dex_file, proguard_mapping,
+                     dexdump_path, output_filename):
+  """Helper method for obfuscating a profile.
+
+  Args:
+    nonobfuscated_profile: a profile with nonobfuscated symbols.
+    dex_file: path to the dex file matching the mapping.
+    proguard_mapping: a mapping from nonobfuscated to obfuscated symbols used
+      in the dex file.
+    dexdump_path: path to the dexdump utility.
+    output_filename: output filename in which to write the obfuscated profile.
+  """
+  dexinfo = ProcessDex(_RunDexDump(dexdump_path, dex_file))
+  _, reverse_mapping = ProcessProguardMapping(
+      _ReadFile(proguard_mapping), dexinfo)
+  obfuscated_profile = ProcessProfile(
+      _ReadFile(nonobfuscated_profile), reverse_mapping)
+  obfuscated_profile.WriteToFile(output_filename)
+
+
 def main(args):
   parser = argparse.ArgumentParser()
   parser.add_argument(

build/android/convert_dex_profile_tests.py

@@ -2,12 +2,31 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import unittest
+"""Tests for convert_dex_profile.
+
+Can be run from build/android/:
+  $ cd build/android
+  $ python convert_dex_profile_tests.py
+"""
+
+import os
+import sys
 import tempfile
+import unittest
+
 import convert_dex_profile as cp
 
+sys.path.insert(1, os.path.join(os.path.dirname(__file__), 'gyp'))
+from util import build_utils
+
 cp.logging.disable(cp.logging.CRITICAL)
 
+# There are two obfuscations used in the tests below, each with the same
+# unobfuscated profile. The first, corresponding to DEX_DUMP, PROGUARD_MAPPING,
+# and OBFUSCATED_PROFILE, has an ambiguous method a() which is mapped to both
+# getInstance and initialize. The second, corresponding to DEX_DUMP_2,
+# PROGUARD_MAPPING_2 and OBFUSCATED_PROFILE_2, removes the ambiguity.
+
 DEX_DUMP = """
 
 Class descriptor  : 'La;'
@@ -76,6 +95,67 @@ PLa;->b()La;
 SLa;->a(Ljava/lang/Object;)I
 HPLa;->a(Ljava/lang/String;)I"""
 
+DEX_DUMP_2 = """
+
+Class descriptor  : 'La;'
+  Direct methods    -
+      #0              : (in La;)
+        name          : '<clinit>'
+        type          : '(Ljava/lang/String;)V'
+        code          -
+        catches       : 1
+                0x000f - 0x001e
+                  <any> -> 0x0093
+        positions     :
+                0x0001 line=310
+                0x0057 line=313
+        locals        : 
+      #1              : (in La;)
+        name          : '<init>'
+        type          : '()V'
+        positions     :
+        locals        : 
+  Virtual methods   -
+      #0              : (in La;)
+        name          : 'a'
+        type          : '(Ljava/lang/String;)I'
+        positions     : 
+          0x0000 line=2
+          0x0003 line=3
+          0x001b line=8
+        locals        : 
+          0x0000 - 0x0021 reg=3 this La; 
+      #1              : (in La;)
+        name          : 'c'
+        type          : '(Ljava/lang/Object;)I'
+        positions     : 
+          0x0000 line=8
+          0x0003 line=9
+        locals        : 
+          0x0000 - 0x0021 reg=3 this La; 
+      #2              : (in La;)
+        name          : 'b'
+        type          : '()La;'
+        positions     : 
+          0x0000 line=1
+        locals        : 
+"""
+
+# pylint: disable=line-too-long
+PROGUARD_MAPPING_2 = \
+"""org.chromium.Original -> a:
+    org.chromium.Original sDisplayAndroidManager -> e
+    org.chromium.Original another() -> b
+    void initialize() -> c
+    org.chromium.Original getInstance():203 -> a
+    4:4:void inlined():237:237 -> a"""
+
+OBFUSCATED_PROFILE_2 = \
+"""La;
+PLa;->b()La;
+HPSLa;->a()La;
+HPLa;->c()V"""
+
 UNOBFUSCATED_PROFILE = \
 """Lorg/chromium/Original;
 PLorg/chromium/Original;->another()Lorg/chromium/Original;
@@ -169,6 +249,28 @@ class GenerateProfileTests(unittest.TestCase):
         for a, b in zip(sorted(f), sorted(UNOBFUSCATED_PROFILE.splitlines())):
           self.assertEquals(a.strip(), b.strip())
 
+  def testObfuscateProfile(self):
+    with build_utils.TempDir() as temp_dir:
+      # The dex dump is used as the dexfile, by passing /bin/cat as the dexdump
+      # program.
+      dex_path = os.path.join(temp_dir, 'dexdump')
+      with open(dex_path, 'w') as dex_file:
+        dex_file.write(DEX_DUMP_2)
+      mapping_path = os.path.join(temp_dir, 'mapping')
+      with open(mapping_path, 'w') as mapping_file:
+        mapping_file.write(PROGUARD_MAPPING_2)
+      unobfuscated_path = os.path.join(temp_dir, 'unobfuscated')
+      with open(unobfuscated_path, 'w') as unobfuscated_file:
+        unobfuscated_file.write(UNOBFUSCATED_PROFILE)
+      obfuscated_path = os.path.join(temp_dir, 'obfuscated')
+      cp.ObfuscateProfile(unobfuscated_path, dex_path, mapping_path, '/bin/cat',
+                          obfuscated_path)
+      with open(obfuscated_path) as obfuscated_file:
+        obfuscated_profile = sorted(obfuscated_file.readlines())
+      for a, b in zip(
+          sorted(OBFUSCATED_PROFILE_2.splitlines()), obfuscated_profile):
+        self.assertEquals(a.strip(), b.strip())
+
 
 if __name__ == '__main__':
   unittest.main()

build/android/gyp/dex.py

@@ -16,6 +16,10 @@ import zipfile
 
 from util import build_utils
 
+sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.path.pardir))
+
+import convert_dex_profile
+
 
 def _CheckFilePathEndsWithJar(parser, file_path):
   if not file_path.endswith(".jar"):
@@ -76,7 +80,12 @@ def _ParseArgs(args):
                     help=('Path to ART dexlayout binary. There should be a '
                           'lib/ directory at the same path containing shared '
                           'libraries (shared with dexlayout).'))
-
+  parser.add_option('--dexdump-path', help='Path to dexdump binary.')
+  parser.add_option(
+      '--proguard-mapping-path',
+      help=('Path to proguard map from obfuscated symbols in the jar to '
+            'unobfuscated symbols present in the code. If not '
+            'present, the jar is assumed not to be obfuscated.'))
 
   options, paths = parser.parse_args(args)
 
@@ -84,8 +93,12 @@ def _ParseArgs(args):
   build_utils.CheckOptions(options, parser, required=required_options)
 
   if options.dexlayout_profile:
-    build_utils.CheckOptions(options, parser, required=('profman_path',
-                                                        'dexlayout_path'))
+    build_utils.CheckOptions(
+        options,
+        parser,
+        required=('profman_path', 'dexlayout_path', 'dexdump_path'))
+  elif options.proguard_mapping_path is not None:
+    raise Exception('Unexpected proguard mapping without dexlayout')
 
   if options.multidex_configuration_path:
     with open(options.multidex_configuration_path) as multidex_config_file:
@@ -274,6 +287,16 @@ def _ZipMultidex(file_dir, dex_files):
   return zip_name
 
 
+def _ZipSingleDex(dex_file, zip_name):
+  """Zip up a single dex file.
+
+  Args:
+    dex_file: A dexfile whose name is ignored.
+    zip_name: The output file in which to write the zip.
+  """
+  build_utils.DoZip([('classes.dex', dex_file)], zip_name)
+
+
 def main(args):
   options, paths = _ParseArgs(args)
   if ((options.proguard_enabled == 'true'
@@ -306,38 +329,56 @@ def main(args):
   is_dex = options.dex_path.endswith('.dex')
   is_jar = options.dex_path.endswith('.jar')
 
-  if is_jar and _NoClassFiles(paths):
-    # Handle case where no classfiles are specified in inputs
-    # by creating an empty JAR
-    with zipfile.ZipFile(options.dex_path, 'w') as outfile:
-      outfile.comment = 'empty'
-  else:
-    # .dex files can't specify a name for D8. Instead, we output them to a
-    # temp directory then move them after the command has finished running
-    # (see _MoveTempDexFile). For other files, tmp_dex_dir is None.
-    with build_utils.TempDir() as tmp_dex_dir:
+  with build_utils.TempDir() as tmp_dir:
+    tmp_dex_dir = os.path.join(tmp_dir, 'tmp_dex_dir')
+    os.mkdir(tmp_dex_dir)
+    if is_jar and _NoClassFiles(paths):
+      # Handle case where no classfiles are specified in inputs
+      # by creating an empty JAR
+      with zipfile.ZipFile(options.dex_path, 'w') as outfile:
+        outfile.comment = 'empty'
+    else:
+      # .dex files can't specify a name for D8. Instead, we output them to a
+      # temp directory then move them after the command has finished running
+      # (see _MoveTempDexFile). For other files, tmp_dex_dir is None.
       _RunD8(dex_cmd, paths, tmp_dex_dir)
-      if is_dex:
-        _MoveTempDexFile(tmp_dex_dir, options.dex_path)
-      else:
-        # d8 supports outputting to a .zip, but does not have deterministic file
-        # ordering: https://issuetracker.google.com/issues/119945929
-        build_utils.ZipDir(options.dex_path, tmp_dex_dir)
 
-  if options.dexlayout_profile:
-    with build_utils.TempDir() as temp_dir:
-      binary_profile = _CreateBinaryProfile(options.dexlayout_profile,
-                                            options.dex_path,
-                                            options.profman_path, temp_dir)
-      output_files = _LayoutDex(binary_profile, options.dex_path,
-                                options.dexlayout_path, temp_dir)
+    tmp_dex_output = os.path.join(tmp_dir, 'tmp_dex_output')
+    if is_dex:
+      _MoveTempDexFile(tmp_dex_dir, tmp_dex_output)
+    else:
+      # d8 supports outputting to a .zip, but does not have deterministic file
+      # ordering: https://issuetracker.google.com/issues/119945929
+      build_utils.ZipDir(tmp_dex_output, tmp_dex_dir)
+
+    if options.dexlayout_profile:
+      if options.proguard_mapping_path is not None:
+        matching_profile = os.path.join(tmp_dir, 'obfuscated_profile')
+        convert_dex_profile.ObfuscateProfile(
+            options.dexlayout_profile, tmp_dex_output,
+            options.proguard_mapping_path, options.dexdump_path,
+            matching_profile)
+      else:
+        logging.warning('No obfuscation for %s', options.dexlayout_profile)
+        matching_profile = options.dexlayout_profile
+      binary_profile = _CreateBinaryProfile(matching_profile, tmp_dex_output,
+                                            options.profman_path, tmp_dir)
+      output_files = _LayoutDex(binary_profile, tmp_dex_output,
+                                options.dexlayout_path, tmp_dir)
       target = None
       if len(output_files) > 1:
-        target = _ZipMultidex(temp_dir, output_files)
+        target = _ZipMultidex(tmp_dir, output_files)
       else:
-        target = output_files[0]
-      shutil.move(os.path.join(temp_dir, target), options.dex_path)
-
+        output = output_files[0]
+        if not zipfile.is_zipfile(output):
+          target = os.path.join(tmp_dir, 'dex_classes.zip')
+          _ZipSingleDex(output, target)
+        else:
+          target = output
+      shutil.move(os.path.join(tmp_dir, target), tmp_dex_output)
+
+    # The dex file is complete and can be moved out of tmp_dir.
+    shutil.move(tmp_dex_output, options.dex_path)
 
   build_utils.WriteDepfile(
       options.depfile, options.dex_path, input_paths, add_pydeps=False)

build/android/gyp/dex.pydeps

 # Generated by running:
 #   build/print_python_deps.py --root build/android/gyp --output build/android/gyp/dex.pydeps build/android/gyp/dex.py
 ../../gn_helpers.py
+../convert_dex_profile.py
 dex.py
 util/__init__.py
 util/build_utils.py

build/config/android/internal_rules.gni

@@ -43,6 +43,7 @@ _java_target_blacklist = [ "*:*_unpack_aar" ]
 _default_proguard_jar_path = "//third_party/proguard/lib/proguard.jar"
 _r8_path = "//third_party/r8/lib/r8.jar"
 
+_dexdump_path = "$android_sdk_build_tools/dexdump"
 _dexlayout_path = "//third_party/android_build_tools/art/dexlayout"
 _profman_path = "//third_party/android_build_tools/art/profman"
 _art_lib_file_names = [
@@ -1288,13 +1289,23 @@ if (enable_java_templates) {
             rebase_path(_dexlayout_path, root_build_dir),
             "--profman-path",
             rebase_path(_profman_path, root_build_dir),
+            "--dexdump-path",
+            rebase_path(_dexdump_path, root_build_dir),
           ]
           inputs += [
             _dexlayout_path,
             _profman_path,
+            _dexdump_path,
             invoker.dexlayout_profile,
           ]
           inputs += _default_art_libs
+          if (_proguard_enabled) {
+            args += [
+              "--proguard-mapping-path",
+              rebase_path(invoker.proguard_mapping_path, root_build_dir),
+            ]
+            inputs += [ invoker.proguard_mapping_path ]
+          }
         }
 
         if (!is_java_debug) {