Revert of Android: Delete rezip in favor of zipalign -p (patchset #2 id:210001...

Revert of Android: Delete rezip in favor of zipalign -p (patchset #2 id:210001 of https://codereview.chromium.org/2612773005/ )

Reason for revert:
Broke system webview merging of debug builds.

Original issue's description:
> Reland of Android: Delete rezip in favor of zipalign -p
>
> There's no need for a custom alignment tool now that zipalign supports
> page-aligning libraries.
>
> Previously reverted in:
> https://codereview.chromium.org/2595233003/
>
> Reason for reland:
> Contains fix for monochrome (tested merging locally).
>
> BUG=676029,676589
>
> Review-Url: https://codereview.chromium.org/2612773005
> Cr-Commit-Position: refs/heads/master@{#442906}
> Committed: https://chromium.googlesource.com/chromium/src/+/c3fd406f53369c1714d3587087146ff6ead4adab

TBR=michaelbai@chromium.org,jbudorick@chromium.org
NOTRY=true
BUG=676029,676589

Review-Url: https://codereview.chromium.org/2621353004
Cr-Commit-Position: refs/heads/master@{#443333}

BUILD.gn

@@ -401,6 +401,7 @@ group("both_gn_and_gyp") {
       "//base:base_junit_tests",
       "//base/android/linker:chromium_android_linker",
       "//build/android/gyp/test:hello_world",
+      "//build/android/rezip",
       "//chrome/android/webapk/shell_apk:webapk",
       "//components/invalidation/impl:components_invalidation_impl_junit_tests",
       "//components/policy/android:components_policy_junit_tests",

android_webview/tools/apk_merger.py

@@ -135,7 +135,8 @@ def AddDiffFiles(diff_files, tmp_dir_32, out_zip, expected_files,
 
 
 def SignAndAlignApk(tmp_apk, signed_tmp_apk, new_apk, zipalign_path,
-                    keystore_path, key_name, key_password):
+                    keystore_path, key_name, key_password,
+                    page_align_shared_libraries):
   try:
     finalize_apk.JarSigner(
         keystore_path,
@@ -148,6 +149,7 @@ def SignAndAlignApk(tmp_apk, signed_tmp_apk, new_apk, zipalign_path,
 
   try:
     finalize_apk.AlignApk(zipalign_path,
+                          page_align_shared_libraries,
                           signed_tmp_apk,
                           new_apk)
   except build_utils.CalledProcessError as e:
@@ -222,8 +224,7 @@ def main():
   parser.add_argument('--key_name', required=True)
   parser.add_argument('--key_password', required=True)
   parser.add_argument('--shared_library')
-  parser.add_argument('--page-align-shared-libraries', action='store_true',
-                      help='Obsolete, but remains for backwards compatibility')
+  parser.add_argument('--page-align-shared-libraries', action='store_true')
   parser.add_argument('--uncompress-shared-libraries', action='store_true')
   parser.add_argument('--debug', action='store_true')
   # This option shall only used in debug build, see http://crbug.com/631494.
@@ -249,7 +250,8 @@ def main():
     MergeApk(args, tmp_apk, tmp_dir_32, tmp_dir_64)
 
     SignAndAlignApk(tmp_apk, signed_tmp_apk, new_apk, args.zipalign_path,
-                    args.keystore_path, args.key_name, args.key_password)
+                    args.keystore_path, args.key_name, args.key_password,
+                    args.page_align_shared_libraries)
 
   except ApkMergeFailure as e:
     print e

build/android/gyp/apkbuilder.py

@@ -161,20 +161,14 @@ def _CreateAssetsList(path_tuples):
 
 def _AddNativeLibraries(out_apk, native_libs, android_abi, uncompress):
   """Add native libraries to APK."""
-  has_crazy_linker = any('android_linker' in os.path.basename(p)
-                         for p in native_libs)
   for path in native_libs:
     basename = os.path.basename(path)
+    apk_path = 'lib/%s/%s' % (android_abi, basename)
 
     compress = None
-    if (uncompress and os.path.splitext(basename)[1] == '.so'
-        and 'android_linker' not in basename):
+    if (uncompress and os.path.splitext(basename)[1] == '.so'):
       compress = False
-      # Add prefix to prevent android install from extracting upon install.
-      if has_crazy_linker:
-        basename = 'crazy.' + basename
 
-    apk_path = 'lib/%s/%s' % (android_abi, basename)
     build_utils.AddToZipHermetic(out_apk,
                                  apk_path,
                                  src_path=path,
@@ -271,9 +265,8 @@ def main(args):
                               options.uncompress_shared_libraries)
 
         for name in sorted(options.native_lib_placeholders):
-          # Note: Empty libs files are ignored by md5check (can cause issues
-          # with stale builds when the only change is adding/removing
-          # placeholders).
+          # Empty libs files are ignored by md5check, but rezip requires them
+          # to be empty in order to identify them as placeholders.
           apk_path = 'lib/%s/%s' % (options.android_abi, name)
           build_utils.AddToZipHermetic(out_apk, apk_path, data='')
 

build/android/gyp/finalize_apk.py

@@ -21,6 +21,32 @@ import resource_sizes  # pylint: disable=unused-import
 
 from util import build_utils
 
+def RenameInflateAndAddPageAlignment(
+    rezip_apk_jar_path, in_zip_file, out_zip_file):
+  rezip_apk_cmd = [
+      'java',
+      '-classpath',
+      rezip_apk_jar_path,
+      'RezipApk',
+      'renamealign',
+      in_zip_file,
+      out_zip_file,
+    ]
+  build_utils.CheckOutput(rezip_apk_cmd)
+
+
+def ReorderAndAlignApk(rezip_apk_jar_path, in_zip_file, out_zip_file):
+  rezip_apk_cmd = [
+      'java',
+      '-classpath',
+      rezip_apk_jar_path,
+      'RezipApk',
+      'reorder',
+      in_zip_file,
+      out_zip_file,
+    ]
+  build_utils.CheckOutput(rezip_apk_cmd)
+
 
 def JarSigner(key_path, key_name, key_passwd, unsigned_path, signed_path):
   shutil.copy(unsigned_path, signed_path)
@@ -36,15 +62,14 @@ def JarSigner(key_path, key_name, key_passwd, unsigned_path, signed_path):
   build_utils.CheckOutput(sign_cmd)
 
 
-def AlignApk(zipalign_path, unaligned_path, final_path):
-  # Note -p will page align native libraries (files ending with .so), but
-  # only those that are stored uncompressed.
+def AlignApk(zipalign_path, package_align, unaligned_path, final_path):
   align_cmd = [
       zipalign_path,
-      '-p',
-      '-f',
+      '-f'
       ]
 
+  if package_align:
+    align_cmd += ['-p']
 
   align_cmd += [
       '4',  # 4 bytes
@@ -60,13 +85,23 @@ def main(args):
   parser = optparse.OptionParser()
   build_utils.AddDepfileOption(parser)
 
+  parser.add_option('--rezip-apk-jar-path',
+                    help='Path to the RezipApk jar file.')
   parser.add_option('--zipalign-path', help='Path to the zipalign tool.')
+  parser.add_option('--page-align-shared-libraries',
+                    action='store_true',
+                    help='Page align shared libraries.')
   parser.add_option('--unsigned-apk-path', help='Path to input unsigned APK.')
   parser.add_option('--final-apk-path',
       help='Path to output signed and aligned APK.')
   parser.add_option('--key-path', help='Path to keystore for signing.')
   parser.add_option('--key-passwd', help='Keystore password')
   parser.add_option('--key-name', help='Keystore name')
+  parser.add_option('--stamp', help='Path to touch on success.')
+  parser.add_option('--load-library-from-zip', type='int',
+      help='If non-zero, build the APK such that the library can be loaded ' +
+           'directly from the zip file using the crazy linker. The library ' +
+           'will be renamed, uncompressed and page aligned.')
 
   options, _ = parser.parse_args()
 
@@ -75,9 +110,14 @@ def main(args):
     options.key_path,
   ]
 
+  if options.load_library_from_zip:
+    input_paths.append(options.rezip_apk_jar_path)
+
   input_strings = [
+    options.load_library_from_zip,
     options.key_name,
     options.key_passwd,
+    options.page_align_shared_libraries,
   ]
 
   build_utils.CallAndWriteDepfileIfStale(
@@ -89,34 +129,57 @@ def main(args):
       output_paths=[options.final_apk_path])
 
 
-def _NormalizeZip(path):
-  with tempfile.NamedTemporaryFile(suffix='.zip') as hermetic_signed_apk:
-    with zipfile.ZipFile(path, 'r') as zi:
-      with zipfile.ZipFile(hermetic_signed_apk, 'w') as zo:
-        for info in zi.infolist():
-          # Ignore 'extended local file headers'. Python doesn't write them
-          # properly (see https://bugs.python.org/issue1742205) which causes
-          # zipalign to miscalculate alignment. Since we don't use them except
-          # for alignment anyway, we write a stripped file here and let
-          # zipalign add them properly later. eLFHs are controlled by 'general
-          # purpose bit flag 03' (0x08) so we mask that out.
-          info.flag_bits = info.flag_bits & 0xF7
-
-          info.date_time = build_utils.HERMETIC_TIMESTAMP
-          zo.writestr(info, zi.read(info.filename))
-
-    shutil.copy(hermetic_signed_apk.name, path)
-
-
 def FinalizeApk(options):
-  with tempfile.NamedTemporaryFile() as signed_apk_path_tmp:
+  with tempfile.NamedTemporaryFile() as signed_apk_path_tmp, \
+      tempfile.NamedTemporaryFile() as apk_to_sign_tmp:
+
+    if options.load_library_from_zip:
+      # We alter the name of the library so that the Android Package Manager
+      # does not extract it into a separate file. This must be done before
+      # signing, as the filename is part of the signed manifest. At the same
+      # time we uncompress the library, which is necessary so that it can be
+      # loaded directly from the APK.
+      # Move the library to a page boundary by adding a page alignment file.
+      apk_to_sign = apk_to_sign_tmp.name
+      RenameInflateAndAddPageAlignment(
+          options.rezip_apk_jar_path, options.unsigned_apk_path, apk_to_sign)
+    else:
+      apk_to_sign = options.unsigned_apk_path
+
     signed_apk_path = signed_apk_path_tmp.name
     JarSigner(options.key_path, options.key_name, options.key_passwd,
-              options.unsigned_apk_path, signed_apk_path)
-    # Make the newly added signing files hermetic.
-    _NormalizeZip(signed_apk_path)
-
-    AlignApk(options.zipalign_path, signed_apk_path, options.final_apk_path)
+              apk_to_sign, signed_apk_path)
+
+    # Make the signing files hermetic.
+    with tempfile.NamedTemporaryFile(suffix='.zip') as hermetic_signed_apk:
+      with zipfile.ZipFile(signed_apk_path, 'r') as zi:
+        with zipfile.ZipFile(hermetic_signed_apk, 'w') as zo:
+          for info in zi.infolist():
+            # Ignore 'extended local file headers'. Python doesn't write them
+            # properly (see https://bugs.python.org/issue1742205) which causes
+            # zipalign to miscalculate alignment. Since we don't use them except
+            # for alignment anyway, we write a stripped file here and let
+            # zipalign add them properly later. eLFHs are controlled by 'general
+            # purpose bit flag 03' (0x08) so we mask that out.
+            info.flag_bits = info.flag_bits & 0xF7
+
+            info.date_time = build_utils.HERMETIC_TIMESTAMP
+            zo.writestr(info, zi.read(info.filename))
+
+      shutil.copy(hermetic_signed_apk.name, signed_apk_path)
+
+    if options.load_library_from_zip:
+      # Reorder the contents of the APK. This re-establishes the canonical
+      # order which means the library will be back at its page aligned location.
+      # This step also aligns uncompressed items to 4 bytes.
+      ReorderAndAlignApk(
+          options.rezip_apk_jar_path, signed_apk_path, options.final_apk_path)
+    else:
+      # Align uncompressed items to 4 bytes
+      AlignApk(options.zipalign_path,
+               options.page_align_shared_libraries,
+               signed_apk_path,
+               options.final_apk_path)
 
 
 if __name__ == '__main__':

build/android/rezip/BUILD.gn

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build/config/android/rules.gni")
+
+java_library("rezip") {
+  jar_path = "$root_build_dir/lib.java/rezip_apk.jar"
+  java_files = [ "RezipApk.java" ]
+}

build/config/android/internal_rules.gni

@@ -32,6 +32,7 @@ _java_target_whitelist = [
 
   # TODO(agrieve): Rename targets below to match above patterns.
   "*android_webview/glue:glue",
+  "//build/android/rezip:rezip",
   "//chrome/test/android/cast_emulator:cast_emulator",
 ]
 
@@ -1391,6 +1392,7 @@ if (enable_java_templates) {
   #   keystore_path: Path to keystore to use for signing.
   #   keystore_name: Key alias to use.
   #   keystore_password: Keystore password.
+  #   rezip_apk: Whether to add crazy-linker alignment.
   template("finalize_apk") {
     action(target_name) {
       deps = []
@@ -1433,6 +1435,20 @@ if (enable_java_templates) {
         "--key-passwd",
         invoker.keystore_password,
       ]
+      if (defined(invoker.rezip_apk) && invoker.rezip_apk) {
+        deps += [ "//build/android/rezip" ]
+        _rezip_jar_path = "$root_build_dir/lib.java/rezip_apk.jar"
+        args += [
+          "--load-library-from-zip=1",
+          "--rezip-apk-jar-path",
+          rebase_path(_rezip_jar_path, root_build_dir),
+        ]
+      }
+
+      if (defined(invoker.page_align_shared_libraries) &&
+          invoker.page_align_shared_libraries) {
+        args += [ "--page-align-shared-libraries" ]
+      }
     }
   }
 
@@ -1680,9 +1696,6 @@ if (enable_java_templates) {
                                "uncompress_shared_libraries",
                                "write_asset_list",
                              ])
-      if (!defined(uncompress_shared_libraries)) {
-        uncompress_shared_libraries = _load_library_from_apk
-      }
       deps = _deps + [ ":${_package_resources_target_name}" ]
       native_libs = _native_libs + _native_libs_even_when_incremental
 
@@ -1703,9 +1716,6 @@ if (enable_java_templates) {
                                "secondary_native_libs",
                                "uncompress_shared_libraries",
                              ])
-      if (!defined(uncompress_shared_libraries)) {
-        uncompress_shared_libraries = _load_library_from_apk
-      }
       _dex_target = "//build/android/incremental_install:bootstrap_java__dex"
       deps = _incremental_deps + [
                ":${_incremental_package_resources_target_name}",
@@ -1732,11 +1742,14 @@ if (enable_java_templates) {
 
     _finalize_apk_rule_name = "${target_name}__finalize"
     finalize_apk(_finalize_apk_rule_name) {
+      forward_variables_from(invoker, [ "page_align_shared_libraries" ])
+
       input_apk_path = _packaged_apk_path
       output_apk_path = _final_apk_path
       keystore_path = _keystore_path
       keystore_name = _keystore_name
       keystore_password = _keystore_password
+      rezip_apk = _load_library_from_apk
 
       public_deps = [
         # Generator of the _packaged_apk_path this target takes as input.

build/config/android/rules.gni

@@ -2009,7 +2009,15 @@ if (enable_java_templates) {
     _extra_native_libs_deps = []
     assert(_extra_native_libs_deps == [])  # Mark as used.
     _extra_native_libs_even_when_incremental = []
+    _extra_native_libs_even_when_incremental_deps = []
+    assert(_extra_native_libs_even_when_incremental_deps == [])  # Mark as used.
     if (_native_libs_deps != []) {
+      # zipalign can't align gdb_server, don't pack gdbserver temporarily.
+      if (is_debug && (!defined(invoker.page_align_shared_libraries) ||
+                       !invoker.page_align_shared_libraries)) {
+        _extra_native_libs_even_when_incremental = [ android_gdbserver ]
+      }
+
       if (_use_chromium_linker) {
         _extra_native_libs =
             [ "$root_shlib_dir/libchromium_android_linker$shlib_extension" ]
@@ -2032,6 +2040,7 @@ if (enable_java_templates) {
                                "deps",
                                "extensions_to_not_compress",
                                "language_splits",
+                               "page_align_shared_libraries",
                                "public_deps",
                                "secondary_native_libs",
                                "shared_resources",
@@ -2081,6 +2090,7 @@ if (enable_java_templates) {
            _extra_native_libs_even_when_incremental != []) &&
           !_create_abi_split) {
         deps += _native_libs_deps + _extra_native_libs_deps +
+                _extra_native_libs_even_when_incremental_deps +
                 [ _native_libs_file_arg_dep ]
         native_libs_filearg = _native_libs_file_arg
         native_libs = _extra_native_libs
@@ -2141,7 +2151,9 @@ if (enable_java_templates) {
                                  "public_deps",
                                ])
 
-        incremental_deps = deps + [ ":$_manifest_rule" ]
+        incremental_deps =
+            deps + _extra_native_libs_even_when_incremental_deps +
+            [ ":$_manifest_rule" ]
         deps = []
         deps = incremental_deps + _native_libs_deps + _extra_native_libs_deps +
                [ _native_libs_file_arg_dep ]

chrome/android/chrome_public_apk_tmpl.gni

@@ -111,6 +111,7 @@ template("monochrome_public_apk_tmpl") {
 
     # Configrations to make android load shared library from APK.
     uncompress_shared_libraries = true
+    page_align_shared_libraries = true
 
     forward_variables_from(invoker, "*")