Reland: Android: Zipalign library dex files for incremental install

Also relands follow-up: Remove obsolete flags, code clean-up.

Original reviews:
https://chromium-review.googlesource.com/c/chromium/src/+/1694090
https://chromium-review.googlesource.com/c/chromium/src/+/1692492

Reason for reland:
* Now applies zipfile work-around to necessary scripts

Bug: 981974, 983519
Change-Id: Idce36ea6102302444bd77bebec4e3c1058db3161
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1711817
Commit-Queue: Andrew Grieve <agrieve@chromium.org>
Reviewed-by: Sam Maier <smaier@chromium.org>
Cr-Commit-Position: refs/heads/master@{#679861}

build/android/gyp/apkbuilder.py

@@ -17,6 +17,10 @@ import zipfile
 import finalize_apk
 
 from util import build_utils
+from util import zipalign
+
+# Input dex.jar files are zipaligned.
+zipalign.ApplyZipFileZipAlignFix()
 
 
 # Taken from aapt's Package.cpp:

build/android/gyp/apkbuilder.pydeps

@@ -6,3 +6,4 @@ finalize_apk.py
 util/__init__.py
 util/build_utils.py
 util/md5_check.py
+util/zipalign.py

build/android/gyp/dex.pydeps

@@ -6,3 +6,4 @@ dex.py
 util/__init__.py
 util/build_utils.py
 util/md5_check.py
+util/zipalign.py

build/android/gyp/util/build_utils.py

@@ -34,10 +34,6 @@ DIR_SOURCE_ROOT = os.environ.get('CHECKOUT_SOURCE_ROOT',
     os.path.abspath(os.path.join(os.path.dirname(__file__),
                                  os.pardir, os.pardir, os.pardir, os.pardir)))
 
-HERMETIC_TIMESTAMP = (2001, 1, 1, 0, 0, 0)
-_HERMETIC_FILE_ATTR = (0o644 << 16)
-
-
 try:
   string_types = basestring
 except NameError:
@@ -314,13 +310,24 @@ def ExtractAll(zip_path, path=None, no_clobber=True, pattern=None,
   return extracted
 
 
-def AddToZipHermetic(zip_file, zip_path, src_path=None, data=None,
+def HermeticZipInfo(*args, **kwargs):
+  """Creates a ZipInfo with a constant timestamp and external_attr."""
+  ret = zipfile.ZipInfo(*args, **kwargs)
+  ret.date_time = (2001, 1, 1, 0, 0, 0)
+  ret.external_attr = (0o644 << 16)
+  return ret
+
+
+def AddToZipHermetic(zip_file,
+                     zip_path,
+                     src_path=None,
+                     data=None,
                      compress=None):
   """Adds a file to the given ZipFile with a hard-coded modified time.
 
   Args:
     zip_file: ZipFile instance to add the file to.
-    zip_path: Destination path within the zip file.
+    zip_path: Destination path within the zip file (or ZipInfo instance).
     src_path: Path of the source file. Mutually exclusive with |data|.
     data: File data as a string.
     compress: Whether to enable compression. Default is taken from ZipFile
@@ -328,9 +335,13 @@ def AddToZipHermetic(zip_file, zip_path, src_path=None, data=None,
   """
   assert (src_path is None) != (data is None), (
       '|src_path| and |data| are mutually exclusive.')
+  if isinstance(zip_path, zipfile.ZipInfo):
+    zipinfo = zip_path
+    zip_path = zipinfo.filename
+  else:
+    zipinfo = HermeticZipInfo(filename=zip_path)
+
   _CheckZipPath(zip_path)
-  zipinfo = zipfile.ZipInfo(filename=zip_path, date_time=HERMETIC_TIMESTAMP)
-  zipinfo.external_attr = _HERMETIC_FILE_ATTR
 
   if src_path and os.path.islink(src_path):
     zipinfo.filename = zip_path

build/android/gyp/util/zipalign.py

+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import struct
+import sys
+import zipfile
+
+from util import build_utils
+
+_FIXED_ZIP_HEADER_LEN = 30
+
+
+def _PatchedDecodeExtra(self):
+  # Try to decode the extra field.
+  extra = self.extra
+  unpack = struct.unpack
+  while len(extra) >= 4:
+    tp, ln = unpack('<HH', extra[:4])
+    if tp == 1:
+      if ln >= 24:
+        counts = unpack('<QQQ', extra[4:28])
+      elif ln == 16:
+        counts = unpack('<QQ', extra[4:20])
+      elif ln == 8:
+        counts = unpack('<Q', extra[4:12])
+      elif ln == 0:
+        counts = ()
+      else:
+        raise RuntimeError, "Corrupt extra field %s" % (ln, )
+
+      idx = 0
+
+      # ZIP64 extension (large files and/or large archives)
+      if self.file_size in (0xffffffffffffffffL, 0xffffffffL):
+        self.file_size = counts[idx]
+        idx += 1
+
+      if self.compress_size == 0xFFFFFFFFL:
+        self.compress_size = counts[idx]
+        idx += 1
+
+      if self.header_offset == 0xffffffffL:
+        self.header_offset = counts[idx]
+        idx += 1
+
+    extra = extra[ln + 4:]
+
+
+def ApplyZipFileZipAlignFix():
+  """Fix zipfile.ZipFile() to be able to open zipaligned .zip files.
+
+  Android's zip alignment uses not-quite-valid zip headers to perform alignment.
+  Python < 3.4 crashes when trying to load them.
+  https://bugs.python.org/issue14315
+  """
+  if sys.version_info < (3, 4):
+    zipfile.ZipInfo._decodeExtra = (  # pylint: disable=protected-access
+        _PatchedDecodeExtra)
+
+
+def _SetAlignment(zip_obj, zip_info, alignment):
+  """Sets a ZipInfo's extra field such that the file will be aligned.
+
+  Args:
+    zip_obj: The ZipFile object that is being written.
+    zip_info: The ZipInfo object about to be written.
+    alignment: The amount of alignment (e.g. 4, or 4*1024).
+  """
+  cur_offset = zip_obj.fp.tell()
+  header_size = _FIXED_ZIP_HEADER_LEN + len(zip_info.filename)
+  padding_needed = (header_size - cur_offset) % alignment
+
+  # Extra field used to 4-byte align classes.dex. Alignment speeds up
+  # execution when dex files are used via incremental install.
+  zip_info.extra = b'\0' * padding_needed
+
+
+def AddToZipHermetic(zip_file,
+                     zip_path,
+                     src_path=None,
+                     data=None,
+                     compress=None,
+                     alignment=None):
+  """Same as build_utils.AddToZipHermetic(), but with alignment.
+
+  Args:
+    alignment: If set, align the data of the entry to this many bytes.
+  """
+  zipinfo = build_utils.HermeticZipInfo(filename=zip_path)
+  if alignment:
+    _SetAlignment(zip_file, zipinfo, alignment)
+  build_utils.AddToZipHermetic(
+      zip_file, zipinfo, src_path=src_path, data=data, compress=compress)

build/android/resource_sizes.py

@@ -18,7 +18,6 @@ import logging
 import os
 import posixpath
 import re
-import struct
 import sys
 import tempfile
 import zipfile
@@ -47,50 +46,13 @@ with host_paths.SysPath(host_paths.TRACING_PATH):
 
 with host_paths.SysPath(_BUILD_UTILS_PATH, 0):
   from util import build_utils # pylint: disable=import-error
+  from util import zipalign  # pylint: disable=import-error
 
 with host_paths.SysPath(_APK_PATCH_SIZE_ESTIMATOR_PATH):
   import apk_patch_size_estimator # pylint: disable=import-error
 
 
-# Python had a bug in zipinfo parsing that triggers on ChromeModern.apk
-# https://bugs.python.org/issue14315
-def _PatchedDecodeExtra(self):
-  # Try to decode the extra field.
-  extra = self.extra
-  unpack = struct.unpack
-  while len(extra) >= 4:
-    tp, ln = unpack('<HH', extra[:4])
-    if tp == 1:
-      if ln >= 24:
-        counts = unpack('<QQQ', extra[4:28])
-      elif ln == 16:
-        counts = unpack('<QQ', extra[4:20])
-      elif ln == 8:
-        counts = unpack('<Q', extra[4:12])
-      elif ln == 0:
-        counts = ()
-      else:
-        raise RuntimeError, "Corrupt extra field %s"%(ln,)
-
-      idx = 0
-
-      # ZIP64 extension (large files and/or large archives)
-      if self.file_size in (0xffffffffffffffffL, 0xffffffffL):
-        self.file_size = counts[idx]
-        idx += 1
-
-      if self.compress_size == 0xFFFFFFFFL:
-        self.compress_size = counts[idx]
-        idx += 1
-
-      if self.header_offset == 0xffffffffL:
-        self.header_offset = counts[idx]
-        idx += 1
-
-    extra = extra[ln + 4:]
-
-zipfile.ZipInfo._decodeExtra = (  # pylint: disable=protected-access
-    _PatchedDecodeExtra)
+zipalign.ApplyZipFileZipAlignFix()
 
 # Captures an entire config from aapt output.
 _AAPT_CONFIG_PATTERN = r'config %s:(.*?)config [a-zA-Z-]+:'

build/android/resource_sizes.pydeps

@@ -55,6 +55,7 @@ devil_chromium.py
 gyp/util/__init__.py
 gyp/util/build_utils.py
 gyp/util/md5_check.py
+gyp/util/zipalign.py
 method_count.py
 pylib/__init__.py
 pylib/constants/__init__.py

build/config/android/internal_rules.gni

@@ -1301,13 +1301,11 @@ if (enable_java_templates) {
           invoker.output,
         ]
 
-        _rebased_output = rebase_path(invoker.output, root_build_dir)
-
         args = [
           "--depfile",
           rebase_path(depfile, root_build_dir),
-          "--dex-path",
-          _rebased_output,
+          "--output",
+          rebase_path(outputs[0], root_build_dir),
         ]
 
         if (_proguard_enabled) {
@@ -1328,7 +1326,7 @@ if (enable_java_templates) {
 
         if (defined(invoker.input_dex_classpath)) {
           inputs += [ invoker.build_config ]
-          args += [ "--inputs=@FileArg(${invoker.input_dex_classpath})" ]
+          args += [ "--input-list=@FileArg(${invoker.input_dex_classpath})" ]
         }
 
         inputs += _dexing_jars