Build: dexlayout build rules

This adds dexlayout to the build system. The dexlayout binary is from a
host build of ART (the Android RunTime) and will be populated by
.DEPS.

Bug: 875276
Change-Id: I1931bb2effe893b9e7962777b5e776e6e2490e8b
Reviewed-on: https://chromium-review.googlesource.com/1199583Reviewed-by: agrieve <agrieve@chromium.org>
Commit-Queue: Matthew Cary <mattcary@chromium.org>
Cr-Commit-Position: refs/heads/master@{#594658}

build/android/gyp/dex.py

@@ -8,8 +8,10 @@ import json
 import logging
 import optparse
 import os
+import re
 import shutil
 import sys
+import tempfile
 import zipfile
 
 from util import build_utils
@@ -57,11 +59,28 @@ def _ParseArgs(args):
   parser.add_option('--d8-jar-path',
                     help='Path to D8 jar.')
 
+  parser.add_option('--dexlayout-profile',
+                    help=('Text profile for dexlayout. If present, a dexlayout '
+                          'pass will happen'))
+  parser.add_option('--profman-path',
+                    help=('Path to ART profman binary. There should be a '
+                          'lib/ directory at the same path containing shared '
+                          'libraries (shared with dexlayout).'))
+  parser.add_option('--dexlayout-path',
+                    help=('Path to ART dexlayout binary. There should be a '
+                          'lib/ directory at the same path containing shared '
+                          'libraries (shared with dexlayout).'))
+
+
   options, paths = parser.parse_args(args)
 
   required_options = ('d8_jar_path',)
   build_utils.CheckOptions(options, parser, required=required_options)
 
+  if options.dexlayout_profile:
+    build_utils.CheckOptions(options, parser, required=('profman_path',
+                                                        'dexlayout_path'))
+
   if options.multidex_configuration_path:
     with open(options.multidex_configuration_path) as multidex_config_file:
       multidex_config = json.loads(multidex_config_file.read())
@@ -127,6 +146,153 @@ def _RunD8(dex_cmd, input_paths, output_path):
   build_utils.CheckOutput(dex_cmd, print_stderr=False)
 
 
+def _EnvWithArtLibPath(binary_path):
+  """Return an environment dictionary for ART host shared libraries.
+
+  Args:
+    binary_path: the path to an ART host binary.
+
+  Returns:
+    An environment dictionary where LD_LIBRARY_PATH has been augmented with the
+    shared library path for the binary. This assumes that there is a lib/
+    directory in the same location as the binary.
+  """
+  lib_path = os.path.join(os.path.dirname(binary_path), 'lib')
+  env = os.environ.copy()
+  libraries = [l for l in env.get('LD_LIBRARY_PATH', '').split(':') if l]
+  libraries.append(lib_path)
+  env['LD_LIBRARY_PATH'] = ':'.join(libraries)
+  return env
+
+
+def _FilterOutput(output, filter_strings):
+  """Output filter from build_utils.CheckOutput.
+
+  Args:
+    output: Executable output as from build_utils.CheckOutput.
+    filter_strings: List of RE strings that will filter (remove) matching
+        lines from |output|.
+
+  Returns:
+    The filtered output, as a single string.
+  """
+  filters = [re.compile(f) for f in filter_strings]
+  filtered_output = []
+  for line in output.splitlines():
+    if any(filter.search(line) for filter in filters):
+      continue
+    else:
+      filtered_output.append(line)
+  return '\n'.join(filtered_output)
+
+
+def _FilterProfmanStderr(output):
+  return _FilterOutput(output, [
+      r'Could not find (method_id|proto_id|name):',
+      r'Could not create type list',
+  ])
+
+
+def _FilterDexlayoutStderr(output):
+  return _FilterOutput(output, [
+      r'Can.t mmap dex file.*please zipalign',
+  ])
+
+
+def _CreateBinaryProfile(text_profile, input_dex, profman_path, temp_dir):
+  """Create a binary profile for dexlayout.
+
+  Args:
+    text_profile: The ART text profile that will be converted to a binary
+        profile.
+    input_dex: The input dex file to layout.
+    profman_path: Path to the profman binary.
+    temp_dir: Directory to work in.
+
+  Returns:
+    The name of the binary profile, which will live in temp_dir.
+  """
+  binary_profile = os.path.join(
+      temp_dir, 'binary_profile-for-' + os.path.basename(text_profile))
+  open(binary_profile, 'w').close()  # Touch binary_profile.
+  profman_cmd = [profman_path,
+                 '--apk=' + input_dex,
+                 '--dex-location=' + input_dex,
+                 '--create-profile-from=' + text_profile,
+                 '--reference-profile-file=' + binary_profile]
+  build_utils.CheckOutput(profman_cmd, env=_EnvWithArtLibPath(profman_path),
+                          stderr_filter=_FilterProfmanStderr)
+  return binary_profile
+
+
+def _LayoutDex(binary_profile, input_dex, dexlayout_path, temp_dir):
+  """Layout a dexfile using a profile.
+
+  Args:
+    binary_profile: An ART binary profile, eg output from _CreateBinaryProfile.
+    input_dex: The dex file used to create the binary profile.
+    dexlayout_path: Path to the dexlayout binary.
+    temp_dir: Directory to work in.
+
+  Returns:
+    List of output files produced by dexlayout. This will be one if the input
+    was a single dexfile, or multiple files if the input was a multidex
+    zip. These output files are located in temp_dir.
+  """
+  dexlayout_output_dir = os.path.join(temp_dir, 'dexlayout_output')
+  os.mkdir(dexlayout_output_dir)
+  dexlayout_cmd = [ dexlayout_path,
+                    '-u',  # Update checksum
+                    '-p', binary_profile,
+                    '-w', dexlayout_output_dir,
+                    input_dex ]
+  build_utils.CheckOutput(dexlayout_cmd, env=_EnvWithArtLibPath(dexlayout_path),
+                          stderr_filter=_FilterDexlayoutStderr)
+  output_files = os.listdir(dexlayout_output_dir)
+  if not output_files:
+    raise Exception('dexlayout unexpectedly produced no output')
+  return [os.path.join(dexlayout_output_dir, f) for f in output_files]
+
+
+def _ZipMultidex(file_dir, dex_files):
+  """Zip dex files into a multidex.
+
+  Args:
+    file_dir: The directory into which to write the output.
+    dex_files: The dexfiles forming the multizip. Their names must end with
+      classes.dex, classes2.dex, ...
+
+  Returns:
+    The name of the multidex file, which will live in file_dir.
+  """
+  ordered_files = []  # List of (archive name, file name)
+  for f in dex_files:
+    if f.endswith('classes.dex.zip'):
+      ordered_files.append(('classes.dex', f))
+      break
+  if not ordered_files:
+    raise Exception('Could not find classes.dex multidex file in %s',
+                    dex_files)
+  for dex_idx in xrange(2, len(dex_files) + 1):
+    archive_name = 'classes%d.dex' % dex_idx
+    for f in dex_files:
+      if f.endswith(archive_name):
+        ordered_files.append((archive_name, f))
+        break
+    else:
+      raise Exception('Could not find classes%d.dex multidex file in %s',
+                      dex_files)
+  if len(set(f[1] for f in ordered_files)) != len(ordered_files):
+    raise Exception('Unexpected clashing filenames for multidex in %s',
+                    dex_files)
+
+  zip_name = os.path.join(file_dir, 'multidex_classes.zip')
+  build_utils.DoZip(((archive_name, os.path.join(file_dir, file_name))
+                     for archive_name, file_name in ordered_files),
+                    zip_name)
+  return zip_name
+
+
 def main(args):
   options, paths = _ParseArgs(args)
   if ((options.proguard_enabled == 'true'
@@ -170,6 +336,21 @@ def main(args):
   else:
     _RunD8(dex_cmd, paths, options.dex_path)
 
+  if options.dexlayout_profile:
+    with build_utils.TempDir() as temp_dir:
+      binary_profile = _CreateBinaryProfile(options.dexlayout_profile,
+                                            options.dex_path,
+                                            options.profman_path, temp_dir)
+      output_files = _LayoutDex(binary_profile, options.dex_path,
+                                options.dexlayout_path, temp_dir)
+      target = None
+      if len(output_files) > 1:
+        target = _ZipMultidex(temp_dir, output_files)
+      else:
+        target = output_files[0]
+      shutil.move(os.path.join(temp_dir, target), options.dex_path)
+
+
   build_utils.WriteDepfile(
       options.depfile, options.dex_path, input_paths, add_pydeps=False)
 

build/config/android/internal_rules.gni

@@ -45,6 +45,64 @@ _java_target_blacklist = [
 
 _default_proguard_jar_path = "//third_party/proguard/lib/proguard.jar"
 
+_dexlayout_path = "//third_party/android_build_tools/art/dexlayout"
+_profman_path = "//third_party/android_build_tools/art/profman"
+_art_lib_file_names = [
+  "libartbased.so",
+  "libartbase.so",
+  "libart-compiler.so",
+  "libartd-compiler.so",
+  "libartd-dexlayout.so",
+  "libartd-disassembler.so",
+  "libart-dexlayout.so",
+  "libart-disassembler.so",
+  "libartd-simulator-container.so",
+  "libartd-simulator.so",
+  "libartd.so",
+  "libart-gtest.so",
+  "libart.so",
+  "libbacktrace.so",
+  "libbase.so",
+  "libcrypto-host.so",
+  "libc++.so",
+  "libcutils.so",
+  "libdexfiled.so",
+  "libdexfile.so",
+  "libexpat-host.so",
+  "libicui18n-host.so",
+  "libicuuc-host.so",
+  "libjavacore.so",
+  "libjavacrypto.so",
+  "liblog.so",
+  "liblz4.so",
+  "liblzma.so",
+  "libnativebridge.so",
+  "libnativehelper.so",
+  "libnativeloader.so",
+  "libopenjdkd.so",
+  "libopenjdkjvmd.so",
+  "libopenjdkjvm.so",
+  "libopenjdkjvmtid.so",
+  "libopenjdkjvmti.so",
+  "libopenjdk.so",
+  "libprofiled.so",
+  "libprofile.so",
+  "libsigchain.so",
+  "libssl-host.so",
+  "libunwindstack.so",
+  "libvixl-arm64.so",
+  "libvixl-arm.so",
+  "libvixld-arm64.so",
+  "libvixld-arm.so",
+  "libz-host.so",
+  "libziparchive.so",
+  "slicer.so",
+]
+_default_art_libs = []
+foreach(lib, _art_lib_file_names) {
+  _default_art_libs += [ "//third_party/android_build_tools/art/lib/$lib" ]
+}
+
 # Write the target's .build_config file. This is a json file that contains a
 # dictionary of information about how to build this target (things that
 # require knowledge about this target's dependencies and cannot be calculated
@@ -1119,6 +1177,23 @@ if (enable_java_templates) {
         _rebased_output,
       ]
 
+      if (defined(invoker.dexlayout_profile)) {
+        args += [
+          "--dexlayout_profile",
+          rebase_path(invoker.dexlayout_profile, root_build_dir),
+          "--dexlayout_path",
+          rebase_path(_dexlayout_path, root_build_dir),
+          "--profman_path",
+          rebase_path(_profman_path, root_build_dir),
+        ]
+        inputs += [
+          _dexlayout_path,
+          _profman_path,
+          invoker.dexlayout_profile,
+        ]
+        inputs += _default_art_libs
+      }
+
       if (_enable_multidex) {
         args += [
           "--multi-dex",

build/config/android/rules.gni

@@ -2511,6 +2511,8 @@ if (enable_java_templates) {
           deps += [ ":$_compile_resources_target" ]
         }
 
+        forward_variables_from(invoker, [ "dexlayout_profile" ])
+
         # All deps are already included in _dex_sources when proguard is used.
         if (!_proguard_enabled) {
           if (_enable_multidex) {
@@ -2871,6 +2873,7 @@ if (enable_java_templates) {
                                "data",
                                "data_deps",
                                "deps",
+                               "dexlayout_profile",
                                "dist_ijar_path",
                                "dont_load_shared_libraries",
                                "emma_never_instrument",