android: Do not generate .apk.jar.info files in javac.py

When compiling Java sources into .jar files, the javac.py
script will also generate a .jar.info file, used to map
fully-qualified Java class names to the path of the
corresponding source or .srcjar file.

These are stored along the generated .jar file, e.g.:

  $OUT/gen/base/base_java.javac.jar
  $OUT/gen/base/base_java.javac.jar.info

In addition to this, each APK has a final .apk.jar.info
file stored under $OUT_DIR/size-info/ that corresponds to the
union of all the .jar.info files of its dependencies, e.g.:

  $OUT/size-info/ChromePublic.apk.jar.info

Before this CL, the .apk.jar.info was generated by javac.py
when compiling the APK's own source files (which include things
like the auto-generated resource-related .srcjar files), by
passing the --apk-jar-info option to the script.

Unfortunately, this made the implementation of the android_apk(),
java_library_impl() and compile_java() GN templates slightly more
complex (tightly coupled) than needed, making future changes more
difficult.

This CL tries to untangle the coupling as follows:

- Add a new script (merge_jar_info_files.py) to merge one or
  more .jar.info file into a final .apk.jar.info.

- Remove --apk-jar-info option from javac.py. The latter is now
  only responsible for generating the .jar.info of its final
  .jar file, and doesn't need to know anything about that may
  or may not embed it later.

  Similarly, remove the 'apk_name' variable from compile_java()
  and java_library_impl().

- Ensure that the android_apk() generates the final .apk.jar.info
  file by calling merge_jar_info_files.py.

+ Move common functionality to build/android/gyp/util/jar_info_utils.py

To test the result, I manually checked that all .apk.jar.info files
are identical after sorting (since they are a dump of a Python dictionary,
with no guaranteed ordering:

BUG=830641
R=agrieve@chromium.org, yfriedman@chromium.org

Change-Id: I2051bc35aba2931947d2a30db507a4d437998238
Reviewed-on: https://chromium-review.googlesource.com/1047269
Commit-Queue: David Turner <digit@chromium.org>
Reviewed-by: Peter Wen <wnwen@chromium.org>
Reviewed-by: agrieve <agrieve@chromium.org>
Cr-Commit-Position: refs/heads/master@{#556770}

build/android/gyp/javac.py

@@ -14,6 +14,7 @@ import zipfile
 
 from util import build_utils
 from util import md5_check
+from util import jar_info_utils
 
 import jar
 
@@ -232,44 +233,6 @@ def _CheckPathMatchesClassName(java_file, package_name, class_name):
                     (java_file, expected_path_suffix))
 
 
-def _ParseInfoFile(info_path):
-  info_data = dict()
-  if os.path.exists(info_path):
-    with open(info_path, 'r') as info_file:
-      for line in info_file:
-        line = line.strip()
-        if line:
-          fully_qualified_name, path = line.split(',', 1)
-          info_data[fully_qualified_name] = path
-  return info_data
-
-
-def _WriteInfoFile(info_path, info_data, srcjar_files):
-  with open(info_path, 'w') as info_file:
-    for fully_qualified_name, path in info_data.iteritems():
-      if path in srcjar_files:
-        path = srcjar_files[path]
-      assert not path.startswith('/tmp'), (
-          'Java file path should not be in temp dir: {}'.format(path))
-      info_file.write('{},{}\n'.format(fully_qualified_name, path))
-
-
-def _FullJavaNameFromClassFilePath(path):
-  # Input:  base/android/java/src/org/chromium/Foo.class
-  # Output: base.android.java.src.org.chromium.Foo
-  if not path.endswith('.class'):
-    return ''
-  path = os.path.splitext(path)[0]
-  parts = []
-  while path:
-    # Use split to be platform independent.
-    head, tail = os.path.split(path)
-    path = head
-    parts.append(tail)
-  parts.reverse()  # Package comes first
-  return '.'.join(parts)
-
-
 def _CreateInfoFile(java_files, options, srcjar_files):
   """Writes a .jar.info file.
 
@@ -293,27 +256,8 @@ def _CreateInfoFile(java_files, options, srcjar_files):
         'Chromium java files must only have one class: {}'.format(source))
     if options.chromium_code:
       _CheckPathMatchesClassName(java_file, package_name, class_names[0])
-  _WriteInfoFile(options.jar_path + '.info', info_data, srcjar_files)
-
-  # Collect all the info files for transitive dependencies of the apk.
-  if options.apk_jar_info_path:
-    for jar_path in options.full_classpath:
-      # android_java_prebuilt adds jar files in the src directory (relative to
-      #     the output directory, usually ../../third_party/example.jar).
-      # android_aar_prebuilt collects jar files in the aar file and uses the
-      #     java_prebuilt rule to generate gen/example/classes.jar files.
-      # We scan these prebuilt jars to parse each class path for the FQN. This
-      #     allows us to later map these classes back to their respective src
-      #     directories.
-      if jar_path.startswith('..') or jar_path.endswith('classes.jar'):
-        with zipfile.ZipFile(jar_path) as zip_info:
-          for path in zip_info.namelist():
-            fully_qualified_name = _FullJavaNameFromClassFilePath(path)
-            if fully_qualified_name:
-              info_data[fully_qualified_name] = jar_path
-      else:
-        info_data.update(_ParseInfoFile(jar_path + '.info'))
-    _WriteInfoFile(options.apk_jar_info_path, info_data, srcjar_files)
+  jar_info_utils.WriteJarInfoFile(options.jar_path + '.info', info_data,
+                                  srcjar_files)
 
 
 def _OnStaleMd5(changes, options, javac_cmd, java_files, classpath_inputs,
@@ -519,9 +463,6 @@ def _ParseOptions(argv):
       action='append',
       default=[],
       help='Additional arguments to pass to javac.')
-  parser.add_option(
-      '--apk-jar-info-path',
-      help='Coalesced jar.info files for the apk')
 
   options, args = parser.parse_args(argv)
   build_utils.CheckOptions(options, parser, required=('jar_path',))
@@ -639,8 +580,6 @@ def main(argv):
   ]
   if options.incremental:
     output_paths.append(options.jar_path + '.pdb')
-  if options.apk_jar_info_path:
-    output_paths.append(options.apk_jar_info_path)
 
   # An escape hatch to be able to check if incremental compiles are causing
   # problems.

build/android/gyp/merge_jar_info_files.py

+#!/usr/bin/env python
+
+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Merges .jar.info files into one for APKs."""
+
+import argparse
+import os
+import shutil
+import sys
+import tempfile
+import zipfile
+
+from util import build_utils
+from util import jar_info_utils
+
+
+def _FullJavaNameFromClassFilePath(path):
+  # Input:  base/android/java/src/org/chromium/Foo.class
+  # Output: base.android.java.src.org.chromium.Foo
+  if not path.endswith('.class'):
+    return ''
+  path = os.path.splitext(path)[0]
+  parts = []
+  while path:
+    # Use split to be platform independent.
+    head, tail = os.path.split(path)
+    path = head
+    parts.append(tail)
+  parts.reverse()  # Package comes first
+  return '.'.join(parts)
+
+
+def _MergeInfoFiles(output, jar_paths):
+  """Merge several .jar.info files to generate an .apk.jar.info.
+
+  Args:
+    output: output file path.
+    jar_paths: List of .jar file paths for the target apk.
+  """
+  info_data = dict()
+  for jar_path in jar_paths:
+    # android_java_prebuilt adds jar files in the src directory (relative to
+    #     the output directory, usually ../../third_party/example.jar).
+    # android_aar_prebuilt collects jar files in the aar file and uses the
+    #     java_prebuilt rule to generate gen/example/classes.jar files.
+    # We scan these prebuilt jars to parse each class path for the FQN. This
+    #     allows us to later map these classes back to their respective src
+    #     directories.
+    jar_info_path = jar_path + '.info'
+    if os.path.exists(jar_info_path):
+      info_data.update(jar_info_utils.ParseJarInfoFile(jar_path + '.info'))
+    else:
+      with zipfile.ZipFile(jar_path) as zip_info:
+        for path in zip_info.namelist():
+          fully_qualified_name = _FullJavaNameFromClassFilePath(path)
+          if fully_qualified_name:
+            info_data[fully_qualified_name] = jar_path
+
+  jar_info_utils.WriteJarInfoFile(output, info_data)
+
+
+def main(args):
+  args = build_utils.ExpandFileArgs(args)
+  parser = argparse.ArgumentParser(description=__doc__)
+  build_utils.AddDepfileOption(parser)
+  parser.add_argument('--output', required=True,
+                      help='Output .apk.jar.info file')
+  parser.add_argument('--apk-jar-file', required=True,
+                      help='Path to main .jar file for this APK.')
+  parser.add_argument('--dep-jar-files', required=True,
+                      help='GN-list of dependent .jar file paths')
+
+  options = parser.parse_args(args)
+  options.dep_jar_files = build_utils.ParseGnList(options.dep_jar_files)
+  jar_files = [ options.apk_jar_file ] + options.dep_jar_files
+
+  def _OnStaleMd5():
+    with tempfile.NamedTemporaryFile() as tmp_file:
+      _MergeInfoFiles(tmp_file.name, jar_files)
+      shutil.move(tmp_file.name, options.output)
+      tmp_file.delete = False
+
+  build_utils.CallAndWriteDepfileIfStale(
+      _OnStaleMd5, options,
+      input_paths=jar_files,
+      output_paths=options.output)
+
+
+if __name__ == '__main__':
+  main(sys.argv[1:])

build/android/gyp/util/jar_info_utils.py

+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+
+# Utilities to read and write .jar.info files.
+#
+# A .jar.info file contains a simple mapping from fully-qualified Java class
+# names to the source file that actually defines it.
+#
+# For APKs, the .jar.info maps the class names to the .jar file that which
+# contains its .class definition instead.
+
+
+def ParseJarInfoFile(info_path):
+  """Parse a given .jar.info file as a dictionary.
+
+  Args:
+    info_path: input .jar.info file path.
+  Returns:
+    A new dictionary mapping fully-qualified Java class names to file paths.
+  """
+  info_data = dict()
+  if os.path.exists(info_path):
+    with open(info_path, 'r') as info_file:
+      for line in info_file:
+        line = line.strip()
+        if line:
+          fully_qualified_name, path = line.split(',', 1)
+          info_data[fully_qualified_name] = path
+  return info_data
+
+
+def WriteJarInfoFile(info_path, info_data, source_file_map=None):
+  """Generate a .jar.info file from a given dictionary.
+
+  Args:
+    info_path: output file path.
+    info_data: a mapping of fully qualified Java class names to filepaths.
+    source_file_map: an optional mapping from java source file paths to the
+      corresponding source .srcjar. This is because info_data may contain the
+      path of Java source files that where extracted from an .srcjar into a
+      temporary location.
+  """
+  with open(info_path, 'w') as info_file:
+    for fully_qualified_name, path in info_data.iteritems():
+      if source_file_map and path in source_file_map:
+        path = source_file_map[path]
+        assert not path.startswith('/tmp'), (
+            'Java file path should not be in temp dir: {}'.format(path))
+      info_file.write('{},{}\n'.format(fully_qualified_name, path))

build/config/android/internal_rules.gni

@@ -1976,6 +1976,43 @@ if (enable_java_templates) {
     }
   }
 
+  # Create an apk.jar.info file by merging several .jar.info files into one.
+  #
+  # Variables:
+  #   apk_build_config: Path to APK's build config file. Used to extract the
+  #       list of input .jar files from its dependencies.
+  #   output: Output file path.
+  #
+  template("create_apk_jar_info") {
+    _output = invoker.output
+    _build_config = invoker.apk_build_config
+    _rebased_build_config = rebase_path(_build_config, root_build_dir)
+    action(target_name) {
+      forward_variables_from(invoker,
+                             [
+                               "testonly",
+                               "deps",
+                             ])
+      script = "//build/android/gyp/merge_jar_info_files.py"
+      inputs = [
+        _build_config,
+      ]
+      outputs = [
+        _output,
+      ]
+      depfile = "$target_gen_dir/$target_name.d"
+      args = [
+        "--depfile",
+        rebase_path(depfile, root_build_dir),
+        "--output",
+        rebase_path(_output, root_build_dir),
+        "--apk-jar-file=@FileArg($_rebased_build_config:deps_info:jar_path)",
+        "--dep-jar-files=@FileArg(" +
+            "$_rebased_build_config:deps_info:javac_full_classpath)",
+      ]
+    }
+  }
+
   # Creates an unsigned .apk.
   #
   # Variables
@@ -2525,15 +2562,6 @@ if (enable_java_templates) {
           "--processorpath=$_rebased_errorprone_processorpath",
         ]
       }
-      if (defined(invoker.apk_name)) {
-        # The supersize tool will search in this directory for each apk.
-        _apk_jar_info_path = "size-info/${invoker.apk_name}.apk.jar.info"
-        args += [
-          "--apk-jar-info-path",
-          _apk_jar_info_path,
-        ]
-        outputs += [ "$root_build_dir/$_apk_jar_info_path" ]
-      }
       foreach(e, _provider_configurations) {
         args += [ "--provider-configuration=" + rebase_path(e, root_build_dir) ]
       }

build/config/android/rules.gni

@@ -2213,7 +2213,6 @@ if (enable_java_templates) {
     java_library_impl(_java_target) {
       forward_variables_from(invoker,
                              [
-                               "apk_name",
                                "android_manifest",
                                "android_manifest_dep",
                                "apk_under_test",
@@ -2448,6 +2447,20 @@ if (enable_java_templates) {
       _extra_native_libs_even_when_incremental += invoker.loadable_modules
     }
 
+    # Generate .apk.jar.info files if needed.
+    if (defined(invoker.apk_name)) {
+      _apk_jar_info_target = "${target_name}__apk_jar_info"
+      create_apk_jar_info(_apk_jar_info_target) {
+        output = "$root_build_dir/size-info/${invoker.apk_name}.apk.jar.info"
+        apk_build_config = _build_config
+        deps = [
+          ":$_build_config_target",
+          ":$_java_target",
+        ]
+      }
+      _deps += [ ":$_apk_jar_info_target" ]
+    }
+
     _final_deps += [ ":${_template_name}__create" ]
     create_apk("${_template_name}__create") {
       forward_variables_from(invoker,