Android: Use turbine to generate java headers (Reland)

See go/faster-javac for a detailed analysis of why turbine was chosen to
speed up our java builds. The current implementation is an initial
implementation that can be further optimized. It already gives savings
of 10-30% of our java incremental builds depending on the target in
question (see go/project-turbo for timing details).

This implementation does remove some of the benefits of the previous way
that java_library_impl was implemented, namely when a target's
.interface.jar file changes, it no longer short-circuits dependent
targets from being built since we now use group(target_name). This is
arguably more correct as prebuilts depend on javac jars directly. In the
future (after the androidx migration and other build improvements) this
can be improved by having prebuilts depend on the interface jars
instead.

Original CL: https://crrev.com/c/2105415

Fixed jacoco instrument not having header deps.

Bug: 1056299, 1062704
Change-Id: I9a6c3242a04c6c57bb3c8f5bb7542c5a4301d1b5
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2110431Reviewed-by: Peter Wen <wnwen@chromium.org>
Reviewed-by: Andrew Grieve <agrieve@chromium.org>
Commit-Queue: Peter Wen <wnwen@chromium.org>
Auto-Submit: Peter Wen <wnwen@chromium.org>
Cr-Commit-Position: refs/heads/master@{#751777}

PRESUBMIT.py

@@ -1337,6 +1337,7 @@ _ANDROID_SPECIFIC_PYDEPS_FILES = [
     'build/android/gyp/merge_manifest.pydeps',
     'build/android/gyp/prepare_resources.pydeps',
     'build/android/gyp/proguard.pydeps',
+    'build/android/gyp/turbine.pydeps',
     'build/android/gyp/validate_static_library_dex_references.pydeps',
     'build/android/gyp/write_build_config.pydeps',
     'build/android/gyp/write_native_libraries_java.pydeps',

build/android/gyp/turbine.py

+#!/usr/bin/env python
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Wraps bin/helper/turbine and expands @FileArgs."""
+
+import argparse
+import logging
+import os
+import shutil
+import subprocess
+import sys
+import time
+
+from util import build_utils
+from util import md5_check
+
+
+def _OnStaleMd5(options, cmd, javac_cmd, files, classpath):
+  if classpath:
+    cmd += ['--classpath']
+    cmd += classpath
+
+  if options.java_srcjars:
+    cmd += ['--source_jars']
+    cmd += options.java_srcjars
+
+  if files:
+    # Use jar_path to ensure paths are relative (needed for goma).
+    files_rsp_path = options.jar_path + '.files_list.txt'
+    with open(files_rsp_path, 'w') as f:
+      f.write(' '.join(files))
+    # Pass source paths as response files to avoid extremely long command lines
+    # that are tedius to debug.
+    cmd += ['--sources']
+    cmd += ['@' + files_rsp_path]
+
+  if javac_cmd:
+    cmd.append('--javacopts')
+    cmd += javac_cmd
+    cmd.append('--')  # Terminate javacopts
+
+  # Use AtomicOutput so that output timestamps are not updated when outputs
+  # are not changed.
+  with build_utils.AtomicOutput(options.jar_path) as f:
+    cmd += ['--output', f.name]
+    logging.info('Command: %s' % ' '.join(cmd))
+    start = time.time()
+    subprocess.check_call(cmd)
+    end = time.time() - start
+    logging.info('Header compilation took %ss', end)
+
+  logging.info('Completed all steps in _OnStaleMd5')
+
+
+def main(argv):
+  build_utils.InitLogging('TURBINE_DEBUG')
+  argv = build_utils.ExpandFileArgs(argv[1:])
+  parser = argparse.ArgumentParser()
+  build_utils.AddDepfileOption(parser)
+  parser.add_argument(
+      '--turbine-jar-path', required=True, help='Path to the turbine jar file.')
+  parser.add_argument(
+      '--java-srcjars',
+      action='append',
+      default=[],
+      help='List of srcjars to include in compilation.')
+  parser.add_argument(
+      '--bootclasspath',
+      action='append',
+      default=[],
+      help='Boot classpath for javac. If this is specified multiple times, '
+      'they will all be appended to construct the classpath.')
+  parser.add_argument(
+      '--java-version',
+      help='Java language version to use in -source and -target args to javac.')
+  parser.add_argument('--classpath', action='append', help='Classpath to use.')
+  parser.add_argument(
+      '--processors',
+      action='append',
+      help='GN list of annotation processor main classes.')
+  parser.add_argument(
+      '--processorpath',
+      action='append',
+      help='GN list of jars that comprise the classpath used for Annotation '
+      'Processors.')
+  parser.add_argument(
+      '--processor-args',
+      action='append',
+      help='key=value arguments for the annotation processors.')
+  parser.add_argument('--jar-path', help='Jar output path.', required=True)
+  options, unknown_args = parser.parse_known_args(argv)
+
+  options.bootclasspath = build_utils.ParseGnList(options.bootclasspath)
+  options.classpath = build_utils.ParseGnList(options.classpath)
+  options.processorpath = build_utils.ParseGnList(options.processorpath)
+  options.processors = build_utils.ParseGnList(options.processors)
+  options.java_srcjars = build_utils.ParseGnList(options.java_srcjars)
+
+  files = []
+  for arg in unknown_args:
+    # Interpret a path prefixed with @ as a file containing a list of sources.
+    if arg.startswith('@'):
+      files.extend(build_utils.ReadSourcesList(arg[1:]))
+
+  cmd = [
+      build_utils.JAVA_PATH, '-classpath', options.turbine_jar_path,
+      'com.google.turbine.main.Main'
+  ]
+  javac_cmd = []
+
+  # Turbine reads lists from command line args by consuming args until one
+  # starts with double dash (--). Thus command line args should be grouped
+  # together and passed in together.
+  if options.processors:
+    cmd += ['--processors']
+    cmd += options.processors
+
+  if options.java_version:
+    javac_cmd.extend([
+        '-source',
+        options.java_version,
+        '-target',
+        options.java_version,
+    ])
+  if options.java_version == '1.8':
+    # Android's boot jar doesn't contain all java 8 classes.
+    options.bootclasspath.append(build_utils.RT_JAR_PATH)
+
+  if options.bootclasspath:
+    cmd += ['--bootclasspath']
+    for bootclasspath in options.bootclasspath:
+      cmd += bootclasspath.split(':')
+
+  if options.processorpath:
+    cmd += ['--processorpath']
+    cmd += options.processorpath
+
+  if options.processor_args:
+    for arg in options.processor_args:
+      javac_cmd.extend(['-A%s' % arg])
+
+  classpath_inputs = (
+      options.bootclasspath + options.classpath + options.processorpath)
+
+  # GN already knows of the java files, so avoid listing individual java files
+  # in the depfile.
+  depfile_deps = classpath_inputs + options.java_srcjars
+  input_paths = depfile_deps + files
+
+  output_paths = [
+      options.jar_path,
+  ]
+
+  input_strings = cmd + options.classpath + files
+
+  md5_check.CallAndWriteDepfileIfStale(
+      lambda: _OnStaleMd5(options, cmd, javac_cmd, files, options.classpath),
+      options,
+      depfile_deps=depfile_deps,
+      input_paths=input_paths,
+      input_strings=input_strings,
+      output_paths=output_paths)
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))

build/android/gyp/turbine.pydeps

+# Generated by running:
+#   build/print_python_deps.py --root build/android/gyp --output build/android/gyp/turbine.pydeps build/android/gyp/turbine.py
+../../gn_helpers.py
+turbine.py
+util/__init__.py
+util/build_utils.py
+util/md5_check.py

build/android/gyp/write_build_config.py

@@ -248,8 +248,8 @@ Path to a file that is the result of processing
 
 * `deps_info['interface_jar_path']:
 Path to the interface jar generated for this library. This corresponds to
-a jar file that only contains declarations. Generated by running the `ijar`
-tool on `deps_info['jar_path']`
+a jar file that only contains declarations. Generated by running the `ijar` on
+`deps_info['jar_path']` or the `turbine` tool on source files.
 
 * `deps_info['dex_path']`:
 Path to the `.dex` file generated for this target, from `deps_info['jar_path']`

build/config/android/rules.gni

@@ -1130,6 +1130,9 @@ if (enable_java_templates) {
         possible_config_deps = invoker.deps
       }
     }
+    java_header_group("${target_name}__header") {
+      forward_variables_from(invoker, [ "deps" ])
+    }
     group(target_name) {
       forward_variables_from(invoker, "*")
       if (!defined(deps)) {

chrome/android/third_party/compositor_animator/BUILD.gn

@@ -6,5 +6,11 @@
 import("//build/config/android/rules.gni")
 
 android_library("compositor_animator_java") {
+  # Turbine does not generate bridge methods in the parent class. Thus each
+  # child class generates its own bridge methods and adds a number of methods.
+  # This target would add about 30 methods if it enabled turbine. We are
+  # hoping to turn this back on once r8 can mitigate these extra methods.
+  enable_turbine = false
+
   sources = [ "java/src/org/chromium/chrome/browser/compositor/animation/FloatProperty.java" ]
 }

third_party/robolectric/BUILD.gn

@@ -413,7 +413,6 @@ java_library("robolectric_utils_java") {
     "robolectric/utils/reflector/src/main/java/org/robolectric/util/reflector/WeakerHashMap.java",
     "robolectric/utils/reflector/src/main/java/org/robolectric/util/reflector/WithType.java",
     "robolectric/utils/src/main/java/org/robolectric/AndroidMetadata.java",
-    "robolectric/utils/src/main/java/org/robolectric/AndroidMetadata.java",
     "robolectric/utils/src/main/java/org/robolectric/util/Clock.java",
     "robolectric/utils/src/main/java/org/robolectric/util/Consumer.java",
     "robolectric/utils/src/main/java/org/robolectric/util/Join.java",
@@ -559,6 +558,10 @@ java_library("robolectric_shadowapi_java") {
 java_library("shadows_core_java") {
   output_name = "shadows-core-3.2"
 
+  # Disable java headers since this target fails to compile with:
+  # error: An exception occurred in org.robolectric.annotation.processing.RobolectricProcessor:
+  enable_turbine = false
+
   # Skip platform checks since we must depend on accessibility_test_framework_java
   # here which requires_android.
   bypass_platform_checks = true
@@ -1085,6 +1088,11 @@ java_library("shadows_multidex_java") {
   # Skip platform checks since we must depend on android_support_multidex_java
   # here which requires_android.
   bypass_platform_checks = true
+
+  # Disable java headers since this target fails to compile with:
+  # error: An exception occurred in org.robolectric.annotation.processing.RobolectricProcessor:
+  enable_turbine = false
+
   skip_jetify = true
   testonly = true
   processor_args_javac = [ "org.robolectric.annotation.processing.shadowPackage=org.robolectric.shadows.multidex" ]
@@ -1127,6 +1135,10 @@ java_library("shadows_play_services_java") {
   # and google_play_services_library here which both requires_android.
   bypass_platform_checks = true
 
+  # Disable java headers since this target fails to compile with:
+  # error: An exception occurred in org.robolectric.annotation.processing.RobolectricProcessor:
+  enable_turbine = false
+
   testonly = true
   skip_jetify = true
   processor_args_javac = [ "org.robolectric.annotation.processing.shadowPackage=org.robolectric.shadows.gms" ]