Android: Use fine-grain dependencies for desugar

D8 provides a list of dependent/dependency relationships for desugaring.
When a file changes (whether within the target or on the classpath), the
classes within the target that implement it will need to be re-desugared
even though those classes did not change. Previously we would re-desugar
all classes within the target. Now we can just re-desugar (i.e. re-dex)
only the necessary ones.

This provides a build speed boost primarily due to avoiding the need to
run any desugaring (either bazel desugar or D8 desugar) when a change
does not require it.

A follow-up CL will deal with the DexSplitter issue for monochrome and
trichrome bundle builds.

The stats below are timed with android_fast_local_dev=true.
Before:
chrome_java_nosig: 34.5s avg (34.1s, 35.2s, 34.2s, 34.4s)
chrome_java_sig: 36.2s avg (35.0s, 37.0s, 36.4s, 36.4s)
chrome_java_res: 39.8s avg (39.8s, 39.9s, 39.6s, 40.0s)
base_java_nosig: 46.0s avg (46.9s, 46.0s, 45.0s, 46.1s)
base_java_sig: 107.2s avg (106.8s, 106.9s, 107.7s, 107.3s)

After:
chrome_java_nosig: 31.6s avg (32.5s, 31.1s, 31.7s, 31.2s)
chrome_java_sig: 34.0s avg (34.2s, 34.0s, 33.8s, 34.1s)
chrome_java_res: 39.8s avg (39.8s, 39.6s, 39.9s, 39.8s)
base_java_nosig: 22.7s avg (22.5s, 23.1s, 22.3s, 22.8s)
base_java_sig: 101.3s avg (102.0s, 101.2s, 100.9s, 101.2s)

Bug: 1015559
Change-Id: Idf4e05bf41583b2ea7d4f1d3855b508be56eea94
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2633323
Commit-Queue: Peter Wen <wnwen@chromium.org>
Reviewed-by: Andrew Grieve <agrieve@chromium.org>
Cr-Commit-Position: refs/heads/master@{#844873}

build/android/gyp/dex.py

@@ -5,6 +5,7 @@
 # found in the LICENSE file.
 
 import argparse
+import collections
 import logging
 import os
 import re
@@ -423,11 +424,39 @@ def _DeleteStaleIncrementalDexFiles(dex_dir, dex_files):
       os.unlink(path)
 
 
-def _ExtractClassFiles(changes, tmp_dir, class_inputs):
+def _ParseDesugarDeps(desugar_dependencies_file):
+  dependents_from_dependency = collections.defaultdict(set)
+  if desugar_dependencies_file and os.path.exists(desugar_dependencies_file):
+    with open(desugar_dependencies_file, 'r') as f:
+      for line in f:
+        dependent, dependency = line.rstrip().split(' -> ')
+        dependents_from_dependency[dependency].add(dependent)
+  return dependents_from_dependency
+
+
+def _ComputeRequiredDesugarClasses(changes, desugar_dependencies_file,
+                                   class_inputs, classpath):
+  dependents_from_dependency = _ParseDesugarDeps(desugar_dependencies_file)
+  required_classes = set()
+  # Gather classes that need to be re-desugared from changes in the classpath.
+  for jar in classpath:
+    for subpath in changes.IterChangedSubpaths(jar):
+      dependency = '{}:{}'.format(jar, subpath)
+      required_classes.update(dependents_from_dependency[dependency])
+
+  for jar in class_inputs:
+    for subpath in changes.IterChangedSubpaths(jar):
+      required_classes.update(dependents_from_dependency[subpath])
+
+  return required_classes
+
+
+def _ExtractClassFiles(changes, tmp_dir, class_inputs, required_classes_set):
   classes_list = []
   for jar in class_inputs:
     if changes:
-      changed_class_list = set(changes.IterChangedSubpaths(jar))
+      changed_class_list = (set(changes.IterChangedSubpaths(jar))
+                            | required_classes_set)
       predicate = lambda x: x in changed_class_list and x.endswith('.class')
     else:
       predicate = lambda x: x.endswith('.class')
@@ -442,9 +471,10 @@ def _CreateIntermediateDexFiles(changes, options, tmp_dir, dex_cmd):
   tmp_extract_dir = os.path.join(tmp_dir, 'tmp_extract_dir')
   os.mkdir(tmp_extract_dir)
 
-  # Do a full rebuild when changes are to classpath or other non-input files.
+  # Do a full rebuild when changes occur in non-input files.
   allowed_changed = set(options.class_inputs)
   allowed_changed.update(options.dex_inputs)
+  allowed_changed.update(options.classpath)
   strings_changed = changes.HasStringChanges()
   non_direct_input_changed = next(
       (p for p in changes.IterChangedPaths() if p not in allowed_changed), None)
@@ -453,14 +483,26 @@ def _CreateIntermediateDexFiles(changes, options, tmp_dir, dex_cmd):
     logging.debug('Full dex required: strings_changed=%s path_changed=%s',
                   strings_changed, non_direct_input_changed)
     changes = None
+
+  if changes:
+    required_desugar_classes_set = _ComputeRequiredDesugarClasses(
+        changes, options.desugar_dependencies, options.class_inputs,
+        options.classpath)
+    logging.debug('Class files needing re-desugar: %d',
+                  len(required_desugar_classes_set))
+  else:
+    required_desugar_classes_set = set()
   class_files = _ExtractClassFiles(changes, tmp_extract_dir,
-                                   options.class_inputs)
+                                   options.class_inputs,
+                                   required_desugar_classes_set)
   logging.debug('Extracted class files: %d', len(class_files))
 
   # If the only change is deleting a file, class_files will be empty.
   if class_files:
     # Dex necessary classes into intermediate dex files.
     dex_cmd = dex_cmd + ['--intermediate', '--file-per-class-file']
+    if options.desugar_dependencies:
+      dex_cmd += ['--file-tmp-prefix', tmp_extract_dir]
     _RunD8(dex_cmd, class_files, options.incremental_dir,
            options.warnings_as_errors,
            options.show_desugar_default_interface_warnings)
@@ -510,14 +552,14 @@ def main(args):
 
   output_paths = [options.output]
 
+  track_subpaths_allowlist = []
   if options.incremental_dir:
     final_dex_inputs = _IntermediateDexFilePathsFromInputJars(
         options.class_inputs, options.incremental_dir)
     output_paths += final_dex_inputs
-    track_subpaths_allowlist = options.class_inputs
+    track_subpaths_allowlist += options.class_inputs
   else:
     final_dex_inputs = list(options.class_inputs)
-    track_subpaths_allowlist = None
   final_dex_inputs += options.dex_inputs
 
   dex_cmd = build_utils.JavaCmd(options.warnings_as_errors) + [
@@ -534,18 +576,12 @@ def main(args):
     dex_cmd += ['--no-desugaring']
   elif options.classpath:
     # The classpath is used by D8 to for interface desugaring.
-    classpath_paths = options.classpath
     if options.desugar_dependencies:
       dex_cmd += ['--desugar-dependencies', options.desugar_dependencies]
-      if os.path.exists(options.desugar_dependencies):
-        with open(options.desugar_dependencies, 'r') as f:
-          lines = [line.strip() for line in f.readlines()]
-          # Use a set to deduplicate entries.
-          desugar_dependencies = set(dep for dep in lines if dep)
-        # Desugar dependencies are a subset of classpath.
-        classpath_paths = list(desugar_dependencies)
-    depfile_deps += classpath_paths
-    input_paths += classpath_paths
+      if track_subpaths_allowlist:
+        track_subpaths_allowlist += options.classpath
+    depfile_deps += options.classpath
+    input_paths += options.classpath
     dex_cmd += ['--lib', build_utils.JAVA_HOME]
     for path in options.bootclasspath:
       dex_cmd += ['--lib', path]

third_party/r8/java/src/org/chromium/build/CustomD8.java

@@ -8,6 +8,7 @@ import com.android.tools.r8.CompilationFailedException;
 import com.android.tools.r8.D8;
 import com.android.tools.r8.D8Command;
 import com.android.tools.r8.DesugarGraphConsumer;
+import com.android.tools.r8.internal.FlagFile;
 import com.android.tools.r8.origin.Origin;
 
 import java.io.IOException;
@@ -15,7 +16,9 @@ import java.io.PrintWriter;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 
 public class CustomD8 {
     private static class CommandLineOrigin extends Origin {
@@ -29,32 +32,33 @@ public class CustomD8 {
         }
     }
 
+    private static String parseAndRemoveArg(List<String> args, String name)
+            throws CompilationFailedException {
+        int idx = args.indexOf(name);
+        if (idx == -1) {
+            return null;
+        }
+        if (idx == args.size() - 1) {
+            throw new CompilationFailedException("Missing argument to '" + name + "'");
+        }
+        String value = args.get(idx + 1);
+        args.subList(idx, idx + 2).clear();
+        return value;
+    }
+
     // Entry point for D8 compilation with support for --desugar-dependencies option
     // as well.
     public static void main(String[] args) throws CompilationFailedException, IOException {
-        String desugarDependenciesOptions = "--desugar-dependencies";
-        String desugarDependenciesPath = null;
-        String[] d8Args = null;
-
-        int desugarDepIdx = Arrays.asList(args).indexOf(desugarDependenciesOptions);
-        if (desugarDepIdx != -1) {
-            int numRemainingArgs = args.length - (desugarDepIdx + 2);
-            if (numRemainingArgs < 0) {
-                throw new CompilationFailedException(
-                        "Missing argument to '" + desugarDependenciesOptions + "'");
-            }
-            desugarDependenciesPath = args[desugarDepIdx + 1];
-            d8Args = new String[args.length - 2];
-            // Copy over all other args before and after the desugar dependencies arg.
-            System.arraycopy(args, 0, d8Args, 0, desugarDepIdx);
-            System.arraycopy(args, desugarDepIdx + 2, d8Args, desugarDepIdx, numRemainingArgs);
-        } else {
-            d8Args = args;
-        }
+        // Need to expand argfile arg in case our custom command line args are in the file.
+        String[] expandedArgs = FlagFile.expandFlagFiles(args, null);
+        List<String> argList = new ArrayList<>(Arrays.asList(expandedArgs));
+        String desugarDependenciesPath = parseAndRemoveArg(argList, "--desugar-dependencies");
+        String fileTmpPrefix = parseAndRemoveArg(argList, "--file-tmp-prefix");
 
         // Use D8 command line parser to handle the normal D8 command line.
-        D8Command.Builder builder = D8Command.parse(d8Args, new CommandLineOrigin());
-        // If additional options was passed amend the D8 command builder.
+        D8Command.Builder builder =
+                D8Command.parse(argList.toArray(new String[0]), new CommandLineOrigin());
+
         if (desugarDependenciesPath != null) {
             final Path desugarDependencies = Paths.get(desugarDependenciesPath);
             PrintWriter desugarDependenciesPrintWriter =
@@ -63,13 +67,25 @@ public class CustomD8 {
                 throw new CompilationFailedException("Too many desugar graph consumers.");
             }
             builder.setDesugarGraphConsumer(new DesugarGraphConsumer() {
+                private String formatOrigin(Origin origin) {
+                    String path = origin.toString();
+                    // Class files are extracted to a temporary directory for incremental dexing.
+                    // Remove the prefix of the path corresponding to the temporary directory so
+                    // that these paths are consistent between builds.
+                    if (fileTmpPrefix != null && path.startsWith(fileTmpPrefix)) {
+                        return path.substring(fileTmpPrefix.length());
+                    }
+                    return path;
+                }
+
                 @Override
                 public void accept(Origin dependent, Origin dependency) {
-                    // The target's class files have root as their parent.
-                    if (dependency.parent().equals(Origin.root())) {
-                        return;
+                    String dependentPath = formatOrigin(dependent);
+                    String dependencyPath = formatOrigin(dependency);
+                    synchronized (desugarDependenciesPrintWriter) {
+                        desugarDependenciesPrintWriter.println(
+                                dependentPath + " -> " + dependencyPath);
                     }
-                    desugarDependenciesPrintWriter.println(dependency.parent());
                 }
 
                 @Override