Allow R8 to run concurrently with lld on low-ram machines

Adds a secondary ninja pool for R8 / lint for machines with
not much RAM. For machines with more ram, R8 / lint continue
to use the same pool as links.

Speed up compiles on android-binary-size by ~90 seconds.

Example:
A bot has 18GB of RAM, and it takes 10GB to link:
 * concurrent_links = 1, with 8GB leftover
 * So add an java_cmd_pool of size 8/2 = 4.

A bot has 22GB of RAM, and it takes 10GB to link:
 * concurrent_links = 2, with 2GB leftover
 * A secondary pool would be of size=1, so in this
   case have java commands use the link_pool directly
   so that size=2 rather than size=1.


Bug: 1078460
Change-Id: I7e4ecfeb04d1c215b2ff27b6a0c438c0e8e3dd88
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2347960
Commit-Queue: Andrew Grieve <agrieve@chromium.org>
Reviewed-by: Peter Wen <wnwen@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@google.com>
Cr-Commit-Position: refs/heads/master@{#799095}

build/config/android/BUILD.gn

@@ -7,6 +7,10 @@ import("//build/config/c++/c++.gni")
 import("//build/config/compiler/compiler.gni")
 import("//build/config/sanitizers/sanitizers.gni")
 
+if (current_toolchain == default_toolchain) {
+  import("//build/toolchain/concurrent_links.gni")
+}
+
 assert(is_android)
 
 # This is included by reference in the //build/config/compiler config that
@@ -159,4 +163,12 @@ if (current_toolchain == default_toolchain) {
     # Override action_pool when goma is enabled for javac.
     depth = 10000
   }
+
+  # When defined, this pool should be used instead of link_pool for command
+  # that need 1-2GB of RAM. https://crbug.com/1078460
+  if (defined(java_cmd_pool_size)) {
+    pool("java_cmd_pool") {
+      depth = java_cmd_pool_size
+    }
+  }
 }

build/config/android/internal_rules.gni

@@ -16,6 +16,9 @@ import("//build/toolchain/goma.gni")
 import("//build/toolchain/kythe.gni")
 import("//build/util/generate_wrapper.gni")
 import("//build_overrides/build.gni")
+if (current_toolchain == default_toolchain) {
+  import("//build/toolchain/concurrent_links.gni")
+}
 assert(is_android)
 
 # The following _java_*_types variables capture all the existing target types.
@@ -970,7 +973,11 @@ if (enable_java_templates) {
       }
 
       # https://crbug.com/1098752 Fix for bot OOM (https://crbug.com/1098333).
-      pool = "//build/toolchain:link_pool($default_toolchain)"
+      if (defined(java_cmd_pool_size)) {
+        pool = "//build/config/android:java_cmd_pool($default_toolchain)"
+      } else {
+        pool = "//build/toolchain:link_pool($default_toolchain)"
+      }
 
       # Lint only requires generated sources and generated resources from the
       # build. Since turbine __header targets already depend on and generate all
@@ -1118,9 +1125,6 @@ if (enable_java_templates) {
     _script = "//build/android/gyp/proguard.py"
     _deps = invoker.deps
 
-    # http://crbug.com/725224. Fix for bots running out of memory.
-    _pool = "//build/toolchain:link_pool($default_toolchain)"
-
     _inputs = [
       invoker.build_config,
       _r8_path,
@@ -1307,7 +1311,6 @@ if (enable_java_templates) {
     action_with_pydeps(target_name) {
       script = _script
       deps = _deps
-      pool = _pool
       inputs = _inputs
       outputs = _outputs
       depfile = "${target_gen_dir}/${target_name}.d"
@@ -1315,6 +1318,13 @@ if (enable_java_templates) {
                "--depfile",
                rebase_path(depfile, root_build_dir),
              ]
+
+      # http://crbug.com/725224. Fix for bots running out of memory.
+      if (defined(java_cmd_pool_size)) {
+        pool = "//build/config/android:java_cmd_pool($default_toolchain)"
+      } else {
+        pool = "//build/toolchain:link_pool($default_toolchain)"
+      }
     }
   }
 
@@ -1539,7 +1549,11 @@ if (enable_java_templates) {
           deps = _deps
 
           # http://crbug.com/725224. Fix for bots running out of memory.
-          pool = "//build/toolchain:link_pool($default_toolchain)"
+          if (defined(java_cmd_pool_size)) {
+            pool = "//build/config/android:java_cmd_pool($default_toolchain)"
+          } else {
+            pool = "//build/toolchain:link_pool($default_toolchain)"
+          }
 
           _shrinked_android = "$android_sdk_build_tools/lib/shrinkedAndroid.jar"
           _dx = "$android_sdk_build_tools/lib/dx.jar"

build/toolchain/concurrent_links.gni

@@ -50,12 +50,6 @@ if (concurrent_links == -1) {
     _args = [ "--mem_per_link_gb=25" ]
   } else if (is_android && !is_debug && !using_sanitizer && is_java_debug &&
              disable_android_lint && symbol_level < 2) {
-    # The link pool is also shared with:
-    # * ProGuard / R8
-    # * Android Lint
-    # Increase the number of concurrent links for release bots that have
-    # ProGuard and Lint disabled, and that do not enable sanitizers (which use a
-    # lot of RAM).
     if (symbol_level == 1) {
       _args = [ "--mem_per_link_gb=6" ]
     } else {
@@ -68,12 +62,29 @@ if (concurrent_links == -1) {
     _args = []
   }
 
+  # For Android builds, we also need to be wary of:
+  # * ProGuard / R8
+  # * Android Lint
+  # These both have a peak usage of < 2GB, but that is still large enough for
+  # them to need to use a pool since they both typically happen at the
+  # same time as linking.
+  if (is_android) {
+    _args += [ "--secondary_mem_per_link=2" ]
+  }
+
   # TODO(crbug.com/617429) Pass more build configuration info to the script
   # so that we can compute better values.
   _command_dict = exec_script("get_concurrent_links.py", _args, "scope")
 
-  concurrent_links = _command_dict.concurrent_links
+  concurrent_links = _command_dict.primary_pool_size
   concurrent_links_logs = _command_dict.explanation
+  if (_command_dict.secondary_pool_size >= concurrent_links) {
+    # Have R8 / Lint share the link pool unless we would safely get more
+    # concurrency out of using a separate one.
+    # On low-RAM machines, this allows an apk's native library to link at the
+    # same time as its java is optimized with R8.
+    java_cmd_pool_size = _command_dict.secondary_pool_size
+  }
 } else {
   concurrent_links_logs =
       [ "concurrent_links set by GN arg (value=$concurrent_links)" ]

build/toolchain/get_concurrent_links.py

@@ -57,18 +57,19 @@ def _GetTotalMemoryInBytes():
   return 0
 
 
-def _GetDefaultConcurrentLinks(mem_per_link_gb, reserve_mem_gb):
+def _GetDefaultConcurrentLinks(per_link_gb, reserve_gb, secondary_per_link_gb):
   explanation = []
-  explanation.append('mem_per_link_gb={} reserve_mem_gb={}'.format(
-      mem_per_link_gb, reserve_mem_gb))
+  explanation.append(
+      'per_link_gb={} reserve_gb={} secondary_per_link_gb={}'.format(
+          per_link_gb, reserve_gb, secondary_per_link_gb))
   # Inherit the legacy environment variable for people that have set it in GYP.
   num_links = int(os.getenv('GYP_LINK_CONCURRENCY', 0))
   if num_links:
     reason = 'GYP_LINK_CONCURRENCY'
   else:
-    mem_total_bytes = _GetTotalMemoryInBytes()
-    mem_total_bytes = max(0, mem_total_bytes - reserve_mem_gb * 2**30)
-    mem_cap = int(max(1, mem_total_bytes / mem_per_link_gb / 2**30))
+    mem_total_gb = float(_GetTotalMemoryInBytes()) / 2**30
+    mem_total_gb = max(0, mem_total_gb - reserve_gb)
+    mem_cap = int(max(1, mem_total_gb / per_link_gb))
     hard_cap = max(1, int(os.getenv('GYP_LINK_CONCURRENCY_MAX', 2**32)))
 
     try:
@@ -76,8 +77,8 @@ def _GetDefaultConcurrentLinks(mem_per_link_gb, reserve_mem_gb):
     except:
       cpu_cap = 1
 
-    explanation.append('cpu_count={} mem_total_bytes={:.1f}GiB'.format(
-        cpu_cap, mem_total_bytes / 2**30))
+    explanation.append('cpu_count={} mem_total_gb={:.1f}GiB'.format(
+        cpu_cap, mem_total_gb))
 
     num_links = min(mem_cap, hard_cap, cpu_cap)
     if num_links == cpu_cap:
@@ -89,20 +90,34 @@ def _GetDefaultConcurrentLinks(mem_per_link_gb, reserve_mem_gb):
 
   explanation.append('concurrent_links={}  (reason: {})'.format(
       num_links, reason))
-  return num_links, explanation
+
+  # See if there is RAM leftover for a secondary pool.
+  if secondary_per_link_gb and num_links == mem_cap:
+    mem_remaining = mem_total_gb - mem_cap * per_link_gb
+    secondary_size = int(max(0, mem_remaining / secondary_per_link_gb))
+    explanation.append('secondary_size={} (mem_remaining={:.1f}GiB)'.format(
+        secondary_size, mem_remaining))
+  else:
+    secondary_size = 0
+
+  return num_links, secondary_size, explanation
 
 
 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument('--mem_per_link_gb', type=int, default=8)
   parser.add_argument('--reserve_mem_gb', type=int, default=0)
+  parser.add_argument('--secondary_mem_per_link', type=int, default=0)
   options = parser.parse_args()
 
-  num_links, explanation = _GetDefaultConcurrentLinks(options.mem_per_link_gb,
-                                                      options.reserve_mem_gb)
+  primary_pool_size, secondary_pool_size, explanation = (
+      _GetDefaultConcurrentLinks(options.mem_per_link_gb,
+                                 options.reserve_mem_gb,
+                                 options.secondary_mem_per_link))
   sys.stdout.write(
       gn_helpers.ToGNString({
-          'concurrent_links': num_links,
+          'primary_pool_size': primary_pool_size,
+          'secondary_pool_size': secondary_pool_size,
           'explanation': explanation,
       }))
   return 0