Commit e445f27f authored by Andrew Grieve's avatar Andrew Grieve Committed by Commit Bot

Allow R8 to run concurrently with lld on low-ram machines

Adds a secondary ninja pool for R8 / lint for machines with
not much RAM. For machines with more ram, R8 / lint continue
to use the same pool as links.

Speed up compiles on android-binary-size by ~90 seconds.

Example:
A bot has 18GB of RAM, and it takes 10GB to link:
 * concurrent_links = 1, with 8GB leftover
 * So add an java_cmd_pool of size 8/2 = 4.

A bot has 22GB of RAM, and it takes 10GB to link:
 * concurrent_links = 2, with 2GB leftover
 * A secondary pool would be of size=1, so in this
   case have java commands use the link_pool directly
   so that size=2 rather than size=1.


Bug: 1078460
Change-Id: I7e4ecfeb04d1c215b2ff27b6a0c438c0e8e3dd88
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2347960
Commit-Queue: Andrew Grieve <agrieve@chromium.org>
Reviewed-by: default avatarPeter Wen <wnwen@chromium.org>
Reviewed-by: default avatarDirk Pranke <dpranke@google.com>
Cr-Commit-Position: refs/heads/master@{#799095}
parent 56203474
...@@ -7,6 +7,10 @@ import("//build/config/c++/c++.gni") ...@@ -7,6 +7,10 @@ import("//build/config/c++/c++.gni")
import("//build/config/compiler/compiler.gni") import("//build/config/compiler/compiler.gni")
import("//build/config/sanitizers/sanitizers.gni") import("//build/config/sanitizers/sanitizers.gni")
if (current_toolchain == default_toolchain) {
import("//build/toolchain/concurrent_links.gni")
}
assert(is_android) assert(is_android)
# This is included by reference in the //build/config/compiler config that # This is included by reference in the //build/config/compiler config that
...@@ -159,4 +163,12 @@ if (current_toolchain == default_toolchain) { ...@@ -159,4 +163,12 @@ if (current_toolchain == default_toolchain) {
# Override action_pool when goma is enabled for javac. # Override action_pool when goma is enabled for javac.
depth = 10000 depth = 10000
} }
# When defined, this pool should be used instead of link_pool for command
# that need 1-2GB of RAM. https://crbug.com/1078460
if (defined(java_cmd_pool_size)) {
pool("java_cmd_pool") {
depth = java_cmd_pool_size
}
}
} }
...@@ -16,6 +16,9 @@ import("//build/toolchain/goma.gni") ...@@ -16,6 +16,9 @@ import("//build/toolchain/goma.gni")
import("//build/toolchain/kythe.gni") import("//build/toolchain/kythe.gni")
import("//build/util/generate_wrapper.gni") import("//build/util/generate_wrapper.gni")
import("//build_overrides/build.gni") import("//build_overrides/build.gni")
if (current_toolchain == default_toolchain) {
import("//build/toolchain/concurrent_links.gni")
}
assert(is_android) assert(is_android)
# The following _java_*_types variables capture all the existing target types. # The following _java_*_types variables capture all the existing target types.
...@@ -970,7 +973,11 @@ if (enable_java_templates) { ...@@ -970,7 +973,11 @@ if (enable_java_templates) {
} }
# https://crbug.com/1098752 Fix for bot OOM (https://crbug.com/1098333). # https://crbug.com/1098752 Fix for bot OOM (https://crbug.com/1098333).
pool = "//build/toolchain:link_pool($default_toolchain)" if (defined(java_cmd_pool_size)) {
pool = "//build/config/android:java_cmd_pool($default_toolchain)"
} else {
pool = "//build/toolchain:link_pool($default_toolchain)"
}
# Lint only requires generated sources and generated resources from the # Lint only requires generated sources and generated resources from the
# build. Since turbine __header targets already depend on and generate all # build. Since turbine __header targets already depend on and generate all
...@@ -1118,9 +1125,6 @@ if (enable_java_templates) { ...@@ -1118,9 +1125,6 @@ if (enable_java_templates) {
_script = "//build/android/gyp/proguard.py" _script = "//build/android/gyp/proguard.py"
_deps = invoker.deps _deps = invoker.deps
# http://crbug.com/725224. Fix for bots running out of memory.
_pool = "//build/toolchain:link_pool($default_toolchain)"
_inputs = [ _inputs = [
invoker.build_config, invoker.build_config,
_r8_path, _r8_path,
...@@ -1307,7 +1311,6 @@ if (enable_java_templates) { ...@@ -1307,7 +1311,6 @@ if (enable_java_templates) {
action_with_pydeps(target_name) { action_with_pydeps(target_name) {
script = _script script = _script
deps = _deps deps = _deps
pool = _pool
inputs = _inputs inputs = _inputs
outputs = _outputs outputs = _outputs
depfile = "${target_gen_dir}/${target_name}.d" depfile = "${target_gen_dir}/${target_name}.d"
...@@ -1315,6 +1318,13 @@ if (enable_java_templates) { ...@@ -1315,6 +1318,13 @@ if (enable_java_templates) {
"--depfile", "--depfile",
rebase_path(depfile, root_build_dir), rebase_path(depfile, root_build_dir),
] ]
# http://crbug.com/725224. Fix for bots running out of memory.
if (defined(java_cmd_pool_size)) {
pool = "//build/config/android:java_cmd_pool($default_toolchain)"
} else {
pool = "//build/toolchain:link_pool($default_toolchain)"
}
} }
} }
...@@ -1539,7 +1549,11 @@ if (enable_java_templates) { ...@@ -1539,7 +1549,11 @@ if (enable_java_templates) {
deps = _deps deps = _deps
# http://crbug.com/725224. Fix for bots running out of memory. # http://crbug.com/725224. Fix for bots running out of memory.
pool = "//build/toolchain:link_pool($default_toolchain)" if (defined(java_cmd_pool_size)) {
pool = "//build/config/android:java_cmd_pool($default_toolchain)"
} else {
pool = "//build/toolchain:link_pool($default_toolchain)"
}
_shrinked_android = "$android_sdk_build_tools/lib/shrinkedAndroid.jar" _shrinked_android = "$android_sdk_build_tools/lib/shrinkedAndroid.jar"
_dx = "$android_sdk_build_tools/lib/dx.jar" _dx = "$android_sdk_build_tools/lib/dx.jar"
......
...@@ -50,12 +50,6 @@ if (concurrent_links == -1) { ...@@ -50,12 +50,6 @@ if (concurrent_links == -1) {
_args = [ "--mem_per_link_gb=25" ] _args = [ "--mem_per_link_gb=25" ]
} else if (is_android && !is_debug && !using_sanitizer && is_java_debug && } else if (is_android && !is_debug && !using_sanitizer && is_java_debug &&
disable_android_lint && symbol_level < 2) { disable_android_lint && symbol_level < 2) {
# The link pool is also shared with:
# * ProGuard / R8
# * Android Lint
# Increase the number of concurrent links for release bots that have
# ProGuard and Lint disabled, and that do not enable sanitizers (which use a
# lot of RAM).
if (symbol_level == 1) { if (symbol_level == 1) {
_args = [ "--mem_per_link_gb=6" ] _args = [ "--mem_per_link_gb=6" ]
} else { } else {
...@@ -68,12 +62,29 @@ if (concurrent_links == -1) { ...@@ -68,12 +62,29 @@ if (concurrent_links == -1) {
_args = [] _args = []
} }
# For Android builds, we also need to be wary of:
# * ProGuard / R8
# * Android Lint
# These both have a peak usage of < 2GB, but that is still large enough for
# them to need to use a pool since they both typically happen at the
# same time as linking.
if (is_android) {
_args += [ "--secondary_mem_per_link=2" ]
}
# TODO(crbug.com/617429) Pass more build configuration info to the script # TODO(crbug.com/617429) Pass more build configuration info to the script
# so that we can compute better values. # so that we can compute better values.
_command_dict = exec_script("get_concurrent_links.py", _args, "scope") _command_dict = exec_script("get_concurrent_links.py", _args, "scope")
concurrent_links = _command_dict.concurrent_links concurrent_links = _command_dict.primary_pool_size
concurrent_links_logs = _command_dict.explanation concurrent_links_logs = _command_dict.explanation
if (_command_dict.secondary_pool_size >= concurrent_links) {
# Have R8 / Lint share the link pool unless we would safely get more
# concurrency out of using a separate one.
# On low-RAM machines, this allows an apk's native library to link at the
# same time as its java is optimized with R8.
java_cmd_pool_size = _command_dict.secondary_pool_size
}
} else { } else {
concurrent_links_logs = concurrent_links_logs =
[ "concurrent_links set by GN arg (value=$concurrent_links)" ] [ "concurrent_links set by GN arg (value=$concurrent_links)" ]
......
...@@ -57,18 +57,19 @@ def _GetTotalMemoryInBytes(): ...@@ -57,18 +57,19 @@ def _GetTotalMemoryInBytes():
return 0 return 0
def _GetDefaultConcurrentLinks(mem_per_link_gb, reserve_mem_gb): def _GetDefaultConcurrentLinks(per_link_gb, reserve_gb, secondary_per_link_gb):
explanation = [] explanation = []
explanation.append('mem_per_link_gb={} reserve_mem_gb={}'.format( explanation.append(
mem_per_link_gb, reserve_mem_gb)) 'per_link_gb={} reserve_gb={} secondary_per_link_gb={}'.format(
per_link_gb, reserve_gb, secondary_per_link_gb))
# Inherit the legacy environment variable for people that have set it in GYP. # Inherit the legacy environment variable for people that have set it in GYP.
num_links = int(os.getenv('GYP_LINK_CONCURRENCY', 0)) num_links = int(os.getenv('GYP_LINK_CONCURRENCY', 0))
if num_links: if num_links:
reason = 'GYP_LINK_CONCURRENCY' reason = 'GYP_LINK_CONCURRENCY'
else: else:
mem_total_bytes = _GetTotalMemoryInBytes() mem_total_gb = float(_GetTotalMemoryInBytes()) / 2**30
mem_total_bytes = max(0, mem_total_bytes - reserve_mem_gb * 2**30) mem_total_gb = max(0, mem_total_gb - reserve_gb)
mem_cap = int(max(1, mem_total_bytes / mem_per_link_gb / 2**30)) mem_cap = int(max(1, mem_total_gb / per_link_gb))
hard_cap = max(1, int(os.getenv('GYP_LINK_CONCURRENCY_MAX', 2**32))) hard_cap = max(1, int(os.getenv('GYP_LINK_CONCURRENCY_MAX', 2**32)))
try: try:
...@@ -76,8 +77,8 @@ def _GetDefaultConcurrentLinks(mem_per_link_gb, reserve_mem_gb): ...@@ -76,8 +77,8 @@ def _GetDefaultConcurrentLinks(mem_per_link_gb, reserve_mem_gb):
except: except:
cpu_cap = 1 cpu_cap = 1
explanation.append('cpu_count={} mem_total_bytes={:.1f}GiB'.format( explanation.append('cpu_count={} mem_total_gb={:.1f}GiB'.format(
cpu_cap, mem_total_bytes / 2**30)) cpu_cap, mem_total_gb))
num_links = min(mem_cap, hard_cap, cpu_cap) num_links = min(mem_cap, hard_cap, cpu_cap)
if num_links == cpu_cap: if num_links == cpu_cap:
...@@ -89,20 +90,34 @@ def _GetDefaultConcurrentLinks(mem_per_link_gb, reserve_mem_gb): ...@@ -89,20 +90,34 @@ def _GetDefaultConcurrentLinks(mem_per_link_gb, reserve_mem_gb):
explanation.append('concurrent_links={} (reason: {})'.format( explanation.append('concurrent_links={} (reason: {})'.format(
num_links, reason)) num_links, reason))
return num_links, explanation
# See if there is RAM leftover for a secondary pool.
if secondary_per_link_gb and num_links == mem_cap:
mem_remaining = mem_total_gb - mem_cap * per_link_gb
secondary_size = int(max(0, mem_remaining / secondary_per_link_gb))
explanation.append('secondary_size={} (mem_remaining={:.1f}GiB)'.format(
secondary_size, mem_remaining))
else:
secondary_size = 0
return num_links, secondary_size, explanation
def main(): def main():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('--mem_per_link_gb', type=int, default=8) parser.add_argument('--mem_per_link_gb', type=int, default=8)
parser.add_argument('--reserve_mem_gb', type=int, default=0) parser.add_argument('--reserve_mem_gb', type=int, default=0)
parser.add_argument('--secondary_mem_per_link', type=int, default=0)
options = parser.parse_args() options = parser.parse_args()
num_links, explanation = _GetDefaultConcurrentLinks(options.mem_per_link_gb, primary_pool_size, secondary_pool_size, explanation = (
options.reserve_mem_gb) _GetDefaultConcurrentLinks(options.mem_per_link_gb,
options.reserve_mem_gb,
options.secondary_mem_per_link))
sys.stdout.write( sys.stdout.write(
gn_helpers.ToGNString({ gn_helpers.ToGNString({
'concurrent_links': num_links, 'primary_pool_size': primary_pool_size,
'secondary_pool_size': secondary_pool_size,
'explanation': explanation, 'explanation': explanation,
})) }))
return 0 return 0
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment