Android: Build .pak resource whitelist without linking lib(mono)chrome

Allows R8 and lld to run concurrently, saving on build times.

Bug: 1078460
Change-Id: I9ee978800b649634461c49ced4a26ca5015869cb
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2259292
Commit-Queue: Andrew Grieve <agrieve@chromium.org>
Reviewed-by: Peter Wen <wnwen@chromium.org>
Cr-Commit-Position: refs/heads/master@{#782274}

build/android/gyp/write_native_libraries_java.py

@@ -26,6 +26,7 @@ def _FormatLibraryName(library_name):
 def main():
   parser = argparse.ArgumentParser()
 
+  build_utils.AddDepfileOption(parser)
   parser.add_argument('--final', action='store_true', help='Use final fields.')
   parser.add_argument(
       '--enable-chromium-linker',
@@ -97,6 +98,13 @@ def main():
           zip_path='org/chromium/base/library_loader/NativeLibraries.java',
           data=NATIVE_LIBRARIES_TEMPLATE.format(**format_dict))
 
+  if options.depfile:
+    assert options.native_libraries_list
+    build_utils.WriteDepfile(options.depfile,
+                             options.output,
+                             inputs=[options.native_libraries_list],
+                             add_pydeps=False)
+
 
 if __name__ == '__main__':
   sys.exit(main())

build/config/android/rules.gni

@@ -708,12 +708,16 @@ if (enable_java_templates) {
         _cpu_family,
       ]
       if (invoker.use_final_fields) {
+        # Write native_libraries_list_file via depfile rather than specifyin it
+        # as a dep in order allow R8 to run in parallel with native compilation.
+        depfile = "$target_gen_dir/$target_name.d"
         args += [
           "--final",
+          "--depfile",
+          rebase_path(depfile, root_build_dir),
           "--native-libraries-list",
           rebase_path(invoker.native_libraries_list_file, root_build_dir),
         ]
-        inputs = [ invoker.native_libraries_list_file ]
         if (defined(invoker.version_number)) {
           args += [
             "--version-number",
@@ -2633,16 +2637,17 @@ if (enable_java_templates) {
     if (_generate_native_libraries_java) {
       write_native_libraries_java("${_template_name}__native_libraries") {
         forward_variables_from(invoker, [ "main_component_library" ])
-        deps = [
-          ":${_template_name}__secondary_abi_shared_library_list",
-          ":${_template_name}__shared_library_list",
-        ]
+        deps = []
         if (defined(invoker.native_lib_version_rule)) {
           deps += [ invoker.native_lib_version_rule ]
         }
         if (defined(invoker.native_lib_version_arg)) {
           version_number = invoker.native_lib_version_arg
         }
+
+        # Do not add a dep on the generated_file target in order to avoid having
+        # to build the native libraries before this target. The dependency is
+        # instead captured via a depfile.
         if (_native_libs_deps != []) {
           native_libraries_list_file = _shared_library_list_file
         } else {
@@ -2833,10 +2838,9 @@ if (enable_java_templates) {
         }
       }
 
-      # Don't depend on the runtime_deps target in order to avoid having to
-      # build the native libraries just to create the .build_config file.
-      # The dep is unnecessary since the runtime_deps file is created by gn gen
-      # and the runtime_deps file is added to write_build_config.py's depfile.
+      # Do not add a dep on the generated_file target in order to avoid having
+      # to build the native libraries before this target. The dependency is
+      # instead captured via a depfile.
       if (_native_libs_deps != []) {
         shared_libraries_runtime_deps_file = _shared_library_list_file
       }
@@ -3391,14 +3395,16 @@ if (enable_java_templates) {
         data_deps = []
       }
 
+      # Include unstripped native libraries so tests can symbolize stacks.
+      data_deps += _all_native_libs_deps
+
       if (_enable_lint) {
         data_deps += [ ":$_android_lint_target" ]
       }
 
       if (_incremental_apk) {
         # device/commands is used by the installer script to push files via .zip.
-        data_deps +=
-            [ "//build/android/pylib/device/commands" ] + _native_libs_deps
+        data_deps += [ "//build/android/pylib/device/commands" ]
       }
       if (_uses_static_library) {
         data_deps += [ invoker.static_library_provider ]

chrome/android/BUILD.gn

@@ -1397,18 +1397,38 @@ jinja_template_resources("chrome_test_apk_template_resources") {
 }
 
 # Chrome APK's native library.
-chrome_common_shared_library("libchrome") {
-  sources = [
-    "../browser/android/chrome_entry_point.cc",
-    chrome_jni_registration_header,
-  ]
-  deps = [ ":chrome_jni_registration($default_toolchain)" ]
+template("libchrome_impl") {
+  chrome_common_shared_library(target_name) {
+    sources = [
+      "../browser/android/chrome_entry_point.cc",
+      chrome_jni_registration_header,
+    ]
+    deps = [ ":chrome_jni_registration($default_toolchain)" ]
+    if (defined(invoker.deps)) {
+      deps += invoker.deps
+    }
 
-  if (enable_vr) {
-    deps += [ "//chrome/browser/android/vr:module_factory" ]
+    if (enable_vr) {
+      deps += [ "//chrome/browser/android/vr:module_factory" ]
+    }
+
+    module_descs = chrome_modern_module_descs
+    forward_variables_from(invoker, "*", [ "deps" ])
   }
+}
 
-  module_descs = chrome_modern_module_descs
+if (enable_resource_whitelist_generation) {
+  libchrome_impl("libchrome_pak_whitelist_inputs") {
+    collect_inputs_only = true
+  }
+}
+
+libchrome_impl("libchrome") {
+  if (enable_resource_whitelist_generation) {
+    # Make sure whitelist_inputs is built first so when concurrent_links == 1
+    # it comes before the actual (very slow) link step.
+    deps = [ ":libchrome_pak_whitelist_inputs" ]
+  }
 }
 
 chrome_common_shared_library("libchromefortest") {
@@ -1495,9 +1515,9 @@ if (current_toolchain == default_toolchain) {
 
     if (enable_resource_whitelist_generation) {
       if (_is_trichrome || _is_monochrome) {
-        _lib = "libmonochrome"
+        _lib = "libmonochrome_pak_whitelist_inputs"
       } else {
-        _lib = "libchrome"
+        _lib = "libchrome_pak_whitelist_inputs"
       }
       _resource_whitelist_target = "${_variant}_resource_whitelist"
       _resource_whitelist_file =
@@ -1683,13 +1703,15 @@ if (current_toolchain == default_toolchain) {
 # Monochrome equivalent of Chrome's APK or bundle library template.
 template("libmonochrome_apk_or_bundle_tmpl") {
   chrome_common_shared_library(target_name) {
-    forward_variables_from(invoker, "*")
     sources = [ "../browser/android/monochrome_entry_point.cc" ]
     deps = [
       "//android_webview/lib",
       "//android_webview/nonembedded",
       "//components/crash/android:crashpad_main",
     ]
+    if (defined(invoker.deps)) {
+      deps += invoker.deps
+    }
 
     if (webview_includes_weblayer) {
       defines = [ "WEBVIEW_INCLUDES_WEBLAYER" ]
@@ -1702,13 +1724,26 @@ template("libmonochrome_apk_or_bundle_tmpl") {
 
     is_monochrome = true
     module_descs = monochrome_module_descs
+
+    forward_variables_from(invoker, "*", [ "deps" ])
   }
 }
 
 if (!android_64bit_target_cpu ||
     current_toolchain == android_secondary_abi_toolchain) {
-  # Monochrome APK native library.
+  if (enable_resource_whitelist_generation) {
+    # Avoiding a real link for this step allows the actual libmonochrome to
+    # run at the same time as R8.
+    libmonochrome_apk_or_bundle_tmpl("libmonochrome_pak_whitelist_inputs") {
+      collect_inputs_only = true
+    }
+  }
   libmonochrome_apk_or_bundle_tmpl("libmonochrome") {
+    if (enable_resource_whitelist_generation) {
+      # Make sure whitelist_inputs is built first so when concurrent_links == 1
+      # it comes before the actual (very slow) link step.
+      deps = [ ":libmonochrome_pak_whitelist_inputs" ]
+    }
   }
 
   if (android_64bit_target_cpu) {

chrome/android/chrome_common_shared_library.gni

@@ -33,28 +33,33 @@ template("chrome_common_shared_library") {
   _is_monochrome = defined(invoker.is_monochrome) && invoker.is_monochrome
   _is_webview = defined(invoker.is_webview) && invoker.is_webview
   _export_java_symbols = _is_monochrome || _is_webview
-
-  _linker_script = "$target_gen_dir/${target_name}_linker_script.txt"
-  _linker_script_target = "${target_name}_linker_script"
+  _collect_inputs_only =
+      defined(invoker.collect_inputs_only) && invoker.collect_inputs_only
 
   # Create a partitioned libraries if the build config supports it, and the
   # invoker has supplied module descriptors.
-  _generate_partitions = defined(invoker.module_descs) && use_native_partitions
+  _generate_partitions = defined(invoker.module_descs) &&
+                         use_native_partitions && !_collect_inputs_only
   _module_descs = []
   if (defined(invoker.module_descs)) {
     _module_descs = invoker.module_descs
   }
 
-  # Create a custom linker script based on JNI and feature module requirements.
-  generate_linker_version_script(_linker_script_target) {
-    linker_script = _linker_script
-    export_java_symbols = _export_java_symbols
-    export_feature_registrations = true
-    if (_generate_partitions) {
-      export_symbol_allowlist_files = []
-      foreach(_module_desc, invoker.module_descs) {
-        if (defined(_module_desc.native_entrypoints)) {
-          export_symbol_allowlist_files += [ _module_desc.native_entrypoints ]
+  if (!_collect_inputs_only) {
+    _linker_script_target = "${target_name}_linker_script"
+    _linker_script = "$target_gen_dir/${target_name}_linker_script.txt"
+
+    # Create a custom linker script based on JNI and feature module requirements.
+    generate_linker_version_script(_linker_script_target) {
+      linker_script = _linker_script
+      export_java_symbols = _export_java_symbols
+      export_feature_registrations = true
+      if (_generate_partitions) {
+        export_symbol_allowlist_files = []
+        foreach(_module_desc, invoker.module_descs) {
+          if (defined(_module_desc.native_entrypoints)) {
+            export_symbol_allowlist_files += [ _module_desc.native_entrypoints ]
+          }
         }
       }
     }
@@ -77,13 +82,19 @@ template("chrome_common_shared_library") {
 
     # Use a dynamically-generated linker script.
     configs -= [ "//build/config/android:hide_all_but_jni_onload" ]
-    deps += [ ":$_linker_script_target" ]
-    inputs = [ "$_linker_script" ]
     if (!defined(ldflags)) {
       ldflags = []
     }
-    ldflags += [ "-Wl,--version-script=" +
-                 rebase_path(_linker_script, root_build_dir) ]
+    if (_collect_inputs_only) {
+      metadata = {
+      }
+      ldflags += [ "--collect-inputs-only" ]
+    } else {
+      deps += [ ":$_linker_script_target" ]
+      inputs = [ "$_linker_script" ]
+      ldflags += [ "-Wl,--version-script=" +
+                   rebase_path(_linker_script, root_build_dir) ]
+    }
 
     # Handle VR JNI registration and dependencies.
     if (!_is_webview && enable_vr) {
@@ -107,6 +118,12 @@ template("chrome_common_shared_library") {
           deps += _module_desc.native_deps
         }
       }
+    } else if (_collect_inputs_only) {
+      foreach(_module_desc, _module_descs) {
+        if (defined(_module_desc.native_deps)) {
+          deps += _module_desc.native_deps
+        }
+      }
     }
 
     # Compress relocations if needed.
@@ -132,7 +149,7 @@ template("chrome_common_shared_library") {
     }
   }
 
-  if (!_generate_partitions) {
+  if (!_generate_partitions && !_collect_inputs_only) {
     # Make helper targets so that we always have module native targets no matter
     # whether partitions are enabled.
     foreach(_module_desc, _module_descs) {

components/module_installer/android/module_desc_java.gni

@@ -23,32 +23,36 @@ import("//build/config/python.gni")
 template("module_desc_java") {
   _target_name = target_name
 
-  _libraries = "${target_gen_dir}/${_target_name}.libraries"
-  _rebased_libraries = rebase_path(_libraries, root_out_dir)
+  _libraries_file = "${target_gen_dir}/${_target_name}.libraries"
   generated_file("${_target_name}__libraries") {
     if (defined(invoker.shared_libraries)) {
       deps = invoker.shared_libraries
     }
-    outputs = [ _libraries ]
+    outputs = [ _libraries_file ]
     data_keys = [ "shared_libraries" ]
     walk_keys = [ "shared_libraries_barrier" ]
     rebase = root_build_dir
-    output_conversion = "json"
+    output_conversion = "value"
   }
 
   _srcjar = "$target_gen_dir/${_target_name}__srcjar.srcjar"
   action_with_pydeps("${_target_name}__srcjar") {
     script = "//components/module_installer/android/module_desc_java.py"
-    deps = [ ":${_target_name}__libraries" ]
-    inputs = [ _libraries ]
     outputs = [ _srcjar ]
+
+    # Do not add a dep on the generated_file target in order to avoid having
+    # to build the native libraries before this target. The dependency is
+    # instead captured via a depfile.
+    depfile = "$target_gen_dir/$target_name.d"
     args = [
       "--module",
       invoker.module_name,
-      "--libraries",
-      "@FileArg($_rebased_libraries)",
+      "--libraries-file",
+      rebase_path(_libraries_file, root_out_dir),
       "--output",
       rebase_path(_srcjar, root_out_dir),
+      "--depfile",
+      rebase_path(depfile, root_out_dir),
     ]
 
     if (defined(invoker.load_native_on_get_impl) &&

components/module_installer/android/module_desc_java.py

@@ -48,45 +48,55 @@ public class ModuleDescriptor_{MODULE} implements ModuleDescriptor {{
 
 
 def main():
-  parser = argparse.ArgumentParser()
-  parser.add_argument('--module', required=True, help='The module name.')
-  parser.add_argument(
-      '--libraries', required=True, help='GN list of native library paths.')
-  parser.add_argument('--paks', help='GN list of PAK file paths')
-  parser.add_argument(
-      '--output', required=True, help='Path to the generated srcjar file.')
-  parser.add_argument('--load-native-on-get-impl', action='store_true',
-      default=False,
-      help='Load module automatically on calling Module.getImpl().')
-  options = parser.parse_args(build_utils.ExpandFileArgs(sys.argv[1:]))
-  options.libraries = build_utils.ParseGnList(options.libraries)
-  options.paks = build_utils.ParseGnList(options.paks)
-
-  libraries = []
-  for path in options.libraries:
-    path = path.strip()
-    filename = os.path.split(path)[1]
-    assert filename.startswith('lib')
-    assert filename.endswith('.so')
-    # Remove lib prefix and .so suffix.
-    libraries += [filename[3:-3]]
-  paks = options.paks if options.paks else []
-
-  format_dict = {
-      'MODULE': options.module,
-      'LIBRARIES': ','.join(['"%s"' % l for l in libraries]),
-      'PAKS': ','.join(['"%s"' % os.path.basename(p) for p in paks]),
-      'LOAD_NATIVE_ON_GET_IMPL': (
-          'true' if options.load_native_on_get_impl else 'false'),
-  }
-  with build_utils.AtomicOutput(options.output) as f:
-    with zipfile.ZipFile(f.name, 'w') as srcjar_file:
-      build_utils.AddToZipHermetic(
-          srcjar_file,
-          'org/chromium/components/module_installer/builder/'
-          'ModuleDescriptor_%s.java' % options.module,
-          data=_TEMPLATE.format(**format_dict))
+    parser = argparse.ArgumentParser()
+    build_utils.AddDepfileOption(parser)
+    parser.add_argument('--module', required=True, help='The module name.')
+    parser.add_argument('--libraries-file',
+                        required=True,
+                        help='Path to file with GN list of library paths')
+    parser.add_argument('--paks', help='GN list of PAK file paths')
+    parser.add_argument(
+        '--output', required=True, help='Path to the generated srcjar file.')
+    parser.add_argument('--load-native-on-get-impl', action='store_true',
+        default=False,
+        help='Load module automatically on calling Module.getImpl().')
+    options = parser.parse_args()
+    options.paks = build_utils.ParseGnList(options.paks)
+
+    with open(options.libraries_file) as f:
+        libraries_list = build_utils.ParseGnList(f.read())
+
+    libraries = []
+    for path in libraries_list:
+        path = path.strip()
+        filename = os.path.split(path)[1]
+        assert filename.startswith('lib')
+        assert filename.endswith('.so')
+        # Remove lib prefix and .so suffix.
+        libraries += [filename[3:-3]]
+    paks = options.paks if options.paks else []
+
+    format_dict = {
+        'MODULE': options.module,
+        'LIBRARIES': ','.join(['"%s"' % l for l in libraries]),
+        'PAKS': ','.join(['"%s"' % os.path.basename(p) for p in paks]),
+        'LOAD_NATIVE_ON_GET_IMPL': (
+            'true' if options.load_native_on_get_impl else 'false'),
+    }
+    with build_utils.AtomicOutput(options.output) as f:
+        with zipfile.ZipFile(f.name, 'w') as srcjar_file:
+            build_utils.AddToZipHermetic(
+                srcjar_file,
+                'org/chromium/components/module_installer/builder/'
+                'ModuleDescriptor_%s.java' % options.module,
+                data=_TEMPLATE.format(**format_dict))
+
+    if options.depfile:
+        build_utils.WriteDepfile(options.depfile,
+                                 options.output,
+                                 inputs=[options.libraries_file],
+                                 add_pydeps=False)
 
 
 if __name__ == '__main__':
-  sys.exit(main())
+    sys.exit(main())

tools/resources/generate_resource_whitelist.py

@@ -130,14 +130,19 @@ def WriteResourceWhitelist(args):
   for input in args.inputs:
     with open(input, 'r') as f:
       magic = f.read(4)
+      chunk = f.read(60)
     if magic == '\x7fELF':
-      resource_ids = resource_ids.union(GetResourceWhitelistELF(input))
+      func = GetResourceWhitelistELF
     elif magic == 'Micr':
-      resource_ids = resource_ids.union(GetResourceWhitelistPDB(input))
-    elif magic == 'obj/':
-      resource_ids = resource_ids.union(GetResourceWhitelistFileList(input))
+      func = GetResourceWhitelistPDB
+    elif magic == 'obj/' or '/obj/' in chunk:
+      # For secondary toolchain, path will look like android_clang_arm/obj/...
+      func = GetResourceWhitelistFileList
     else:
       raise Exception('unknown file format')
+
+    resource_ids.update(func(input))
+
   if len(resource_ids) == 0:
     raise Exception('No debug info was dumped. Ensure GN arg "symbol_level" '
                     '!= 0 and that the file is not stripped.')