[build] Use metadata to collect APK's shared libraries

Previously, we used runtime deps to collect the shared libraries. That
worked due to some special runtime dependency propagation rules around
shared_library targets. However, this will stop working with the new
linker assisted native feature module approach where some shared
libraries will become action targets. To solve this, this CL

- adds the shared library output file to metadata of each shared_library
  target,

- adds the new partitioned shared library output files to metadata as
  well,

- adds barriers where propagation should be interrupted, and

- adds generated_file targets to read the metadata.



Bug: 980569
Change-Id: Ib3f49a00d04d066a2fc1f0bb916d469adabac447
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1693023
Commit-Queue: Tibor Goldschwendt <tiborg@chromium.org>
Reviewed-by: Andrew Grieve <agrieve@chromium.org>
Cr-Commit-Position: refs/heads/master@{#675773}

build/android/gyp/write_ordered_libraries.py

@@ -83,12 +83,11 @@ def main():
     exclude_shared_libraries = options.exclude_shared_libraries.split(',')
   for f in open(options.runtime_deps):
     f = f[:-1]
-    if f.endswith('.so'):
-      p = f.replace('lib.unstripped/', '')
-      if os.path.basename(p) in exclude_shared_libraries:
-        continue
-      unsorted_lib_paths.append(p)
-      _library_path_map[os.path.basename(p)] = p
+    assert f.endswith('.so')
+    if os.path.basename(f) in exclude_shared_libraries:
+      continue
+    unsorted_lib_paths.append(f)
+    _library_path_map[os.path.basename(f)] = f
 
   lib_paths = GetSortedTransitiveDependencies(unsorted_lib_paths)
 

build/config/BUILDCONFIG.gn

@@ -561,7 +561,8 @@ foreach(_target_type,
           "shared_library",
         ]) {
   template(_target_type) {
-    target(_target_type, target_name) {
+    _target_name = target_name
+    target(_target_type, _target_name) {
       forward_variables_from(invoker, "*", [ "no_default_deps" ])
       if (!defined(deps)) {
         deps = []
@@ -569,6 +570,35 @@ foreach(_target_type,
       if (!defined(invoker.no_default_deps) || !invoker.no_default_deps) {
         deps += [ "//build/config:${_target_type}_deps" ]
       }
+
+      # On Android, write shared library output file to metadata. We will use
+      # this information to, for instance, collect all shared libraries that
+      # should be packaged into an APK.
+      if (!defined(invoker.metadata) && is_android &&
+          _target_type == "shared_library") {
+        _output_name = _target_name
+        if (defined(invoker.output_name)) {
+          _output_name = invoker.output_name
+        }
+
+        # Remove 'lib' prefix from output name if it exists.
+        _magic_prefix = "$0x01$0x01"
+        _output_name = string_replace("${_magic_prefix}${_output_name}",
+                                      "${_magic_prefix}lib",
+                                      _magic_prefix,
+                                      1)
+        _output_name = string_replace(_output_name, _magic_prefix, "", 1)
+
+        _shlib_extension = ".so"
+        if (is_component_build) {
+          _shlib_extension = ".cr.so"
+        }
+
+        metadata = {
+          shared_libraries =
+              [ "$root_out_dir/lib${_output_name}${_shlib_extension}" ]
+        }
+      }
     }
   }
 }

build/config/android/rules.gni

@@ -62,27 +62,31 @@ template("create_native_executable_dist") {
 
   _libraries_list = "${target_gen_dir}/${target_name}_library_dependencies.list"
 
-  _runtime_deps_file = "$target_gen_dir/${target_name}.runtimedeps"
-  _runtime_deps_target_name = "${target_name}__runtime_deps"
-  group(_runtime_deps_target_name) {
-    data = _sanitizer_runtimes
-    data_deps = []
-    if (defined(invoker.deps)) {
-      data_deps += invoker.deps
+  _shared_library_list_file = "$target_gen_dir/${target_name}.native_libs"
+  _shared_library_list_target_name = "${target_name}__shared_library_list"
+  generated_file(_shared_library_list_target_name) {
+    forward_variables_from(invoker, [ "deps" ])
+    metadata = {
+      shared_libraries = _sanitizer_runtimes
     }
-    write_runtime_deps = _runtime_deps_file
+    outputs = [
+      _shared_library_list_file,
+    ]
+    data_keys = [ "shared_libraries" ]
+    walk_keys = [ "shared_libraries_barrier" ]
+    rebase = root_build_dir
   }
 
   _find_deps_target_name = "${target_name}__find_library_dependencies"
 
   # TODO(agrieve): Extract dependent libs from GN rather than readelf.
   action_with_pydeps(_find_deps_target_name) {
-    deps = invoker.deps + [ ":$_runtime_deps_target_name" ]
+    deps = invoker.deps + [ ":$_shared_library_list_target_name" ]
     script = "//build/android/gyp/write_ordered_libraries.py"
     depfile = "$target_gen_dir/$target_name.d"
     inputs = [
       invoker.binary,
-      _runtime_deps_file,
+      _shared_library_list_file,
       android_readelf,
     ]
     outputs = [
@@ -92,7 +96,7 @@ template("create_native_executable_dist") {
       "--depfile",
       rebase_path(depfile, root_build_dir),
       "--runtime-deps",
-      rebase_path(_runtime_deps_file, root_build_dir),
+      rebase_path(_shared_library_list_file, root_build_dir),
       "--output",
       rebase_path(_libraries_list, root_build_dir),
       "--readelf",
@@ -2203,38 +2207,44 @@ if (enable_java_templates) {
     if (_shared_libraries_is_valid) {
       _native_libs_deps += invoker.shared_libraries
 
-      # To determine the filenames of all dependent shared libraries, write the
-      # runtime deps of |shared_libraries| to a file during "gn gen".
-      # write_build_config.py will then grep this file for *.so to obtain the
-      # complete list.
-      _runtime_deps_file =
-          "$target_gen_dir/${_template_name}.native.runtimedeps"
-      group("${_template_name}__runtime_deps") {
+      # Write shared library output files of all dependencies to a file. Those
+      # will be the shared libraries packaged into the APK.
+      _shared_library_list_file =
+          "$target_gen_dir/${_template_name}.native_libs"
+      generated_file("${_template_name}__shared_library_list") {
         deps = _native_libs_deps
-        write_runtime_deps = _runtime_deps_file
+        outputs = [
+          _shared_library_list_file,
+        ]
+        data_keys = [ "shared_libraries" ]
+        walk_keys = [ "shared_libraries_barrier" ]
+        rebase = root_build_dir
       }
     } else {
       # Must exist for instrumentation_test_apk() to depend on.
-      group("${_template_name}__runtime_deps") {
+      group("${_template_name}__shared_library_list") {
       }
     }
 
     if (_secondary_abi_shared_libraries_is_valid) {
       _secondary_abi_native_libs_deps += invoker.secondary_abi_shared_libraries
 
-      # To determine the filenames of all dependent shared libraries, write the
-      # runtime deps of |shared_libraries| to a file during "gn gen".
-      # write_build_config.py will then grep this file for *.so to obtain the
-      # complete list.
-      _secondary_abi_runtime_deps_file =
-          "$target_gen_dir/${_template_name}.secondary.abi.native.runtimedeps"
-      group("${_template_name}__secondary_abi__runtime_deps") {
+      # Write shared library output files of all dependencies to a file. Those
+      # will be the shared libraries packaged into the APK.
+      _secondary_abi_shared_library_list_file =
+          "$target_gen_dir/${_template_name}.secondary_abi_native_libs"
+      generated_file("${_template_name}__secondary_abi_shared_library_list") {
         deps = _secondary_abi_native_libs_deps
-        write_runtime_deps = _secondary_abi_runtime_deps_file
+        outputs = [
+          _secondary_abi_shared_library_list_file,
+        ]
+        data_keys = [ "shared_libraries" ]
+        walk_keys = [ "shared_libraries_barrier" ]
+        rebase = root_build_dir
       }
     } else {
       # Must exist for instrumentation_test_apk() to depend on.
-      group("${_template_name}__secondary_abi__runtime_deps") {
+      group("${_template_name}__secondary_abi_shared_library_list") {
       }
     }
 
@@ -2558,13 +2568,13 @@ if (enable_java_templates) {
         script = "//build/android/gyp/write_ordered_libraries.py"
         deps = [
           ":$_build_config_target",
-          ":${_template_name}__runtime_deps",
-          ":${_template_name}__secondary_abi__runtime_deps",
+          ":${_template_name}__secondary_abi_shared_library_list",
+          ":${_template_name}__shared_library_list",
         ]
         if (_native_libs_deps != []) {
-          _deps_file_to_use = _runtime_deps_file
+          _deps_file_to_use = _shared_library_list_file
         } else {
-          _deps_file_to_use = _secondary_abi_runtime_deps_file
+          _deps_file_to_use = _secondary_abi_shared_library_list_file
         }
         inputs = [
           _deps_file_to_use,
@@ -2772,11 +2782,11 @@ if (enable_java_templates) {
       # The dep is unnecessary since the runtime_deps file is created by gn gen
       # and the runtime_deps file is added to write_build_config.py's depfile.
       if (_native_libs_deps != []) {
-        shared_libraries_runtime_deps_file = _runtime_deps_file
+        shared_libraries_runtime_deps_file = _shared_library_list_file
       }
       if (_secondary_abi_native_libs_deps != []) {
         secondary_abi_shared_libraries_runtime_deps_file =
-            _secondary_abi_runtime_deps_file
+            _secondary_abi_shared_library_list_file
       }
 
       extra_shared_libraries = _extra_native_libs
@@ -3680,15 +3690,15 @@ if (enable_java_templates) {
       # Ensure unstripped libraries are included in runtime deps so that
       # symbolization can be done.
       deps = [
-        ":${_apk_target_name}__runtime_deps",
-        ":${_apk_target_name}__secondary_abi__runtime_deps",
+        ":${_apk_target_name}__secondary_abi_shared_library_list",
+        ":${_apk_target_name}__shared_library_list",
       ]
       if (defined(invoker.apk_under_test)) {
         _under_test_label =
             get_label_info(invoker.apk_under_test, "label_no_toolchain")
         deps += [
-          "${_under_test_label}__runtime_deps",
-          "${_under_test_label}__secondary_abi__runtime_deps",
+          "${_under_test_label}__secondary_abi_shared_library_list",
+          "${_under_test_label}__shared_library_list",
         ]
       }
     }
@@ -3775,8 +3785,8 @@ if (enable_java_templates) {
       }
       shared_libraries = [ invoker.shared_library ]
       deps += [
-        ":${target_name}__runtime_deps",
-        ":${target_name}__secondary_abi__runtime_deps",
+        ":${target_name}__secondary_abi_shared_library_list",
+        ":${target_name}__shared_library_list",
         "//base:base_java",
         "//testing/android/reporter:reporter_java",
       ]

build/partitioned_shared_library.gni

@@ -64,6 +64,11 @@ template("partitioned_shared_library") {
       "-Wl,-soname,lib${_output_name}.so",
       "--link-only",
     ]
+
+    # This shared library is an intermediate artifact that should not packaged
+    # into the final build. Therefore, reset metadata.
+    metadata = {
+    }
   }
 
   template("partition_action") {
@@ -79,9 +84,9 @@ template("partitioned_shared_library") {
         invoker.unstripped_output,
         invoker.stripped_output,
       ]
-      data = [
-        invoker.stripped_output,
-      ]
+      metadata = {
+        shared_libraries = [ invoker.stripped_output ]
+      }
       args = [
         "--objcopy",
         rebase_path("$clang_base_path/bin/llvm-objcopy", root_build_dir),

tools/resources/generate_resource_whitelist.gni

@@ -38,5 +38,11 @@ template("generate_resource_whitelist") {
     foreach(input, invoker.inputs) {
       args += [ rebase_path(input, root_build_dir) ]
     }
+
+    # This target's dependencies are needed at build time only. Make sure shared
+    # library dependencies don't get packaged into the final build.
+    metadata = {
+      shared_libraries_barrier = []
+    }
   }
 }