Commit 723c1509 authored by Andrew Grieve's avatar Andrew Grieve Committed by Commit Bot

Android: Add enable_jdk_library_desugaring GN arg

When enabled, it will cause some jdk library symbols to be desugared
when dexing, and adds an extra classes.dex file to every .apk and .aab.

Bug: 1056751
Change-Id: Ib51b00f93fbe6120d44553782ce56ae60f7e6bb9
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2144357
Commit-Queue: Andrew Grieve <agrieve@chromium.org>
Reviewed-by: default avatarPeter Wen <wnwen@chromium.org>
Cr-Commit-Position: refs/heads/master@{#761957}
parent b26f53a8
......@@ -1323,6 +1323,7 @@ _ANDROID_SPECIFIC_PYDEPS_FILES = [
'build/android/gyp/desugar.pydeps',
'build/android/gyp/dexsplitter.pydeps',
'build/android/gyp/dex.pydeps',
'build/android/gyp/dex_jdk_libs.pydeps',
'build/android/gyp/dist_aar.pydeps',
'build/android/gyp/filter_zip.pydeps',
'build/android/gyp/gcc_preprocess.pydeps',
......
......@@ -16,6 +16,13 @@ if (enable_java_templates) {
create_cache = true
}
if (enable_jdk_library_desugaring) {
dex_jdk_libs("all_jdk_libs") {
output = "$target_out_dir/$target_name.l8.dex"
min_sdk_version = default_min_sdk_version
}
}
# Write to a file some GN vars that are useful to scripts that use the output
# directory. Format is chosen as easliy importable by both python and bash.
_lines = [
......
......@@ -58,6 +58,9 @@ def _ParseArgs(args):
default='apk', help='Specify output format.')
parser.add_argument('--dex-file',
help='Path to the classes.dex to use')
parser.add_argument(
'--jdk-libs-dex-file',
help='Path to classes.dex created by dex_jdk_libs.py')
parser.add_argument('--uncompress-dex', action='store_true',
help='Store .dex files uncompressed in the APK')
parser.add_argument('--native-libs',
......@@ -425,19 +428,29 @@ def main(args):
if options.dex_file:
with open(options.dex_file) as dex_file_obj:
if options.dex_file.endswith('.dex'):
max_dex_number = 1
# This is the case for incremental_install=true.
add_to_zip(
apk_dex_dir + 'classes.dex',
dex_file_obj.read(),
compress=not options.uncompress_dex)
else:
max_dex_number = 0
with zipfile.ZipFile(dex_file_obj) as dex_zip:
for dex in (d for d in dex_zip.namelist() if d.endswith('.dex')):
max_dex_number += 1
add_to_zip(
apk_dex_dir + dex,
dex_zip.read(dex),
compress=not options.uncompress_dex)
if options.jdk_libs_dex_file:
with open(options.jdk_libs_dex_file) as dex_file_obj:
add_to_zip(
apk_dex_dir + 'classes{}.dex'.format(max_dex_number + 1),
dex_file_obj.read(),
compress=not options.uncompress_dex)
# 4. Native libraries.
logging.debug('Adding lib/')
added_libs = _AddNativeLibraries(
......
......@@ -68,6 +68,8 @@ def _ParseArgs(args):
'--bootclasspath',
action='append',
help='GN-list of bootclasspath. Needed for --desugar')
parser.add_argument(
'--desugar-jdk-libs-json', help='Path to desugar_jdk_libs.json.')
parser.add_argument(
'--classpath',
action='append',
......@@ -498,6 +500,8 @@ def main(args):
input_paths += options.classpath
input_paths += options.bootclasspath
if options.desugar_jdk_libs_json:
dex_cmd += ['--desugared-lib', options.desugar_jdk_libs_json]
if options.force_enable_assertions:
dex_cmd += ['--force-enable-assertions']
......
#!/usr/bin/env python
#
# Copyright 2020 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import argparse
import os
import shutil
import subprocess
import sys
import zipfile
from util import build_utils
def _ParseArgs(args):
args = build_utils.ExpandFileArgs(args)
parser = argparse.ArgumentParser()
parser.add_argument('--output', required=True, help='Dex output path.')
parser.add_argument('--r8-path', required=True, help='Path to R8 jar.')
parser.add_argument(
'--desugar-jdk-libs-json', help='Path to desugar_jdk_libs.json.')
parser.add_argument(
'--desugar-jdk-libs-jar', help='Path to desugar_jdk_libs.jar.')
parser.add_argument('--min-api', help='minSdkVersion', required=True)
options = parser.parse_args(args)
return options
def main(args):
options = _ParseArgs(args)
# TODO(agrieve): Spews a lot of stderr about missing classes.
with build_utils.TempDir() as tmp_dir:
cmd = [
build_utils.JAVA_PATH,
'-jar',
options.r8_path,
'l8',
'--min-api',
options.min_api,
#'--lib', build_utils.JAVA_HOME,
'--desugared-lib',
options.desugar_jdk_libs_json,
'--output',
tmp_dir,
options.desugar_jdk_libs_jar
]
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
if os.path.exists(os.path.join(tmp_dir, 'classes2.dex')):
raise Exception('Achievement unlocked: desugar_jdk_libs is multidex!')
shutil.move(os.path.join(tmp_dir, 'classes.dex'), options.output)
if __name__ == '__main__':
main(sys.argv[1:])
# Generated by running:
# build/print_python_deps.py --root build/android/gyp --output build/android/gyp/dex_jdk_libs.pydeps build/android/gyp/dex_jdk_libs.py
../../gn_helpers.py
dex_jdk_libs.py
util/__init__.py
util/build_utils.py
......@@ -109,6 +109,8 @@ def _ParseOptions():
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('--proguard-path', help='Path to the proguard.jar to use.')
group.add_argument('--r8-path', help='Path to the R8.jar to use.')
parser.add_argument(
'--desugar-jdk-libs-json', help='Path to desugar_jdk_libs.json.')
parser.add_argument(
'--input-paths', required=True, help='GN-list of .jar files to optimize.')
parser.add_argument(
......@@ -235,11 +237,8 @@ def _OptimizeWithR8(options,
tmp_mapping_path,
]
for lib in libraries:
cmd += ['--lib', lib]
for config_file in config_paths:
cmd += ['--pg-conf', config_file]
if options.desugar_jdk_libs_json:
cmd += ['--desugared-lib', options.desugar_jdk_libs_json]
if options.min_api:
cmd += ['--min-api', options.min_api]
......@@ -247,6 +246,12 @@ def _OptimizeWithR8(options,
if options.force_enable_assertions:
cmd += ['--force-enable-assertions']
for lib in libraries:
cmd += ['--lib', lib]
for config_file in config_paths:
cmd += ['--pg-conf', config_file]
if options.main_dex_rules_path:
for main_dex_rule in options.main_dex_rules_path:
cmd += ['--main-dex-rules', main_dex_rule]
......
......@@ -237,6 +237,10 @@ if (is_android || is_chromeos) {
# Desugar lambdas and interfaces methods using Desugar.jar rather than
# D8/R8. D8/R8 will still be used for backported method desugaring.
enable_bazel_desugar = true
# Enables Java library desugaring.
# This will cause an extra 1MB classes.dex file to appear in every apk.
enable_jdk_library_desugaring = false
}
# Path to where selected build variables are written to.
......
......@@ -55,6 +55,8 @@ _java_lib_exceptions =
]
_r8_path = "//third_party/r8/lib/r8.jar"
_desugar_jdk_libs_json = "//third_party/r8/desugar_jdk_libs.json"
_desugar_jdk_libs_jar = "//third_party/android_deps/libs/com_android_tools_desugar_jdk_libs/desugar_jdk_libs-1.0.5.jar"
_dexdump_path = "$android_sdk_build_tools/dexdump"
_dexlayout_path = "//third_party/android_build_tools/art/dexlayout"
......@@ -1067,6 +1069,13 @@ if (enable_java_templates) {
rebase_path(_r8_path, root_build_dir),
]
inputs += [ _r8_path ]
if (enable_jdk_library_desugaring) {
args += [
"--desugar-jdk-libs-json",
rebase_path(_desugar_jdk_libs_json, root_build_dir),
]
inputs += [ _desugar_jdk_libs_json ]
}
} else {
_proguard_jar_path = invoker.proguard_jar_path
args += [
......@@ -1455,6 +1464,15 @@ if (enable_java_templates) {
if (_enable_desugar) {
args += [ "--desugar" ]
# Passing the flag for dex merging causes invalid dex files to be created.
if (enable_jdk_library_desugaring && !_is_dex_merging) {
inputs += [ _desugar_jdk_libs_json ]
args += [
"--desugar-jdk-libs-json",
rebase_path(_desugar_jdk_libs_json, root_build_dir),
]
}
}
if (_desugar_needs_classpath) {
args += [
......@@ -1482,6 +1500,32 @@ if (enable_java_templates) {
}
}
# Variables
# output: Path to output ".l8.dex".
# min_sdk_version: The minimum Android SDK version this target supports.
template("dex_jdk_libs") {
action_with_pydeps(target_name) {
script = "//build/android/gyp/dex_jdk_libs.py"
inputs = [
_r8_path,
_desugar_jdk_libs_json,
_desugar_jdk_libs_jar,
]
outputs = [ invoker.output ]
args = [
"--r8-path",
rebase_path(_r8_path, root_build_dir),
"--desugar-jdk-libs-json",
rebase_path(_desugar_jdk_libs_json, root_build_dir),
"--desugar-jdk-libs-jar",
rebase_path(_desugar_jdk_libs_jar, root_build_dir),
"--output",
rebase_path(invoker.output, root_build_dir),
"--min-api=${invoker.min_sdk_version}",
]
}
}
template("jacoco_instr") {
action_with_pydeps(target_name) {
forward_variables_from(invoker,
......@@ -2555,6 +2599,7 @@ if (enable_java_templates) {
# asset information.
# deps: Specifies the dependencies of this target.
# dex_path: Path to classes.dex file to include (optional).
# jdk_libs_dex: Path to classes.dex for desugar_jdk_libs.
# packaged_resources_path: Path to .ap_ to use.
# output_apk_path: Output path for the generated .apk.
# min_sdk_version: The minimum Android SDK version this target supports.
......@@ -2655,8 +2700,18 @@ if (enable_java_templates) {
args += [ "--java-resources=@FileArg($_rebased_build_config:java_resources_jars)" ]
}
if (defined(invoker.dex_path)) {
_rebased_dex_path = rebase_path(invoker.dex_path, root_build_dir)
args += [ "--dex-file=$_rebased_dex_path" ]
inputs += [ invoker.dex_path ]
args += [
"--dex-file",
rebase_path(invoker.dex_path, root_build_dir),
]
}
if (defined(invoker.jdk_libs_dex)) {
inputs += [ invoker.jdk_libs_dex ]
args += [
"--jdk-libs-dex-file",
rebase_path(invoker.jdk_libs_dex, root_build_dir),
]
}
if ((defined(invoker.loadable_modules) &&
invoker.loadable_modules != []) ||
......@@ -3897,6 +3952,19 @@ template("create_android_app_bundle_module") {
]
}
}
# TODO(https://crbug.com/1056751): Add support for proguarding jdk libs.
if (enable_jdk_library_desugaring && invoker.module_name == "base") {
_all_jdk_libs = "//build/android:all_jdk_libs"
deps += [ _all_jdk_libs ]
_jdk_libs_dex = get_label_info(_all_jdk_libs, "target_out_dir") +
"/all_jdk_libs.l8.dex"
inputs += [ _jdk_libs_dex ]
args += [
"--jdk-libs-dex-file",
rebase_path(_jdk_libs_dex, root_build_dir),
]
}
}
}
......
......@@ -3184,6 +3184,14 @@ if (enable_java_templates) {
deps = _deps + [ ":$_build_config_target" ]
# TODO(https://crbug.com/1056751): Add support for proguarding jdk libs.
if (enable_jdk_library_desugaring) {
_all_jdk_libs = "//build/android:all_jdk_libs"
deps += [ _all_jdk_libs ]
jdk_libs_dex = get_label_info(_all_jdk_libs, "target_out_dir") +
"/all_jdk_libs.l8.dex"
}
if (_incremental_apk) {
_dex_target =
"//build/android/incremental_install:bootstrap_java__dex"
......
......@@ -7,6 +7,14 @@ License File: NOT_SHIPPED
Security Critical: no
Description:
lib/r8.jar
- Contains R8 (proguard replacement) as well as D8 (dexer).
desugar_jdk_libs.json
- Configuration for Java library desugaring.
- Taken from fa020a2164ea087c1d79c8c6b0f435f199f8d196
- This file should be kept in sync with the prebuilt shims that live
at //third_party/android_deps/libs/com_android_tools_desugar_jdk_libs.
R8 is a proguard-like optimizer that also has the ability to dex.
Local Modifications:
......@@ -31,6 +39,9 @@ cp r8.jar $CHROMIUM_SRC/third_party/r8/libs/r8.jar
# Upload to CIPD:
cipd create --pkg-def cipd.yaml # Make note of the instance ID
# Copy over desugar config:
cp src/library_desugar/desugar_jdk_libs.json $CHROMIUM_SRC/third_party/r8
# Manually update:
* README.chromium (version number via "java -jar lib/r8.jar --version")
* //DEPS (instance ID output by cipd create)
......
{
"configuration_format_version": 3,
"version": "0.11.1",
"required_compilation_api_level": 26,
"synthesized_library_classes_package_prefix": "j$.",
"library_flags": [
{
"api_level_below_or_equal": 25,
"rewrite_prefix": {
"j$.time.": "java.time.",
"java.time.": "j$.time.",
"java.util.Desugar": "j$.util.Desugar"
},
"backport": {
"java.lang.Double8": "java.lang.Double",
"java.lang.Integer8": "java.lang.Integer",
"java.lang.Long8": "java.lang.Long",
"java.lang.Math8": "java.lang.Math"
},
"retarget_lib_member": {
"java.util.Date#toInstant": "java.util.DesugarDate",
"java.util.GregorianCalendar#toZonedDateTime": "java.util.DesugarGregorianCalendar"
}
},
{
"api_level_below_or_equal": 23,
"rewrite_prefix": {
"j$.util.Optional": "java.util.Optional",
"j$.util.LongSummaryStatistics": "java.util.LongSummaryStatistics",
"j$.util.IntSummaryStatistics": "java.util.IntSummaryStatistics",
"j$.util.DoubleSummaryStatistics": "java.util.DoubleSummaryStatistics",
"java.util.stream.": "j$.util.stream.",
"java.util.function.": "j$.util.function.",
"java.util.Comparators": "j$.util.Comparators",
"java.util.DoubleSummaryStatistics": "j$.util.DoubleSummaryStatistics",
"java.util.IntSummaryStatistics": "j$.util.IntSummaryStatistics",
"java.util.LongSummaryStatistics": "j$.util.LongSummaryStatistics",
"java.util.Objects": "j$.util.Objects",
"java.util.Optional": "j$.util.Optional",
"java.util.PrimitiveIterator": "j$.util.PrimitiveIterator",
"java.util.SortedSet$1": "j$.util.SortedSet$1",
"java.util.Spliterator": "j$.util.Spliterator",
"java.util.StringJoiner": "j$.util.StringJoiner",
"java.util.Tripwire": "j$.util.Tripwire",
"java.util.concurrent.DesugarUnsafe": "j$.util.concurrent.DesugarUnsafe",
"java.util.concurrent.ThreadLocalRandom": "j$.util.concurrent.ThreadLocalRandom",
"java.util.concurrent.atomic.DesugarAtomic": "j$.util.concurrent.atomic.DesugarAtomic",
"java.util.concurrent.ConcurrentHashMap": "j$.util.concurrent.ConcurrentHashMap"
},
"retarget_lib_member": {
"java.util.Arrays#stream": "java.util.DesugarArrays",
"java.util.Arrays#spliterator": "java.util.DesugarArrays",
"java.util.LinkedHashSet#spliterator": "java.util.DesugarLinkedHashSet"
},
"dont_rewrite": [
"java.util.Iterator#remove"
],
"emulate_interface": {
"java.lang.Iterable": "j$.lang.Iterable",
"java.util.Map$Entry": "j$.util.Map$Entry",
"java.util.Collection": "j$.util.Collection",
"java.util.Map": "j$.util.Map",
"java.util.Iterator": "j$.util.Iterator",
"java.util.Comparator": "j$.util.Comparator",
"java.util.List": "j$.util.List",
"java.util.SortedSet": "j$.util.SortedSet",
"java.util.Set": "j$.util.Set",
"java.util.concurrent.ConcurrentMap": "j$.util.concurrent.ConcurrentMap"
},
"custom_conversion": {
"java.util.Optional": "java.util.OptionalConversions",
"java.util.OptionalDouble": "java.util.OptionalConversions",
"java.util.OptionalInt": "java.util.OptionalConversions",
"java.util.OptionalLong": "java.util.OptionalConversions",
"java.util.LongSummaryStatistics": "java.util.LongSummaryStatisticsConversions",
"java.util.IntSummaryStatistics": "java.util.IntSummaryStatisticsConversions",
"java.util.DoubleSummaryStatistics": "java.util.DoubleSummaryStatisticsConversions"
}
}
],
"program_flags": [
{
"api_level_below_or_equal": 25,
"rewrite_prefix": {
"java.time.": "j$.time.",
"java.util.Desugar": "j$.util.Desugar"
},
"retarget_lib_member": {
"java.util.Calendar#toInstant": "java.util.DesugarCalendar",
"java.util.Date#from": "java.util.DesugarDate",
"java.util.Date#toInstant": "java.util.DesugarDate",
"java.util.GregorianCalendar#from": "java.util.DesugarGregorianCalendar",
"java.util.GregorianCalendar#toZonedDateTime": "java.util.DesugarGregorianCalendar"
},
"custom_conversion": {
"java.time.ZonedDateTime": "java.time.TimeConversions",
"java.time.LocalDate": "java.time.TimeConversions",
"java.time.Duration": "java.time.TimeConversions",
"java.time.ZoneId": "java.time.TimeConversions",
"java.time.MonthDay": "java.time.TimeConversions",
"java.time.Instant": "java.time.TimeConversions"
}
},
{
"api_level_below_or_equal": 23,
"rewrite_prefix": {
"java.util.stream.": "j$.util.stream.",
"java.util.function.": "j$.util.function.",
"java.util.DoubleSummaryStatistics": "j$.util.DoubleSummaryStatistics",
"java.util.IntSummaryStatistics": "j$.util.IntSummaryStatistics",
"java.util.LongSummaryStatistics": "j$.util.LongSummaryStatistics",
"java.util.Optional": "j$.util.Optional",
"java.util.PrimitiveIterator": "j$.util.PrimitiveIterator",
"java.util.Spliterator": "j$.util.Spliterator",
"java.util.StringJoiner": "j$.util.StringJoiner",
"java.util.concurrent.ThreadLocalRandom": "j$.util.concurrent.ThreadLocalRandom",
"java.util.concurrent.atomic.DesugarAtomic": "j$.util.concurrent.atomic.DesugarAtomic",
"java.util.concurrent.ConcurrentHashMap": "j$.util.concurrent.ConcurrentHashMap"
},
"retarget_lib_member": {
"java.util.Arrays#stream": "java.util.DesugarArrays",
"java.util.Arrays#spliterator": "java.util.DesugarArrays",
"java.util.LinkedHashSet#spliterator": "java.util.DesugarLinkedHashSet",
"java.util.concurrent.atomic.AtomicInteger#getAndUpdate": "java.util.concurrent.atomic.DesugarAtomicInteger",
"java.util.concurrent.atomic.AtomicInteger#updateAndGet": "java.util.concurrent.atomic.DesugarAtomicInteger",
"java.util.concurrent.atomic.AtomicInteger#getAndAccumulate": "java.util.concurrent.atomic.DesugarAtomicInteger",
"java.util.concurrent.atomic.AtomicInteger#accumulateAndGet": "java.util.concurrent.atomic.DesugarAtomicInteger",
"java.util.concurrent.atomic.AtomicLong#getAndUpdate": "java.util.concurrent.atomic.DesugarAtomicLong",
"java.util.concurrent.atomic.AtomicLong#updateAndGet": "java.util.concurrent.atomic.DesugarAtomicLong",
"java.util.concurrent.atomic.AtomicLong#getAndAccumulate": "java.util.concurrent.atomic.DesugarAtomicLong",
"java.util.concurrent.atomic.AtomicLong#accumulateAndGet": "java.util.concurrent.atomic.DesugarAtomicLong",
"java.util.concurrent.atomic.AtomicReference#getAndUpdate": "java.util.concurrent.atomic.DesugarAtomicReference",
"java.util.concurrent.atomic.AtomicReference#updateAndGet": "java.util.concurrent.atomic.DesugarAtomicReference",
"java.util.concurrent.atomic.AtomicReference#getAndAccumulate": "java.util.concurrent.atomic.DesugarAtomicReference",
"java.util.concurrent.atomic.AtomicReference#accumulateAndGet": "java.util.concurrent.atomic.DesugarAtomicReference",
"java.util.Collections#synchronizedMap": "java.util.DesugarCollections",
"java.util.Collections#synchronizedSortedMap": "java.util.DesugarCollections"
},
"dont_rewrite": [
"java.util.Iterator#remove"
],
"emulate_interface": {
"java.lang.Iterable": "j$.lang.Iterable",
"java.util.Map$Entry": "j$.util.Map$Entry",
"java.util.Collection": "j$.util.Collection",
"java.util.Map": "j$.util.Map",
"java.util.Iterator": "j$.util.Iterator",
"java.util.Comparator": "j$.util.Comparator",
"java.util.List": "j$.util.List",
"java.util.SortedSet": "j$.util.SortedSet",
"java.util.Set": "j$.util.Set",
"java.util.concurrent.ConcurrentMap": "j$.util.concurrent.ConcurrentMap"
},
"custom_conversion": {
"java.util.Optional": "java.util.OptionalConversions",
"java.util.OptionalDouble": "java.util.OptionalConversions",
"java.util.OptionalInt": "java.util.OptionalConversions",
"java.util.OptionalLong": "java.util.OptionalConversions",
"java.util.LongSummaryStatistics": "java.util.LongSummaryStatisticsConversions",
"java.util.IntSummaryStatistics": "java.util.IntSummaryStatisticsConversions",
"java.util.DoubleSummaryStatistics": "java.util.DoubleSummaryStatisticsConversions"
}
}
],
"shrinker_config": [
"-keepclassmembers class j$.util.concurrent.ConcurrentHashMap$TreeBin { int lockState; }",
"-keepclassmembers class j$.util.concurrent.ConcurrentHashMap { int sizeCtl; int transferIndex; long baseCount; int cellsBusy; }",
"-keepclassmembers class j$.util.concurrent.ConcurrentHashMap$CounterCell { long value; }",
"-keepclassmembers enum * { public static **[] values(); public static ** valueOf(java.lang.String); }",
"-keeppackagenames j$",
"-keepclassmembers class j$.util.IntSummaryStatistics { long count; long sum; int min; int max; }",
"-keepclassmembers class j$.util.LongSummaryStatistics { long count; long sum; long min; long max; }",
"-keepclassmembers class j$.util.DoubleSummaryStatistics { long count; double sum; double min; double max; }"
]
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment