Commit 2b85bd3d authored by Peter Kotwicz's avatar Peter Kotwicz Committed by Commit Bot

Use androidx snapshot builds in android_deps/fetch_all.py

Design doc: https://docs.google.com/document/d/1SNrFzWCXb4KaegXH8nokilOHGSkTFvLfhK75ZwmIKaE/edit?usp=sharing

This CL introduces //third_party/androidx/fetch_all_androidx.py which
retrieves androidx dependencies from snapshots in
https://androidx.dev/snapshots/latest/artifacts and runs
//third_party/android_deps/fetch_all.py

The version of the androidx dependencies is generated based on
information in https://androidx.dev/snapshots/latest/artifacts/BUILD_INFO

The CL also:
- Fixes fetch_all.py --ignore-vulnerabilities command line parameter to
  skip 'dependencyCheckAnalyze' step.
- Adds the 'useDedicatedAndroidxDir' parameter to build.gradle to
replace androidx dependencies in //third_party/android_deps with a
java_group() which points to the target declaration in
//third_party/androidx

BUG=1064277

Change-Id: I652e56a9721154487c10fafaa2008fc728b9d8b1
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2399773
Commit-Queue: Peter Kotwicz <pkotwicz@chromium.org>
Reviewed-by: default avatarRobert Sesek <rsesek@chromium.org>
Reviewed-by: default avatarYaron Friedman <yfriedman@chromium.org>
Reviewed-by: default avatarAndrew Grieve <agrieve@chromium.org>
Cr-Commit-Position: refs/heads/master@{#812961}
parent 52abe908
......@@ -244,6 +244,48 @@ _RE_CIPD_CREATE = re.compile('cipd create --pkg-def cipd.yaml -tag (\S*)')
_RE_CIPD_PACKAGE = re.compile('package: (\S*)')
def _CheckVulnerabilities(build_android_deps_dir, report_dst):
logging.info(
'Running Gradle dependencyCheckAnalyze. This may take a few minutes the first time.'
)
abs_gradle_wrapper_path = os.path.join(_CHROMIUM_SRC, 'third_party',
'gradle_wrapper', 'gradlew')
# Separate command from main gradle command so that we can provide specific
# diagnostics in case of failure of this step.
gradle_cmd = [
abs_gradle_wrapper_path,
'-b',
os.path.join(build_android_deps_dir, _BUILD_GRADLE),
'dependencyCheckAnalyze',
]
report_src = os.path.join(build_android_deps_dir, 'build', 'reports')
if os.path.exists(report_dst):
shutil.rmtree(report_dst)
try:
subprocess.run(gradle_cmd, check=True)
except subprocess.CalledProcessError:
report_path = os.path.join(report_dst, 'dependency-check-report.html')
logging.error(
textwrap.dedent("""
=============================================================================
A package has a known vulnerability. It may not be in a package or packages
which you just added, but you need to resolve the problem before proceeding.
If you can't easily fix it by rolling the package to a fixed version now,
please file a crbug of type= Bug-Security providing all relevant information,
and then rerun this command with --ignore-vulnerabilities.
The html version of the report is avialable at: {}
=============================================================================
""".format(report_path)))
raise
finally:
if os.path.exists(report_src):
CopyFileOrDirectory(report_src, report_dst)
def GetCipdPackageInfo(cipd_yaml_path):
"""Returns the CIPD package name corresponding to a given cipd.yaml file.
......@@ -449,11 +491,6 @@ def main():
raise Exception('Missing files from {}: {}'.format(
_CHROMIUM_SRC, missing_files))
abs_build_gradle_path = os.path.join(abs_android_deps_dir, _BUILD_GRADLE)
# Path to the gradlew script used to run build.gradle.
abs_gradle_wrapper_path = os.path.join(_CHROMIUM_SRC, 'third_party',
'gradle_wrapper', 'gradlew')
with BuildDir(args.build_dir) as build_dir:
build_android_deps_dir = os.path.join(build_dir, args.android_deps_dir)
......@@ -467,57 +504,27 @@ def main():
CopyFileOrDirectory(os.path.join(_CHROMIUM_SRC, path),
os.path.join(build_dir, dest))
logging.info(
'Running Gradle dependencyCheckAnalyze. This may take a few minutes the first time.'
)
# Not run as part of the main gradle command below
# such that we can provide specific diagnostics in case
# of failure of this build stage.
gradle_cmd = [
abs_gradle_wrapper_path,
'-b',
os.path.join(build_dir, abs_build_gradle_path),
'dependencyCheckAnalyze',
]
if debug:
gradle_cmd.append('--debug')
report_src = os.path.join(build_android_deps_dir, 'build', 'reports')
if not args.ignore_vulnerabilities:
report_dst = os.path.join(abs_android_deps_dir,
'vulnerability_reports')
if os.path.exists(report_dst):
shutil.rmtree(report_dst)
try:
subprocess.run(gradle_cmd, check=True)
except subprocess.CalledProcessError:
report_path = os.path.join(report_dst,
'dependency-check-report.html')
logging.error(
textwrap.dedent("""
=============================================================================
A package has a known vulnerability. It may not be in a package or packages
which you just added, but you need to resolve the problem before proceeding.
If you can't easily fix it by rolling the package to a fixed version now,
please file a crbug of type= Bug-Security providing all relevant information,
and then rerun this command with --ignore-vulnerabilities.
The html version of the report is avialable at: {}
=============================================================================
""".format(report_path)))
if not args.ignore_vulnerabilities:
raise
finally:
if os.path.exists(report_src):
CopyFileOrDirectory(report_src, report_dst)
_CheckVulnerabilities(build_android_deps_dir, report_dst)
logging.info('Running Gradle.')
# Path to the gradlew script used to run build.gradle.
abs_gradle_wrapper_path = os.path.join(_CHROMIUM_SRC, 'third_party',
'gradle_wrapper', 'gradlew')
# This gradle command generates the new DEPS and BUILD.gn files, it can
# also handle special cases.
# Edit BuildConfigGenerator.groovy#addSpecialTreatment for such cases.
gradle_cmd = [
abs_gradle_wrapper_path,
'-b',
os.path.join(build_dir, abs_build_gradle_path),
os.path.join(build_android_deps_dir, _BUILD_GRADLE),
'setupRepository',
'--stacktrace',
]
......
.gradle/
.idea/
.settings/
build.gradle
local.properties
# JAR and AAR files will be fetched via CIPD.
libs/*/*.jar
libs/*/*.aar
[style]
based_on_style = pep8
file://third_party/android_deps/OWNERS
# OS: Android
# COMPONENT: Tools
Name: Androidx Repository
Short Name: androidx
Version: 1
License: Refer to additional_readme_paths.json
License Android Compatible: yes
License File: NOT_SHIPPED
Security Critical: yes
Description:
A gradle-specified repository for Androidx dependencies.
# Copyright 2019 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Help R8 to remove debug related coded. All of these static constants are
# initialized to: Log.isLoggable(DEBUG, TAG).
-assumevalues class androidx.mediarouter.** {
static boolean DEBUG return false;
}
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
apply plugin: ChromiumPlugin
repositories {
mavenCentral()
maven {
url 'https://maven.google.com'
}
maven {
url '{{androidx_repository_url}}'
}
}
dependencies {
// Note about the configuration names: they are defined in buildSrc/ChromiumPlugin
compile "androidx.lifecycle:lifecycle-runtime:{{androidx_dependency_version}}"
compile "androidx.lifecycle:lifecycle-common:{{androidx_dependency_version}}"
compile "androidx.lifecycle:lifecycle-common-java8:{{androidx_dependency_version}}"
compile "androidx.lifecycle:lifecycle-livedata-core:{{androidx_dependency_version}}"
compile "androidx.lifecycle:lifecycle-viewmodel:{{androidx_dependency_version}}"
compile "androidx.core:core:{{androidx_dependency_version}}"
compile "androidx.activity:activity:{{androidx_dependency_version}}"
compile "androidx.annotation:annotation:{{androidx_dependency_version}}"
compile "androidx.appcompat:appcompat:{{androidx_dependency_version}}"
compile "androidx.appcompat:appcompat-resources:{{androidx_dependency_version}}"
compile "androidx.asynclayoutinflater:asynclayoutinflater:{{androidx_dependency_version}}"
compile "androidx.cardview:cardview:{{androidx_dependency_version}}"
compile "androidx.concurrent:concurrent-futures:{{androidx_dependency_version}}"
compile "androidx.collection:collection:{{androidx_dependency_version}}"
compile "androidx.coordinatorlayout:coordinatorlayout:{{androidx_dependency_version}}"
compile "androidx.customview:customview:{{androidx_dependency_version}}"
compile "androidx.drawerlayout:drawerlayout:{{androidx_dependency_version}}"
compile "androidx.fragment:fragment:{{androidx_dependency_version}}"
compile "androidx.gridlayout:gridlayout:{{androidx_dependency_version}}"
compile "androidx.interpolator:interpolator:{{androidx_dependency_version}}"
compile "androidx.leanback:leanback:{{androidx_dependency_version}}"
compile "androidx.leanback:leanback-preference:{{androidx_dependency_version}}"
compile "androidx.localbroadcastmanager:localbroadcastmanager:{{androidx_dependency_version}}"
compile "androidx.media:media:{{androidx_dependency_version}}"
compile "androidx.mediarouter:mediarouter:{{androidx_dependency_version}}"
compile "androidx.preference:preference:{{androidx_dependency_version}}"
compile "androidx.slice:slice-core:{{androidx_dependency_version}}"
compile "androidx.slice:slice-builders:{{androidx_dependency_version}}"
compile "androidx.recyclerview:recyclerview:{{androidx_dependency_version}}"
compile "androidx.savedstate:savedstate:{{androidx_dependency_version}}"
compile "androidx.swiperefreshlayout:swiperefreshlayout:{{androidx_dependency_version}}"
compile "androidx.vectordrawable:vectordrawable:{{androidx_dependency_version}}"
compile "androidx.vectordrawable:vectordrawable-animated:{{androidx_dependency_version}}"
compile "androidx.tvprovider:tvprovider:{{androidx_dependency_version}}"
compile "androidx.viewpager:viewpager:{{androidx_dependency_version}}"
compile "androidx.window:window:{{androidx_dependency_version}}"
compile "androidx.exifinterface:exifinterface:{{androidx_dependency_version}}"
// Those are for use by doubledown libraries.
compile "androidx.arch.core:core-common:{{androidx_dependency_version}}"
compile "androidx.annotation:annotation-experimental:{{androidx_dependency_version}}"
compile "androidx.annotation:annotation:{{androidx_dependency_version}}"
compile "androidx.lifecycle:lifecycle-runtime:{{androidx_dependency_version}}"
compile "androidx.transition:transition:{{androidx_dependency_version}}"
compile "androidx.viewpager2:viewpager2:{{androidx_dependency_version}}"
compile "androidx.multidex:multidex:2.0.0"
compile "androidx.webkit:webkit:{{androidx_dependency_version}}"
// Used by 1p Play Services.
compile "androidx.legacy:legacy-support-v4:1.0.0"
// testCompile targets have test_only = true.
androidTestCompile "androidx.test:core:1.2.0"
androidTestCompile "androidx.test:monitor:1.2.0"
androidTestCompile "androidx.test:rules:1.2.0"
androidTestCompile "androidx.test:runner:1.2.0"
androidTestCompile "androidx.test.espresso:espresso-contrib:3.2.0"
androidTestCompile "androidx.test.espresso:espresso-core:3.2.0"
androidTestCompile "androidx.test.espresso:espresso-intents:3.2.0"
androidTestCompile "androidx.test.espresso:espresso-web:3.2.0"
androidTestCompile "androidx.test.ext:junit:1.1.1"
androidTestCompile "androidx.test.uiautomator:uiautomator:2.2.0"
}
task setUpRepository(type: BuildConfigGenerator) {
// Paths are relative to the chromium source root.
repositoryPath 'third_party/androidx'
chromiumSourceRoot '../..'
cipdBucket 'chromium'
internalTargetVisibility = [ '//third_party/android_deps:*', ':*' ]
skipLicenses true
}
#!/usr/bin/env python3
# Copyright 2020 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""A script to generate build.gradle from template and run fetch_all.py
More specifically, to generate build.gradle:
- It downloads the BUILD_INFO file for the latest androidx snapshot from
https://androidx.dev/snapshots/
- It replaces {{androidx_repository_url}} with the URL for the latest snapshot
- For each dependency, it looks up the version in the BUILD_INFO file and
substitutes the version into {{androidx_dependency_version}}.
"""
import contextlib
import json
import os
import re
import requests
import shutil
import subprocess
import tempfile
_ANDROIDX_PATH = os.path.normpath(os.path.join(__file__, '..'))
_FETCH_ALL_PATH = os.path.normpath(
os.path.join(_ANDROIDX_PATH, '..', 'android_deps', 'fetch_all.py'))
# URL to BUILD_INFO in latest androidx snapshot.
_ANDROIDX_LATEST_SNAPSHOT_BUILD_INFO_URL = 'https://androidx.dev/snapshots/latest/artifacts/BUILD_INFO'
def _parse_dir_list(dir_list):
"""Computes 'library_group:library_name'->library_version mapping.
Args:
dir_list: List of androidx library directories.
"""
dependency_version_map = dict()
for dir_entry in dir_list:
stripped_dir = dir_entry.strip()
if not stripped_dir.startswith('repository/androidx/'):
continue
dir_components = stripped_dir.split('/')
# Expected format:
# "repository/androidx/library_group/library_name/library_version/pom_or_jar"
if len(dir_components) < 6:
continue
dependency_module = 'androidx.{}:{}'.format(dir_components[2],
dir_components[3])
if dependency_module not in dependency_version_map:
dependency_version_map[dependency_module] = dir_components[4]
return dependency_version_map
def _compute_replacement(dependency_version_map, androidx_repository_url,
line):
"""Computes output line for build.gradle from build.gradle.template line.
Replaces {{android_repository_url}} and {{androidx_dependency_version}}.
Args:
dependency_version_map: An "dependency_group:dependency_name"->dependency_version mapping.
androidx_repository_url: URL of the maven repository.
line: Input line from the build.gradle.template.
"""
line = line.replace('{{androidx_repository_url}}', androidx_repository_url)
match = re.search(r'"(\S+):{{androidx_dependency_version}}"', line)
if not match:
return line
version = dependency_version_map.get(match.group(1))
if not version:
return line
return line.replace('{{androidx_dependency_version}}', version)
@contextlib.contextmanager
def _build_dir():
dirname = tempfile.mkdtemp()
try:
yield dirname
finally:
shutil.rmtree(dirname)
def _download_and_parse_build_info():
"""Downloads and parses BUILD_INFO file."""
with _build_dir() as build_dir:
androidx_build_info_response = requests.get(
_ANDROIDX_LATEST_SNAPSHOT_BUILD_INFO_URL)
androidx_build_info_path = os.path.join(build_dir, 'BUILD_INFO')
with open(androidx_build_info_path, 'w') as f:
f.write(androidx_build_info_response.text)
# Compute repository URL from resolved BUILD_INFO url in case 'latest' redirect changes.
androidx_snapshot_repository_url = androidx_build_info_response.url.rsplit(
'/', 1)[0] + '/repository'
with open(androidx_build_info_path, 'r') as f:
build_info_dict = json.loads(f.read())
dir_list = build_info_dict['target']['dir_list']
dependency_version_map = _parse_dir_list(dir_list)
return (dependency_version_map, androidx_snapshot_repository_url)
def _process_build_gradle(dependency_version_map, androidx_repository_url):
"""Generates build.gradle from template.
Args:
dependency_version_map: An "dependency_group:dependency_name"->dependency_version mapping.
androidx_repository_url: URL of the maven repository.
"""
build_gradle_template_path = os.path.join(_ANDROIDX_PATH,
'build.gradle.template')
build_gradle_out_path = os.path.join(_ANDROIDX_PATH, 'build.gradle')
# |build_gradle_out_path| is not deleted after script has finished running. The file is in
# .gitignore and thus will be excluded from uploaded CLs.
with open(build_gradle_template_path, 'r') as template_f, \
open(build_gradle_out_path, 'w') as out:
for template_line in template_f:
replacement = _compute_replacement(dependency_version_map,
androidx_repository_url,
template_line)
out.write(replacement)
def main():
dependency_version_map, androidx_snapshot_repository_url = (
_download_and_parse_build_info())
_process_build_gradle(dependency_version_map,
androidx_snapshot_repository_url)
fetch_all_cmd = [
_FETCH_ALL_PATH, '--android-deps-dir',
os.path.join('third_party', 'androidx'), '--ignore-vulnerabilities'
]
subprocess.run(fetch_all_cmd)
if __name__ == '__main__':
main()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment