Use androidx snapshot builds in android_deps/fetch_all.py

Design doc: https://docs.google.com/document/d/1SNrFzWCXb4KaegXH8nokilOHGSkTFvLfhK75ZwmIKaE/edit?usp=sharing

This CL introduces //third_party/androidx/fetch_all_androidx.py which
retrieves androidx dependencies from snapshots in
https://androidx.dev/snapshots/latest/artifacts and runs
//third_party/android_deps/fetch_all.py

The version of the androidx dependencies is generated based on
information in https://androidx.dev/snapshots/latest/artifacts/BUILD_INFO

The CL also:
- Fixes fetch_all.py --ignore-vulnerabilities command line parameter to
  skip 'dependencyCheckAnalyze' step.
- Adds the 'useDedicatedAndroidxDir' parameter to build.gradle to
replace androidx dependencies in //third_party/android_deps with a
java_group() which points to the target declaration in
//third_party/androidx

BUG=1064277

Change-Id: I652e56a9721154487c10fafaa2008fc728b9d8b1
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2399773
Commit-Queue: Peter Kotwicz <pkotwicz@chromium.org>
Reviewed-by: Robert Sesek <rsesek@chromium.org>
Reviewed-by: Yaron Friedman <yfriedman@chromium.org>
Reviewed-by: Andrew Grieve <agrieve@chromium.org>
Cr-Commit-Position: refs/heads/master@{#812961}

third_party/android_deps/fetch_all.py

@@ -244,6 +244,48 @@ _RE_CIPD_CREATE = re.compile('cipd create --pkg-def cipd.yaml -tag (\S*)')
 _RE_CIPD_PACKAGE = re.compile('package: (\S*)')
 
 
+def _CheckVulnerabilities(build_android_deps_dir, report_dst):
+    logging.info(
+        'Running Gradle dependencyCheckAnalyze. This may take a few minutes the first time.'
+    )
+
+    abs_gradle_wrapper_path = os.path.join(_CHROMIUM_SRC, 'third_party',
+                                           'gradle_wrapper', 'gradlew')
+
+    # Separate command from main gradle command so that we can provide specific
+    # diagnostics in case of failure of this step.
+    gradle_cmd = [
+        abs_gradle_wrapper_path,
+        '-b',
+        os.path.join(build_android_deps_dir, _BUILD_GRADLE),
+        'dependencyCheckAnalyze',
+    ]
+
+    report_src = os.path.join(build_android_deps_dir, 'build', 'reports')
+    if os.path.exists(report_dst):
+        shutil.rmtree(report_dst)
+
+    try:
+        subprocess.run(gradle_cmd, check=True)
+    except subprocess.CalledProcessError:
+        report_path = os.path.join(report_dst, 'dependency-check-report.html')
+        logging.error(
+            textwrap.dedent("""
+               =============================================================================
+               A package has a known vulnerability. It may not be in a package or packages
+               which you just added, but you need to resolve the problem before proceeding.
+               If you can't easily fix it by rolling the package to a fixed version now,
+               please file a crbug of type= Bug-Security providing all relevant information,
+               and then rerun this command with --ignore-vulnerabilities.
+               The html version of the report is avialable at: {}
+               =============================================================================
+               """.format(report_path)))
+        raise
+    finally:
+        if os.path.exists(report_src):
+            CopyFileOrDirectory(report_src, report_dst)
+
+
 def GetCipdPackageInfo(cipd_yaml_path):
     """Returns the CIPD package name corresponding to a given cipd.yaml file.
 
@@ -449,11 +491,6 @@ def main():
         raise Exception('Missing files from {}: {}'.format(
             _CHROMIUM_SRC, missing_files))
 
-    abs_build_gradle_path = os.path.join(abs_android_deps_dir, _BUILD_GRADLE)
-    # Path to the gradlew script used to run build.gradle.
-    abs_gradle_wrapper_path = os.path.join(_CHROMIUM_SRC, 'third_party',
-                                           'gradle_wrapper', 'gradlew')
-
     with BuildDir(args.build_dir) as build_dir:
         build_android_deps_dir = os.path.join(build_dir, args.android_deps_dir)
 
@@ -467,57 +504,27 @@ def main():
             CopyFileOrDirectory(os.path.join(_CHROMIUM_SRC, path),
                                 os.path.join(build_dir, dest))
 
-        logging.info(
-            'Running Gradle dependencyCheckAnalyze. This may take a few minutes the first time.'
-        )
-        # Not run as part of the main gradle command below
-        # such that we can provide specific diagnostics in case
-        # of failure of this build stage.
-        gradle_cmd = [
-            abs_gradle_wrapper_path,
-            '-b',
-            os.path.join(build_dir, abs_build_gradle_path),
-            'dependencyCheckAnalyze',
-        ]
         if debug:
             gradle_cmd.append('--debug')
 
-        report_src = os.path.join(build_android_deps_dir, 'build', 'reports')
-        report_dst = os.path.join(abs_android_deps_dir,
-                                  'vulnerability_reports')
-        if os.path.exists(report_dst):
-            shutil.rmtree(report_dst)
-
-        try:
-            subprocess.run(gradle_cmd, check=True)
-        except subprocess.CalledProcessError:
-            report_path = os.path.join(report_dst,
-                                       'dependency-check-report.html')
-            logging.error(
-                textwrap.dedent("""
-                   =============================================================================
-                   A package has a known vulnerability. It may not be in a package or packages
-                   which you just added, but you need to resolve the problem before proceeding.
-                   If you can't easily fix it by rolling the package to a fixed version now,
-                   please file a crbug of type= Bug-Security providing all relevant information,
-                   and then rerun this command with --ignore-vulnerabilities.
-                   The html version of the report is avialable at: {}
-                   =============================================================================
-                   """.format(report_path)))
-            if not args.ignore_vulnerabilities:
-                raise
-        finally:
-            if os.path.exists(report_src):
-                CopyFileOrDirectory(report_src, report_dst)
+        if not args.ignore_vulnerabilities:
+            report_dst = os.path.join(abs_android_deps_dir,
+                                      'vulnerability_reports')
+            _CheckVulnerabilities(build_android_deps_dir, report_dst)
 
         logging.info('Running Gradle.')
+
+        # Path to the gradlew script used to run build.gradle.
+        abs_gradle_wrapper_path = os.path.join(_CHROMIUM_SRC, 'third_party',
+                                               'gradle_wrapper', 'gradlew')
+
         # This gradle command generates the new DEPS and BUILD.gn files, it can
         # also handle special cases.
         # Edit BuildConfigGenerator.groovy#addSpecialTreatment for such cases.
         gradle_cmd = [
             abs_gradle_wrapper_path,
             '-b',
-            os.path.join(build_dir, abs_build_gradle_path),
+            os.path.join(build_android_deps_dir, _BUILD_GRADLE),
             'setupRepository',
             '--stacktrace',
         ]

third_party/androidx/.gitignore

+.gradle/
+.idea/
+.settings/
+build.gradle
+local.properties
+
+# JAR and AAR files will be fetched via CIPD.
+libs/*/*.jar
+libs/*/*.aar

third_party/androidx/.style.yapf

+[style]
+based_on_style = pep8

third_party/androidx/OWNERS

+file://third_party/android_deps/OWNERS
+
+# OS: Android
+# COMPONENT: Tools

third_party/androidx/README.chromium

+Name: Androidx Repository
+Short Name: androidx
+Version: 1
+License: Refer to additional_readme_paths.json
+License Android Compatible: yes
+License File: NOT_SHIPPED
+Security Critical: yes
+
+Description:
+A gradle-specified repository for Androidx dependencies.

third_party/androidx/additional_readme_paths.json

+[
+]

third_party/androidx/androidx_mediarouter.flags

+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Help R8 to remove debug related coded. All of these static constants are
+# initialized to: Log.isLoggable(DEBUG, TAG).
+
+-assumevalues class androidx.mediarouter.** {
+  static boolean DEBUG return false;
+}

third_party/androidx/build.gradle.template

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+apply plugin: ChromiumPlugin
+
+repositories {
+    mavenCentral()
+    maven {
+        url 'https://maven.google.com'
+    }
+    maven {
+        url '{{androidx_repository_url}}'
+    }
+}
+
+dependencies {
+    // Note about the configuration names: they are defined in buildSrc/ChromiumPlugin
+
+    compile "androidx.lifecycle:lifecycle-runtime:{{androidx_dependency_version}}"
+    compile "androidx.lifecycle:lifecycle-common:{{androidx_dependency_version}}"
+    compile "androidx.lifecycle:lifecycle-common-java8:{{androidx_dependency_version}}"
+    compile "androidx.lifecycle:lifecycle-livedata-core:{{androidx_dependency_version}}"
+    compile "androidx.lifecycle:lifecycle-viewmodel:{{androidx_dependency_version}}"
+
+    compile "androidx.core:core:{{androidx_dependency_version}}"
+    compile "androidx.activity:activity:{{androidx_dependency_version}}"
+    compile "androidx.annotation:annotation:{{androidx_dependency_version}}"
+    compile "androidx.appcompat:appcompat:{{androidx_dependency_version}}"
+    compile "androidx.appcompat:appcompat-resources:{{androidx_dependency_version}}"
+    compile "androidx.asynclayoutinflater:asynclayoutinflater:{{androidx_dependency_version}}"
+    compile "androidx.cardview:cardview:{{androidx_dependency_version}}"
+    compile "androidx.concurrent:concurrent-futures:{{androidx_dependency_version}}"
+    compile "androidx.collection:collection:{{androidx_dependency_version}}"
+    compile "androidx.coordinatorlayout:coordinatorlayout:{{androidx_dependency_version}}"
+    compile "androidx.customview:customview:{{androidx_dependency_version}}"
+    compile "androidx.drawerlayout:drawerlayout:{{androidx_dependency_version}}"
+    compile "androidx.fragment:fragment:{{androidx_dependency_version}}"
+    compile "androidx.gridlayout:gridlayout:{{androidx_dependency_version}}"
+    compile "androidx.interpolator:interpolator:{{androidx_dependency_version}}"
+    compile "androidx.leanback:leanback:{{androidx_dependency_version}}"
+    compile "androidx.leanback:leanback-preference:{{androidx_dependency_version}}"
+    compile "androidx.localbroadcastmanager:localbroadcastmanager:{{androidx_dependency_version}}"
+    compile "androidx.media:media:{{androidx_dependency_version}}"
+    compile "androidx.mediarouter:mediarouter:{{androidx_dependency_version}}"
+    compile "androidx.preference:preference:{{androidx_dependency_version}}"
+    compile "androidx.slice:slice-core:{{androidx_dependency_version}}"
+    compile "androidx.slice:slice-builders:{{androidx_dependency_version}}"
+    compile "androidx.recyclerview:recyclerview:{{androidx_dependency_version}}"
+    compile "androidx.savedstate:savedstate:{{androidx_dependency_version}}"
+    compile "androidx.swiperefreshlayout:swiperefreshlayout:{{androidx_dependency_version}}"
+    compile "androidx.vectordrawable:vectordrawable:{{androidx_dependency_version}}"
+    compile "androidx.vectordrawable:vectordrawable-animated:{{androidx_dependency_version}}"
+    compile "androidx.tvprovider:tvprovider:{{androidx_dependency_version}}"
+    compile "androidx.viewpager:viewpager:{{androidx_dependency_version}}"
+    compile "androidx.window:window:{{androidx_dependency_version}}"
+    compile "androidx.exifinterface:exifinterface:{{androidx_dependency_version}}"
+
+    // Those are for use by doubledown libraries.
+    compile "androidx.arch.core:core-common:{{androidx_dependency_version}}"
+    compile "androidx.annotation:annotation-experimental:{{androidx_dependency_version}}"
+    compile "androidx.annotation:annotation:{{androidx_dependency_version}}"
+    compile "androidx.lifecycle:lifecycle-runtime:{{androidx_dependency_version}}"
+    compile "androidx.transition:transition:{{androidx_dependency_version}}"
+    compile "androidx.viewpager2:viewpager2:{{androidx_dependency_version}}"
+
+    compile "androidx.multidex:multidex:2.0.0"
+    compile "androidx.webkit:webkit:{{androidx_dependency_version}}"
+
+    // Used by 1p Play Services.
+    compile "androidx.legacy:legacy-support-v4:1.0.0"
+
+    // testCompile targets have test_only = true.
+    androidTestCompile "androidx.test:core:1.2.0"
+    androidTestCompile "androidx.test:monitor:1.2.0"
+    androidTestCompile "androidx.test:rules:1.2.0"
+    androidTestCompile "androidx.test:runner:1.2.0"
+    androidTestCompile "androidx.test.espresso:espresso-contrib:3.2.0"
+    androidTestCompile "androidx.test.espresso:espresso-core:3.2.0"
+    androidTestCompile "androidx.test.espresso:espresso-intents:3.2.0"
+    androidTestCompile "androidx.test.espresso:espresso-web:3.2.0"
+    androidTestCompile "androidx.test.ext:junit:1.1.1"
+    androidTestCompile "androidx.test.uiautomator:uiautomator:2.2.0"
+}
+
+task setUpRepository(type: BuildConfigGenerator) {
+    // Paths are relative to the chromium source root.
+    repositoryPath 'third_party/androidx'
+    chromiumSourceRoot '../..'
+    cipdBucket 'chromium'
+    internalTargetVisibility = [ '//third_party/android_deps:*', ':*' ]
+    skipLicenses true
+}

third_party/androidx/fetch_all_androidx.py

+#!/usr/bin/env python3
+
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""A script to generate build.gradle from template and run fetch_all.py
+
+More specifically, to generate build.gradle:
+  - It downloads the BUILD_INFO file for the latest androidx snapshot from
+    https://androidx.dev/snapshots/
+  - It replaces {{androidx_repository_url}} with the URL for the latest snapshot
+  - For each dependency, it looks up the version in the BUILD_INFO file and
+    substitutes the version into {{androidx_dependency_version}}.
+"""
+
+import contextlib
+import json
+import os
+import re
+import requests
+import shutil
+import subprocess
+import tempfile
+
+_ANDROIDX_PATH = os.path.normpath(os.path.join(__file__, '..'))
+
+_FETCH_ALL_PATH = os.path.normpath(
+    os.path.join(_ANDROIDX_PATH, '..', 'android_deps', 'fetch_all.py'))
+
+# URL to BUILD_INFO in latest androidx snapshot.
+_ANDROIDX_LATEST_SNAPSHOT_BUILD_INFO_URL = 'https://androidx.dev/snapshots/latest/artifacts/BUILD_INFO'
+
+
+def _parse_dir_list(dir_list):
+    """Computes 'library_group:library_name'->library_version mapping.
+
+    Args:
+      dir_list: List of androidx library directories.
+    """
+    dependency_version_map = dict()
+    for dir_entry in dir_list:
+        stripped_dir = dir_entry.strip()
+        if not stripped_dir.startswith('repository/androidx/'):
+            continue
+        dir_components = stripped_dir.split('/')
+        # Expected format:
+        # "repository/androidx/library_group/library_name/library_version/pom_or_jar"
+        if len(dir_components) < 6:
+            continue
+        dependency_module = 'androidx.{}:{}'.format(dir_components[2],
+                                                    dir_components[3])
+        if dependency_module not in dependency_version_map:
+            dependency_version_map[dependency_module] = dir_components[4]
+    return dependency_version_map
+
+
+def _compute_replacement(dependency_version_map, androidx_repository_url,
+                         line):
+    """Computes output line for build.gradle from build.gradle.template line.
+
+    Replaces {{android_repository_url}} and {{androidx_dependency_version}}.
+
+    Args:
+      dependency_version_map: An "dependency_group:dependency_name"->dependency_version mapping.
+      androidx_repository_url: URL of the maven repository.
+      line: Input line from the build.gradle.template.
+    """
+    line = line.replace('{{androidx_repository_url}}', androidx_repository_url)
+
+    match = re.search(r'"(\S+):{{androidx_dependency_version}}"', line)
+    if not match:
+        return line
+
+    version = dependency_version_map.get(match.group(1))
+    if not version:
+        return line
+
+    return line.replace('{{androidx_dependency_version}}', version)
+
+
+@contextlib.contextmanager
+def _build_dir():
+    dirname = tempfile.mkdtemp()
+    try:
+        yield dirname
+    finally:
+        shutil.rmtree(dirname)
+
+
+def _download_and_parse_build_info():
+    """Downloads and parses BUILD_INFO file."""
+    with _build_dir() as build_dir:
+        androidx_build_info_response = requests.get(
+            _ANDROIDX_LATEST_SNAPSHOT_BUILD_INFO_URL)
+        androidx_build_info_path = os.path.join(build_dir, 'BUILD_INFO')
+        with open(androidx_build_info_path, 'w') as f:
+            f.write(androidx_build_info_response.text)
+
+        # Compute repository URL from resolved BUILD_INFO url in case 'latest' redirect changes.
+        androidx_snapshot_repository_url = androidx_build_info_response.url.rsplit(
+            '/', 1)[0] + '/repository'
+
+        with open(androidx_build_info_path, 'r') as f:
+            build_info_dict = json.loads(f.read())
+        dir_list = build_info_dict['target']['dir_list']
+
+        dependency_version_map = _parse_dir_list(dir_list)
+        return (dependency_version_map, androidx_snapshot_repository_url)
+
+
+def _process_build_gradle(dependency_version_map, androidx_repository_url):
+    """Generates build.gradle from template.
+
+    Args:
+      dependency_version_map: An "dependency_group:dependency_name"->dependency_version mapping.
+      androidx_repository_url: URL of the maven repository.
+    """
+    build_gradle_template_path = os.path.join(_ANDROIDX_PATH,
+                                              'build.gradle.template')
+    build_gradle_out_path = os.path.join(_ANDROIDX_PATH, 'build.gradle')
+    # |build_gradle_out_path| is not deleted after script has finished running. The file is in
+    # .gitignore and thus will be excluded from uploaded CLs.
+    with open(build_gradle_template_path, 'r') as template_f, \
+        open(build_gradle_out_path, 'w') as out:
+        for template_line in template_f:
+            replacement = _compute_replacement(dependency_version_map,
+                                               androidx_repository_url,
+                                               template_line)
+            out.write(replacement)
+
+
+def main():
+    dependency_version_map, androidx_snapshot_repository_url = (
+        _download_and_parse_build_info())
+    _process_build_gradle(dependency_version_map,
+                          androidx_snapshot_repository_url)
+
+    fetch_all_cmd = [
+        _FETCH_ALL_PATH, '--android-deps-dir',
+        os.path.join('third_party', 'androidx'), '--ignore-vulnerabilities'
+    ]
+    subprocess.run(fetch_all_cmd)
+
+
+if __name__ == '__main__':
+    main()