Skip to content

GitLab

T
tangled
Commit 3d6123e3 authored by Anthony Fandrianto's avatar Anthony Fandrianto Committed by Commit Bot
Browse files
Options

[cipd] Refactor CIPD template 

Currently, a CIPD staging directory is populated by copying specified
`sources` into it, and producing a .yaml manifest which reflects the
copied files.

This change refactors the template to invoke a pytho script to do the
above instead of in GN, using the `yaml` library to write the file data
in the proper format.

Also make the license filepath explicitly passed through sources.

Renames "archive" -> "package" and "manifest" -> "package definition" to
be consistent with CIPD semantics (crbug.com/1042819#c8).

Finally, have the script emit a depfile for all files tracked by the
.yaml manifest.

This refactor is desirable, as we want to create CIPD packages with more
complicated rules than simply "copy and include these files" (see
crrev.com/c/1925446, which extends the functionality of the template to
allow for the inclusion of directories). Wriiting yamls with a library
reduces the error-proneness of trying to recreate yaml's format in pure
GN.

Also, emitting a depfile ensure that we track and rebuild if the CIPD
package is modified.

Currently the potential danger of this change is limited in scope to
fuchsia, as they are the only users of the template. I've checked
locally that the produced CIPD packages for fuchsia are identical prior
to this change.

Bug: fuchsia:41443
Change-Id: If1b27d6a8fbbffdf767aaa4230bf7b527a553170
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2135186
Commit-Queue: Anthony Fandrianto <atyfto@google.com>
Reviewed-by: default avatarBruce Dawson <brucedawson@chromium.org>
Reviewed-by: default avatarKevin Marshall <kmarshall@chromium.org>
Cr-Commit-Position: refs/heads/master@{#758433}
parent 28816f56
Hide whitespace changes
Inline Side-by-side
Showing with 174 additions and 61 deletions
build/cipd/cipd.gni
View file @ 3d6123e3
......@@ -4,12 +4,12 @@
# Build targets for constructing CIPD packages.
#
# Prepares a CIPD archive and generates a manifest file.
# Prepares a CIPD package and generates a package definition.
#
# TODO(crbug.com/1042819): Add support for including directories.
#
# Parameters:
# package_definition_yaml: CIPD package definition filename. "cipd.yaml"
# package_definition_name: CIPD package definition filename. "cipd.yaml"
# if unspecified.
# package: The path where the package will be located inside the CIPD
# repository.
......@@ -47,27 +47,33 @@ template("cipd_package_definition") {
assert(_install_mode == "copy" || _install_mode == "symlink",
"\"install_mode\" arg should be either \"copy\" or \"symlink\".")
_cipd_definition_yaml = "cipd.yaml"
if (defined(invoker.package_definition_yaml)) {
_cipd_definition_yaml = invoker.package_definition_yaml
_package_definition_name = "cipd.yaml"
if (defined(invoker.package_definition_name)) {
_package_definition_name = invoker.package_definition_name
}
_package_staging_dir = "${target_gen_dir}/${target_name}"
_package_root = "${target_gen_dir}/${target_name}"
_yaml_contents = [
"package: ${invoker.package}",
"description: ${invoker.description}",
"root: \${outdir}/" + rebase_path(_package_staging_dir, root_build_dir),
"install_mode: ${_install_mode}",
"data:",
]
foreach(source, sources) {
_yaml_contents += [ " - file: " + get_path_info(source, "file") ]
}
write_file("${_package_staging_dir}/${_cipd_definition_yaml}", _yaml_contents)
action(target_name) {
script = "//build/cipd/prepare_cipd_package_definition.py"
depfile = "$target_gen_dir/$target_name.d"
_definition_path = "${_package_root}/${_package_definition_name}"
outputs = [ _definition_path ]
copy(target_name) {
outputs = [ "${_package_staging_dir}/{{source_file_part}}" ]
args = [
"--pkg-name",
invoker.package,
"--description",
invoker.description,
"--pkg-root",
rebase_path(_package_root, root_build_dir),
"--install-mode",
_install_mode,
"--pkg-def",
rebase_path(_definition_path, root_build_dir),
"--depfile",
rebase_path(depfile, root_build_dir),
]
args += [ "--copy-files" ] + rebase_path(sources, root_build_dir)
}
}
build/cipd/prepare_cipd_package_definition.py 0 → 100755
View file @ 3d6123e3
#!/usr/bin/env python
#
# Copyright 2020 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Prepares a directory and a corresponding package definition which can be
used to create a CIPD package."""
import argparse
import errno
import os
import sys
import yaml
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument('--pkg-name',
type=str,
required=True,
help='Name of the CIPD package.')
parser.add_argument('--description',
type=str,
required=True,
help='Description of the CIPD package.')
parser.add_argument('--pkg-root',
type=str,
required=True,
help='Path to the package root.')
parser.add_argument('--install-mode',
type=str,
choices=['copy', 'symlink'],
required=True,
help='CIPD install mode.')
parser.add_argument('--pkg-def',
type=str,
required=True,
help='Path to the output package definition.')
parser.add_argument('--depfile',
type=str,
required=True,
help='Path to the depfile.')
parser.add_argument('--copy-files',
nargs='+',
help='Files to be copied into --pkg-root and included '
'in the package definition.')
args = parser.parse_args(args)
pkg_def = {
'package': args.pkg_name,
'description': args.description,
'root': '${outdir}/%s' % os.path.join(args.pkg_root),
'install_mode': args.install_mode,
'data': [],
}
deps = set()
# Copy files into the root.
for filepath in args.copy_files:
basename = os.path.basename(filepath)
dest = os.path.join(args.pkg_root, basename)
try:
os.link(filepath, dest)
except OSError as e:
if e.errno != errno.EEXIST:
raise
pkg_def['data'].append({'file': basename})
deps.add(dest)
with open(args.pkg_def, 'w') as f:
yaml.dump(pkg_def, f)
with open(args.depfile, 'w') as f:
f.writelines('%s: %s\n' % (args.pkg_def, ' '.join(sorted(deps))))
return 0
if __name__ == '__main__':
sys.exit(main(sys.argv[1:]))
fuchsia/cipd/BUILD.gn
View file @ 3d6123e3
......@@ -2,7 +2,7 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Build targets for constructing CIPD release archives.
# Build targets for constructing CIPD release packages.
assert(is_fuchsia)
......@@ -16,9 +16,10 @@ if (host_os == "mac") {
_gn_path = "//buildtools/linux64/gn"
}
_license_path = "${target_gen_dir}/LICENSE"
# Produces a consolidated license file.
action("license") {
_license_path = "${target_gen_dir}/LICENSE"
script = "//tools/licenses.py"
inputs = [ "$_gn_path" ]
outputs = [ _license_path ]
......@@ -45,18 +46,18 @@ process_version("build_id") {
process_only = true
}
# Prepares a CIPD archive and generates a manifest file.
# Prepares a CIPD package and generates a package definition.
#
# Parameters:
# package_definition_yaml: The filename to use for the generated CIPD YAML
# file.
# package_definition_name: CIPD package definition filename. "cipd.yaml"
# if unspecified.
# package: The path where the package will be located inside the CIPD
# repository.
# description: Sets the "description" field in CIPD metadata.
# install_mode: String, should be either "symlink" or "copy".
# deps: A list of targets to build prior to copying files.
# sources: A list of files to copy into the staging root.
template("cipd_archive") {
template("cipd_package") {
forward_variables_from(invoker,
[
"deps",
......@@ -66,15 +67,7 @@ template("cipd_archive") {
if (!defined(deps)) {
deps = []
}
deps += [
":build_id",
":license",
]
if (!defined(sources)) {
sources = []
}
sources += get_target_outputs(":license")
deps += [ ":build_id" ]
cipd_package_definition(target_name) {
testonly = true
......@@ -83,17 +76,18 @@ template("cipd_archive") {
"description",
"install_mode",
"package",
"package_definition_yaml",
"package_definition_name",
])
}
}
cipd_archive("webrunner") {
package_definition_yaml = "webrunner.yaml"
cipd_package("webrunner") {
package_definition_name = "webrunner.yaml"
package = "chromium/fuchsia/webrunner-\${targetarch}"
description = "Prebuilt Chrome and Web Runner binaries for Fuchsia."
deps = [
":license",
"//fuchsia/engine:web_engine",
"//fuchsia/runners:web_runner_pkg",
]
......@@ -101,27 +95,40 @@ cipd_archive("webrunner") {
sources = [
"${root_gen_dir}/fuchsia/engine/web_engine/web_engine.far",
"${root_gen_dir}/fuchsia/runners/web_runner/web_runner.far",
_license_path,
]
}
cipd_archive("castrunner") {
package_definition_yaml = "castrunner.yaml"
cipd_package("castrunner") {
package_definition_name = "castrunner.yaml"
package = "chromium/fuchsia/castrunner-\${targetarch}"
description = "Prebuilt Cast application Runner binaries for Fuchsia."
deps = [ "//fuchsia/runners:cast_runner_pkg" ]
deps = [
":license",
"//fuchsia/runners:cast_runner_pkg",
]
sources = [ "${root_gen_dir}/fuchsia/runners/cast_runner/cast_runner.far" ]
sources = [
"${root_gen_dir}/fuchsia/runners/cast_runner/cast_runner.far",
_license_path,
]
}
cipd_archive("http") {
package_definition_yaml = "http.yaml"
cipd_package("http") {
package_definition_name = "http.yaml"
package = "chromium/fuchsia/http-\${targetarch}"
description = "Prebuilt HTTP service binary for Fuchsia."
deps = [ "//fuchsia/http:http_pkg" ]
deps = [
":license",
"//fuchsia/http:http_pkg",
]
sources = [ "${root_gen_dir}/fuchsia/http/http/http.far" ]
sources = [
"${root_gen_dir}/fuchsia/http/http/http.far",
_license_path,
]
}
_stripped_chromedriver_file = "${root_out_dir}/clang_x64/stripped/chromedriver"
......@@ -149,23 +156,31 @@ action("strip_chromedriver_binary") {
visibility = [ ":*" ]
}
cipd_archive("chromedriver") {
package_definition_yaml = "chromedriver.yaml"
cipd_package("chromedriver") {
package_definition_name = "chromedriver.yaml"
package = "chromium/fuchsia/chromedriver/\${os}-\${arch}"
description = "Prebuilt Chromedriver binary for Fuchsia host."
install_mode = "copy"
deps = [ ":strip_chromedriver_binary" ]
sources = [ _stripped_chromedriver_file ]
deps = [
":license",
":strip_chromedriver_binary",
]
sources = [
_license_path,
_stripped_chromedriver_file,
]
}
cipd_archive("tests") {
cipd_package("tests") {
_manifest_path = "${target_gen_dir}/test_manifest.json"
package_definition_yaml = "tests.yaml"
package_definition_name = "tests.yaml"
package = "chromium/fuchsia/tests-\${targetarch}"
description = "Prebuilt Chromium tests for Fuchsia."
deps = [
":license",
"//base:base_unittests_pkg",
"//fuchsia/runners:cast_runner_integration_tests_pkg",
"//fuchsia/runners:web_runner_integration_tests_pkg",
......@@ -187,7 +202,7 @@ cipd_archive("tests") {
"${root_gen_dir}/third_party/blink/common/blink_common_unittests/blink_common_unittests.far",
]
# Build a JSON manifest of the tests and include it in the archive.
# Build a JSON manifest of the tests and include it in the package.
_manifest_contents = []
foreach(source, far_sources) {
package_name = get_path_info(source, "name")
......@@ -201,11 +216,14 @@ cipd_archive("tests") {
}
write_file(_manifest_path, _manifest_contents, "json")
sources = far_sources + [ _manifest_path ]
sources = far_sources + [
_manifest_path,
_license_path,
]
}
cipd_archive("debug_symbols") {
package_definition_yaml = "debug_symbols.yaml"
cipd_package("debug_symbols") {
package_definition_name = "debug_symbols.yaml"
package = "chromium/fuchsia/debug-symbols-\${targetarch}"
description = "Debugging symbols for prebuilt binaries from Chromium."
......@@ -223,21 +241,31 @@ cipd_archive("debug_symbols") {
write_file(_symbol_manifest, _symbol_manifest_contents, "json")
deps = [
":license",
"//fuchsia/engine:symbol_archive",
"//fuchsia/runners:cast_runner_symbol_archive",
"//fuchsia/runners:web_runner_symbol_archive",
]
sources = [ _symbol_manifest ] + _symbol_tarballs
sources = _symbol_tarballs + [
_symbol_manifest,
_license_path,
]
}
cipd_archive("clear_key_cdm") {
package_definition_yaml = "clear_key_cdm.yaml"
cipd_package("clear_key_cdm") {
package_definition_name = "clear_key_cdm.yaml"
package = "chromium/fuchsia/libclearkeycdm-\${targetarch}"
description = "Prebuilt libclearkeycdm.so binary for Fuchsia."
deps = [ "//media/cdm/library_cdm/clear_key_cdm:clear_key_cdm" ]
deps = [
":license",
"//media/cdm/library_cdm/clear_key_cdm:clear_key_cdm",
]
sources = [ "${root_out_dir}/lib/libclearkeycdm.so" ]
sources = [
"${root_out_dir}/lib/libclearkeycdm.so",
_license_path,
]
}
group("cipd") {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment