Revert "[cipd] Refactor CIPD template"

This reverts commit 3d6123e3.

Reason for revert: Broke official desktop continuous builder, at least for Fuchsia (see https://crbug.com/1070174).

Original change's description:
> [cipd] Refactor CIPD template
> 
> Currently, a CIPD staging directory is populated by copying specified
> `sources` into it, and producing a .yaml manifest which reflects the
> copied files.
> 
> This change refactors the template to invoke a pytho script to do the
> above instead of in GN, using the `yaml` library to write the file data
> in the proper format.
> 
> Also make the license filepath explicitly passed through sources.
> 
> Renames "archive" -> "package" and "manifest" -> "package definition" to
> be consistent with CIPD semantics (crbug.com/1042819#c8).
> 
> Finally, have the script emit a depfile for all files tracked by the
> .yaml manifest.
> 
> This refactor is desirable, as we want to create CIPD packages with more
> complicated rules than simply "copy and include these files" (see
> crrev.com/c/1925446, which extends the functionality of the template to
> allow for the inclusion of directories). Wriiting yamls with a library
> reduces the error-proneness of trying to recreate yaml's format in pure
> GN.
> 
> Also, emitting a depfile ensure that we track and rebuild if the CIPD
> package is modified.
> 
> Currently the potential danger of this change is limited in scope to
> fuchsia, as they are the only users of the template. I've checked
> locally that the produced CIPD packages for fuchsia are identical prior
> to this change.
> 
> Bug: fuchsia:41443
> Change-Id: If1b27d6a8fbbffdf767aaa4230bf7b527a553170
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2135186
> Commit-Queue: Anthony Fandrianto <atyfto@google.com>
> Reviewed-by: Bruce Dawson <brucedawson@chromium.org>
> Reviewed-by: Kevin Marshall <kmarshall@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#758433}

TBR=kmarshall@chromium.org,brucedawson@chromium.org,joshuaseaton@google.com,atyfto@google.com

Change-Id: I68f850f43ca121c3dfb7f87d0ec68e7f96448d94
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: fuchsia:41443
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2144183Reviewed-by: Wez <wez@chromium.org>
Commit-Queue: Wez <wez@chromium.org>
Cr-Commit-Position: refs/heads/master@{#758489}

build/cipd/cipd.gni

@@ -4,12 +4,12 @@
 
 # Build targets for constructing CIPD packages.
 #
-# Prepares a CIPD package and generates a package definition.
+# Prepares a CIPD archive and generates a manifest file.
 #
 # TODO(crbug.com/1042819): Add support for including directories.
 #
 # Parameters:
-#   package_definition_name: CIPD package definition filename. "cipd.yaml"
+#   package_definition_yaml: CIPD package definition filename. "cipd.yaml"
 #                            if unspecified.
 #   package: The path where the package will be located inside the CIPD
 #            repository.
@@ -47,33 +47,27 @@ template("cipd_package_definition") {
   assert(_install_mode == "copy" || _install_mode == "symlink",
          "\"install_mode\" arg should be either \"copy\" or \"symlink\".")
 
-  _package_definition_name = "cipd.yaml"
-  if (defined(invoker.package_definition_name)) {
-    _package_definition_name = invoker.package_definition_name
+  _cipd_definition_yaml = "cipd.yaml"
+  if (defined(invoker.package_definition_yaml)) {
+    _cipd_definition_yaml = invoker.package_definition_yaml
   }
 
-  _package_root = "${target_gen_dir}/${target_name}"
+  _package_staging_dir = "${target_gen_dir}/${target_name}"
 
-  action(target_name) {
-    script = "//build/cipd/prepare_cipd_package_definition.py"
-    depfile = "$target_gen_dir/$target_name.d"
-    _definition_path = "${_package_root}/${_package_definition_name}"
-    outputs = [ _definition_path ]
+  _yaml_contents = [
+    "package: ${invoker.package}",
+    "description: ${invoker.description}",
+    "root: \${outdir}/" + rebase_path(_package_staging_dir, root_build_dir),
+    "install_mode: ${_install_mode}",
+    "data:",
+  ]
+  foreach(source, sources) {
+    _yaml_contents += [ "  - file: " + get_path_info(source, "file") ]
+  }
+
+  write_file("${_package_staging_dir}/${_cipd_definition_yaml}", _yaml_contents)
 
-    args = [
-      "--pkg-name",
-      invoker.package,
-      "--description",
-      invoker.description,
-      "--pkg-root",
-      rebase_path(_package_root, root_build_dir),
-      "--install-mode",
-      _install_mode,
-      "--pkg-def",
-      rebase_path(_definition_path, root_build_dir),
-      "--depfile",
-      rebase_path(depfile, root_build_dir),
-    ]
-    args += [ "--copy-files" ] + rebase_path(sources, root_build_dir)
+  copy(target_name) {
+    outputs = [ "${_package_staging_dir}/{{source_file_part}}" ]
   }
 }

build/cipd/prepare_cipd_package_definition.py

-#!/usr/bin/env python
-#
-# Copyright 2020 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-"""Prepares a directory and a corresponding package definition which can be
-used to create a CIPD package."""
-
-import argparse
-import errno
-import os
-import sys
-import yaml
-
-
-def main(args):
-  parser = argparse.ArgumentParser()
-  parser.add_argument('--pkg-name',
-                      type=str,
-                      required=True,
-                      help='Name of the CIPD package.')
-  parser.add_argument('--description',
-                      type=str,
-                      required=True,
-                      help='Description of the CIPD package.')
-  parser.add_argument('--pkg-root',
-                      type=str,
-                      required=True,
-                      help='Path to the package root.')
-  parser.add_argument('--install-mode',
-                      type=str,
-                      choices=['copy', 'symlink'],
-                      required=True,
-                      help='CIPD install mode.')
-  parser.add_argument('--pkg-def',
-                      type=str,
-                      required=True,
-                      help='Path to the output package definition.')
-  parser.add_argument('--depfile',
-                      type=str,
-                      required=True,
-                      help='Path to the depfile.')
-  parser.add_argument('--copy-files',
-                      nargs='+',
-                      help='Files to be copied into --pkg-root and included '
-                      'in the package definition.')
-  args = parser.parse_args(args)
-
-  pkg_def = {
-      'package': args.pkg_name,
-      'description': args.description,
-      'root': '${outdir}/%s' % os.path.join(args.pkg_root),
-      'install_mode': args.install_mode,
-      'data': [],
-  }
-
-  deps = set()
-  # Copy files into the root.
-  for filepath in args.copy_files:
-    basename = os.path.basename(filepath)
-    dest = os.path.join(args.pkg_root, basename)
-    try:
-      os.link(filepath, dest)
-    except OSError as e:
-      if e.errno != errno.EEXIST:
-        raise
-    pkg_def['data'].append({'file': basename})
-    deps.add(dest)
-
-  with open(args.pkg_def, 'w') as f:
-    yaml.dump(pkg_def, f)
-  with open(args.depfile, 'w') as f:
-    f.writelines('%s: %s\n' % (args.pkg_def, ' '.join(sorted(deps))))
-
-  return 0
-
-
-if __name__ == '__main__':
-  sys.exit(main(sys.argv[1:]))

fuchsia/cipd/BUILD.gn

@@ -2,7 +2,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-# Build targets for constructing CIPD release packages.
+# Build targets for constructing CIPD release archives.
 
 assert(is_fuchsia)
 
@@ -16,10 +16,9 @@ if (host_os == "mac") {
   _gn_path = "//buildtools/linux64/gn"
 }
 
-_license_path = "${target_gen_dir}/LICENSE"
-
 # Produces a consolidated license file.
 action("license") {
+  _license_path = "${target_gen_dir}/LICENSE"
   script = "//tools/licenses.py"
   inputs = [ "$_gn_path" ]
   outputs = [ _license_path ]
@@ -46,18 +45,18 @@ process_version("build_id") {
   process_only = true
 }
 
-# Prepares a CIPD package and generates a package definition.
+# Prepares a CIPD archive and generates a manifest file.
 #
 # Parameters:
-#   package_definition_name: CIPD package definition filename. "cipd.yaml"
-#                            if unspecified.
+#   package_definition_yaml: The filename to use for the generated CIPD YAML
+#                            file.
 #   package: The path where the package will be located inside the CIPD
 #            repository.
 #   description: Sets the "description" field in CIPD metadata.
 #   install_mode: String, should be either "symlink" or "copy".
 #   deps: A list of targets to build prior to copying files.
 #   sources: A list of files to copy into the staging root.
-template("cipd_package") {
+template("cipd_archive") {
   forward_variables_from(invoker,
                          [
                            "deps",
@@ -67,7 +66,15 @@ template("cipd_package") {
   if (!defined(deps)) {
     deps = []
   }
-  deps += [ ":build_id" ]
+  deps += [
+    ":build_id",
+    ":license",
+  ]
+
+  if (!defined(sources)) {
+    sources = []
+  }
+  sources += get_target_outputs(":license")
 
   cipd_package_definition(target_name) {
     testonly = true
@@ -76,18 +83,17 @@ template("cipd_package") {
                              "description",
                              "install_mode",
                              "package",
-                             "package_definition_name",
+                             "package_definition_yaml",
                            ])
   }
 }
 
-cipd_package("webrunner") {
-  package_definition_name = "webrunner.yaml"
+cipd_archive("webrunner") {
+  package_definition_yaml = "webrunner.yaml"
   package = "chromium/fuchsia/webrunner-\${targetarch}"
   description = "Prebuilt Chrome and Web Runner binaries for Fuchsia."
 
   deps = [
-    ":license",
     "//fuchsia/engine:web_engine",
     "//fuchsia/runners:web_runner_pkg",
   ]
@@ -95,40 +101,27 @@ cipd_package("webrunner") {
   sources = [
     "${root_gen_dir}/fuchsia/engine/web_engine/web_engine.far",
     "${root_gen_dir}/fuchsia/runners/web_runner/web_runner.far",
-    _license_path,
   ]
 }
 
-cipd_package("castrunner") {
-  package_definition_name = "castrunner.yaml"
+cipd_archive("castrunner") {
+  package_definition_yaml = "castrunner.yaml"
   package = "chromium/fuchsia/castrunner-\${targetarch}"
   description = "Prebuilt Cast application Runner binaries for Fuchsia."
 
-  deps = [
-    ":license",
-    "//fuchsia/runners:cast_runner_pkg",
-  ]
+  deps = [ "//fuchsia/runners:cast_runner_pkg" ]
 
-  sources = [
-    "${root_gen_dir}/fuchsia/runners/cast_runner/cast_runner.far",
-    _license_path,
-  ]
+  sources = [ "${root_gen_dir}/fuchsia/runners/cast_runner/cast_runner.far" ]
 }
 
-cipd_package("http") {
-  package_definition_name = "http.yaml"
+cipd_archive("http") {
+  package_definition_yaml = "http.yaml"
   package = "chromium/fuchsia/http-\${targetarch}"
   description = "Prebuilt HTTP service binary for Fuchsia."
 
-  deps = [
-    ":license",
-    "//fuchsia/http:http_pkg",
-  ]
+  deps = [ "//fuchsia/http:http_pkg" ]
 
-  sources = [
-    "${root_gen_dir}/fuchsia/http/http/http.far",
-    _license_path,
-  ]
+  sources = [ "${root_gen_dir}/fuchsia/http/http/http.far" ]
 }
 
 _stripped_chromedriver_file = "${root_out_dir}/clang_x64/stripped/chromedriver"
@@ -156,31 +149,23 @@ action("strip_chromedriver_binary") {
   visibility = [ ":*" ]
 }
 
-cipd_package("chromedriver") {
-  package_definition_name = "chromedriver.yaml"
+cipd_archive("chromedriver") {
+  package_definition_yaml = "chromedriver.yaml"
   package = "chromium/fuchsia/chromedriver/\${os}-\${arch}"
   description = "Prebuilt Chromedriver binary for Fuchsia host."
   install_mode = "copy"
+  deps = [ ":strip_chromedriver_binary" ]
 
-  deps = [
-    ":license",
-    ":strip_chromedriver_binary",
-  ]
-
-  sources = [
-    _license_path,
-    _stripped_chromedriver_file,
-  ]
+  sources = [ _stripped_chromedriver_file ]
 }
 
-cipd_package("tests") {
+cipd_archive("tests") {
   _manifest_path = "${target_gen_dir}/test_manifest.json"
-  package_definition_name = "tests.yaml"
+  package_definition_yaml = "tests.yaml"
   package = "chromium/fuchsia/tests-\${targetarch}"
   description = "Prebuilt Chromium tests for Fuchsia."
 
   deps = [
-    ":license",
     "//base:base_unittests_pkg",
     "//fuchsia/runners:cast_runner_integration_tests_pkg",
     "//fuchsia/runners:web_runner_integration_tests_pkg",
@@ -202,7 +187,7 @@ cipd_package("tests") {
     "${root_gen_dir}/third_party/blink/common/blink_common_unittests/blink_common_unittests.far",
   ]
 
-  # Build a JSON manifest of the tests and include it in the package.
+  # Build a JSON manifest of the tests and include it in the archive.
   _manifest_contents = []
   foreach(source, far_sources) {
     package_name = get_path_info(source, "name")
@@ -216,14 +201,11 @@ cipd_package("tests") {
   }
   write_file(_manifest_path, _manifest_contents, "json")
 
-  sources = far_sources + [
-              _manifest_path,
-              _license_path,
-            ]
+  sources = far_sources + [ _manifest_path ]
 }
 
-cipd_package("debug_symbols") {
-  package_definition_name = "debug_symbols.yaml"
+cipd_archive("debug_symbols") {
+  package_definition_yaml = "debug_symbols.yaml"
   package = "chromium/fuchsia/debug-symbols-\${targetarch}"
   description = "Debugging symbols for prebuilt binaries from Chromium."
 
@@ -241,31 +223,21 @@ cipd_package("debug_symbols") {
   write_file(_symbol_manifest, _symbol_manifest_contents, "json")
 
   deps = [
-    ":license",
     "//fuchsia/engine:symbol_archive",
     "//fuchsia/runners:cast_runner_symbol_archive",
     "//fuchsia/runners:web_runner_symbol_archive",
   ]
-  sources = _symbol_tarballs + [
-              _symbol_manifest,
-              _license_path,
-            ]
+  sources = [ _symbol_manifest ] + _symbol_tarballs
 }
 
-cipd_package("clear_key_cdm") {
-  package_definition_name = "clear_key_cdm.yaml"
+cipd_archive("clear_key_cdm") {
+  package_definition_yaml = "clear_key_cdm.yaml"
   package = "chromium/fuchsia/libclearkeycdm-\${targetarch}"
   description = "Prebuilt libclearkeycdm.so binary for Fuchsia."
 
-  deps = [
-    ":license",
-    "//media/cdm/library_cdm/clear_key_cdm:clear_key_cdm",
-  ]
+  deps = [ "//media/cdm/library_cdm/clear_key_cdm:clear_key_cdm" ]
 
-  sources = [
-    "${root_out_dir}/lib/libclearkeycdm.so",
-    _license_path,
-  ]
+  sources = [ "${root_out_dir}/lib/libclearkeycdm.so" ]
 }
 
 group("cipd") {