Template to package test dependencies for chromecast builds.

Bug: b/132795052
Test: Manual and CQ

Change-Id: I57deb7bd5867f57226436f05185be6a1903550c5
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1611312Reviewed-by: Luke Halliwell <halliwell@chromium.org>
Commit-Queue: Godofredo Contreras <godofredoc@chromium.org>
Cr-Commit-Position: refs/heads/master@{#661041}

chromecast/build/tests/cast_test.gni

@@ -456,3 +456,88 @@ template("cast_test_group_list") {
     }
   }
 }
+
+# This template creates a tar.gz file with test dependencies.
+#
+# Parameters
+#   output (required)
+#       The absolute path to the tar.gz file to be created.
+#
+#   deps_list_path (required)
+#       Absolute path to runtime dependencies file. This has to point to the
+#       same file runtime_deps_path of cast_test_group_list is pointing to.
+#
+#   exclude_deps (optional)
+#       gn list of paths to filter out from the dependencies list.
+#       This is used to remove folders with huge files that are not needed.
+#
+#   additional_deps (optional)
+#       gn list of paths to be added to the list of dependencies
+#       before creating the tar.gz file.
+#
+# This template has to be used in combination with cast_test_group_list
+# that generates the runtime dependencies file.
+#
+# Example usage:
+#
+# cast_test_group_list("chromecast_test_lists") {
+#  build_list_path = "$root_out_dir/tests/build_test_list.txt"
+#  runtime_deps_path = "$root_out_dir/tests/runtime_deps.json"
+#  run_list_path = "$root_out_dir/tests/run_test_list.txt"
+#  test_groups = [ ":chromecast_test_group" ]
+# }
+#
+# cast_test_deps_archive("chromecast_test_deps_archive") {
+#  output = "$root_out_dir/test_deps.tar.gz"
+#  deps_list_path = "$root_out_dir/tests/runtime_deps.json"
+#  exclude_deps = [
+#    "exe.unstripped",
+#    "lib.unstripped"
+#  ]
+#  additional_deps = [
+#    "tests",
+#  ]
+#  deps = [
+#    ":chromecast_test_lists"
+#  ]
+# }
+#
+template("cast_test_deps_archive") {
+  assert(defined(invoker.output), "$target_name needs 'output'")
+  assert(defined(invoker.deps_list_path),
+         "$target_name needs 'deps_list_path'")
+
+  deps = invoker.deps
+
+  action(target_name) {
+    testonly = true
+
+    script = "//chromecast/tools/build/package_test_deps.py"
+    outputs = [
+      invoker.output
+    ]
+
+    # Generate a comma separated list of filters.
+    _exclude_deps = ""
+    foreach(_exclude_dep, invoker.exclude_deps) {
+      _exclude_deps += "," + _exclude_dep
+    }
+
+    # Generate a comma separated list of includes.
+    _additional_deps = ""
+    foreach(_additional_dep, invoker.additional_deps) {
+      _additional_deps += "," + _additional_dep
+    }
+
+    args = [
+      "--output",
+      rebase_path(invoker.output, root_build_dir),
+      "--deps_list_path",
+      rebase_path(invoker.deps_list_path, root_build_dir),
+      "--exclude_deps",
+      _exclude_deps,
+      "--additional_deps",
+      _additional_deps,
+    ]
+  }
+}

chromecast/tools/build/package_test_deps.py

+#!/usr/bin/env python
+#
+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Packages test dependencies as tar.gz file."""
+
+import argparse
+import json
+import logging
+import os
+import sys
+import tarfile
+
+
+parser = argparse.ArgumentParser(
+    description='Package test dependencies as tar.gz files.')
+parser.add_argument('--output', required=True,
+                    help='Full path to the output file.')
+parser.add_argument('--deps_list_path', required=True,
+                    help='Full path to the json dependencies file.')
+parser.add_argument('--exclude_deps', required=False,
+                    default='',
+                    help=('Comma separated list of dependencies to exclude'
+                          ' from tar.gz file.'))
+parser.add_argument('--additional_deps', required=False,
+                    default='',
+                    help=('Comma separated list of additional deps'
+                          ' to include in tar.gz.'))
+
+
+def read_dependencies(file_path):
+  """Reads a json file and creates an iterable of unique dependencies.
+
+  Args:
+    file_path: The path to the runtime dependencies file.
+
+  Returns:
+    An iterable with unique dependencies.
+  """
+  deps = None
+  with open(file_path) as deps_file:
+    deps = json.load(deps_file)
+  deps_set = set()
+  for _, dep_list in deps.items():
+    deps_set.update(dep_list)
+  return deps_set
+
+
+def filter_dependencies(dependencies, filters):
+  """Filters out dependencies from a dependencies iterable.
+
+  Args:
+    dependencies: An iterable with the full list of dependencies.
+    filters: A list of dependencies to remove.
+
+  Returns:
+    An iterable with the filtered dependencies.
+  """
+  filters_list = filters.strip(',').split(',')
+  logging.info('Filtering: %s', filters_list)
+  filtered_deps = set()
+  for dep in dependencies:
+    norm_dep = os.path.normpath(dep)
+    if not any(norm_dep.startswith(f) for f in filters_list):
+      filtered_deps.add(norm_dep)
+  return filtered_deps
+
+
+def create_tarfile(output_path, dependencies):
+  """Creates a tar.gz file and saves it to output_path.
+
+  Args:
+    output_path: A string with the path to where tar.gz file will be saved to.
+    dependencies: An iterable with file/folders test dependencies.
+  """
+  total_deps = len(dependencies)
+  if total_deps < 1:
+    logging.error('There are no dependencies to archive')
+    sys.exit(1)
+  step = (total_deps / 10) or 1
+  logging.info('Adding %s files', total_deps)
+  with tarfile.open(output_path, 'w:gz') as tar_file:
+    for idx, dep in enumerate(dependencies):
+      dep = os.path.normpath(dep)
+      archive_name = os.path.join('fuchsia/release', dep)
+      archive_name = os.path.normpath(archive_name)
+      tar_file.add(dep, arcname=archive_name)
+      if idx % step == 0 or idx == (total_deps - 1):
+        logging.info('Progress: %s percent', int(round(100.0/total_deps * idx)))
+
+
+def main():
+  logging.basicConfig(level=logging.INFO)
+  args = parser.parse_args()
+  dependencies = read_dependencies(args.deps_list_path)
+  if args.additional_deps:
+    to_include = args.additional_deps.strip(',').split(',')
+    logging.info('Including: %s', to_include)
+    dependencies.update(to_include)
+  if args.exclude_deps:
+    dependencies = filter_dependencies(dependencies, args.exclude_deps)
+  create_tarfile(args.output, dependencies)
+
+
+if __name__ == '__main__':
+  main()