Build Fuchsia packages for test and executable targets.

All targets that use the test() or fuchsia_executable_runner rules
will produce Fuchsia packages, which contain the test executable and
their runtime dependencies.

The packages are modules which can be deployed in an already-running
system, as opposed to the current scheme which bakes the files into
the system boot image.

Deployment and support for dynamic libraries will be addressed
in followup CLs.

This CL also:
* Fixes PathService logic for packaged invocations.
* Adds a new "build_manifest.py" GN helper script.
* Fixes the dependency graph for Fuchsia runner script generation
  so that it accurately captures the real dependency ordering.


Bug: 796779
Change-Id: I4e3a38e0075573b328ad98425cc493e28cff1ca4
Reviewed-on: https://chromium-review.googlesource.com/841749
Commit-Queue: Kevin Marshall <kmarshall@chromium.org>
Reviewed-by: Wez <wez@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Reviewed-by: Sergey Ulanov <sergeyu@chromium.org>
Cr-Commit-Position: refs/heads/master@{#531288}

base/BUILD.gn

@@ -1278,6 +1278,7 @@ jumbo_component("base") {
 
     sources += [
       "base_paths_fuchsia.cc",
+      "base_paths_fuchsia.h",
       "debug/stack_trace_fuchsia.cc",
       "files/file_path_watcher_fuchsia.cc",
       "fuchsia/default_job.cc",

base/base_paths_fuchsia.cc

@@ -6,41 +6,54 @@
 
 #include <stdlib.h>
 
+#include "base/base_paths_fuchsia.h"
 #include "base/command_line.h"
-#include "base/files/file_path.h"
+#include "base/files/file_util.h"
+#include "base/path_service.h"
+#include "base/process/process.h"
 
 namespace base {
+namespace {
+
+constexpr char kPackageRoot[] = "/pkg";
+
+}  // namespace
+
+base::FilePath GetPackageRoot() {
+  base::FilePath path_obj(kPackageRoot);
+  if (PathExists(path_obj)) {
+    return path_obj;
+  } else {
+    return base::FilePath();
+  }
+}
 
 bool PathProviderFuchsia(int key, FilePath* result) {
   switch (key) {
     case FILE_MODULE:
-    // Not supported in debug or component builds. Fall back on using the EXE
-    // path for now.
-    // TODO(fuchsia): Get this value from an API. See crbug.com/726124
-    case FILE_EXE: {
-      // Use the binary name as specified on the command line.
-      // TODO(fuchsia): It would be nice to get the canonical executable path
-      // from a kernel API. See https://crbug.com/726124
-      char bin_dir[PATH_MAX + 1];
-      if (realpath(base::CommandLine::ForCurrentProcess()
-                       ->GetProgram()
-                       .AsUTF8Unsafe()
-                       .c_str(),
-                   bin_dir) == NULL) {
+      NOTIMPLEMENTED();
       return false;
-      }
-      *result = FilePath(bin_dir);
+    case FILE_EXE: {
+      *result = base::MakeAbsoluteFilePath(base::FilePath(
+          base::CommandLine::ForCurrentProcess()->GetProgram().AsUTF8Unsafe()));
       return true;
     }
     case DIR_SOURCE_ROOT:
-      // This is only used for tests, so we return the binary location for now.
+      *result = GetPackageRoot();
+      if (result->empty()) {
         *result = FilePath("/system");
+      }
       return true;
     case DIR_CACHE:
       *result = FilePath("/data");
       return true;
+    case DIR_FUCHSIA_RESOURCES:
+      *result = GetPackageRoot();
+      if (result->empty()) {
+        PathService::Get(DIR_EXE, result);
+      }
+      return true;
   }
-
   return false;
 }
 

base/base_paths_fuchsia.h

+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef BASE_BASE_PATHS_FUCHSIA_H_
+#define BASE_BASE_PATHS_FUCHSIA_H_
+
+#include "base/base_export.h"
+#include "base/files/file_path.h"
+
+namespace base {
+
+// These can be used with the PathService to access various special
+// directories and files.
+enum {
+  PATH_FUCHSIA_START = 1200,
+
+  // Path to the directory which contains application libraries and resources.
+  DIR_FUCHSIA_RESOURCES,
+
+  PATH_FUCHSIA_END,
+};
+
+// If running inside a package, returns a FilePath of the root path
+// of the currently deployed package.
+// Otherwise returns an empty FilePath.
+BASE_EXPORT base::FilePath GetPackageRoot();
+
+}  // namespace base
+
+#endif  // BASE_BASE_PATHS_FUCHSIA_H_

base/i18n/icu_util.cc

@@ -37,6 +37,10 @@
 #include "base/mac/foundation_util.h"
 #endif
 
+#if defined(OS_FUCHSIA)
+#include "base/base_paths_fuchsia.h"
+#endif
+
 namespace base {
 namespace i18n {
 
@@ -107,6 +111,8 @@ void LazyInitIcuDataFile() {
   CHECK(path_ok);  // TODO(scottmg): http://crbug.com/445616
 #elif defined(OS_ANDROID)
   bool path_ok = PathService::Get(DIR_ANDROID_APP_DATA, &data_path);
+#elif defined(OS_FUCHSIA)
+  bool path_ok = PathService::Get(DIR_FUCHSIA_RESOURCES, &data_path);
 #else
   // For now, expect the data file to be alongside the executable.
   // This is sufficient while we work on unit tests, but will eventually

build/config/fuchsia/build_manifest.py

+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Creates a archive manifest used for Fuchsia package generation.
+
+Arguments:
+  root_dir: The absolute path to the Chromium source tree root.
+
+  out_dir: The absolute path to the Chromium build directory.
+
+  app_name: The filename of the package's executable target.
+
+  runtime_deps: The path to the GN runtime deps file.
+
+  output_path: The path of the manifest file which will be written.
+"""
+
+import json
+import os
+import sys
+import tempfile
+
+
+# Path to file describing the services to be made available to the process.
+SANDBOX_POLICY_PATH = 'build/config/fuchsia/sandbox_policy'
+
+
+def MakePackagePath(file_path, roots):
+  """Computes a path for |file_path| that is relative to one of the directory
+  paths in |roots|.
+
+  file_path: The absolute file path to relativize.
+  roots: A list of absolute directory paths which may serve as a relative root
+         for |file_path|. At least one path must contain |file_path|.
+         Overlapping roots are permitted; the deepest matching root will be
+         chosen.
+
+  Examples:
+
+  >>> MakePackagePath('/foo/bar.txt', ['/foo/'])
+  'bar.txt'
+
+  >>> MakePackagePath('/foo/dir/bar.txt', ['/foo/'])
+  'dir/bar.txt'
+
+  >>> MakePackagePath('/foo/out/Debug/bar.exe', ['/foo/', '/foo/out/Debug/'])
+  'bar.exe'
+  """
+
+  # Prevents greedily matching against a shallow path when a deeper, better
+  # matching path exists.
+  roots.sort(key=len, reverse=True)
+
+  for next_root in roots:
+    if not next_root.endswith(os.sep):
+      next_root += os.sep
+
+    if file_path.startswith(next_root):
+      relative_path = file_path[len(next_root):]
+
+      # TODO(fuchsia): The requirements for finding/loading .so are in flux, so
+      # this ought to be reconsidered at some point.
+      # See https://crbug.com/732897.
+      if file_path.endswith('.so'):
+        relative_path = 'lib/' + os.path.basename(relative_path)
+
+      return relative_path
+
+  sys.stderr.write(
+      'Error: no matching root paths found for \'%s\'.' % file_path)
+  assert False
+
+
+def BuildManifest(root_dir, out_dir, app_name, runtime_deps_file, output_path):
+  with open(output_path, "w") as output:
+    # Process the runtime deps file for file paths, recursively walking
+    # directories as needed.
+    # runtime_deps may contain duplicate paths, so use a set for
+    # de-duplication.
+    expanded_files = set()
+    for next_path in open(runtime_deps_file, 'r'):
+      next_path = next_path.strip()
+      if os.path.isdir(next_path):
+        for root, _, files in os.walk(next_path):
+          for next_file in files:
+            expanded_files.add(os.path.abspath(os.path.join(root, next_file)))
+      else:
+        expanded_files.add(os.path.abspath(next_path))
+
+    # Format and write out the manifest contents.
+    app_found = False
+    for next_file in expanded_files:
+      in_package_path = MakePackagePath(os.path.join(out_dir, next_file),
+                                        [root_dir, out_dir])
+      if in_package_path == app_name:
+        in_package_path = 'bin/app'
+        app_found = True
+      output.write('%s=%s\n' % (in_package_path, next_file))
+    assert app_found
+    output.write('meta/sandbox=%s%s' % (root_dir, SANDBOX_POLICY_PATH))
+
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(BuildManifest(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4],
+                         sys.argv[5]))

build/config/fuchsia/package.gni

+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Creates a Fuchsia .far package file.
+#
+# Parameters are:
+# target_name: The name of the package to build.
+# binary: the executable target which should be launched by the package.
+#         Will be renamed as "bin/app" in the package contents.
+template("package") {
+  pkg = {
+    package_name = target_name
+    forward_variables_from(invoker, "*")
+  }
+  assert(defined(pkg.binary))
+
+  _runtime_deps_file = "$root_out_dir/gen.runtime/${pkg.package_name}/" +
+                       "${pkg.package_name}.runtime_deps"
+  _manifest_file = "$root_out_dir/gen.runtime/${pkg.package_name}/" +
+                   "${pkg.package_name}.archive_manifest"
+  _write_manifest_target = "${pkg.package_name}__write_manifest"
+  _write_archive_target = "${pkg.package_name}__write_archive"
+  _pkg_out_dir = "$root_build_dir/package"
+
+  # Generates a manifest file based on the GN runtime deps
+  # suitable for "far" tool consumption.
+  action(_write_manifest_target) {
+    forward_variables_from(invoker,
+                           [
+                             "data",
+                             "data_deps",
+                             "deps",
+                             "public_deps",
+                             "testonly",
+                           ])
+
+    script = "//build/config/fuchsia/build_manifest.py"
+    inputs = [
+      _runtime_deps_file,
+    ]
+    outputs = [
+      _manifest_file,
+    ]
+
+    args = [
+      rebase_path("//"),
+      rebase_path(root_out_dir),
+      pkg.binary,
+      rebase_path(_runtime_deps_file),
+      rebase_path(_manifest_file),
+    ]
+
+    write_runtime_deps = _runtime_deps_file
+  }
+
+  # Packages an executable target and its dependencies into a Fuchsia archive
+  # file (.far).
+  action(_write_archive_target) {
+    forward_variables_from(invoker,
+                           [
+                             "testonly",
+                             "data_deps",
+                           ])
+
+    far_tool_path = "//third_party/fuchsia-sdk/tools/far"
+    archive_path = "$_pkg_out_dir/${pkg.package_name}.far"
+    script = "//build/gn_run_binary.py"
+    depfile = "$target_gen_dir/$target_name.d"
+
+    deps = [
+      ":$_write_manifest_target",
+    ]
+
+    outputs = [
+      archive_path,
+    ]
+
+    args = [
+      rebase_path(far_tool_path, root_build_dir),
+      "create",
+      "--archive=" + rebase_path(archive_path),
+      "--manifest=" + rebase_path(_manifest_file),
+    ]
+  }
+
+  group(target_name) {
+    deps = []
+    forward_variables_from(invoker, "*")
+    deps += [
+      ":$_write_archive_target",
+      ":$_write_manifest_target",
+    ]
+  }
+}

build/config/fuchsia/rules.gni

-# Copyright 2017 The Chromium Authors. All rights reserved.
+# Copyright 2018 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 assert(is_fuchsia)
 
+import("//build/config/fuchsia/package.gni")
+
 template("generate_runner_script") {
   # This runtime_deps file is used at runtime and thus cannot go in
   # target_gen_dir.
-  _target_dir_name = get_label_info(invoker.exe_target, "dir")
-  _target_shortname = get_label_info(invoker.exe_target, "name")
-  _runtime_deps_file = "$root_out_dir/gen.runtime/$_target_dir_name/$_target_shortname.runtime_deps"
-  _runtime_deps_target = "${target_name}__write_deps"
+  _runtime_deps_target = "${target_name}__deps"
+  _runtime_deps_file =
+      "$root_out_dir/gen.runtime/" +
+      get_label_info(invoker.root_target_name, "dir") + "/" +
+      get_label_info(invoker.root_target_name, "name") + ".runtime_deps"
+
   group(_runtime_deps_target) {
     forward_variables_from(invoker,
                            [
@@ -67,7 +71,7 @@ template("generate_runner_script") {
     ]
 
     deps += [
-      ":$_runtime_deps_target",
+      ":${_runtime_deps_target}",
       "//testing/buildbot/filters:fuchsia_filters",
     ]
     data += [ _runtime_deps_file ]
@@ -85,7 +89,7 @@ template("generate_runner_script") {
       "--script-output-path",
       rebase_path(invoker.generated_script, root_build_dir),
       "--exe-name",
-      _target_shortname,
+      rebase_path(invoker.exe_path, root_build_dir),
     ]
 
     if (defined(invoker.use_test_server) && invoker.use_test_server) {
@@ -102,47 +106,75 @@ template("test_runner_script") {
   generate_runner_script(target_name) {
     testonly = true
     runner_script = "test_runner.py"
-    exe_target = invoker.test_name
+    exe_path = invoker.exe_path
+    root_target_name = invoker.test_name
     generated_script =
-        "$root_build_dir/bin/run_" + get_label_info(exe_target, "name")
+        "$root_build_dir/bin/run_" + get_label_info(invoker.test_name, "name")
     forward_variables_from(invoker, "*")
   }
 }
 
 # This template is used to generate a runner script for arbitrary executables
 # into the build dir for Fuchsia. The template should reference an "executable"
-# target using the "exe_target" attribute.
+# path using the "exe_path" attribute.
 #
 # Example usage:
 #
+#   _exe_path = "$root_out_dir/foo_fuchsia"
 #   executable("foo") {
 #     sources = [ "foo_main.cc" ]
+#     output_name = _exe_path
 #   }
 #   fuchsia_executable_runner("foo_fuchsia") {
-#     exe_target = ":foo"
+#     exe_path = _exe_path
 #   }
 template("fuchsia_executable_runner") {
-  generate_runner_script(target_name) {
+  forward_variables_from(invoker, [ "exe_target" ])
+
+  _pkg_target = "${target_name}_pkg"
+  _gen_runner_target = "${target_name}_runner"
+  _archive_target = "${target_name}_archive"
+  _exe_name = get_label_info(exe_target, "name")
+  _exe_path = "${root_out_dir}/${_exe_name}"
+
+  package(_pkg_target) {
+    forward_variables_from(invoker, [ "testonly" ])
+    package_name = _exe_name
+    binary = _exe_name
+    data_deps = [
+      exe_target,
+    ]
+  }
+
+  generate_runner_script(_gen_runner_target) {
     forward_variables_from(invoker,
                            [
                              "testonly",
-                             "exe_target",
                              "data_deps",
                            ])
     runner_script = "exe_runner.py"
-    generated_script =
-        "$root_build_dir/bin/run_" + get_label_info(exe_target, "name")
+    generated_script = "$root_build_dir/bin/run_${_exe_name}"
     if (!defined(data_deps)) {
       data_deps = []
     }
     data_deps += [ exe_target ]
+    exe_path = _exe_path
+    root_target_name = invoker.target_name
+  }
+
+  group(target_name) {
+    forward_variables_from(invoker, [ "testonly" ])
+    deps = [
+      ":${_archive_target}",
+      ":${_gen_runner_target}",
+      ":${_pkg_target}",
+    ]
   }
 
-  generate_runner_script(target_name + "_archive") {
+  generate_runner_script(_archive_target) {
     forward_variables_from(invoker,
                            [
                              "testonly",
-                             "exe_target",
                              "data_deps",
                            ])
     runner_script = "archive_builder.py"
@@ -152,5 +184,7 @@ template("fuchsia_executable_runner") {
       data_deps = []
     }
     data_deps += [ exe_target ]
+    exe_path = _exe_path
+    root_target_name = invoker.target_name
   }
 }

build/config/fuchsia/sandbox_policy

+{
+  "features": [ "persistent-storage",
+                "shell",
+                "system-temp" ]
+}
+

testing/test.gni

@@ -14,6 +14,7 @@ if (is_android) {
 
 if (is_fuchsia) {
   import("//build/config/fuchsia/rules.gni")
+  import("//build/config/fuchsia/package.gni")
 }
 
 # Define a test as an executable (or apk on Android) with the "testonly" flag
@@ -226,29 +227,43 @@ template("test") {
     }
   } else if (is_fuchsia) {
     _output_name = invoker.target_name
-    _test_runner_target = "${_output_name}__test_runner_script"
+    _pkg_target = "${_output_name}_pkg"
+    _gen_runner_target = "${_output_name}_runner"
+    _exec_target = "${_output_name}__exec"
 
-    test_runner_script(_test_runner_target) {
-      forward_variables_from(invoker,
-                             [
-                               "data",
-                               "data_deps",
-                               "deps",
-                               "public_deps",
-                               "use_test_server",
-                             ])
+    group(target_name) {
+      testonly = true
+      data_deps = [
+        ":$_gen_runner_target",
+      ]
+    }
+
+    # Makes the script which invokes the executable.
+    test_runner_script(_gen_runner_target) {
+      forward_variables_from(invoker, [ "use_test_server" ])
+      data_deps = [
+        ":$_exec_target",
+        ":$_pkg_target",
+      ]
       test_name = _output_name
+      exe_path = "$root_out_dir/" + get_label_info(_exec_target, "name")
     }
 
-    executable(target_name) {
+    executable(_exec_target) {
       testonly = true
       forward_variables_from(invoker, "*")
-      if (!defined(data_deps)) {
-        data_deps = []
-      }
-      data_deps += [ ":$_test_runner_target" ]
+      output_name = _exec_target
       deps += [ "//build/config:exe_and_shlib_deps" ]
     }
+
+    package(_pkg_target) {
+      testonly = true
+      package_name = _output_name
+      binary = get_label_info(_exec_target, "name")
+      data_deps = [
+        ":$_exec_target",
+      ]
+    }
   } else if (is_ios) {
     import("//build/config/ios/ios_sdk.gni")
     import("//build/config/ios/rules.gni")

tools/mb/mb.py

@@ -711,6 +711,7 @@ class MetaBuildWrapper(object):
         return ret
 
     android = 'target_os="android"' in vals['gn_args']
+    fuchsia = 'target_os="fuchsia"' in vals['gn_args']
     for target in swarming_targets:
       if android:
         # Android targets may be either android_apk or executable. The former
@@ -720,6 +721,11 @@ class MetaBuildWrapper(object):
         runtime_deps_targets = [
             target + '.runtime_deps',
             'obj/%s.stamp.runtime_deps' % label.replace(':', '/')]
+      elif fuchsia:
+        # Only emit a runtime deps file for the group() target on Fuchsia.
+        label = isolate_map[target]['label']
+        runtime_deps_targets = [
+          'obj/%s.stamp.runtime_deps' % label.replace(':', '/')]
       elif (isolate_map[target]['type'] == 'script' or
             isolate_map[target].get('label_type') == 'group'):
         # For script targets, the build target is usually a group,