Fuchsia: Produce signed metadata+blob packages at build time.

* Remove FAR to package conversion logic.
* Migrate to "pm install" for installing packages on the deployment
  target.
* Plumb the unadorned application name through the various scripting
  layers, so that it can be run like "run foo_unittests".
* Roll SDK to 6e46feb3b26db267c65ea0923426a16f4da835bb.

Bug: 707030,798851

Change-Id: I4c8cc439a8be7a8d158df87200d0db6d67e6c9df
Reviewed-on: https://chromium-review.googlesource.com/963763
Commit-Queue: Kevin Marshall <kmarshall@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Reviewed-by: Sergey Ulanov <sergeyu@chromium.org>
Cr-Commit-Position: refs/heads/master@{#544130}

build/config/fuchsia/build_manifest.py

@@ -92,7 +92,8 @@ def _IsBinary(path):
   return file_tag == '\x7fELF'
 
 
-def BuildManifest(root_dir, out_dir, app_name, runtime_deps_file, output_path):
+def BuildManifest(root_dir, out_dir, app_name, app_filename,
+                  runtime_deps_file, output_path):
   with open(output_path, 'w') as output:
     # Process the runtime deps file for file paths, recursively walking
     # directories as needed.
@@ -118,7 +119,7 @@ def BuildManifest(root_dir, out_dir, app_name, runtime_deps_file, output_path):
 
       in_package_path = MakePackagePath(os.path.join(out_dir, next_file),
                                         [root_dir, out_dir])
-      if in_package_path == app_name:
+      if in_package_path == app_filename:
         in_package_path = 'bin/app'
         app_found = True
 
@@ -129,7 +130,14 @@ def BuildManifest(root_dir, out_dir, app_name, runtime_deps_file, output_path):
                                 os.path.relpath(next_file, out_dir)))
     if not app_found:
       raise Exception('Could not locate executable inside runtime_deps.')
-    output.write('meta/sandbox=' +
+
+    with open(os.path.join(os.path.dirname(output_path), 'package'), 'w') \
+        as package_json:
+      json.dump({'version': '0', 'name': app_name}, package_json)
+      output.write('meta/package=%s\n' %
+                   os.path.relpath(package_json.name, out_dir))
+
+    output.write('meta/sandbox=%s\n' %
                  os.path.relpath(os.path.join(root_dir, SANDBOX_POLICY_PATH),
                                  out_dir))
 
@@ -137,5 +145,4 @@ def BuildManifest(root_dir, out_dir, app_name, runtime_deps_file, output_path):
 
 
 if __name__ == '__main__':
-  sys.exit(BuildManifest(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4],
-                         sys.argv[5]))
+  sys.exit(BuildManifest(*sys.argv[1:]))

build/config/fuchsia/package.gni

@@ -16,11 +16,20 @@ template("package") {
   }
   assert(defined(pkg.binary))
 
+  _pm_tool_path = "//third_party/fuchsia-sdk/tools/pm"
+
   _pkg_out_dir = "$root_out_dir/gen/" + get_label_info(pkg.package_name, "dir")
   _runtime_deps_file = "$_pkg_out_dir/${pkg.package_name}.runtime_deps"
   _manifest_file = "$_pkg_out_dir/${pkg.package_name}.archive_manifest"
-  _archive_file = "$_pkg_out_dir/${pkg.package_name}.far"
+  _key_file = "$_pkg_out_dir/signing-key"
+  _meta_far_file = "$_pkg_out_dir/meta.far"
+  _combined_far_file = "$_pkg_out_dir/${pkg.package_name}-0.far"
+  _final_far_file = "$_pkg_out_dir/${pkg.package_name}.far"
+
   _write_manifest_target = "${pkg.package_name}__write_manifest"
+  _generate_key_target = "${pkg.package_name}__genkey"
+  _package_target = "${pkg.package_name}__pkg"
+  _bundle_target = "${pkg.package_name}__bundle"
 
   # Generates a manifest file based on the GN runtime deps
   # suitable for "far" tool consumption.
@@ -47,6 +56,7 @@ template("package") {
     args = [
       rebase_path("//"),
       rebase_path(root_out_dir),
+      pkg.package_name,
       pkg.binary,
       rebase_path(_runtime_deps_file),
       rebase_path(_manifest_file),
@@ -55,31 +65,104 @@ template("package") {
     write_runtime_deps = _runtime_deps_file
   }
 
-  # Packages an executable target and its dependencies into a Fuchsia archive
-  # file (.far).
-  action(target_name) {
+  # Generates a signing key to use for building the package.
+  action(_generate_key_target) {
+    forward_variables_from(invoker, [ "testonly" ])
+
+    script = "//build/gn_run_binary.py"
+
+    outputs = [
+      _key_file,
+    ]
+
+    args = [
+      rebase_path(_pm_tool_path, root_build_dir),
+      "-k",
+      rebase_path(_key_file),
+      "genkey",
+    ]
+  }
+
+  # Creates a signed Fuchsia metadata package.
+  action(_package_target) {
     forward_variables_from(invoker, [ "testonly" ])
 
-    far_tool_path = "//third_party/fuchsia-sdk/tools/far"
     script = "//build/gn_run_binary.py"
 
     deps = [
+      ":$_generate_key_target",
       ":$_write_manifest_target",
     ]
 
+    inputs = [
+      _key_file,
+    ]
+
     outputs = [
-      _archive_file,
+      _meta_far_file,
     ]
 
-    data = [
-      _archive_file,
+    args = [
+      rebase_path(_pm_tool_path, root_build_dir),
+      "-o",
+      rebase_path(_pkg_out_dir),
+      "-k",
+      rebase_path(_key_file),
+      "-m",
+      rebase_path(_manifest_file),
+      "build",
+    ]
+  }
+
+  # Creates a package containing the metadata archive and blob data.
+  action(_bundle_target) {
+    forward_variables_from(invoker, [ "testonly" ])
+
+    script = "//build/gn_run_binary.py"
+
+    deps = [
+      ":$_package_target",
+      ":$_write_manifest_target",
+    ]
+
+    inputs = [
+      _meta_far_file,
+      _manifest_file,
+    ]
+
+    outputs = [
+      _combined_far_file,
     ]
 
     args = [
-      rebase_path(far_tool_path, root_build_dir),
-      "create",
-      "--archive=" + rebase_path(_archive_file),
-      "--manifest=" + rebase_path(_manifest_file),
+      rebase_path(_pm_tool_path, root_build_dir),
+      "-o",
+      rebase_path(_pkg_out_dir),
+      "-m",
+      rebase_path(_manifest_file),
+      "archive",
+    ]
+  }
+
+  # Copies the archive to a well-known path.
+  # TODO(kmarshall): Use a 'pm' output flag to write directly to the desired
+  # file path instead.
+  copy(target_name) {
+    forward_variables_from(invoker, [ "testonly" ])
+
+    deps = [
+      ":$_bundle_target",
+    ]
+
+    data = [
+      _final_far_file,
+    ]
+
+    sources = [
+      _combined_far_file,
+    ]
+    outputs = [
+      _final_far_file,
     ]
   }
 }

build/config/fuchsia/rules.gni

@@ -126,10 +126,6 @@ template("generate_runner_script") {
     args = [
       "--script-output-path",
       rebase_path(_generated_script, root_build_dir, root_out_dir),
-      "--package",
-      rebase_path(_package_path, root_out_dir, root_build_dir),
-      "--package-manifest",
-      rebase_path(_manifest_path, root_out_dir, root_build_dir),
     ]
 
     if (defined(invoker.use_test_server) && invoker.use_test_server) {
@@ -144,6 +140,10 @@ template("generate_runner_script") {
       rebase_path(root_build_dir, root_build_dir),
       "--target-cpu",
       target_cpu,
+      "--package",
+      rebase_path(_package_path, root_out_dir, root_build_dir),
+      "--package-name",
+      invoker.package_name,
     ]
   }
 }

build/fuchsia/runner_common.py

@@ -733,7 +733,7 @@ def RunFuchsia(bootfs_data, use_device, kernel_path, dry_run,
     # The precise root cause is still nebulous, but this fix works.
     # See crbug.com/741194.
     process = subprocess.Popen(
-        qemu_command, stdout=subprocess.PIPE, stdin=open(os.devnull))
+        qemu_command, stdout=subprocess.PIPE)
 
   success = _HandleOutputFromProcess(process,
                                      bootfs_data.symbols_mapping)

build/fuchsia/runner_v2/common_args.py

@@ -18,9 +18,9 @@ def AddCommonArgs(arg_parser):
   common_args.add_argument('--package',
                            type=os.path.realpath, required=True,
                            help='Path to the package to execute.')
-  common_args.add_argument('--package-manifest',
-                           type=os.path.realpath, required=True,
-                           help='Path to the Fuchsia package manifest file.')
+  common_args.add_argument('--package-name', required=True,
+                           help='Name of the package to execute, defined in ' +
+                                'package metadata.')
   common_args.add_argument('--output-directory',
                            type=os.path.realpath, required=True,
                            help=('Path to the directory in which build files are'

build/fuchsia/runner_v2/create_runner_script.py

@@ -55,7 +55,6 @@ def main(args):
   group = parser.add_argument_group('Test runner path arguments.')
   group.add_argument('--output-directory')
   group.add_argument('--package')
-  group.add_argument('--package-manifest')
   args, runner_args = parser.parse_known_args(args)
 
   def RelativizePathToScript(path):
@@ -71,8 +70,6 @@ def main(args):
       ('--output-directory', RelativizePathToScript(args.output_directory)))
   runner_path_args.append(
       ('--package', RelativizePathToScript(args.package)))
-  runner_path_args.append(
-      ('--package-manifest', RelativizePathToScript(args.package_manifest)))
 
   with open(args.script_output_path, 'w') as script:
     script.write(SCRIPT_TEMPLATE.format(

build/fuchsia/runner_v2/exe_runner.py

@@ -28,7 +28,7 @@ def main():
   with GetDeploymentTargetForArgs(args) as target:
     target.Start()
     RunPackage(args.output_directory, target, args.package,
-               args.child_args, args.package_manifest)
+               args.package_name, args.child_args)
 
 
 if __name__ == '__main__':

build/fuchsia/runner_v2/qemu_target.py

@@ -126,10 +126,14 @@ class QemuTarget(target.Target):
     # See crbug.com/741194.
     logging.debug('Launching QEMU.')
     logging.debug(' '.join(qemu_command))
-    self._qemu_process = subprocess.Popen(
-        qemu_command, stdout=open(os.devnull), stdin=open(os.devnull),
-        stderr=open(os.devnull))
 
+    stdio_flags = {'stdin': open(os.devnull)}
+    if logging.getLogger().getEffectiveLevel() != logging.DEBUG:
+      # Output the Fuchsia debug log.
+      stdio_flags['stdout'] = open(os.devnull)
+      stdio_flags['stderr'] = open(os.devnull)
+
+    self._qemu_process = subprocess.Popen(qemu_command, **stdio_flags)
     self._WaitUntilReady();
 
   def Shutdown(self):

build/fuchsia/runner_v2/run_package.py

@@ -20,124 +20,7 @@ FAR = os.path.join(common.SDK_ROOT, 'tools', 'far')
 PM = os.path.join(common.SDK_ROOT, 'tools', 'pm')
 
 
-def _Deploy(target, output_dir, archive_path):
-  """Converts the FAR archive at |archive_path| into a Fuchsia package
-  and deploys it to |target|'s blobstore. If target.IsNewInstance() is not set,
-  then the remote target's blobstore is queried and only the changed blobs
-  are copied.
-
-  Returns: the name of the package, which can be run by executing
-           'run <package_name>' on the target."""
-
-  logging.info('Deploying package to target.')
-
-  staging_path = None
-  blob_link_dir = None
-  far_contents_dir = None
-  try:
-    package_name = os.path.basename(archive_path)
-
-    logging.debug('Extracting archive contents.')
-    far_contents_dir = tempfile.mkdtemp()
-    subprocess.check_call([FAR, 'extract', '--archive=%s' % archive_path,
-                           '--output=%s' % far_contents_dir])
-
-    logging.debug('Building package metadata.')
-    with open(os.path.join(far_contents_dir, 'meta', 'package'), 'w') \
-        as package_json:
-      json.dump({'version': '0', 'name': package_name}, package_json)
-    manifest = tempfile.NamedTemporaryFile()
-    for root, _, files in os.walk(far_contents_dir):
-      for f in files:
-        path = os.path.join(root, f)
-        manifest.write('%s=%s\n' %
-                       (os.path.relpath(path, far_contents_dir), path))
-    manifest.flush()
-
-    # Generate the package signing key.
-    signing_key_path = os.path.join(output_dir, 'signing-key')
-    if not os.path.exists(signing_key_path):
-      subprocess.check_call([PM, '-k', signing_key_path, 'genkey'])
-
-    # Build the package metadata archive.
-    staging_path = tempfile.mkdtemp()
-    subprocess.check_call([PM, '-o', staging_path, '-k', signing_key_path,
-                           '-m', manifest.name, 'build'])
-
-    # If the target is already warm, then it's possible that some time can be
-    # saved by only sending the sending the changed blobs.
-    # If the target is newly booted, however, it can be safely assumed that the
-    # target doesn't contain the blobs, so the query step can be skipped.
-    existing_blobs = set()
-    if not target.IsNewInstance():
-      logging.debug('Querying the target blobstore\'s state.')
-      ls = target.RunCommandPiped(['ls', '/blob'], stdout=subprocess.PIPE)
-      for blob in ls.stdout:
-        existing_blobs.add(blob.strip())
-      ls.wait()
-
-    # Locally stage the blobs and metadata into a local temp dir to prepare for
-    # a one-shot 'scp' copy.
-    blob_link_dir = tempfile.mkdtemp()
-    os.symlink(os.path.join(staging_path, 'meta.far'),
-               os.path.join(blob_link_dir, 'meta.far'))
-    for next_line in open(os.path.join(staging_path, 'meta', 'contents')):
-      rel_path, blob = next_line.strip().split('=')
-      dest_path = os.path.join(blob_link_dir, blob)
-      if blob not in existing_blobs:
-        existing_blobs.add(blob)
-        os.symlink(os.path.join(far_contents_dir, rel_path),
-                   os.path.join(blob_link_dir, blob))
-
-    # Copy the package metadata and contents into tmpfs, and then execute
-    # remote commands to copy the data into pkgfs/incoming.
-    # We can't use 'scp' to copy directly into /pkgfs/incoming due to a bug in
-    # Fuchsia (see bug PKG-9).
-    logging.debug('Deploying to target blobstore.')
-    target.PutFile(blob_link_dir, '/tmp', recursive=True)
-
-    # Copy everything into target's /pkgfs/incoming and then clean up.
-    # The command is joined with double ampersands so that all the steps can
-    # be done over the lifetime of a single SSH connection.
-    # Note that 'mv' moves to pkgfs are unsupported (Fuchsia bug PKG-9).
-    blob_tmpdir = os.path.join('/tmp/', os.path.basename(blob_link_dir))
-    result = target.RunCommand([
-        # Speed things up by using builtin commands, which don't suffer the
-        # dynamic library loading speed penalty that's present in the
-        # executables.
-        'unset PATH', '&&'
-
-        # Separate meta.far from the blobs so we can add the blobs in
-        # bulk in advance of registering the package (faster).
-        'mv', os.path.join(blob_tmpdir, 'meta.far'), '/tmp', '&&',
-
-        # Register the package.
-        'cp', '/tmp/meta.far', '/pkgfs/incoming', '&&',
-
-        # Install the blobs into /pkgfs.
-        'for', 'f', 'in', os.path.join(blob_tmpdir, '*'), ';', 'do',
-               'cp', '$f', '/pkgfs/incoming', ';',
-        'done', '&&',
-
-        'rm', '-rf', blob_tmpdir, '&&',
-
-        'rm', '/tmp/meta.far'])
-
-    if result != 0:
-      raise Exception('Deployment failed.')
-
-    return package_name
-
-  finally:
-    if blob_link_dir:
-      shutil.rmtree(blob_link_dir)
-    if staging_path:
-      shutil.rmtree(staging_path)
-    if far_contents_dir:
-      shutil.rmtree(far_contents_dir)
-
-
-def RunPackage(output_dir, target, package_path, run_args,
+def RunPackage(output_dir, target, package_path, package_name, run_args,
                symbolizer_config=None):
   """Copies the Fuchsia package at |package_path| to the target,
   executes it with |run_args|, and symbolizes its output.
@@ -145,14 +28,30 @@ def RunPackage(output_dir, target, package_path, run_args,
   output_dir: The path containing the build output files.
   target: The deployment Target object that will run the package.
   package_path: The path to the .far package file.
-  run_args: The command-linearguments which will be passed to the Fuchsia process.
-  symbolizer_config: A newline delimited list of source files contained in the
-                     package. Omitting this parameter will disable symbolization.
+  package_name: The name of app specified by package metadata.
+  run_args: The arguments which will be passed to the Fuchsia process.
+  symbolizer_config: A newline delimited list of source files contained
+                     in the package. Omitting this parameter will disable
+                     symbolization.
 
   Returns the exit code of the remote package process."""
 
 
-  package_name = _Deploy(target, output_dir, package_path)
+  logging.debug('Copying package to target.')
+  tmp_path = os.path.join('/tmp', os.path.basename(package_path))
+  target.PutFile(package_path, tmp_path)
+
+  logging.debug('Installing package.')
+  p = target.RunCommandPiped(['pm', 'install', tmp_path],
+                             stderr=subprocess.PIPE)
+  output = p.stderr.readlines()
+  p.wait()
+  if p.returncode != 0:
+    # Don't error out if the package already exists on the device.
+    if len(output) != 1 or 'ErrAlreadyExists' not in output[0]:
+      raise Exception('Error when installing package: %s' % '\n'.join(output))
+
+  logging.debug('Running package.')
   command = ['run', package_name] + run_args
   process = target.RunCommandPiped(command,
                                    stdin=open(os.devnull, 'r'),
@@ -167,7 +66,7 @@ def RunPackage(output_dir, target, package_path, run_args,
     output = process.stdout
 
   for next_line in output:
-    print next_line
+    print next_line.strip()
 
   process.wait()
   if process.returncode != 0:

build/fuchsia/runner_v2/test_runner.py

@@ -100,8 +100,8 @@ def main():
     if args.enable_test_server:
       test_server, forwarder = SetupTestServer(target, test_concurrency)
 
-    RunPackage(args.output_directory, target, args.package,
-               child_args, args.package_manifest)
+    RunPackage(args.output_directory, target, args.package, args.package_name,
+               child_args)
 
     if forwarder:
       forwarder.terminate()

build/fuchsia/update_sdk.py

@@ -13,7 +13,7 @@ import sys
 import tarfile
 import tempfile
 
-SDK_HASH = 'c277a8a3c81b8012cfe03bff945cf15f945dfd63'
+SDK_HASH = '6e46feb3b26db267c65ea0923426a16f4da835bb'
 
 REPOSITORY_ROOT = os.path.abspath(os.path.join(
     os.path.dirname(__file__), '..', '..'))