Fuchsia: switch new runner scripts to use FVM disks instead of bootfs.

Switching to FVM will give us a functioning blobstore, a prerequisite
for installing packages, and allows the Fuchsia team to follow through
with bootfs deprecation.

Also added a missing package reference to local-sdk.py that caused it
to skip building the Fuchsia userspace.

Bug: 802331
Change-Id: I6cfc45c51dae97bef3c9f7a202f74fff483d5c50
Reviewed-on: https://chromium-review.googlesource.com/919266
Commit-Queue: Kevin Marshall <kmarshall@chromium.org>
Reviewed-by: Scott Graham <scottmg@chromium.org>
Cr-Commit-Position: refs/heads/master@{#539512}

build/config/fuchsia/package.gni

@@ -20,7 +20,6 @@ template("package") {
   _runtime_deps_file = "$_pkg_out_dir/${pkg.package_name}.runtime_deps"
   _manifest_file = "$_pkg_out_dir/${pkg.package_name}.archive_manifest"
   _archive_file = "$_pkg_out_dir/${pkg.package_name}.far"
-  _write_archive_target = "${pkg.package_name}__write_archive"
   _write_manifest_target = "${pkg.package_name}__write_manifest"
 
   # Generates a manifest file based on the GN runtime deps
@@ -58,7 +57,7 @@ template("package") {
 
   # Packages an executable target and its dependencies into a Fuchsia archive
   # file (.far).
-  action(_write_archive_target) {
+  action(target_name) {
     forward_variables_from(invoker, [ "testonly" ])
 
     far_tool_path = "//third_party/fuchsia-sdk/tools/far"
@@ -83,11 +82,4 @@ template("package") {
       "--manifest=" + rebase_path(_manifest_file),
     ]
   }
-
-  group(target_name) {
-    forward_variables_from(invoker, [ "testonly" ])
-    deps = [
-      ":$_write_archive_target",
-    ]
-  }
 }

build/fuchsia/runner_v2/boot_data.py

@@ -29,6 +29,7 @@ Host *
 
 def _TargetCpuToSdkBinPath(target_arch):
   """Returns the path to the kernel & bootfs .bin files for |target_cpu|."""
+
   return os.path.join(common.SDK_ROOT, 'target', target_arch)
 
 
@@ -62,45 +63,69 @@ def _ProvisionSSH(output_dir):
         _SSH_CONFIG_TEMPLATE.format(identity=id_key_path,
                                     known_hosts=known_hosts_path))
 
+  if os.path.exists(known_hosts_path):
+    os.remove(known_hosts_path)
+
   return (
       ssh_config_path,
-      (('data/ssh/ssh_host_ed25519_key', host_key_path),
-       ('data/ssh/ssh_host_ed25519_key.pub', host_pubkey_path),
-       ('data/ssh/authorized_keys', id_pubkey_path))
+      (('ssh/ssh_host_ed25519_key', host_key_path),
+       ('ssh/ssh_host_ed25519_key.pub', host_pubkey_path),
+       ('ssh/authorized_keys', id_pubkey_path))
   )
 
 
-def GetKernelPath(target_arch):
-  return os.path.join(_TargetCpuToSdkBinPath(target_arch), 'zircon.bin')
+def GetTargetFile(target_arch, filename):
+  """Computes a path to |filename| in the Fuchsia target directory specific to
+  |target_arch|."""
+
+  return os.path.join(_TargetCpuToSdkBinPath(target_arch), filename)
 
 
 def GetSSHConfigPath(output_dir):
   return output_dir + '/ssh_config'
 
 
-def CreateBootdata(output_dir, target_arch):
-  """Creates a bootdata image ready for SSH remote access.
+def ConfigureDataFVM(output_dir, sparse):
+  """Builds the FVM image for the persistent /data volume and prepopulates it
+  with SSH keys.
+
+  output_dir: Path to the output directory which will contain the FVM file.
+  sparse: If true, then a netboot-friendly sparse file will be generated.
+
+  Returns the path to the new FVM file."""
+
+  logging.debug('Building persistent data FVM file.')
+  data_file = os.path.join(output_dir, 'data.minfs.bin')
+  try:
+    # Build up the minfs partition data and install keys into it.
+    ssh_config, ssh_data = _ProvisionSSH(output_dir)
+    manifest = tempfile.NamedTemporaryFile()
+    for dest, src in ssh_data:
+      manifest.write('%s=%s\n' % (dest, src))
+    manifest.flush()
+    minfs_path = os.path.join(common.SDK_ROOT, 'tools', 'minfs')
+    subprocess.check_call([minfs_path, '%s@10m' % data_file, 'create'])
+    subprocess.check_call([minfs_path, data_file, 'manifest', manifest.name])
+
+    # Wrap the minfs partition in a FVM container.
+    fvm_path = os.path.join(common.SDK_ROOT, 'tools', 'fvm')
+    fvm_output_path = os.path.join(output_dir, 'fvm.data.blk')
+    if os.path.exists(fvm_output_path):
+      os.remove(fvm_output_path)
 
-  Returns a path to the bootdata.bin file."""
+    if sparse:
+      cmd = [fvm_path, fvm_output_path, 'sparse', '--compress', 'lz4',
+             '--data', data_file]
+    else:
+      cmd = [fvm_path, fvm_output_path, 'create', '--data', data_file]
 
-  base_boot_data = os.path.join(
-      _TargetCpuToSdkBinPath(target_arch), 'bootdata.bin')
-  ssh_config, ssh_data = _ProvisionSSH(output_dir)
-  ssh_manifest = tempfile.NamedTemporaryFile(delete=False)
-  for key, val in ssh_data:
-    ssh_manifest.write("%s=%s\n" % (key, val))
-  ssh_manifest.close()
-  mkbootfs_path = os.path.join(common.SDK_ROOT, 'tools', 'mkbootfs')
-  bootfs_path = output_dir + '/image.bootfs'
-  args = [mkbootfs_path, '-o', bootfs_path,
-          '--target=boot', base_boot_data,
-          '--target=system', ssh_manifest.name]
+    logging.debug(' '.join(cmd))
 
-  logging.debug(' '.join(args))
-  subprocess.check_call(args)
-  os.remove(ssh_manifest.name)
+    subprocess.check_call(cmd)
+    return fvm_output_path
 
-  return bootfs_path
+  finally:
+    os.remove(data_file)
 
 
 def GetNodeName(output_dir):

build/fuchsia/runner_v2/device_target.py

@@ -66,12 +66,23 @@ class DeviceTarget(target.Target):
 
       logging.info('Netbooting Fuchsia. ' +
                    'Please ensure that your device is in bootloader mode.')
-      boot_data_path = boot_data.CreateBootdata(
-          self._output_dir, self._GetTargetSdkArch())
       bootserver_path = os.path.join(common.SDK_ROOT, 'tools', 'bootserver')
-      bootserver_command = [bootserver_path, '-1',
-                            boot_data.GetKernelPath(self._GetTargetSdkArch()),
-                            boot_data_path, '--'] + \
+      data_fvm_path = boot_data.ConfigureDataFVM(self._output_dir, True)
+      bootserver_command = [bootserver_path,
+                            '-1',
+                            '--efi',
+                            boot_data.GetTargetFile(self._GetTargetSdkArch(),
+                                                    'local.esp.blk'),
+                            '--fvm',
+                            boot_data.GetTargetFile(self._GetTargetSdkArch(),
+                                                    'fvm.sparse.blk'),
+                            '--fvm',
+                            data_fvm_path,
+                            boot_data.GetTargetFile(self._GetTargetSdkArch(),
+                                                    'zircon.bin'),
+                            boot_data.GetTargetFile(self._GetTargetSdkArch(),
+                                                    'bootdata-blobstore.bin'),
+                            '--'] + \
                             boot_data.GetKernelArgs(self._output_dir)
       logging.debug(' '.join(bootserver_command))
       subprocess.check_call(bootserver_command)

build/fuchsia/runner_v2/qemu_target.py

@@ -51,8 +51,6 @@ class QemuTarget(target.Target):
       self.Shutdown()
 
   def Start(self):
-    boot_data_path = boot_data.CreateBootdata(
-        self._output_dir, self._GetTargetSdkArch())
     qemu_path = os.path.join(
         common.SDK_ROOT, 'qemu', 'bin',
         'qemu-system-' + self._GetTargetSdkArch())
@@ -61,10 +59,23 @@ class QemuTarget(target.Target):
     qemu_command = [qemu_path,
         '-m', str(self._ram_size_mb),
         '-nographic',
-        '-kernel', boot_data.GetKernelPath(self._GetTargetSdkArch()),
-        '-initrd', boot_data_path,
+        '-kernel', boot_data.GetTargetFile(self._GetTargetSdkArch(),
+                                           'zircon.bin'),
+        '-initrd', boot_data.GetTargetFile(self._GetTargetSdkArch(),
+                                           'bootdata-blobstore.bin'),
         '-smp', '4',
 
+        # Attach the blobstore and data volumes.
+        '-drive', 'file=%s,format=qcow2,if=none,id=data' %
+            self._MakeQcowDisk(
+                boot_data.GetTargetFile(self._GetTargetSdkArch(),
+                                        'fvm.blk')),
+        '-drive', 'file=%s,format=qcow2,if=none,id=blobstore' %
+            self._MakeQcowDisk(boot_data.ConfigureDataFVM(self._output_dir,
+                                                          False)),
+        '-device', 'virtio-blk-pci,drive=data',
+        '-device', 'virtio-blk-pci,drive=blobstore',
+
         # Use stdio for the guest OS only; don't attach the QEMU interactive
         # monitor.
         '-serial', 'stdio',
@@ -132,3 +143,14 @@ class QemuTarget(target.Target):
 
   def _GetSshConfigPath(self):
     return boot_data.GetSSHConfigPath(self._output_dir)
+
+  def _MakeQcowDisk(self, disk_path):
+    """Creates a QEMU copy-on-write version of |disk_path| in the output
+    directory."""
+
+    qimg_path = os.path.join(common.SDK_ROOT, 'qemu', 'bin', 'qemu-img')
+    output_path = os.path.join(self._output_dir,
+                               os.path.basename(disk_path) + '.qcow2')
+    subprocess.check_call([qimg_path, 'create', '-q', '-f', 'qcow2',
+                           '-b', disk_path, output_path])
+    return output_path

build/fuchsia/runner_v2/target.py

@@ -143,7 +143,7 @@ class Target(object):
         return True
       time.sleep(_ATTACH_RETRY_INTERVAL)
     sys.stderr.write(' timeout limit reached.\n')
-    raise FuchsiaTargetException('Couldn\'t connect to QEMU using SSH.')
+    raise FuchsiaTargetException('Couldn\'t connect using SSH.')
 
   def _GetSshConfigPath(self, path):
     raise NotImplementedError