[fuchsia] Add support for running in Amber repos in fuchsia-out-dir.

If an executable is run with "-d" and either the flag --fuchsia-out-dir
or the GN arg default_fuchsia_build_dir_for_installation are set,
the runner script will publish to the Fuchsia output directory's
Amber repository and rely on an existing instance of "fx serve" for
delivering the package to the target.

Encapsulates Amber repo interactions into a new abstract class, AmberRepo.
Adds two concrete classes, one for ephemeral repos (used by emulator runs),
and one for external repos (used for publishing to devices or persistent
Fuchsia AEMU instances.)

Bug: 1022002
Change-Id: Ieb330dd8cd92239c1c5341741ef559483af3285b
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1907488
Commit-Queue: Kevin Marshall <kmarshall@chromium.org>
Reviewed-by: David Dorwin <ddorwin@chromium.org>
Cr-Commit-Position: refs/heads/master@{#714636}

build/config/fuchsia/generate_runner_scripts.gni

@@ -150,6 +150,13 @@ template("fuchsia_package_runner") {
     if (defined(invoker.use_test_server) && invoker.use_test_server) {
       executable_args += [ "--enable-test-server" ]
     }
+
+    if (default_fuchsia_build_dir_for_installation != "") {
+      executable_args += [
+        "--fuchsia-out-dir",
+        default_fuchsia_build_dir_for_installation,
+      ]
+    }
   }
 
   # Produces a script which installs a package and its dependencies into the

build/fuchsia/amber_repo.py

+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import common
+import json
+import logging
+import os
+import shutil
+import subprocess
+import tempfile
+import time
+import urllib2
+
+
+# Maximum amount of time to block while waiting for "pm serve" to come up.
+_PM_SERVE_LIVENESS_TIMEOUT_SECS = 10
+
+_MANAGED_REPO_NAME = 'chrome_runner'
+
+
+class AmberRepo(object):
+  """Abstract interface for a repository used to serve packages to devices."""
+
+  def __init__(self, target):
+    self._target = target
+
+  def PublishPackage(self, package_path):
+    pm_tool = common.GetHostToolPathFromPlatform('pm')
+    subprocess.check_call(
+        [pm_tool, 'publish', '-a', '-f', package_path, '-r', self.GetPath(),
+         '-vt', '-v'],
+        stderr=subprocess.STDOUT)
+
+  def GetPath(self):
+    pass
+
+
+class ManagedAmberRepo(AmberRepo):
+  """Creates and serves packages from an ephemeral repository."""
+
+  def __init__(self, target):
+    AmberRepo.__init__(self, target)
+
+    self._amber_root = tempfile.mkdtemp()
+    pm_tool = common.GetHostToolPathFromPlatform('pm')
+    subprocess.check_call([pm_tool, 'newrepo', '-repo', self._amber_root])
+    logging.info('Creating and serving temporary Amber root: {}.'.format(
+        self._amber_root))
+
+    serve_port = common.GetAvailableTcpPort()
+    self._pm_serve_task = subprocess.Popen(
+        [pm_tool, 'serve', '-d', os.path.join(self._amber_root, 'repository'),
+         '-l', ':%d' % serve_port, '-q'])
+
+    # Block until "pm serve" starts serving HTTP traffic at |serve_port|.
+    timeout = time.time() + _PM_SERVE_LIVENESS_TIMEOUT_SECS
+    while True:
+      try:
+        urllib2.urlopen('http://localhost:%d' % serve_port, timeout=1).read()
+        break
+      except urllib2.URLError:
+        logging.info('Waiting until \'pm serve\' is up...')
+
+      if time.time() >= timeout:
+        raise Exception('Timed out while waiting for \'pm serve\'.')
+
+      time.sleep(1)
+
+    remote_port = common.ConnectPortForwardingTask(target, serve_port, 0)
+    self._RegisterAmberRepository(self._amber_root, remote_port)
+
+  def __enter__(self):
+    return self
+
+  def __exit__(self, type, value, tb):
+    """Allows the repository to delete itself when it leaves the scope of a
+    'with' block."""
+    if self._amber_root:
+      logging.info('Cleaning up Amber root: ' + self._amber_root)
+      shutil.rmtree(self._amber_root)
+
+    self._UnregisterAmberRepository()
+    if self._pm_serve_task:
+      self._pm_serve_task.kill()
+
+  def GetPath(self):
+    return self._amber_root
+
+  def _RegisterAmberRepository(self, tuf_repo, remote_port):
+    """Configures a device to use a local TUF repository as an installation
+    source for packages.
+    |tuf_repo|: The host filesystem path to the TUF repository.
+    |remote_port|: The reverse-forwarded port used to connect to instance of
+                   `pm serve` that is serving the contents of |tuf_repo|."""
+
+# Extract the public signing key for inclusion in the config file.
+    root_keys = []
+    root_json_path = os.path.join(tuf_repo, 'repository', 'root.json')
+    root_json = json.load(open(root_json_path, 'r'))
+    for root_key_id in root_json['signed']['roles']['root']['keyids']:
+      root_keys.append({
+          'Type': root_json['signed']['keys'][root_key_id]['keytype'],
+          'Value': root_json['signed']['keys'][root_key_id]['keyval']['public']
+      })
+
+    # "pm serve" can automatically generate a "config.json" file at query time,
+    # but the file is unusable because it specifies URLs with port
+    # numbers that are unreachable from across the port forwarding boundary.
+    # So instead, we generate our own config file with the forwarded port
+    # numbers instead.
+    config_file = open(os.path.join(tuf_repo, 'repository', 'repo_config.json'),
+                       'w')
+    json.dump({
+        'ID': _MANAGED_REPO_NAME,
+        'RepoURL': "http://127.0.0.1:%d" % remote_port,
+        'BlobRepoURL': "http://127.0.0.1:%d/blobs" % remote_port,
+        'RatePeriod': 10,
+        'RootKeys': root_keys,
+        'StatusConfig': {
+            'Enabled': True
+        },
+        'Auto': True
+    }, config_file)
+    config_file.close()
+
+    # Register the repo.
+    return_code = self._target.RunCommand(
+        [('amberctl rm_src -n %s; ' +
+          'amberctl add_src -f http://127.0.0.1:%d/repo_config.json')
+         % (_MANAGED_REPO_NAME, remote_port)])
+    if return_code != 0:
+      raise Exception('Error code %d when running amberctl.' % return_code)
+
+
+  def _UnregisterAmberRepository(self):
+    """Unregisters the Amber repository."""
+
+    logging.debug('Unregistering Amber repository.')
+    self._target.RunCommand(['amberctl', 'rm_src', '-n', _MANAGED_REPO_NAME])
+
+    # Re-enable 'devhost' repo if it's present. This is useful for devices that
+    # were booted with 'fx serve'.
+    self._target.RunCommand(['amberctl', 'enable_src', '-n', 'devhost'],
+                            silent=True)
+
+
+class ExternalAmberRepo(AmberRepo):
+  """Publishes packages to an Amber repository located and served externally
+  (ie. located under a Fuchsia build directory and served by "fx serve"."""
+
+  def __init__(self, amber_root):
+    self._amber_root = amber_root
+    logging.info('Using existing Amber root: {}'.format(amber_root))
+
+#TODO(kmarshall) : Find a way to programatically check if "fx serve" is running.
+    logging.info('Ensure that "fx serve" is running.')
+
+  def GetPath(self):
+    return self._amber_root
+
+  def __enter__(self):
+    return self
+
+  def __exit__(self, type, value, tb):
+    pass

build/fuchsia/common.py

@@ -97,13 +97,3 @@ def GetAvailableTcpPort():
   port = sock.getsockname()[1]
   sock.close()
   return port
-
-
-def PublishPackage(package_path, tuf_root):
-  """Publishes a combined FAR package to a TUF repository root."""
-
-  pm_tool = GetHostToolPathFromPlatform('pm')
-  subprocess.check_call(
-      [pm_tool, 'publish', '-a', '-f', package_path, '-r', tuf_root, '-vt',
-          '-v'],
-      stderr=subprocess.STDOUT)

build/fuchsia/deploy_to_amber_repo.py

@@ -7,12 +7,11 @@
 """Deploys Fuchsia packages to an Amber repository in a Fuchsia
 build output directory."""
 
+import amber_repo
 import argparse
 import os
 import sys
 
-from common import PublishPackage
-
 
 # Populates the GDB-standard symbol directory structure |build_ids_path| with
 # the files and build IDs specified in |ids_txt_path|.
@@ -54,10 +53,12 @@ def main():
     return 1
 
   fuchsia_out_dir = os.path.expanduser(args.fuchsia_out_dir.pop())
-  tuf_root = os.path.join(fuchsia_out_dir, 'amber-files')
-  print('Installing packages and symbols in Amber repo %s...' % tuf_root)
+  repo = amber_repo.ExternalAmberRepo(
+      os.path.join(fuchsia_out_dir, 'amber-files'))
+  print('Installing packages and symbols in Amber repo %s...' % repo.GetPath())
+
   for package in args.package:
-    PublishPackage(package, tuf_root)
+    repo.PublishPackage(package)
     InstallSymbols(os.path.join(os.path.dirname(package), 'ids.txt'),
                    os.path.join(fuchsia_out_dir, '.build-id'))
 

build/fuchsia/device_target.py

@@ -6,6 +6,7 @@
 
 from __future__ import print_function
 
+import amber_repo
 import boot_data
 import filecmp
 import logging
@@ -85,21 +86,21 @@ class DeviceTarget(target.Target):
     self._system_log_file = system_log_file
     self._loglistener = None
     self._host = host
-    self._fuchsia_out_dir = fuchsia_out_dir
+    self._fuchsia_out_dir = os.path.expanduser(fuchsia_out_dir)
     self._node_name = node_name
     self._os_check = os_check,
 
     if self._host and self._node_name:
       raise Exception('Only one of "--host" or "--name" can be specified.')
 
-    if fuchsia_out_dir:
+    if self._fuchsia_out_dir:
       if ssh_config:
         raise Exception('Only one of "--fuchsia-out-dir" or "--ssh_config" can '
                         'be specified.')
 
       # Use SSH keys from the Fuchsia output directory.
-      self._ssh_config_path = os.path.join(os.path.expanduser(fuchsia_out_dir),
-                                           'ssh-keys', 'ssh_config')
+      self._ssh_config_path = os.path.join(self._fuchsia_out_dir, 'ssh-keys',
+                                           'ssh_config')
       self._os_check = 'ignore'
 
     elif ssh_config:
@@ -213,6 +214,15 @@ class DeviceTarget(target.Target):
       assert self._host
 
 
+  def _GetAmberRepo(self):
+    if self._fuchsia_out_dir:
+      # Deploy to an already-booted device running a local Fuchsia build.
+      return amber_repo.ExternalAmberRepo(
+          os.path.join(self._fuchsia_out_dir, 'amber-files'))
+    else:
+      # Pave a Zedbootable device.
+      return amber_repo.ManagedAmberRepo(self)
+
   def __ProvisionDevice(self):
     """Netboots a device with Fuchsia. If |_node_name| is set, then only a
     device with a matching node name is used.

build/fuchsia/emu_target.py

@@ -4,6 +4,7 @@
 
 """Implements commands for running/interacting with Fuchsia on an emulator."""
 
+import amber_repo
 import boot_data
 import logging
 import os
@@ -73,6 +74,9 @@ class EmuTarget(target.Target):
                      open(temporary_system_log_file.name, 'r').read())
       raise
 
+  def _GetAmberRepo(self):
+    return amber_repo.ManagedAmberRepo(self)
+
   def Shutdown(self):
     if self._IsEmuStillRunning():
       logging.info('Shutting down %s' % (self._GetEmulatorName()))
@@ -89,4 +93,4 @@ class EmuTarget(target.Target):
     return ('localhost', self._host_ssh_port)
 
   def _GetSshConfigPath(self):
-    return boot_data.GetSSHConfigPath(self._output_dir)
\ No newline at end of file
+    return boot_data.GetSSHConfigPath(self._output_dir)

build/fuchsia/target.py

@@ -11,7 +11,6 @@ import remote_cmd
 import shutil
 import subprocess
 import sys
-import urllib2
 import tempfile
 import time
 
@@ -20,15 +19,10 @@ _SHUTDOWN_CMD = ['dm', 'poweroff']
 _ATTACH_MAX_RETRIES = 10
 _ATTACH_RETRY_INTERVAL = 1
 
-_REPO_NAME = 'chrome_runner'
-
 # Amount of time to wait for Amber to complete package installation, as a
 # mitigation against hangs due to amber/network-related failures.
 _INSTALL_TIMEOUT_SECS = 5 * 60
 
-# Maximum amount of time to block while waitin for "pm serve" to come up.
-_PM_SERVE_LIVENESS_TIMEOUT_SECS = 10
-
 
 def _GetPackageInfo(package_path):
   """Returns a tuple with the name and version of a package."""
@@ -55,22 +49,6 @@ class _MapIsolatedPathsForPackage:
                                            path[len(isolated_directory):])
     return path
 
-def _WaitForPmServeToBeReady(port):
-  """Blocks until "pm serve" starts serving HTTP traffic at |port|."""
-
-  timeout = time.time() + _PM_SERVE_LIVENESS_TIMEOUT_SECS
-  while True:
-    try:
-      urllib2.urlopen('http://localhost:%d' % port, timeout=1).read()
-      break
-    except urllib2.URLError:
-      logging.info('Waiting until \'pm serve\' is up...')
-
-    if time.time() >= timeout:
-      raise Exception('Timed out while waiting for \'pm serve\'.')
-
-    time.sleep(1)
-
 
 class FuchsiaTargetException(Exception):
   def __init__(self, message):
@@ -257,34 +235,20 @@ class Target(object):
       return 'x86_64'
     raise Exception('Unknown target_cpu %s:' % self._target_cpu)
 
+  def _GetAmberRepo(self):
+    """Returns an AmberRepo instance which serves packages for this Target."""
+    pass
+
   def InstallPackage(self, package_paths):
     """Installs a package and it's dependencies on the device. If the package is
     already installed then it will be updated to the new version.
 
     package_paths: Paths to the .far files to install."""
 
-    try:
-      tuf_root = tempfile.mkdtemp()
-      pm_serve_task = None
-      pm_tool = common.GetHostToolPathFromPlatform('pm')
-
-      subprocess.check_call([pm_tool, 'newrepo', '-repo', tuf_root])
-
-      # Serve the |tuf_root| using 'pm serve' and configure the target to pull
-      # from it.
-      serve_port = common.GetAvailableTcpPort()
-      pm_serve_task = subprocess.Popen(
-          [pm_tool, 'serve', '-d', os.path.join(tuf_root, 'repository'), '-l',
-           ':%d' % serve_port, '-q'])
-
+    with self._GetAmberRepo() as amber_repo:
       # Publish all packages to the serving TUF repository under |tuf_root|.
       for next_package_path in package_paths:
-        common.PublishPackage(next_package_path, tuf_root)
-
-      _WaitForPmServeToBeReady(serve_port)
-
-      remote_port = common.ConnectPortForwardingTask(self, serve_port, 0)
-      self._RegisterAmberRepository(tuf_root, remote_port)
+        amber_repo.PublishPackage(next_package_path)
 
       # Install all packages.
       for next_package_path in package_paths:
@@ -299,65 +263,3 @@ class Target(object):
         if return_code != 0:
           raise Exception('Error while installing %s.' % install_package_name)
 
-    finally:
-      self._UnregisterAmberRepository()
-      if pm_serve_task:
-        pm_serve_task.kill()
-      shutil.rmtree(tuf_root)
-
-
-  def _RegisterAmberRepository(self, tuf_repo, remote_port):
-    """Configures a device to use a local TUF repository as an installation
-    source for packages.
-    |tuf_repo|: The host filesystem path to the TUF repository.
-    |remote_port|: The reverse-forwarded port used to connect to instance of
-                   `pm serve` that is serving the contents of |tuf_repo|."""
-
-    # Extract the public signing key for inclusion in the config file.
-    root_keys = []
-    root_json_path = os.path.join(tuf_repo, 'repository', 'root.json')
-    root_json = json.load(open(root_json_path, 'r'))
-    for root_key_id in root_json['signed']['roles']['root']['keyids']:
-      root_keys.append({
-          'Type': root_json['signed']['keys'][root_key_id]['keytype'],
-          'Value': root_json['signed']['keys'][root_key_id]['keyval']['public']
-      })
-
-    # "pm serve" can automatically generate a "config.json" file at query time,
-    # but the file is unusable because it specifies URLs with port
-    # numbers that are unreachable from across the port forwarding boundary.
-    # So instead, we generate our own config file with the forwarded port
-    # numbers instead.
-    config_file = open(os.path.join(tuf_repo, 'repository', 'repo_config.json'),
-                       'w')
-    json.dump({
-        'ID': _REPO_NAME,
-        'RepoURL': "http://127.0.0.1:%d" % remote_port,
-        'BlobRepoURL': "http://127.0.0.1:%d/blobs" % remote_port,
-        'RatePeriod': 10,
-        'RootKeys': root_keys,
-        'StatusConfig': {
-            'Enabled': True
-        },
-        'Auto': True
-    }, config_file)
-    config_file.close()
-
-    # Register the repo.
-    return_code = self.RunCommand(
-        [('amberctl rm_src -n %s; ' +
-          'amberctl add_src -f http://127.0.0.1:%d/repo_config.json')
-         % (_REPO_NAME, remote_port)])
-    if return_code != 0:
-      raise Exception('Error code %d when running amberctl.' % return_code)
-
-
-  def _UnregisterAmberRepository(self):
-    """Unregisters the Amber repository."""
-
-    logging.debug('Unregistering Amber repository.')
-    self.RunCommand(['amberctl', 'rm_src', '-n', _REPO_NAME])
-
-    # Re-enable 'devhost' repo if it's present. This is useful for devices that
-    # were booted with 'fx serve'.
-    self.RunCommand(['amberctl', 'enable_src', '-n', 'devhost'], silent=True)