Fuchsia: Add net test server support to the new-style runners.

This CL uses SSH reverse port forwarding to dynamically connect the test
suites running on Fuchsia to the Python-based test servers running on
the controlling machine.

Also adds a logging context to the spawning server library, so that the
verbosity of its logging output can be independently adjusted.

Change-Id: I6099ccc21efdd5e2e6b994e863f8d4b3dadaaae1
Reviewed-on: https://chromium-review.googlesource.com/907633
Commit-Queue: Kevin Marshall <kmarshall@chromium.org>
Reviewed-by: Scott Graham <scottmg@chromium.org>
Reviewed-by: Wez <wez@chromium.org>
Reviewed-by: Sergey Ulanov <sergeyu@chromium.org>
Cr-Commit-Position: refs/heads/master@{#535911}

build/fuchsia/runner_v2/common_args.py

@@ -45,6 +45,13 @@ def ConfigureLogging(args):
 
   logging.basicConfig(level=(logging.DEBUG if args.verbose else logging.INFO))
 
+  # The test server spawner is too noisy with INFO level logging, so tweak
+  # its verbosity a bit by adjusting its logging level.
+  if args.verbose:
+    logging.getLogger('chrome_test_server_spawner').setLevel(logging.DEBUG)
+  else:
+    logging.getLogger('chrome_test_server_spawner').setLevel(logging.WARN)
+
 
 def GetDeploymentTargetForArgs(args):
   """Constructs a deployment target object using parameters taken from

build/fuchsia/runner_v2/net_test_server.py

+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import logging
+import os
+import re
+import select
+import socket
+import sys
+import subprocess
+import tempfile
+import time
+
+DIR_SOURCE_ROOT = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir))
+sys.path.append(os.path.join(DIR_SOURCE_ROOT, 'build', 'util', 'lib', 'common'))
+import chrome_test_server_spawner
+
+PORT_MAP_RE = re.compile('Allocated port (?P<port>\d+) for remote')
+GET_PORT_NUM_TIMEOUT_SECS = 5
+
+
+def _ConnectPortForwardingTask(target, local_port):
+  """Establishes a port forwarding SSH task to a localhost TCP endpoint hosted
+  at port |local_port|. Blocks until port forwarding is established.
+
+  Returns a tuple containing the remote port and the SSH task Popen object."""
+
+  forwarding_flags = ['-NT',  # Don't execute command; don't allocate terminal.
+                      '-R', '0:localhost:%d' % local_port]
+  task = target.RunCommandPiped([],
+                                ssh_args=forwarding_flags,
+                                stderr=subprocess.PIPE)
+
+  # SSH reports the remote dynamic port number over stderr.
+  # Unfortunately, the output is incompatible with Python's line buffered
+  # input (or vice versa), so we have to build our own buffered input system to
+  # pull bytes over the pipe.
+  poll_obj = select.poll()
+  poll_obj.register(task.stderr, select.POLLIN)
+  line = ''
+  timeout = time.time() + GET_PORT_NUM_TIMEOUT_SECS
+  while time.time() < timeout:
+    poll_result = poll_obj.poll(max(0, timeout - time.time()))
+    if poll_result:
+      next_char = task.stderr.read(1)
+      if not next_char:
+        break
+      line += next_char
+      if line.endswith('\n'):
+        line.strip()
+        matched = PORT_MAP_RE.match(line)
+        if matched:
+          device_port = int(matched.group('port'))
+          logging.debug('Port forwarding established (local=%d, device=%d)' %
+                        (local_port, device_port))
+          return (device_port, task)
+        line = ''
+
+  raise Exception('Could not establish a port forwarding connection.')
+
+
+# Implementation of chrome_test_server_spawner.PortForwarder that uses SSH's
+# remote port forwarding feature to forward ports.
+class SSHPortForwarder(chrome_test_server_spawner.PortForwarder):
+  def __init__(self, target):
+    self._target = target
+
+    # Maps the host (server) port to a tuple consisting of the device port
+    # number and a subprocess.Popen object for the forwarding SSH process.
+    self._port_mapping = {}
+
+  def Map(self, port_pairs):
+    for p in port_pairs:
+      _, host_port = p
+      self._port_mapping[host_port] = \
+          _ConnectPortForwardingTask(self._target, host_port)
+
+  def GetDevicePortForHostPort(self, host_port):
+    return self._port_mapping[host_port][0]
+
+  def Unmap(self, device_port):
+    for host_port, entry in self._port_mapping.iteritems():
+      if entry[0] == device_port:
+        entry[1].terminate()
+        entry[1].wait()
+        del self._port_mapping[host_port]
+        return
+
+    raise Exception('Unmap called for unknown port: %d' % device_port)
+
+
+def SetupTestServer(target, test_concurrency):
+  """Provisions a forwarding test server and configures |target| to use it.
+
+  Returns a tuple with a Popen opbject for the test server process,
+  and a Popen object for the port forwarding connection."""
+
+  logging.debug('Starting test server.')
+  spawning_server = chrome_test_server_spawner.SpawningServer(
+      0, SSHPortForwarder(target), test_concurrency)
+  forwarded_port, forwarding_process = _ConnectPortForwardingTask(
+      target, spawning_server.server_port)
+  spawning_server.Start()
+
+  logging.debug('Test server listening for connections (port=%d)' %
+                spawning_server.server_port)
+  logging.debug('Forwarded port is %d' % forwarded_port)
+
+  config_file = tempfile.NamedTemporaryFile(delete=True)
+
+  # Clean up the config JSON to only pass ports. See https://crbug.com/810209 .
+  config_file.write(json.dumps({
+    'name': 'testserver',
+    'address': '127.0.0.1',
+    'spawner_url_base': 'http://localhost:%d' % forwarded_port
+  }))
+
+  config_file.flush()
+  target.PutFile(config_file.name, '/system/net-test-server-config')
+
+  return spawning_server, forwarding_process

build/fuchsia/runner_v2/remote_cmd.py

@@ -43,7 +43,8 @@ def RunSsh(config_path, host, port, command, silent):
     return subprocess.call(ssh_command)
 
 
-def RunPipedSsh(config_path, host, port, command, **kwargs):
+def RunPipedSsh(config_path, host, port, command = None, ssh_args = None,
+                **kwargs):
   """Executes an SSH command on the remote host and returns a process object
   with access to the command's stdio streams. Does not block.
 
@@ -51,14 +52,20 @@ def RunPipedSsh(config_path, host, port, command, **kwargs):
   host: The hostname or IP address of the remote host.
   port: The port to connect to.
   command: A list of strings containing the command and its arguments.
+  ssh_args: Arguments that will be passed to SSH.
   kwargs: A dictionary of parameters to be passed to subprocess.Popen().
           The parameters can be used to override stdin and stdout, for example.
 
   Returns a Popen object for the command."""
 
+  if not command:
+    command = []
+  if not ssh_args:
+    ssh_args = []
+
   ssh_command = _SSH + ['-F', config_path,
                         host,
-                        '-p', str(port)] + command
+                        '-p', str(port)] + ssh_args + ['--'] + command
   logging.debug(' '.join(ssh_command))
   return subprocess.Popen(ssh_command, **kwargs)
 

build/fuchsia/runner_v2/test_runner.py

@@ -19,6 +19,7 @@ import tempfile
 import time
 
 from common_args import AddCommonArgs, ConfigureLogging, GetDeploymentTargetForArgs
+from net_test_server import SetupTestServer
 from run_package import RunPackage
 
 DEFAULT_TEST_CONCURRENCY = 4
@@ -56,6 +57,9 @@ def main():
                       help='Sets the number of parallel test jobs.')
   parser.add_argument('--test-launcher-summary-output',
                       help='Where the test launcher will output its json.')
+  parser.add_argument('--enable-test-server', action='store_true',
+                      default=False,
+                      help='Enable Chrome test server spawner.')
   parser.add_argument('child_args', nargs='*',
                       help='Arguments for the test process.')
   args = parser.parse_args()
@@ -86,15 +90,24 @@ def main():
     child_args.append('--test-launcher-summary-output=' + TEST_RESULT_PATH)
 
   with GetDeploymentTargetForArgs(args) as target:
+    target = GetDeploymentTargetForArgs(args)
     target.Start()
 
     if args.test_launcher_filter_file:
       target.PutFile(args.test_launcher_filter_file, TEST_FILTER_PATH)
       child_args.append('--test-launcher-filter-file=' + TEST_FILTER_PATH)
 
+    forwarder = None
+    if args.enable_test_server:
+      test_server, forwarder = SetupTestServer(target, test_concurrency)
+
     RunPackage(args.output_directory, target, args.package,
                child_args, args.package_manifest)
 
+    if forwarder:
+      forwarder.terminate()
+      forwarder.wait()
+
     if args.test_launcher_summary_output:
       target.GetFile(TEST_RESULT_PATH, args.test_launcher_summary_output)