Removing the isolate.py functionality and requiring tasks to be created from...

Removing the isolate.py functionality and requiring tasks to be created from an archive hash rather than an isolate file.

Also adding back the controller_test.py and isolate files which were missed in the refactor cl due to a bad move. The only change to these files are to support isolate hashes rather than isolate files.

BUG=460340

Review URL: https://codereview.chromium.org/943923003

Cr-Commit-Position: refs/heads/master@{#317703}

testing/legion/examples/hello_world/controller_test.isolate

+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+{
+  'includes': [
+    '../../legion.isolate',
+  ],
+  'conditions': [
+    ['multi_machine == 1', {
+      'variables': {
+        'command': [
+          'controller_test.py',
+        ],
+        'files': [
+          'controller_test.py',
+        ],
+      },
+    }],
+  ]
+}

testing/legion/examples/hello_world/controller_test.py

+#!/usr/bin/env python
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""A simple host test module.
+
+This module runs on the host machine and is responsible for creating 2
+task machines, waiting for them, and running RPC calls on them.
+"""
+
+# Map the legion directory so we can import the host controller.
+import sys
+sys.path.append('../../')
+
+import argparse
+import logging
+import time
+
+import test_controller
+
+
+class ExampleTestController(test_controller.TestController):
+  """A simple example controller for a test."""
+
+  def __init__(self):
+    super(ExampleTestController, self).__init__()
+    self.task1 = None
+    self.task2 = None
+
+  def CreateTask(self, isolated_hash):
+    """Create a task object and set the proper values."""
+    task = self.CreateNewTask(
+        isolated_hash=isolated_hash,
+        dimensions={'os': 'legion-linux'}, priority=200,
+        idle_timeout_secs=90, connection_timeout_secs=90,
+        verbosity=logging.INFO,
+        run_id=1)
+    task.Create()
+    return task
+
+  def SetUp(self):
+    """Create the task machines and wait until they connect.
+
+    In this call the actual creation of the task machines is done in parallel
+    by the system. The WaitForConnect calls are performed in series but will
+    return as soon as the tasks connect.
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--task-hash')
+    args, _ = parser.parse_known_args()
+
+    self.task1 = self.CreateTask(args.task_hash)
+    self.task2 = self.CreateTask(args.task_hash)
+    self.task1.WaitForConnection()
+    self.task2.WaitForConnection()
+
+  def RunTest(self):
+    """Main method to run the test code."""
+    self.CallEcho(self.task1)
+    self.CallEcho(self.task2)
+    self.CallTaskTest(self.task1)
+    self.CallTaskTest(self.task2)
+
+  def CallEcho(self, task):
+    """Call rpc.Echo on a task."""
+    logging.info('Calling Echo on %s', task.name)
+    logging.info(task.rpc.Echo(task.name))
+
+  def CallTaskTest(self, task):
+    """Call task_test.py name on a task."""
+    logging.info('Calling Subprocess to run "./task_test.py %s"', task.name)
+    proc = task.rpc.subprocess.Popen(['./task_test.py', task.name])
+    task.rpc.subprocess.Wait(proc)
+    retcode = task.rpc.subprocess.GetReturncode(proc)
+    stdout = task.rpc.subprocess.ReadStdout(proc)
+    stderr = task.rpc.subprocess.ReadStderr(proc)
+    logging.info('retcode: %s, stdout: %s, stderr: %s', retcode, stdout, stderr)
+
+
+if __name__ == '__main__':
+  ExampleTestController().RunController()

testing/legion/examples/subprocess/subprocess_test.isolate

@@ -5,7 +5,6 @@
 {
   'includes': [
     '../../legion.isolate',
-    'task.isolate'
   ],
   'conditions': [
     ['multi_machine == 1', {

testing/legion/examples/subprocess/subprocess_test.py

@@ -9,6 +9,7 @@
 import sys
 sys.path.append('../../')
 
+import argparse
 import logging
 import time
 import xmlrpclib
@@ -25,9 +26,12 @@ class ExampleTestController(test_controller.TestController):
 
   def SetUp(self):
     """Creates the task machine and waits until it connects."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--task-hash')
+    args, _ = parser.parse_known_args()
+
     self.task = self.CreateNewTask(
-        isolate_file='task.isolate',
-        config_vars={'multi_machine': '1'},
+        isolated_hash=args.task_hash,
         dimensions={'os': 'legion-linux'},
         idle_timeout_secs=90, connection_timeout_secs=90,
         verbosity=logging.DEBUG)

testing/legion/task_controller.py

@@ -49,21 +49,18 @@ class TaskController(object):
   _task_count = 0
   _tasks = []
 
-  def __init__(self, isolate_file, config_vars, dimensions, priority=100,
+  def __init__(self, isolated_hash, dimensions, priority=100,
                idle_timeout_secs=common_lib.DEFAULT_TIMEOUT_SECS,
                connection_timeout_secs=common_lib.DEFAULT_TIMEOUT_SECS,
-               verbosity='ERROR', name=None):
-    assert isinstance(config_vars, dict)
+               verbosity='ERROR', name=None, run_id=None):
     assert isinstance(dimensions, dict)
     type(self)._tasks.append(self)
     type(self)._task_count += 1
     self.verbosity = verbosity
     self._name = name or 'Task%d' % type(self)._task_count
     self._priority = priority
-    self._isolate_file = isolate_file
-    self._isolated_file = isolate_file + 'd'
+    self._isolated_hash = isolated_hash
     self._idle_timeout_secs = idle_timeout_secs
-    self._config_vars = config_vars
     self._dimensions = dimensions
     self._connect_event = threading.Event()
     self._connected = False
@@ -71,6 +68,10 @@ class TaskController(object):
     self._otp = self._CreateOTP()
     self._rpc = None
 
+    run_id = run_id or datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
+    self._task_name = '%s/%s/%s' % (
+        os.path.splitext(sys.argv[0])[0], self._name, run_id)
+
     parser = argparse.ArgumentParser()
     parser.add_argument('--isolate-server')
     parser.add_argument('--swarming-server')
@@ -137,7 +138,6 @@ class TaskController(object):
     """Creates the task machine."""
     logging.info('Creating %s', self.name)
     self._connect_event.clear()
-    self._ExecuteIsolate()
     self._ExecuteSwarming()
 
   def WaitForConnection(self):
@@ -163,31 +163,15 @@ class TaskController(object):
       self._rpc = None
       self._connected = False
 
-  def _ExecuteIsolate(self):
-    """Executes isolate.py."""
-    cmd = [
-        'python',
-        ISOLATE_PY,
-        'archive',
-        '--isolate', self._isolate_file,
-        '--isolated', self._isolated_file,
-        ]
-
-    if self._isolate_server:
-      cmd.extend(['--isolate-server', self._isolate_server])
-    for key, value in self._config_vars.iteritems():
-      cmd.extend(['--config-var', key, value])
-
-    self._ExecuteProcess(cmd)
-
   def _ExecuteSwarming(self):
     """Executes swarming.py."""
     cmd = [
         'python',
         SWARMING_PY,
         'trigger',
-        self._isolated_file,
+        self._isolated_hash,
         '--priority', str(self._priority),
+        '--task-name', self._task_name,
         ]
 
     if self._isolate_server: