Update run-swarmed.py to pass cwd and command directly to swarming.

We're removing the relative_cwd and command support in isolate
files in swarming as part of moving to the RBE-CAS backend.

//tools/run-swarmed.py currently depends on that to work.

This CL extends mb.py to be able to easily extract the command line
that the target would use in the isolate, and modifies run-swarmed.py
to use that to pass the command (and the relative_cwd) to the task
instead.

Bug: 1119643
Change-Id: Icf81faa1c1293ffbf13160e4d3ee1b7f2d636797
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2495797Reviewed-by: Takuto Ikuta <tikuta@chromium.org>
Reviewed-by: Stephen Martinis <martiniss@chromium.org>
Commit-Queue: Dirk Pranke <dpranke@google.com>
Cr-Commit-Position: refs/heads/master@{#820947}

tools/mb/mb.py

@@ -204,6 +204,27 @@ class MetaBuildWrapper(object):
                       help='path to goma directory')
     subp.set_defaults(func=self.CmdExport)
 
+    subp = subps.add_parser('get-swarming-command',
+                            description='Get the command needed to run the '
+                            'binary under swarming')
+    AddCommonOptions(subp)
+    subp.add_argument('--no-build',
+                      dest='build',
+                      default=True,
+                      action='store_false',
+                      help='Do not build, just isolate')
+    subp.add_argument('--as-list',
+                      action='store_true',
+                      help='return the command line as a JSON-formatted '
+                      'list of strings instead of single string')
+    subp.add_argument('path',
+                      help=('path to generate build into (or use).'
+                            ' This can be either a regular path or a '
+                            'GN-style source-relative path like '
+                            '//out/Default.'))
+    subp.add_argument('target', help='ninja target to build and run')
+    subp.set_defaults(func=self.CmdGetSwarmingCommand)
+
     subp = subps.add_parser('train',
                             description='Writes the expanded configuration '
                             'for each builder as JSON files to a configured '
@@ -427,6 +448,15 @@ class MetaBuildWrapper(object):
     vals = self.Lookup()
     return self.RunGNGen(vals)
 
+  def CmdGetSwarmingCommand(self):
+    vals = self.GetConfig()
+    command, _ = self.GetSwarmingCommand(self.args.target, vals)
+    if self.args.as_list:
+      self.Print(json.dumps(command))
+    else:
+      self.Print(' '.join(command))
+    return 0
+
   def CmdIsolateEverything(self):
     vals = self.Lookup()
     return self.RunGNGenAllIsolates(vals)
@@ -544,7 +574,7 @@ class MetaBuildWrapper(object):
 
     self.Print('')
     if self.args.swarmed:
-      cmd, _ = self.GetIsolateCommand(self.args.target, vals)
+      cmd, _ = self.GetSwarmingCommand(self.args.target, vals)
       return self._RunUnderSwarming(self.args.path, self.args.target, cmd)
     else:
       return self._RunLocallyIsolated(self.args.path, self.args.target)
@@ -1132,7 +1162,7 @@ class MetaBuildWrapper(object):
       if not found_one:
         raise MBErr('Did not find any of %s' % ', '.join(rpaths))
 
-      command, extra_files = self.GetIsolateCommand(target, vals)
+      command, extra_files = self.GetSwarmingCommand(target, vals)
       runtime_deps = self.ReadFile(path_to_use).splitlines()
 
       canonical_target = target.replace(':','_').replace('/','_')
@@ -1216,7 +1246,7 @@ class MetaBuildWrapper(object):
 
     build_dir = self.args.path
 
-    command, extra_files = self.GetIsolateCommand(target, vals)
+    command, extra_files = self.GetSwarmingCommand(target, vals)
 
     # Any warning for an unused arg will get interleaved into the cmd's
     # stdout. When that happens, the isolate step below will fail with an
@@ -1432,7 +1462,7 @@ class MetaBuildWrapper(object):
 
     return '\n'.join(args_gn_lines)
 
-  def GetIsolateCommand(self, target, vals):
+  def GetSwarmingCommand(self, target, vals):
     isolate_map = self.ReadIsolateMap()
 
     is_android = 'target_os="android"' in vals['gn_args']

tools/run-swarmed.py

@@ -48,7 +48,7 @@ def _Spawn(args):
   - The json file created by triggering and used to collect results;
   - The command line arguments object.
   """
-  index, args, isolated_hash = args
+  index, args, isolated_hash, swarming_command = args
   json_file = os.path.join(args.results, '%d.json' % index)
   trigger_args = [
       'tools/luci-go/swarming',
@@ -110,9 +110,9 @@ def _Spawn(args):
     if os.path.isfile(filter_file):
       runner_args.append('--test-launcher-filter-file=../../' + filter_file)
 
-  if runner_args:
-    trigger_args.append('--')
-    trigger_args.extend(runner_args)
+  trigger_args.extend(['--relative-cwd', args.out_dir, '--raw-cmd', '--'])
+  trigger_args.extend(swarming_command)
+  trigger_args.extend(runner_args)
 
   with open(os.devnull, 'w') as nul:
     subprocess.check_call(trigger_args, stdout=nul)
@@ -249,6 +249,16 @@ def main():
   with open(isolated) as f:
     isolated_hash = hashlib.sha1(f.read()).hexdigest()
 
+  mb_cmd = [
+      sys.executable, 'tools/mb/mb.py', 'get-swarming-command', '--as-list'
+  ]
+  if not args.build:
+    mb_cmd.append('--no-build')
+  if args.isolate_map_file:
+    mb_cmd += ['--isolate-map-file', args.isolate_map_file]
+  mb_cmd += ['//' + args.out_dir, args.target_name]
+  swarming_cmd = json.loads(subprocess.check_output(mb_cmd))
+
   if os.path.isdir(args.results):
     shutil.rmtree(args.results)
   os.makedirs(args.results)
@@ -258,7 +268,8 @@ def main():
     # Use dummy since threadpools give better exception messages
     # than process pools do, and threads work fine for what we're doing.
     pool = multiprocessing.dummy.Pool()
-    spawn_args = map(lambda i: (i, args, isolated_hash), range(args.copies))
+    spawn_args = map(lambda i: (i, args, isolated_hash, swarming_cmd),
+                     range(args.copies))
     spawn_results = pool.imap_unordered(_Spawn, spawn_args)
 
     exit_codes = []