Add support for --runtime-deps-list-file flag to MB.

This allows us to compute what files are needed at runtime
to run each of the specified targets while we are also
generating the .ninja files to build the targets; since GN
has integrated support for this, we don't need to build
.isolate.gen.json or .isolate files at compile time like
we do for GYP.

As far as the implementation goes, while
--runtime-deps-list-file takes a list of GN labels (like
//base:base_unittests), we actually only have a list of ninja
build targets (like 'base_unittests'). So, we need to add a
mapping of ninja target -> GN label somewhere.

I think the best place to do this is in //testing/buildbot ,
and there I've updated the manage.py presubmit checks to
make sure that we have mappings for all of the targets
listed in the *.json config files. We can't easily check
that the mapping is right, but at least we can verify
that it does exist.

R=phajdan.jr@chromium.org, maruel@chromium.org
BUG=480053

Review URL: https://codereview.chromium.org/1169923008

Cr-Commit-Position: refs/heads/master@{#333802}

testing/buildbot/manage.py

@@ -10,6 +10,7 @@ formatted.
 """
 
 import argparse
+import ast
 import collections
 import glob
 import json
@@ -51,6 +52,15 @@ SKIP = {
 }
 
 
+# TODO(GYP): These targets have not been ported to GN yet.
+SKIP_NINJA_TO_GN_TARGETS = {
+  'cast_media_unittests',
+  'cast_shell_browser_test',
+  'chromevox_tests',
+  'nacl_helper_nonsfi_unittests',
+}
+
+
 class Error(Exception):
   """Processing error."""
 
@@ -90,7 +100,8 @@ def process_builder_remaining(data, filename, builder, tests_location):
           filename, []).append(builder)
 
 
-def process_file(mode, test_name, tests_location, filepath):
+def process_file(mode, test_name, tests_location, filepath, ninja_targets,
+                 ninja_targets_seen):
   """Processes a file.
 
   The action depends on mode. Updates tests_location.
@@ -120,6 +131,14 @@ def process_file(mode, test_name, tests_location, filepath):
       raise Error(
           '%s: %s is broken: %s' % (filename, builder, data['gtest_tests']))
 
+    for d in data['gtest_tests']:
+      if (d['test'] not in ninja_targets and
+          d['test'] not in SKIP_NINJA_TO_GN_TARGETS):
+        raise Error('%s: %s / %s is not listed in ninja_to_gn.pyl.' %
+                    (filename, builder, d['test']))
+      elif d['test'] in ninja_targets:
+        ninja_targets_seen.add(d['test'])
+
     config[builder]['gtest_tests'] = sorted(
         data['gtest_tests'], key=lambda x: x['test'])
     if mode == 'remaining':
@@ -222,12 +241,26 @@ def main():
         'count_run_local': 0, 'count_run_on_swarming': 0, 'local_configs': {}
       })
 
+  with open(os.path.join(THIS_DIR, "ninja_to_gn.pyl")) as fp:
+    ninja_targets = ast.literal_eval(fp.read())
+
   try:
     result = 0
+    ninja_targets_seen = set()
     for filepath in glob.glob(os.path.join(THIS_DIR, '*.json')):
-      if not process_file(args.mode, args.test_name, tests_location, filepath):
+      if not process_file(args.mode, args.test_name, tests_location, filepath,
+                          ninja_targets, ninja_targets_seen):
         result = 1
 
+    extra_targets = set(ninja_targets) - ninja_targets_seen
+    if extra_targets:
+      if len(extra_targets) > 1:
+        extra_targets_str = ', '.join(extra_targets) + ' are'
+      else:
+        extra_targets_str = list(extra_targets)[0] + ' is'
+      raise Error('%s listed in ninja_to_gn.pyl but not in any .json files' %
+                  extra_targets_str)
+
     if args.mode == 'remaining':
       print_remaining(args.test_name, tests_location)
     return result

testing/buildbot/ninja_to_gn.pyl

+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# ninja_to_gn - A mapping of Ninja build target names to GN labels for
+# the tests run on the bots.
+#
+# This mapping is used by MB so that we can uniformly refer to test binaries
+# by their Ninja target names in the recipes and not need to worry about how
+# they are referred to in GN or GYP specifically (the GYP target name is pretty
+# much always the same as the Ninja target name, since GYP target names are not
+# hierarchical).
+
+{
+  "base_unittests":                    "//base:base_unittests",
+  "accessibility_unittests":           "//ui/accessibility:accessibility_unittests",
+  "angle_unittests":                   "//gpu:angle_unittests",
+  "app_list_unittests":                "//ui/app_list:app_list_unittests",
+  "app_shell_unittests":               "//extensions/shell:app_shell_unittests",
+  "ash_unittests":                     "//ash:ash_unittests",
+  "aura_unittests":                    "//ui/aura:aura_unittests",
+  "blink_heap_unittests":              "//third_party/WebKit/public:blink_heap_unittests",
+  "blink_platform_unittests":          "//third_party/WebKit/public:blink_platform_unittests",
+  "browser_tests":                     "//chrome/test:browser_tests",
+  "cacheinvalidation_unittests":       "//third_party/cacheinvalidation:cacheinvalidation_unittests",
+  "cast_base_unittests":               "//cast:cast_base_unittests",
+  "cast_unittests":                    "//cast:cast_unittests",
+  "cc_unittests":                      "//cc:cc_unittests",
+  "chromedriver_unittests":            "//chrome/test/chromedriver:chromedriver_unittests",
+  "chrome_elf_unittests":              "//chrome_elf:chrome_elf_unittests",
+  "chromeos_unittests":                "//chromeos:chromeos_unittests",
+  "components_browsertests":           "//components:components_browsertests",
+  "components_unittests":              "//components:components_unittests",
+  "compositor_unittests":              "//ui/compositor:compositor_unittests",
+  "content_browsertests":              "//content/test:content_browsertests",
+  "content_unittests":                 "//content/test:content_unittests",
+  "courgette_unittests":               "//courgette:courgette_unittests",
+  "crypto_unittests":                  "//crypto:crypto_unittests",
+  "dbus_unittests":                    "//dbus:dbus_unittests",
+  "device_unittests":                  "//device:device_unittests",
+  "display_unittests":                 "//ui/display:display_unittests",
+  "events_unittests":                  "//ui/events:events_unittests",
+  "extensions_browsertests":           "//extensions:extensions_browsertests",
+  "extensions_unittests":              "//extensions:extensions_unittests",
+  "gcm_unit_tests":                    "//google_apis/gcm:gcm_unit_tests",
+  "gfx_unittests":                     "//ui/gfx:gfx_unittests",
+  "gl_unittests":                      "//ui/gl:gl_unittests",
+  "gn_unittests":                      "//tools/gn:gn_unittests",
+  "google_apis_unittests":             "//google_apis:google_apis_unittests",
+  "gpu_unittests":                     "//gpu:gpu_unittests",
+  "installer_util_unittests":          "//chrome/installer/util:installer_util_unittests",
+  "interactive_ui_tests":              "//chrome/test:interactive_ui_tests",
+  "ipc_mojo_unittests":                "//ipc/mojo:ipc_mojo_unittests",
+  "ipc_tests":                         "//ipc:ipc_tests",
+  "jingle_unittests":                  "//jingle:jingle_unittests",
+  "keyboard_unittests":                "//ui/keyboard:keyboard_unittests",
+  "media_unittests":                   "//media:media_unittests",
+  "midi_unittests":                    "//media:midi_unittests",
+  "mojo_common_unittests":             "//mojo/common:mojo_common_unittests",
+  "mojo_public_application_unittests": "//third_party/mojo/src/mojo/edk/test:mojo_public_applicaiton_unittests",
+  "mojo_public_bindings_unittests":    "//third_party/mojo/src/mojo/edk/test:mojo_public_bindings_unittests",
+  "mojo_public_environment_unittests": "//third_party/mojo/src/mojo/edk/test:mojo_public_environment_unittests",
+  "mojo_public_system_unittests":      "//third_party/mojo/src/mojo/edk/test:mojo_public_system_unittests",
+  "mojo_public_utility_unittests":     "//third_party/mojo/src/mojo/edk/test:mojo_public_utility_unittests",
+  "mojo_system_unittests":             "//third_party/mojo/src/mojo/edk/system:mojo_system_unittests",
+  "message_center_unittests":          "//ui/message_center:message_center_unittests",
+  "nacl_loader_unittests":             "//components/nacl:nacl_loader_unittests",
+  "net_unittests":                     "//net:net_unittests",
+  "ozone_unittests":                   "//ui/ozone:ozone_unittests",
+  "ppapi_unittests":                   "//ppapi:ppapi_unittests",
+  "printing_unittests":                "//printing:printing_unittests",
+  "remoting_unittests":                "//remoting:remoting_unittests",
+  "sandbox_linux_unittests":           "//sandbox/linux:sandbox_linux_unittests",
+  "sandbox_mac_unittests":             "//sandbox/mac:sandbox_mac_unittests",
+  "sbox_integration_tests":            "//sandbox/win:sbox_integration_tests",
+  "sbox_unittests":                    "//sandbox/win:sbox_unittests",
+  "sbox_validation_tests":             "//sandbox/win:sbox_validation_tests",
+  "skia_unittests":                    "//skia:skia_unittests",
+  "sql_unittests":                     "//sql:sql_unittests",
+  "sync_integration_tests":            "//sync:sync_integration_tests",
+  "sync_unit_tests":                   "//sync:sync_unit_tests",
+  "ui_base_unittests":                 "//ui/base:ui_base_unittests",
+  "ui_chromeos_unittests":             "//ui/chromeos:ui_chromeos_unittests",
+  "ui_touch_selection_unittests":      "//ui/touch_selection:ui_touch_selection_unittests",
+  "unit_tests":                        "//chrome/test:unit_tests",
+  "url_unittests":                     "//url:url_unittests",
+  "views_unittests":                   "//ui/views:views_unittests",
+  "webkit_unit_tests":                 "//third_party/WebKit/public:webkit_unit_tests",
+  "wm_unittests":                      "//ui/wm:wm_unittests",
+  "wtf_unittests":                     "//third_party/WebKit/public:wtf_unittests",
+}

tools/mb/mb.py

@@ -89,6 +89,9 @@ class MetaBuildWrapper(object):
     subp = subps.add_parser('gen',
                             help='generate a new set of build files')
     AddCommonOptions(subp)
+    subp.add_argument('--swarming-targets-file',
+                      help='save runtime dependencies for targets listed '
+                           'in file.')
     subp.add_argument('path', nargs=1,
                       help='path to generate build into')
     subp.set_defaults(func=self.CmdGen)
@@ -336,7 +339,35 @@ class MetaBuildWrapper(object):
 
   def RunGNGen(self, path, vals):
     cmd = self.GNCmd('gen', path, vals['gn_args'])
+
+    swarming_targets = []
+    if self.args.swarming_targets_file:
+      # We need GN to generate the list of runtime dependencies for
+      # the compile targets listed (one per line) in the file so
+      # we can run them via swarming. We use ninja_to_gn.pyl to convert
+      # the compile targets to the matching GN labels.
+      contents = self.ReadFile(self.args.swarming_targets_file)
+      swarming_targets = contents.splitlines()
+      ninja_targets_to_labels = ast.literal_eval(self.ReadFile(os.path.join(
+          self.chromium_src_dir, 'testing', 'buildbot', 'ninja_to_gn.pyl')))
+      gn_labels = []
+      for target in swarming_targets:
+        if not target in ninja_targets_to_labels:
+          raise MBErr('test target "%s"  not found in %s' %
+                      (target, '//testing/buildbot/ninja_to_gn.pyl'))
+        gn_labels.append(ninja_targets_to_labels[target])
+
+      gn_runtime_deps_path = self.ToAbsPath(path, 'runtime_deps')
+      self.WriteFile(gn_runtime_deps_path, '\n'.join(gn_labels) + '\n')
+      cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path)
+
     ret, _, _ = self.Run(cmd)
+
+    for target in swarming_targets:
+      deps_path = self.ToAbsPath(path, target + '.runtime_deps')
+      if not self.Exists(deps_path):
+          raise MBErr('did not generate %s' % deps_path)
+
     return ret
 
   def GNCmd(self, subcommand, path, gn_args=''):
@@ -387,7 +418,7 @@ class MetaBuildWrapper(object):
       outp = json.loads(self.ReadFile(self.args.output_path[0]))
       self.Print()
       self.Print('analyze output:')
-      self.PrintJSON(inp)
+      self.PrintJSON(outp)
       self.Print()
 
     return ret
@@ -504,10 +535,10 @@ class MetaBuildWrapper(object):
 
     return cmdline, extra_files
 
-  def ToAbsPath(self, build_path, relpath):
+  def ToAbsPath(self, build_path, *comps):
     return os.path.join(self.chromium_src_dir,
                         self.ToSrcRelPath(build_path),
-                        relpath)
+                        *comps)
 
   def ToSrcRelPath(self, path):
     """Returns a relative path from the top of the repo."""