Add runtime dependency extraction for GN.

GN will now compute the list of files required for targets to run based on dependencies and data files listed on the targets.

This is exposed via the "gn desc ... runtime_deps" command, and via a new switch that will write runtime deps files for specific targets at generation time.

Review URL: https://codereview.chromium.org/1130183007

Cr-Commit-Position: refs/heads/master@{#329976}

tools/gn/BUILD.gn

@@ -126,6 +126,8 @@ static_library("gn_lib") {
     "path_output.h",
     "pattern.cc",
     "pattern.h",
+    "runtime_deps.cc",
+    "runtime_deps.h",
     "scheduler.cc",
     "scheduler.h",
     "scope.cc",
@@ -258,6 +260,7 @@ test("gn_unittests") {
     "parser_unittest.cc",
     "path_output_unittest.cc",
     "pattern_unittest.cc",
+    "runtime_deps_unittest.cc",
     "scope_per_file_provider_unittest.cc",
     "scope_unittest.cc",
     "source_dir_unittest.cc",

tools/gn/build_settings.h

@@ -18,7 +18,6 @@
 #include "tools/gn/source_file.h"
 
 class Item;
-class OutputFile;
 
 // Settings for one build, which is one toplevel output directory. There
 // may be multiple Settings objects that refer to this, one for each toolchain.

tools/gn/command_desc.cc

@@ -14,6 +14,7 @@
 #include "tools/gn/filesystem_utils.h"
 #include "tools/gn/item.h"
 #include "tools/gn/label.h"
+#include "tools/gn/runtime_deps.h"
 #include "tools/gn/setup.h"
 #include "tools/gn/standard_out.h"
 #include "tools/gn/substitution_writer.h"
@@ -24,6 +25,9 @@ namespace commands {
 
 namespace {
 
+// The switch for displaying blame.
+const char kBlame[] = "blame";
+
 // Prints the given directory in a nice way for the user to view.
 std::string FormatSourceDir(const SourceDir& dir) {
 #if defined(OS_WIN)
@@ -404,7 +408,7 @@ template<typename T> void OutputRecursiveTargetConfig(
     const char* header_name,
     const std::vector<T>& (ConfigValues::* getter)() const) {
   bool display_blame =
-      base::CommandLine::ForCurrentProcess()->HasSwitch("blame");
+      base::CommandLine::ForCurrentProcess()->HasSwitch(kBlame);
 
   DescValueWriter<T> writer;
   std::ostringstream out;
@@ -437,6 +441,30 @@ template<typename T> void OutputRecursiveTargetConfig(
   }
 }
 
+void PrintRuntimeDeps(const Target* target) {
+  bool display_blame =
+      base::CommandLine::ForCurrentProcess()->HasSwitch(kBlame);
+  Label toolchain = target->label().GetToolchainLabel();
+
+  const Target* previous_from = NULL;
+  for (const auto& pair : ComputeRuntimeDeps(target)) {
+    if (display_blame) {
+      // Generally a target's runtime deps will be listed sequentially, so
+      // group them and don't duplicate the "from" label for two in a row.
+      if (previous_from == pair.second) {
+        OutputString("  ");  // Just indent.
+      } else {
+        previous_from = pair.second;
+        OutputString("From ");
+        OutputString(pair.second->label().GetUserVisibleName(toolchain));
+        OutputString("\n  ");  // Make the file name indented.
+      }
+    }
+    OutputString(pair.first.value());
+    OutputString("\n");
+  }
+}
+
 }  // namespace
 
 // desc ------------------------------------------------------------------------
@@ -514,6 +542,18 @@ const char kDesc_Help[] =
     "      Shows the given values taken from the target and all configs\n"
     "      applying. See \"--blame\" below.\n"
     "\n"
+    "  runtime_deps\n"
+    "      Compute all runtime deps for the given target. This is a\n"
+    "      computed list and does not correspond to any GN variable, unlike\n"
+    "      most other values here.\n"
+    "\n"
+    "      The output is a list of file names relative to the build\n"
+    "      directory. See \"gn help runtime_deps\" for how this is computed.\n"
+    "      This also works with \"--blame\" to see the source of the\n"
+    "      dependency.\n"
+    "\n"
+    "Shared flags\n"
+    "\n"
     "  --blame\n"
     "      Used with any value specified by a config, this will name\n"
     "      the config that specified the value. This doesn't currently work\n"
@@ -628,6 +668,8 @@ int RunDesc(const std::vector<std::string>& args) {
       PrintLibDirs(target, false);
     } else if (what == variables::kLibs) {
       PrintLibs(target, false);
+    } else if (what == "runtime_deps") {
+      PrintRuntimeDeps(target);
 
     CONFIG_VALUE_HANDLER(defines, std::string)
     CONFIG_VALUE_HANDLER(include_dirs, SourceDir)

tools/gn/command_gen.cc

@@ -11,6 +11,7 @@
 #include "tools/gn/commands.h"
 #include "tools/gn/ninja_target_writer.h"
 #include "tools/gn/ninja_writer.h"
+#include "tools/gn/runtime_deps.h"
 #include "tools/gn/scheduler.h"
 #include "tools/gn/setup.h"
 #include "tools/gn/standard_out.h"
@@ -101,6 +102,11 @@ int RunGen(const std::vector<std::string>& args) {
     return 1;
   }
 
+  if (!WriteRuntimeDepsFilesIfNecessary(*setup->builder(), &err)) {
+    err.PrintToStdout();
+    return 1;
+  }
+
   base::TimeDelta elapsed_time = timer.Elapsed();
 
   if (!base::CommandLine::ForCurrentProcess()->HasSwitch(switches::kQuiet)) {

tools/gn/command_help.cc

@@ -13,6 +13,7 @@
 #include "tools/gn/input_conversion.h"
 #include "tools/gn/label_pattern.h"
 #include "tools/gn/parser.h"
+#include "tools/gn/runtime_deps.h"
 #include "tools/gn/setup.h"
 #include "tools/gn/standard_out.h"
 #include "tools/gn/substitution_writer.h"
@@ -61,9 +62,10 @@ void PrintToplevelHelp() {
   PrintShortHelp("buildargs: How build arguments work.");
   PrintShortHelp("dotfile: Info about the toplevel .gn file.");
   PrintShortHelp("grammar: Formal grammar for GN build files.");
-  PrintShortHelp("label_pattern: Matching more than one label.");
   PrintShortHelp(
       "input_conversion: Processing input from exec_script and read_file.");
+  PrintShortHelp("label_pattern: Matching more than one label.");
+  PrintShortHelp("runtime_deps: How runtime dependency computation works.");
   PrintShortHelp("source_expansion: Map sources to outputs for scripts.");
   PrintShortHelp("switches: Show available command-line switches.");
 }
@@ -123,6 +125,7 @@ void PrintAllHelp() {
   PrintLongHelp(kGrammar_Help);
   PrintLongHelp(kInputConversion_Help);
   PrintLongHelp(kLabelPattern_Help);
+  PrintLongHelp(kRuntimeDeps_Help);
   PrintLongHelp(kSourceExpansion_Help);
   PrintSwitchHelp();
 }
@@ -236,6 +239,10 @@ int RunHelp(const std::vector<std::string>& args) {
     PrintLongHelp(kLabelPattern_Help);
     return 0;
   }
+  if (what == "runtime_deps") {
+    PrintLongHelp(kRuntimeDeps_Help);
+    return 0;
+  }
   if (what == "source_expansion") {
     PrintLongHelp(kSourceExpansion_Help);
     return 0;

tools/gn/gn.gyp

@@ -128,6 +128,8 @@
         'path_output.h',
         'pattern.cc',
         'pattern.h',
+        'runtime_deps.cc',
+        'runtime_deps.h',
         'scheduler.cc',
         'scheduler.h',
         'scope.cc',
@@ -235,6 +237,7 @@
         'parser_unittest.cc',
         'path_output_unittest.cc',
         'pattern_unittest.cc',
+        'runtime_deps_unittest.cc',
         'scope_per_file_provider_unittest.cc',
         'scope_unittest.cc',
         'source_dir_unittest.cc',

tools/gn/ninja_target_writer.cc

@@ -4,7 +4,6 @@
 
 #include "tools/gn/ninja_target_writer.h"
 
-#include <fstream>
 #include <sstream>
 
 #include "base/files/file_util.h"

tools/gn/runtime_deps.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "tools/gn/runtime_deps.h"
+
+#include <set>
+#include <sstream>
+
+#include "base/command_line.h"
+#include "base/files/file_util.h"
+#include "base/strings/string_split.h"
+#include "tools/gn/build_settings.h"
+#include "tools/gn/builder.h"
+#include "tools/gn/deps_iterator.h"
+#include "tools/gn/filesystem_utils.h"
+#include "tools/gn/loader.h"
+#include "tools/gn/output_file.h"
+#include "tools/gn/settings.h"
+#include "tools/gn/switches.h"
+#include "tools/gn/target.h"
+#include "tools/gn/trace.h"
+
+namespace {
+
+using RuntimeDepsVector = std::vector<std::pair<OutputFile, const Target*>>;
+
+// Adds the given file to the deps list if it hasn't already been listed in
+// the found_files list. Updates the list.
+void AddIfNew(const OutputFile& output_file,
+              const Target* source,
+              RuntimeDepsVector* deps,
+              std::set<OutputFile>* found_file) {
+  if (found_file->find(output_file) != found_file->end())
+    return;  // Already there.
+  deps->push_back(std::make_pair(output_file, source));
+}
+
+// Automatically converts a SourceFile to an OutputFile.
+void AddIfNew(const SourceFile& source_file,
+              const Target* source,
+              RuntimeDepsVector* deps,
+              std::set<OutputFile>* found_file) {
+  AddIfNew(OutputFile(source->settings()->build_settings(), source_file),
+           source, deps, found_file);
+}
+
+// Returns the output file that the runtime deps considers for the given
+// targets. This is weird only for shared libraries.
+const OutputFile& GetMainOutput(const Target* target) {
+  if (target->output_type() == Target::SHARED_LIBRARY)
+    return target->link_output_file();
+  return target->dependency_output_file();
+}
+
+// To avoid duplicate traversals of targets, or duplicating output files that
+// might be listed by more than one target, the set of targets and output files
+// that have been found so far is passed.
+void RecursiveCollectRuntimeDeps(const Target* target,
+                                 bool is_target_data_dep,
+                                 RuntimeDepsVector* deps,
+                                 std::set<const Target*>* seen_targets,
+                                 std::set<OutputFile>* found_files) {
+  if (seen_targets->find(target) != seen_targets->end())
+    return;  // Already checked.
+  seen_targets->insert(target);
+
+  // Add the main output file for executables and shared libraries.
+  if (target->output_type() == Target::EXECUTABLE ||
+      target->output_type() == Target::SHARED_LIBRARY)
+    AddIfNew(GetMainOutput(target), target, deps, found_files);
+
+  // Add all data files.
+  for (const auto& file : target->data())
+    AddIfNew(file, target, deps, found_files);
+
+  // Actions/copy have all outputs considered when the're a data dep.
+  if (is_target_data_dep &&
+      (target->output_type() == Target::ACTION ||
+       target->output_type() == Target::ACTION_FOREACH ||
+       target->output_type() == Target::COPY_FILES)) {
+    std::vector<SourceFile> outputs;
+    target->action_values().GetOutputsAsSourceFiles(target, &outputs);
+    for (const auto& output_file : outputs)
+      AddIfNew(output_file, target, deps, found_files);
+  }
+
+  // Non-data dependencies (both public and private).
+  for (const auto& dep_pair : target->GetDeps(Target::DEPS_LINKED)) {
+    if (dep_pair.ptr->output_type() == Target::EXECUTABLE)
+      continue;  // Skip executables that aren't data deps.
+    RecursiveCollectRuntimeDeps(dep_pair.ptr, false,
+                                deps, seen_targets, found_files);
+  }
+
+  // Data dependencies.
+  for (const auto& dep_pair : target->data_deps()) {
+    RecursiveCollectRuntimeDeps(dep_pair.ptr, true,
+                                deps, seen_targets, found_files);
+  }
+}
+
+bool WriteRuntimeDepsFile(const Target* target) {
+  SourceFile target_output_as_source =
+      GetMainOutput(target).AsSourceFile(target->settings()->build_settings());
+  std::string data_deps_file_as_str = target_output_as_source.value();
+  data_deps_file_as_str.append(".runtime_deps");
+  base::FilePath data_deps_file =
+      target->settings()->build_settings()->GetFullPath(
+          SourceFile(SourceFile::SwapIn(), &data_deps_file_as_str));
+
+  std::stringstream contents;
+  for (const auto& pair : ComputeRuntimeDeps(target))
+    contents << pair.first.value() << std::endl;
+
+  ScopedTrace trace(TraceItem::TRACE_FILE_WRITE, data_deps_file_as_str);
+  base::CreateDirectory(data_deps_file.DirName());
+
+  std::string contents_str = contents.str();
+  return base::WriteFile(data_deps_file, contents_str.c_str(),
+                         static_cast<int>(contents_str.size())) > -1;
+}
+
+}  // namespace
+
+const char kRuntimeDeps_Help[] =
+    "Runtime dependencies\n"
+    "\n"
+    "  Runtime dependencies of a target are exposed via the \"runtime_deps\"\n"
+    "  category of \"gn desc\" (see \"gn help desc\") or they can be written\n"
+    "  at build generation time via \"--runtime-deps-list-file\"\n"
+    "  (see \"gn help --runtime-deps-list-file\").\n"
+    "\n"
+    "  To a first approximation, the runtime dependencies of a target are\n"
+    "  the set of \"data\" files and the shared libraries from all transitive\n"
+    "  dependencies. Executables and shared libraries are considered runtime\n"
+    "  dependencies of themselves.\n"
+    "\n"
+    "Details\n"
+    "\n"
+    "  Executable targets and those executable targets' transitive\n"
+    "  dependencies are not considered unless that executable is listed in\n"
+    "  \"data_deps\". Otherwise, GN assumes that the executable (and\n"
+    "  everything it requires) is a build-time dependency only.\n"
+    "\n"
+    "  Action and copy targets that are listed as \"data_deps\" will have all\n"
+    "  of their outputs and data files considered as runtime dependencies.\n"
+    "  Action and copy targets that are \"deps\" or \"public_deps\" will have\n"
+    "  only their data files considered as runtime dependencies. These\n"
+    "  targets can list an output file in both the \"outputs\" and \"data\"\n"
+    "  lists to force an output file as a runtime dependency in all cases.\n"
+    "\n"
+    "  The results of static_library or source_set targets are not considered\n"
+    "  runtime dependencies since these are assumed to be intermediate\n"
+    "  targets only. If you need to list a static library as a runtime\n"
+    "  dependency, you can manually compute the .a/.lib file name for the\n"
+    "  current platform and list it in the \"data\" list of a target\n"
+    "  (possibly on the static library target itself).\n"
+    "\n"
+    "  When a tool produces more than one output, only the first output\n"
+    "  is considered. For example, a shared library target may produce a\n"
+    "  .dll and a .lib file on Windows. Only the .dll file will be considered\n"
+    "  a runtime dependency.\n";
+
+RuntimeDepsVector ComputeRuntimeDeps(const Target* target) {
+  RuntimeDepsVector result;
+  std::set<const Target*> seen_targets;
+  std::set<OutputFile> found_files;
+
+  // The initial target is not considered a data dependency so that actions's
+  // outputs (if the current target is an action) are not automatically
+  // considered data deps.
+  RecursiveCollectRuntimeDeps(target, false,
+                              &result, &seen_targets, &found_files);
+  return result;
+}
+
+bool WriteRuntimeDepsFilesIfNecessary(const Builder& builder, Err* err) {
+  std::string deps_target_list_file =
+      base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
+          switches::kRuntimeDepsListFile);
+  if (deps_target_list_file.empty())
+    return true;  // Nothing to do.
+
+  std::string list_contents;
+  ScopedTrace load_trace(TraceItem::TRACE_FILE_LOAD, deps_target_list_file);
+  if (!base::ReadFileToString(UTF8ToFilePath(deps_target_list_file),
+                              &list_contents)) {
+    *err = Err(Location(),
+        std::string("File for --") + switches::kRuntimeDepsListFile +
+            " doesn't exist.",
+        "The file given was \"" + deps_target_list_file + "\"");
+    return false;
+  }
+  load_trace.Done();
+
+  std::vector<std::string> lines;
+  base::SplitString(list_contents, '\n', &lines);
+
+  SourceDir root_dir("//");
+  Label default_toolchain_label = builder.loader()->GetDefaultToolchain();
+  for (const auto& line : lines) {
+    if (line.empty())
+      continue;
+    Label label = Label::Resolve(root_dir, default_toolchain_label,
+                                 Value(nullptr, line), err);
+    if (err->has_error())
+      return false;
+
+    const Item* item = builder.GetItem(label);
+    const Target* target = item ? item->AsTarget() : nullptr;
+    if (!target) {
+      *err = Err(Location(), "The label \"" + label.GetUserVisibleName(true) +
+          "\" isn't a target.",
+          "When reading the line:\n  " + line + "\n"
+          "from the --" + switches::kRuntimeDepsListFile + "=" +
+          deps_target_list_file);
+      return false;
+    }
+
+    // Currently this writes all runtime deps files sequentially. We generally
+    // expect few of these. We can run this on the worker pool if it looks
+    // like it's talking a long time.
+    WriteRuntimeDepsFile(target);
+  }
+  return true;
+}

tools/gn/runtime_deps.h

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef TOOLS_GN_RUNTIME_DEPS_H
+#define TOOLS_GN_RUNTIME_DEPS_H
+
+#include <utility>
+#include <vector>
+
+class Builder;
+class Err;
+class OutputFile;
+class Target;
+
+extern const char kRuntimeDeps_Help[];
+
+// Computes the runtime dependencies of the given target. The result is a list
+// of pairs listing the runtime dependency and the target that the runtime
+// dependency is from (for blaming).
+std::vector<std::pair<OutputFile, const Target*>> ComputeRuntimeDeps(
+    const Target* target);
+
+// Writes all runtime deps files requested on the command line, or does nothing
+// if no files were specified.
+bool WriteRuntimeDepsFilesIfNecessary(const Builder& builder, Err* err);
+
+#endif  // TOOLS_GN_RUNTIME_DEPS_H

tools/gn/standard_out.cc

@@ -239,7 +239,7 @@ void PrintLongHelp(const std::string& text) {
       OutputString(line.substr(0, chars_to_highlight), DECORATION_YELLOW);
       OutputString(line.substr(chars_to_highlight) + "\n");
       continue;
-    } else if (!line.empty() && !in_body) {
+    } else if (is_markdown && !line.empty() && !in_body) {
       OutputString("```\n", DECORATION_NONE);
       in_body = true;
     }

tools/gn/switches.cc

@@ -58,9 +58,9 @@ const char kColor_Help[] = COLOR_HELP_LONG;
 
 const char kDotfile[] = "dotfile";
 const char kDotfile_HelpShort[] =
-    "--dotfile: override the name of the \".gn\" file.";
+    "--dotfile: Override the name of the \".gn\" file.";
 const char kDotfile_Help[] =
-    "--dotfile: override the name of the \".gn\" file.\n"
+    "--dotfile: Override the name of the \".gn\" file.\n"
     "\n"
     "  Normally GN loads the \".gn\"file  from the source root for some basic\n"
     "  configuration (see \"gn help dotfile\"). This flag allows you to\n"
@@ -108,6 +108,39 @@ const char kRoot_Help[] =
     "\n"
     "  gn desc //out/Default --root=\"C:\\Users\\BObama\\My Documents\\foo\"\n";
 
+const char kRuntimeDepsListFile[] = "runtime-deps-list-file";
+const char kRuntimeDepsListFile_HelpShort[] =
+    "--runtime-deps-list-file: Save runtime dependencies for targets in file.";
+const char kRuntimeDepsListFile_Help[] =
+    "--runtime-deps-list-file: Save runtime dependencies for targets in file.\n"
+    "\n"
+    "  --runtime-deps-list-file=<filename>\n"
+    "\n"
+    "  Where <filename> is a text file consisting of the labels, one per\n"
+    "  line, of the targets for which runtime dependencies are desired.\n"
+    "\n"
+    "  See \"gn help runtime_deps\" for a description of how runtime\n"
+    "  dependencies are computed.\n"
+    "\n"
+    "Runtime deps output file\n"
+    "\n"
+    "  For each target requested, GN will write a separate runtime dependency\n"
+    "  file. The runtime dependency file will be in the output directory\n"
+    "  alongside the output file of the target, with a \".runtime_deps\"\n"
+    "  extension. For example, if the target \"//foo:bar\" is listed in the\n"
+    "  input file, and that target produces an output file \"bar.so\", GN\n"
+    "  will create a file \"bar.so.runtime_deps\" in the build directory.\n"
+    "\n"
+    "  If a source set, action, copy, or group is listed, the runtime deps\n"
+    "  file will correspond to the .stamp file corresponding to that target.\n"
+    "  This is probably not useful; the use-case for this feature is\n"
+    "  generally executable targets.\n"
+    "\n"
+    "  The runtime dependency file will list one file per line, with no\n"
+    "  escaping. The files will be relative to the root_build_dir. The first\n"
+    "  line of the file will be the main output file of the target itself\n"
+    "  (in the above example, \"bar.so\").\n";
+
 const char kThreads[] = "threads";
 const char kThreads_HelpShort[] =
     "--threads: Specify number of worker threads.";
@@ -194,6 +227,7 @@ const SwitchInfoMap& GetSwitches() {
     INSERT_VARIABLE(NoColor)
     INSERT_VARIABLE(Root)
     INSERT_VARIABLE(Quiet)
+    INSERT_VARIABLE(RuntimeDepsListFile)
     INSERT_VARIABLE(Time)
     INSERT_VARIABLE(Tracelog)
     INSERT_VARIABLE(Verbose)

tools/gn/switches.h

@@ -56,6 +56,10 @@ extern const char kRoot[];
 extern const char kRoot_HelpShort[];
 extern const char kRoot_Help[];
 
+extern const char kRuntimeDepsListFile[];
+extern const char kRuntimeDepsListFile_HelpShort[];
+extern const char kRuntimeDepsListFile_Help[];
+
 extern const char kThreads[];
 extern const char kThreads_HelpShort[];
 extern const char kThreads_Help[];

tools/gn/variables.cc

@@ -510,16 +510,23 @@ const char kData_Help[] =
     "data: Runtime data file dependencies.\n"
     "\n"
     "  Lists files required to run the given target. These are typically\n"
-    "  data files.\n"
+    "  data files. The paths are interpreted as being relative to the current\n"
+    "  build file. Since these are runtime dependencies, they do not affect\n"
+    "  which targets are built or when. To declare input files to a script,\n"
+    "  use \"inputs\".\n"
     "\n"
     "  Appearing in the \"data\" section does not imply any special handling\n"
     "  such as copying them to the output directory. This is just used for\n"
-    "  declaring runtime dependencies. There currently isn't a good use for\n"
-    "  these but it is envisioned that test data can be listed here for use\n"
-    "  running automated tests.\n"
+    "  declaring runtime dependencies. Runtime dependencies can be queried\n"
+    "  using the \"runtime_deps\" category of \"gn desc\" or written during\n"
+    "  build generation via \"--runtime-deps-list-file\".\n"
     "\n"
-    "  See also \"gn help inputs\" and \"gn help data_deps\", both of\n"
-    "  which actually affect the build in concrete ways.\n";
+    "  GN doesn't require data files to exist at build-time. So actions that\n"
+    "  produce files that are in turn runtime dependencies can list those\n"
+    "  generated files both in the \"outputs\" list as well as the \"data\"\n"
+    "  list.\n"
+    "\n"
+    "  See \"gn help runtime_deps\" for how these are used.\n";
 
 const char kDataDeps[] = "data_deps";
 const char kDataDeps_HelpShort[] =