Hook up link pools in GN.

Adds a toolchain variable "concurrent_links". The value of this variable in the default toolchain is used as the depth of the link pool for the build.

Remove the "pool" stuff from the tool definitions that was never hooked up. The link pool is hardcoded in GN and is controlled by the above variable.

R=scottmg@chromium.org

Review URL: https://codereview.chromium.org/508763002

Cr-Commit-Position: refs/heads/master@{#292261}

tools/gn/function_toolchain.cc

@@ -3,6 +3,7 @@
 // found in the LICENSE file.
 
 #include <algorithm>
+#include <limits>
 
 #include "tools/gn/err.h"
 #include "tools/gn/functions.h"
@@ -202,15 +203,43 @@ const char kToolchain_Help[] =
     "  source code. You can have more than one toolchain in use at once in\n"
     "  a build.\n"
     "\n"
-    "  A toolchain specifies the commands to run for various input file\n"
-    "  types via the \"tool\" call (see \"gn help tool\") and specifies\n"
-    "  arguments to be passed to the toolchain build via the\n"
-    "  \"toolchain_args\" call (see \"gn help toolchain_args\").\n"
+    "Functions and variables\n"
     "\n"
-    "  In addition, a toolchain can specify dependencies via the \"deps\"\n"
-    "  variable like a target. These dependencies will be resolved before any\n"
-    "  target in the toolchain is compiled. To avoid circular dependencies\n"
-    "  these must be targets defined in another toolchain.\n"
+    "  tool()\n"
+    "    The tool() function call specifies the commands commands to run for\n"
+    "    a given step. See \"gn help tool\".\n"
+    "\n"
+    "  toolchain_args()\n"
+    "    List of arguments to pass to the toolchain when invoking this\n"
+    "    toolchain. This applies only to non-default toolchains. See\n"
+    "    \"gn help toolchain_args\" for more.\n"
+    "\n"
+    "  deps\n"
+    "    Dependencies of this toolchain. These dependencies will be resolved\n"
+    "    before any target in the toolchain is compiled. To avoid circular\n"
+    "    dependencies these must be targets defined in another toolchain.\n"
+    "\n"
+    "    This is expressed as a list of targets, and generally these targets\n"
+    "    will always specify a toolchain:\n"
+    "      deps = [ \"//foo/bar:baz(//build/toolchain:bootstrap)\" ]\n"
+    "\n"
+    "    This concept is somewhat inefficient to express in Ninja (it\n"
+    "    requires a lot of duplicate of rules) so should only be used when\n"
+    "    absolutely necessary.\n"
+    "\n"
+    "  concurrent_links\n"
+    "    In integer expressing the number of links that Ninja will perform in\n"
+    "    parallel. GN will create a pool for shared library and executable\n"
+    "    link steps with this many processes. Since linking is memory- and\n"
+    "    I/O-intensive, projects with many large targets may want to limit\n"
+    "    the number of parallel steps to avoid overloading the computer.\n"
+    "    Since creating static libraries is generally not as intensive\n"
+    "    there is no limit to \"alink\" steps.\n"
+    "\n"
+    "    Defaults to 0 which Ninja interprets as \"no limit\".\n"
+    "\n"
+    "    The value used will be the one from the default toolchain of the\n"
+    "    current build.\n"
     "\n"
     "Invoking targets in toolchains:\n"
     "\n"
@@ -236,8 +265,11 @@ const char kToolchain_Help[] =
     "\n"
     "Example:\n"
     "  toolchain(\"plugin_toolchain\") {\n"
+    "    concurrent_links = 8\n"
+    "\n"
     "    tool(\"cc\") {\n"
     "      command = \"gcc $in\"\n"
+    "      ...\n"
     "    }\n"
     "\n"
     "    toolchain_args() {\n"
@@ -287,6 +319,21 @@ Value RunToolchain(Scope* scope,
       return Value();
   }
 
+  // Read concurrent_links (if any).
+  const Value* concurrent_links_value =
+      block_scope.GetValue("concurrent_links", true);
+  if (concurrent_links_value) {
+    if (!concurrent_links_value->VerifyTypeIs(Value::INTEGER, err))
+      return Value();
+    if (concurrent_links_value->int_value() < 0 ||
+        concurrent_links_value->int_value() > std::numeric_limits<int>::max()) {
+      *err = Err(*concurrent_links_value, "Value out of range.");
+      return Value();
+    }
+    toolchain->set_concurrent_links(
+        static_cast<int>(concurrent_links_value->int_value()));
+  }
+
   if (!block_scope.CheckForUnusedVars(err))
     return Value();
 
@@ -458,9 +505,6 @@ const char kTool_Help[] =
     "        Posix systems:\n"
     "          output_prefix = \"lib\"\n"
     "\n"
-    // TODO(brettw) document "pool" when it works.
-    //"    pool  [string, optional]\n"
-    //"\n"
     "    restat  [boolean]\n"
     "        Valid for: all tools (optional, defaults to false)\n"
     "\n"
@@ -742,7 +786,6 @@ Value RunTool(Scope* scope,
                    &Tool::set_depend_output, err) ||
       !ReadString(&block_scope, "output_prefix", tool.get(),
                   &Tool::set_output_prefix, err) ||
-      !ReadString(&block_scope, "pool", tool.get(), &Tool::set_pool, err) ||
       !ReadBool(&block_scope, "restat", tool.get(), &Tool::set_restat, err) ||
       !ReadPattern(&block_scope, "rspfile", subst_validator, tool.get(),
                    &Tool::set_rspfile, err) ||

tools/gn/ninja_build_writer.cc

@@ -75,11 +75,13 @@ std::string GetSelfInvocationCommand(const BuildSettings* build_settings) {
 NinjaBuildWriter::NinjaBuildWriter(
     const BuildSettings* build_settings,
     const std::vector<const Settings*>& all_settings,
+    const Toolchain* default_toolchain,
     const std::vector<const Target*>& default_toolchain_targets,
     std::ostream& out,
     std::ostream& dep_out)
     : build_settings_(build_settings),
       all_settings_(all_settings),
+      default_toolchain_(default_toolchain),
       default_toolchain_targets_(default_toolchain_targets),
       out_(out),
       dep_out_(dep_out),
@@ -91,6 +93,7 @@ NinjaBuildWriter::~NinjaBuildWriter() {
 
 void NinjaBuildWriter::Run() {
   WriteNinjaRules();
+  WriteLinkPool();
   WriteSubninjas();
   WritePhonyAndAllRules();
 }
@@ -99,6 +102,7 @@ void NinjaBuildWriter::Run() {
 bool NinjaBuildWriter::RunAndWriteFile(
     const BuildSettings* build_settings,
     const std::vector<const Settings*>& all_settings,
+    const Toolchain* default_toolchain,
     const std::vector<const Target*>& default_toolchain_targets) {
   ScopedTrace trace(TraceItem::TRACE_FILE_WRITE, "build.ninja");
 
@@ -118,7 +122,7 @@ bool NinjaBuildWriter::RunAndWriteFile(
   if (depfile.fail())
     return false;
 
-  NinjaBuildWriter gen(build_settings, all_settings,
+  NinjaBuildWriter gen(build_settings, all_settings, default_toolchain,
                        default_toolchain_targets, file, depfile);
   gen.Run();
   return true;
@@ -152,6 +156,12 @@ void NinjaBuildWriter::WriteNinjaRules() {
   out_ << std::endl;
 }
 
+void NinjaBuildWriter::WriteLinkPool() {
+  out_ << "pool link_pool\n"
+       << "  depth = " << default_toolchain_->concurrent_links() << std::endl
+       << std::endl;
+}
+
 void NinjaBuildWriter::WriteSubninjas() {
   for (size_t i = 0; i < all_settings_.size(); i++) {
     out_ << "subninja ";

tools/gn/ninja_build_writer.h

@@ -13,6 +13,7 @@
 class BuildSettings;
 class Settings;
 class Target;
+class Toolchain;
 
 // Generates the toplevel "build.ninja" file. This references the individual
 // toolchain files and lists all input .gn files as dependencies of the
@@ -22,11 +23,13 @@ class NinjaBuildWriter {
   static bool RunAndWriteFile(
       const BuildSettings* settings,
       const std::vector<const Settings*>& all_settings,
+      const Toolchain* default_toolchain,
       const std::vector<const Target*>& default_toolchain_targets);
 
  private:
   NinjaBuildWriter(const BuildSettings* settings,
                    const std::vector<const Settings*>& all_settings,
+                   const Toolchain* default_toolchain,
                    const std::vector<const Target*>& default_toolchain_targets,
                    std::ostream& out,
                    std::ostream& dep_out);
@@ -35,6 +38,7 @@ class NinjaBuildWriter {
   void Run();
 
   void WriteNinjaRules();
+  void WriteLinkPool();
   void WriteSubninjas();
   void WritePhonyAndAllRules();
 
@@ -43,6 +47,7 @@ class NinjaBuildWriter {
 
   const BuildSettings* build_settings_;
   std::vector<const Settings*> all_settings_;
+  const Toolchain* default_toolchain_;
   std::vector<const Target*> default_toolchain_targets_;
   std::ostream& out_;
   std::ostream& dep_out_;

tools/gn/ninja_toolchain_writer.cc

@@ -105,8 +105,11 @@ void NinjaToolchainWriter::WriteToolRule(const Toolchain::ToolType type,
     out_ << kIndent << "deps = msvc" << std::endl;
   }
 
-  if (!tool->pool().empty())
-    out_ << kIndent << "pool = " << tool->pool() << std::endl;
+  // The link pool applies to linker tools. Don't count TYPE_ALINK since
+  // static libraries are not generally intensive to write.
+  if (type == Toolchain::TYPE_SOLINK || type == Toolchain::TYPE_LINK)
+    out_ << kIndent << "pool = link_pool\n";
+
   if (tool->restat())
     out_ << kIndent << "restat = 1" << std::endl;
 }

tools/gn/ninja_writer.cc

@@ -9,6 +9,7 @@
 #include "tools/gn/location.h"
 #include "tools/gn/ninja_build_writer.h"
 #include "tools/gn/ninja_toolchain_writer.h"
+#include "tools/gn/settings.h"
 
 NinjaWriter::NinjaWriter(const BuildSettings* build_settings,
                          Builder* builder)
@@ -88,9 +89,15 @@ bool NinjaWriter::WriteToolchains(std::vector<const Settings*>* all_settings,
 bool NinjaWriter::WriteRootBuildfiles(
     const std::vector<const Settings*>& all_settings,
     const std::vector<const Target*>& default_targets) {
+  // All Settings objects should have the same default toolchain, and there
+  // should always be at least one settings object in the build.
+  CHECK(!all_settings.empty());
+  const Toolchain* default_toolchain =
+      builder_->GetToolchain(all_settings[0]->default_toolchain_label());
+
   // Write the root buildfile.
   if (!NinjaBuildWriter::RunAndWriteFile(build_settings_, all_settings,
-                                         default_targets)) {
+                                         default_toolchain, default_targets)) {
     Err(Location(),
         "Couldn't open toolchain buildfile(s) for writing").PrintToStdout();
     return false;

tools/gn/tool.h

@@ -120,14 +120,6 @@ class Tool {
     output_prefix_ = s;
   }
 
-  const std::string& pool() const {
-    return pool_;
-  }
-  void set_pool(const std::string& s) {
-    DCHECK(!complete_);
-    pool_ = s;
-  }
-
   bool restat() const {
     return restat_;
   }
@@ -182,7 +174,6 @@ class Tool {
   SubstitutionPattern link_output_;
   SubstitutionPattern depend_output_;
   std::string output_prefix_;
-  std::string pool_;
   bool restat_;
   SubstitutionPattern rspfile_;
   SubstitutionPattern rspfile_content_;

tools/gn/toolchain.cc

@@ -24,6 +24,7 @@ const char* Toolchain::kToolCopy = "copy";
 
 Toolchain::Toolchain(const Settings* settings, const Label& label)
     : Item(settings, label),
+      concurrent_links_(0),
       setup_complete_(false) {
 }
 

tools/gn/toolchain.h

@@ -108,9 +108,16 @@ class Toolchain : public Item {
     return substitution_bits_;
   }
 
+  void set_concurrent_links(int cl) { concurrent_links_ = cl; }
+  int concurrent_links() const { return concurrent_links_; }
+
  private:
   scoped_ptr<Tool> tools_[TYPE_NUMTYPES];
 
+  // How many links to run in parallel. Only the default toolchain's version of
+  // this variable applies.
+  int concurrent_links_;
+
   bool setup_complete_;
 
   // Substitutions used by the tools in this toolchain.