GN: Add ability to specify a depfile for custom targets.

BUG=297671

Review URL: https://codereview.chromium.org/55633002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@233435 0039d316-1c4b-4281-b951-d872f2087c98

tools/gn/ninja_script_target_writer.cc

@@ -38,6 +38,10 @@ void NinjaScriptTargetWriter::Run() {
     // No sources, write a rule that invokes the script once with the
     // outputs as outputs, and the data as inputs.
     out_ << "build";
+    if (target_->script_values().has_depfile()) {
+      out_ << " ";
+      WriteDepfile(SourceFile());
+    }
     const Target::FileList& outputs = target_->script_values().outputs();
     for (size_t i = 0; i < outputs.size(); i++) {
       OutputFile output_path(
@@ -48,6 +52,11 @@ void NinjaScriptTargetWriter::Run() {
       path_output_.WriteFile(out_, output_path);
     }
     out_ << ": " << custom_rule_name << implicit_deps << std::endl;
+    if (target_->script_values().has_depfile()) {
+      out_ << "  depfile = ";
+      WriteDepfile(SourceFile());
+      out_ << std::endl;
+    }
   }
   out_ << std::endl;
 
@@ -145,6 +154,12 @@ void NinjaScriptTargetWriter::WriteSourceRules(
 
     if (args_template.has_substitutions())
       WriteArgsSubstitutions(sources[i], args_template);
+
+    if (target_->script_values().has_depfile()) {
+      out_ << "  depfile = ";
+      WriteDepfile(sources[i]);
+      out_ << std::endl;
+    }
   }
 }
 
@@ -166,6 +181,12 @@ void NinjaScriptTargetWriter::WriteOutputFilesForBuildLine(
     const FileTemplate& output_template,
     const SourceFile& source,
     std::vector<OutputFile>* output_files) {
+  // If there is a depfile specified we need to list it as the first output as
+  // that is what ninja will expect the depfile to refer to itself as.
+  if (target_->script_values().has_depfile()) {
+    out_ << " ";
+    WriteDepfile(source);
+  }
   std::vector<std::string> output_template_result;
   output_template.ApplyString(source.value(), &output_template_result);
   for (size_t out_i = 0; out_i < output_template_result.size(); out_i++) {
@@ -175,3 +196,18 @@ void NinjaScriptTargetWriter::WriteOutputFilesForBuildLine(
     path_output_.WriteFile(out_, output_path);
   }
 }
+
+void NinjaScriptTargetWriter::WriteDepfile(const SourceFile& source) {
+  std::vector<std::string> result;
+  GetDepfileTemplate().ApplyString(source.value(), &result);
+  path_output_.WriteFile(out_, OutputFile(result[0]));
+}
+
+FileTemplate NinjaScriptTargetWriter::GetDepfileTemplate() const {
+  std::vector<std::string> template_args;
+  std::string depfile_relative_to_build_dir =
+      RemovePrefix(target_->script_values().depfile().value(),
+                   settings_->build_settings()->build_dir().value());
+  template_args.push_back(depfile_relative_to_build_dir);
+  return FileTemplate(template_args);
+}

tools/gn/ninja_script_target_writer.h

@@ -27,6 +27,8 @@ class NinjaScriptTargetWriter : public NinjaTargetWriter {
  private:
   FRIEND_TEST_ALL_PREFIXES(NinjaScriptTargetWriter,
                            WriteOutputFilesForBuildLine);
+  FRIEND_TEST_ALL_PREFIXES(NinjaScriptTargetWriter,
+                           WriteOutputFilesForBuildLineWithDepfile);
   FRIEND_TEST_ALL_PREFIXES(NinjaScriptTargetWriter,
                            WriteArgsSubstitutions);
 
@@ -65,6 +67,11 @@ class NinjaScriptTargetWriter : public NinjaTargetWriter {
                                     const SourceFile& source,
                                     std::vector<OutputFile>* output_files);
 
+  void WriteDepfile(const SourceFile& source);
+
+  // Returns the FileTemplate for the depfile variable.
+  FileTemplate GetDepfileTemplate() const;
+
   // Path output writer that doesn't do any escaping or quoting. It does,
   // however, convert slashes.  Used for
   // computing intermediate strings.

tools/gn/ninja_script_target_writer_unittest.cc

@@ -36,6 +36,34 @@ TEST(NinjaScriptTargetWriter, WriteOutputFilesForBuildLine) {
   EXPECT_EQ(" gen/a$ bbar.h gen/bar.cc", out_str);
 }
 
+TEST(NinjaScriptTargetWriter, WriteOutputFilesForBuildLineWithDepfile) {
+  TestWithScope setup;
+  setup.build_settings()->SetBuildDir(SourceDir("//out/Debug/"));
+  Target target(setup.settings(), Label(SourceDir("//foo/"), "bar"));
+
+  target.script_values().set_depfile(
+      SourceFile("//out/Debug/gen/{{source_name_part}}.d"));
+  target.script_values().outputs().push_back(
+      SourceFile("//out/Debug/gen/{{source_name_part}}.h"));
+  target.script_values().outputs().push_back(
+      SourceFile("//out/Debug/gen/{{source_name_part}}.cc"));
+
+  std::ostringstream out;
+  NinjaScriptTargetWriter writer(&target, setup.toolchain(), out);
+
+  FileTemplate output_template = writer.GetOutputTemplate();
+
+  SourceFile source("//foo/bar.in");
+  std::vector<OutputFile> output_files;
+  writer.WriteOutputFilesForBuildLine(output_template, source, &output_files);
+
+  std::string out_str = out.str();
+#if defined(OS_WIN)
+  std::replace(out_str.begin(), out_str.end(), '\\', '/');
+#endif
+  EXPECT_EQ(" gen/bar.d gen/bar.h gen/bar.cc", out_str);
+}
+
 TEST(NinjaScriptTargetWriter, WriteArgsSubstitutions) {
   TestWithScope setup;
   setup.build_settings()->SetBuildDir(SourceDir("//out/Debug/"));
@@ -77,7 +105,8 @@ TEST(NinjaScriptTargetWriter, InvokeOverSources) {
   target.script_values().args().push_back(
       "--out=foo bar{{source_name_part}}.o");
 
-  target.script_values().outputs().push_back(SourceFile("//out/Debug/{{source_name_part}}.out"));
+  target.script_values().outputs().push_back(
+      SourceFile("//out/Debug/{{source_name_part}}.out"));
 
   target.source_prereqs().push_back(SourceFile("//foo/included.txt"));
 
@@ -159,3 +188,110 @@ TEST(NinjaScriptTargetWriter, InvokeOverSources) {
     EXPECT_EQ(expected_win, out_str);
   }
 }
+
+// Tests the "run script over multiple source files" mode, with a depfile.
+TEST(NinjaScriptTargetWriter, InvokeOverSourcesWithDepfile) {
+  TestWithScope setup;
+  setup.build_settings()->SetBuildDir(SourceDir("//out/Debug/"));
+  Target target(setup.settings(), Label(SourceDir("//foo/"), "bar"));
+  target.set_output_type(Target::CUSTOM);
+
+  target.sources().push_back(SourceFile("//foo/input1.txt"));
+  target.sources().push_back(SourceFile("//foo/input2.txt"));
+
+  target.script_values().set_script(SourceFile("//foo/script.py"));
+  target.script_values().set_depfile(
+      SourceFile("//out/Debug/gen/{{source_name_part}}.d"));
+
+  target.script_values().args().push_back("-i");
+  target.script_values().args().push_back("{{source}}");
+  target.script_values().args().push_back(
+      "--out=foo bar{{source_name_part}}.o");
+
+  target.script_values().outputs().push_back(
+      SourceFile("//out/Debug/{{source_name_part}}.out"));
+
+  target.source_prereqs().push_back(SourceFile("//foo/included.txt"));
+
+  // Posix.
+  {
+    setup.settings()->set_target_os(Settings::LINUX);
+    setup.build_settings()->set_python_path(base::FilePath(FILE_PATH_LITERAL(
+        "/usr/bin/python")));
+
+    std::ostringstream out;
+    NinjaScriptTargetWriter writer(&target, setup.toolchain(), out);
+    writer.Run();
+
+    const char expected_linux[] =
+        "rule __foo_bar___rule\n"
+        "  command = /usr/bin/python ../../foo/script.py -i ${source} "
+            "\"--out=foo$ bar${source_name_part}.o\"\n"
+        "  description = CUSTOM //foo:bar()\n"
+        "  restat = 1\n"
+        "\n"
+        "build gen/input1.d input1.out: __foo_bar___rule ../../foo/input1.txt"
+            " | ../../foo/included.txt\n"
+        "  source = ../../foo/input1.txt\n"
+        "  source_name_part = input1\n"
+        "  depfile = gen/input1.d\n"
+        "build gen/input2.d input2.out: __foo_bar___rule ../../foo/input2.txt"
+            " | ../../foo/included.txt\n"
+        "  source = ../../foo/input2.txt\n"
+        "  source_name_part = input2\n"
+        "  depfile = gen/input2.d\n"
+        "\n"
+        "build obj/foo/bar.stamp: stamp input1.out input2.out\n";
+
+    std::string out_str = out.str();
+#if defined(OS_WIN)
+    std::replace(out_str.begin(), out_str.end(), '\\', '/');
+#endif
+    EXPECT_EQ(expected_linux, out_str);
+  }
+
+  // Windows.
+  {
+    // Note: we use forward slashes here so that the output will be the same on
+    // Linux and Windows.
+    setup.build_settings()->set_python_path(base::FilePath(FILE_PATH_LITERAL(
+        "C:/python/python.exe")));
+    setup.settings()->set_target_os(Settings::WIN);
+
+    std::ostringstream out;
+    NinjaScriptTargetWriter writer(&target, setup.toolchain(), out);
+    writer.Run();
+
+    // TODO(brettw) I think we'll need to worry about backslashes here
+    // depending if we're on actual Windows or Linux pretending to be Windows.
+    const char expected_win[] =
+        "rule __foo_bar___rule\n"
+        "  command = C:/python/python.exe gyp-win-tool action-wrapper "
+            "environment.x86 __foo_bar___rule.$unique_name.rsp\n"
+        "  description = CUSTOM //foo:bar()\n"
+        "  restat = 1\n"
+        "  rspfile = __foo_bar___rule.$unique_name.rsp\n"
+        "  rspfile_content = C:/python/python.exe ../../foo/script.py -i "
+            "${source} \"--out=foo$ bar${source_name_part}.o\"\n"
+        "\n"
+        "build gen/input1.d input1.out: __foo_bar___rule ../../foo/input1.txt"
+            " | ../../foo/included.txt\n"
+        "  unique_name = 0\n"
+        "  source = ../../foo/input1.txt\n"
+        "  source_name_part = input1\n"
+        "  depfile = gen/input1.d\n"
+        "build gen/input2.d input2.out: __foo_bar___rule ../../foo/input2.txt"
+            " | ../../foo/included.txt\n"
+        "  unique_name = 1\n"
+        "  source = ../../foo/input2.txt\n"
+        "  source_name_part = input2\n"
+        "  depfile = gen/input2.d\n"
+        "\n"
+        "build obj/foo/bar.stamp: stamp input1.out input2.out\n";
+    std::string out_str = out.str();
+#if defined(OS_WIN)
+    std::replace(out_str.begin(), out_str.end(), '\\', '/');
+#endif
+    EXPECT_EQ(expected_win, out_str);
+  }
+}

tools/gn/script_target_generator.cc

@@ -4,6 +4,7 @@
 
 #include "tools/gn/script_target_generator.h"
 
+#include "tools/gn/build_settings.h"
 #include "tools/gn/err.h"
 #include "tools/gn/filesystem_utils.h"
 #include "tools/gn/scope.h"
@@ -48,6 +49,10 @@ void ScriptTargetGenerator::DoRun() {
   if (err_->has_error())
     return;
 
+  FillDepfile();
+  if (err_->has_error())
+    return;
+
   // Script outputs don't depend on the current toolchain so we can skip adding
   // that dependency.
 }
@@ -77,3 +82,12 @@ void ScriptTargetGenerator::FillScriptArgs() {
     return;
   target_->script_values().swap_in_args(&args);
 }
+
+void ScriptTargetGenerator::FillDepfile() {
+  const Value* value = scope_->GetValue(variables::kDepfile, true);
+  if (!value)
+    return;
+  target_->script_values().set_depfile(
+      scope_->settings()->build_settings()->build_dir().ResolveRelativeFile(
+          value->string_value()));
+}

tools/gn/script_target_generator.h

@@ -12,9 +12,9 @@
 class ScriptTargetGenerator : public TargetGenerator {
  public:
   ScriptTargetGenerator(Target* target,
-                      Scope* scope,
-                      const Token& function_token,
-                      Err* err);
+                        Scope* scope,
+                        const Token& function_token,
+                        Err* err);
   virtual ~ScriptTargetGenerator();
 
  protected:
@@ -23,6 +23,7 @@ class ScriptTargetGenerator : public TargetGenerator {
  private:
   void FillScript();
   void FillScriptArgs();
+  void FillDepfile();
 
   DISALLOW_COPY_AND_ASSIGN(ScriptTargetGenerator);
 };

tools/gn/script_values.h

@@ -32,10 +32,16 @@ class ScriptValues {
   const std::vector<SourceFile>& outputs() const { return outputs_; }
   void swap_in_outputs(std::vector<SourceFile>* op) { outputs_.swap(*op); }
 
+  // Depfile generated by the script.
+  const SourceFile& depfile() const { return depfile_; }
+  bool has_depfile() const { return !depfile_.is_null(); }
+  void set_depfile(const SourceFile& depfile) { depfile_ = depfile; }
+
  private:
   SourceFile script_;
   std::vector<std::string> args_;
   std::vector<SourceFile> outputs_;
+  SourceFile depfile_;
 
   DISALLOW_COPY_AND_ASSIGN(ScriptValues);
 };

tools/gn/value_extractors.cc

@@ -126,3 +126,12 @@ bool ExtractListOfLabels(const Value& value,
                             LabelResolver<Target>(current_dir,
                                                   current_toolchain));
 }
+
+bool ExtractRelativeFile(const BuildSettings* build_settings,
+                         const Value& value,
+                         const SourceDir& current_dir,
+                         SourceFile* file,
+                         Err* err) {
+  RelativeFileConverter converter(build_settings, current_dir);
+  return converter(value, file, err);
+}

tools/gn/value_extractors.h

@@ -65,4 +65,10 @@ bool ExtractListOfLabels(const Value& value,
                          LabelTargetVector* dest,
                          Err* err);
 
+bool ExtractRelativeFile(const BuildSettings* build_settings,
+                         const Value& value,
+                         const SourceDir& current_dir,
+                         SourceFile* file,
+                         Err* err);
+
 #endif  // TOOLS_GN_VALUE_EXTRACTORS_H_

tools/gn/variables.cc

@@ -393,6 +393,38 @@ const char kDefines_Help[] =
     "Example:\n"
     "  defines = [ \"AWESOME_FEATURE\", \"LOG_LEVEL=3\" ]\n";
 
+const char kDepfile[] = "depfile";
+const char kDepfile_HelpShort[] =
+    "depfile: [string] File name for input dependencies for custom targets.";
+const char kDepfile_Help[] =
+    "depfile: [string] File name for input dependencies for custom targets.\n"
+    "\n"
+    "  If nonempty, this string specifies that the current \"custom\" target\n"
+    "  will generate the given \".d\" file containing the dependencies of the\n"
+    "  input. Empty or unset means that the script doesn't generate the\n"
+    "  files.\n"
+    "\n"
+    "  The .d file should go in the target output directory. If you have more\n"
+    "  than one source file that the script is being run over, you can use\n"
+    "  the output file expansions described in \"gn help custom\" to name the\n"
+    "  .d file according to the input."
+    "\n"
+    "  The format is that of a Makefile, and all of the paths should be\n"
+    "  relative to the root build directory.\n"
+    "\n"
+    "Example:\n"
+    "  custom(\"myscript_target\") {\n"
+    "    script = \"myscript.py\"\n"
+    "    sources = [ ... ]\n"
+    "\n"
+    "    # Locate the depfile in the output directory named like the\n"
+    "    # inputs but with a \".d\" appended.\n"
+    "    depfile = \"$relative_target_output_dir/{{source_name}}.d\"\n"
+    "\n"
+    "    # Say our script uses \"-o <d file>\" to indicate the depfile.\n"
+    "    args = [ \"{{source}}\", \"-o\", depfile ]\n"
+    "  }\n";
+
 const char kDeps[] = "deps";
 const char kDeps_HelpShort[] =
     "deps: [label list] Linked dependencies.";
@@ -767,6 +799,7 @@ const VariableInfoMap& GetTargetVariables() {
     INSERT_VARIABLE(Configs)
     INSERT_VARIABLE(Data)
     INSERT_VARIABLE(Datadeps)
+    INSERT_VARIABLE(Depfile)
     INSERT_VARIABLE(Deps)
     INSERT_VARIABLE(DirectDependentConfigs)
     INSERT_VARIABLE(External)

tools/gn/variables.h

@@ -111,6 +111,10 @@ extern const char kDefines[];
 extern const char kDefines_HelpShort[];
 extern const char kDefines_Help[];
 
+extern const char kDepfile[];
+extern const char kDepfile_HelpShort[];
+extern const char kDepfile_Help[];
+
 extern const char kDeps[];
 extern const char kDeps_HelpShort[];
 extern const char kDeps_Help[];