Support for source_target_relative expansion in GN

This can be used to get a path to the source file relative to the target's
directory.

Review-Url: https://codereview.chromium.org/2387763002
Cr-Commit-Position: refs/heads/master@{#439541}

tools/gn/action_values.cc

@@ -22,7 +22,7 @@ void ActionValues::GetOutputsAsSourceFiles(
       target->output_type() == Target::ACTION_FOREACH) {
     // Copy and foreach applies the outputs to the sources.
     SubstitutionWriter::ApplyListToSources(
-        target->settings(), outputs_, target->sources(), result);
+        target, target->settings(), outputs_, target->sources(), result);
   } else {
     // Actions (and anything else that happens to specify an output) just use
     // the output list with no substitution.

tools/gn/bundle_file_rule.cc

@@ -46,7 +46,7 @@ SourceFile BundleFileRule::ApplyPatternToSource(
         break;
       default:
         output_path.append(SubstitutionWriter::GetSourceSubstitution(
-            settings, source_file, subrange.type,
+            target_, target_->settings(), source_file, subrange.type,
             SubstitutionWriter::OUTPUT_ABSOLUTE, SourceDir()));
         break;
     }

tools/gn/desc_builder.cc

@@ -616,8 +616,9 @@ class TargetDescBuilder : public BaseDescBuilder {
         res->SetWithoutPathExpansion("output_patterns", std::move(patterns));
       }
       std::vector<SourceFile> output_files;
-      SubstitutionWriter::ApplyListToSources(target_->settings(), outputs,
-                                             target_->sources(), &output_files);
+      SubstitutionWriter::ApplyListToSources(target_, target_->settings(),
+                                             outputs, target_->sources(),
+                                             &output_files);
       res->SetWithoutPathExpansion(variables::kOutputs,
                                    RenderValue(output_files));
     } else {

tools/gn/docs/reference.md

@@ -6584,6 +6584,13 @@
       build.gn file.
         "//foo/bar/baz.txt" => "obj/foo/bar"
 
+  {{source_target_relative}}\n"
+      The path to the source file relative to the target's directory. This will
+      generally be used for replicating the source directory layout in the
+      output directory. This can only be used in actions and it is an error to
+      use in process_file_template where there is no "target".
+        "//foo/bar/baz.txt" => "baz.txt"
+
 ```
 
 ### **(*) Note on directories**

tools/gn/function_process_file_template.cc

@@ -93,8 +93,15 @@ Value RunProcessFileTemplate(Scope* scope,
     return Value();
   }
 
+  auto& types = subst.required_types();
+  if (std::find(types.begin(), types.end(),
+                SUBSTITUTION_SOURCE_TARGET_RELATIVE) != types.end()) {
+    *err = Err(template_arg, "Not a valid substitution type for the function.");
+    return Value();
+  }
+
   SubstitutionWriter::ApplyListToSourcesAsString(
-      scope->settings(), subst, input_files, &result_files);
+      nullptr, scope->settings(), subst, input_files, &result_files);
 
   // Convert the list of strings to the return Value.
   Value ret(function, Value::LIST);

tools/gn/ninja_action_target_writer.cc

@@ -184,11 +184,11 @@ void NinjaActionTargetWriter::WriteSourceRules(
     // very unusual (normally the substitutions will go in one place or the
     // other) and the redundant assignment won't bother Ninja.
     SubstitutionWriter::WriteNinjaVariablesForSource(
-        settings_, sources[i],
+        target_, settings_, sources[i],
         target_->action_values().args().required_types(),
         args_escape_options, out_);
     SubstitutionWriter::WriteNinjaVariablesForSource(
-        settings_, sources[i],
+        target_, settings_, sources[i],
         target_->action_values().rsp_file_contents().required_types(),
         args_escape_options, out_);
 
@@ -206,7 +206,8 @@ void NinjaActionTargetWriter::WriteOutputFilesForBuildLine(
   size_t first_output_index = output_files->size();
 
   SubstitutionWriter::ApplyListToSourceAsOutputFile(
-      settings_, target_->action_values().outputs(), source, output_files);
+      target_, settings_, target_->action_values().outputs(), source,
+      output_files);
 
   for (size_t i = first_output_index; i < output_files->size(); i++) {
     out_ << " ";
@@ -217,5 +218,5 @@ void NinjaActionTargetWriter::WriteOutputFilesForBuildLine(
 void NinjaActionTargetWriter::WriteDepfile(const SourceFile& source) {
   path_output_.WriteFile(out_,
       SubstitutionWriter::ApplyPatternToSourceAsOutputFile(
-          settings_, target_->action_values().depfile(), source));
+          target_, settings_, target_->action_values().depfile(), source));
 }

tools/gn/ninja_copy_target_writer.cc

@@ -103,7 +103,7 @@ void NinjaCopyTargetWriter::WriteCopyRules(
   for (const auto& input_file : target_->sources()) {
     OutputFile output_file =
         SubstitutionWriter::ApplyPatternToSourceAsOutputFile(
-            target_->settings(), output_subst, input_file);
+            target_, target_->settings(), output_subst, input_file);
     output_files->push_back(output_file);
 
     out_ << "build ";

tools/gn/substitution_type.cc

@@ -21,6 +21,7 @@ const char* kSubstitutionNames[SUBSTITUTION_NUM_TYPES] = {
   "{{source_root_relative_dir}}",  // SUBSTITUTION_SOURCE_ROOT_RELATIVE_DIR
   "{{source_gen_dir}}",  // SUBSTITUTION_SOURCE_GEN_DIR
   "{{source_out_dir}}",  // SUBSTITUTION_SOURCE_OUT_DIR
+  "{{source_target_relative}}",  // SUBSTITUTION_SOURCE_TARGET_RELATIVE
 
   "{{label}}",  // SUBSTITUTION_LABEL
   "{{label_name}}",  // SUBSTITUTION_LABEL_NAME
@@ -70,6 +71,7 @@ const char* kSubstitutionNinjaNames[SUBSTITUTION_NUM_TYPES] = {
     "source_root_relative_dir",  // SUBSTITUTION_SOURCE_ROOT_RELATIVE_DIR
     "source_gen_dir",            // SUBSTITUTION_SOURCE_GEN_DIR
     "source_out_dir",            // SUBSTITUTION_SOURCE_OUT_DIR
+    "source_target_relative",    // SUBSTITUTION_SOURCE_TARGET_RELATIVE
 
     "label",               // SUBSTITUTION_LABEL
     "label_name",          // SUBSTITUTION_LABEL_NAME
@@ -160,7 +162,8 @@ bool IsValidSourceSubstitution(SubstitutionType type) {
          type == SUBSTITUTION_SOURCE_DIR ||
          type == SUBSTITUTION_SOURCE_ROOT_RELATIVE_DIR ||
          type == SUBSTITUTION_SOURCE_GEN_DIR ||
-         type == SUBSTITUTION_SOURCE_OUT_DIR;
+         type == SUBSTITUTION_SOURCE_OUT_DIR ||
+         type == SUBSTITUTION_SOURCE_TARGET_RELATIVE;
 }
 
 bool IsValidToolSubstitution(SubstitutionType type) {

tools/gn/substitution_type.h

@@ -30,6 +30,7 @@ enum SubstitutionType {
   SUBSTITUTION_SOURCE_ROOT_RELATIVE_DIR,  // {{root_relative_dir}}
   SUBSTITUTION_SOURCE_GEN_DIR,  // {{source_gen_dir}}
   SUBSTITUTION_SOURCE_OUT_DIR,  // {{source_out_dir}}
+  SUBSTITUTION_SOURCE_TARGET_RELATIVE,  // {{source_target_relative}}
 
   // Valid for all compiler and linker tools. These depend on the target and
   // do not vary on a per-file basis.

tools/gn/substitution_writer.cc

@@ -98,6 +98,13 @@ Placeholders
       build.gn file.
         "//foo/bar/baz.txt" => "obj/foo/bar"
 
+  {{source_target_relative}}\n"
+      The path to the source file relative to the target's directory. This will
+      generally be used for replicating the source directory layout in the
+      output directory. This can only be used in actions and it is an error to
+      use in process_file_template where there is no "target".
+        "//foo/bar/baz.txt" => "baz.txt"
+
 (*) Note on directories
 
   Paths containing directories (except the source_root_relative_dir) will be
@@ -196,11 +203,12 @@ void SubstitutionWriter::GetListAsOutputFiles(
 
 // static
 SourceFile SubstitutionWriter::ApplyPatternToSource(
+      const Target* target,
       const Settings* settings,
       const SubstitutionPattern& pattern,
       const SourceFile& source) {
   std::string result_value = ApplyPatternToSourceAsString(
-      settings, pattern, source);
+      target, settings, pattern, source);
   CHECK(!result_value.empty() && result_value[0] == '/')
       << "The result of the pattern \""
       << pattern.AsString()
@@ -210,6 +218,7 @@ SourceFile SubstitutionWriter::ApplyPatternToSource(
 
 // static
 std::string SubstitutionWriter::ApplyPatternToSourceAsString(
+    const Target* target,
     const Settings* settings,
     const SubstitutionPattern& pattern,
     const SourceFile& source) {
@@ -219,7 +228,7 @@ std::string SubstitutionWriter::ApplyPatternToSourceAsString(
       result_value.append(subrange.literal);
     } else {
       result_value.append(
-          GetSourceSubstitution(settings, source, subrange.type,
+          GetSourceSubstitution(target, settings, source, subrange.type,
                                 OUTPUT_ABSOLUTE, SourceDir()));
     }
   }
@@ -228,78 +237,89 @@ std::string SubstitutionWriter::ApplyPatternToSourceAsString(
 
 // static
 OutputFile SubstitutionWriter::ApplyPatternToSourceAsOutputFile(
+    const Target* target,
     const Settings* settings,
     const SubstitutionPattern& pattern,
     const SourceFile& source) {
-  SourceFile result_as_source = ApplyPatternToSource(settings, pattern, source);
+  SourceFile result_as_source = ApplyPatternToSource(
+      target, settings, pattern, source);
   return OutputFile(settings->build_settings(), result_as_source);
 }
 
 // static
 void SubstitutionWriter::ApplyListToSource(
+    const Target* target,
     const Settings* settings,
     const SubstitutionList& list,
     const SourceFile& source,
     std::vector<SourceFile>* output) {
   for (const auto& item : list.list())
-    output->push_back(ApplyPatternToSource(settings, item, source));
+    output->push_back(ApplyPatternToSource(target, settings, item, source));
 }
 
 // static
 void SubstitutionWriter::ApplyListToSourceAsString(
+    const Target* target,
     const Settings* settings,
     const SubstitutionList& list,
     const SourceFile& source,
     std::vector<std::string>* output) {
   for (const auto& item : list.list())
-    output->push_back(ApplyPatternToSourceAsString(settings, item, source));
+    output->push_back(ApplyPatternToSourceAsString(
+        target, settings, item, source));
 }
 
 // static
 void SubstitutionWriter::ApplyListToSourceAsOutputFile(
+    const Target* target,
     const Settings* settings,
     const SubstitutionList& list,
     const SourceFile& source,
     std::vector<OutputFile>* output) {
   for (const auto& item : list.list())
-    output->push_back(ApplyPatternToSourceAsOutputFile(settings, item, source));
+    output->push_back(ApplyPatternToSourceAsOutputFile(
+        target, settings, item, source));
 }
 
 // static
 void SubstitutionWriter::ApplyListToSources(
+    const Target* target,
     const Settings* settings,
     const SubstitutionList& list,
     const std::vector<SourceFile>& sources,
     std::vector<SourceFile>* output) {
   output->clear();
   for (const auto& source : sources)
-    ApplyListToSource(settings, list, source, output);
+    ApplyListToSource(target, settings, list, source, output);
 }
 
 // static
 void SubstitutionWriter::ApplyListToSourcesAsString(
+    const Target* target,
     const Settings* settings,
     const SubstitutionList& list,
     const std::vector<SourceFile>& sources,
     std::vector<std::string>* output) {
   output->clear();
   for (const auto& source : sources)
-    ApplyListToSourceAsString(settings, list, source, output);
+    ApplyListToSourceAsString(target, settings, list, source, output);
 }
 
 // static
 void SubstitutionWriter::ApplyListToSourcesAsOutputFile(
+    const Target* target,
     const Settings* settings,
     const SubstitutionList& list,
     const std::vector<SourceFile>& sources,
     std::vector<OutputFile>* output) {
   output->clear();
   for (const auto& source : sources)
-    ApplyListToSourceAsOutputFile(settings, list, source, output);
+    ApplyListToSourceAsOutputFile(target, settings, list, source, output);
 }
 
 // static
 void SubstitutionWriter::WriteNinjaVariablesForSource(
+    const Target* target,
     const Settings* settings,
     const SourceFile& source,
     const std::vector<SubstitutionType>& types,
@@ -314,7 +334,8 @@ void SubstitutionWriter::WriteNinjaVariablesForSource(
       out << "  " << kSubstitutionNinjaNames[type] << " = ";
         EscapeStringToStream(
             out,
-            GetSourceSubstitution(settings, source, type, OUTPUT_RELATIVE,
+            GetSourceSubstitution(target, settings, source, type,
+                                  OUTPUT_RELATIVE,
                                   settings->build_settings()->build_dir()),
             escape_options);
       out << std::endl;
@@ -324,6 +345,7 @@ void SubstitutionWriter::WriteNinjaVariablesForSource(
 
 // static
 std::string SubstitutionWriter::GetSourceSubstitution(
+    const Target* target,
     const Settings* settings,
     const SourceFile& source,
     SubstitutionType type,
@@ -366,6 +388,16 @@ std::string SubstitutionWriter::GetSourceSubstitution(
           BuildDirContext(settings), source.GetDir(), BuildDirType::OBJ));
       break;
 
+    case SUBSTITUTION_SOURCE_TARGET_RELATIVE:
+      if (target) {
+        return RebasePath(source.value(), target->label().dir(),
+            settings->build_settings()->root_path_utf8());
+      }
+      NOTREACHED()
+          << "Cannot use substitution " << kSubstitutionNames[type]
+          << " without target";
+      return std::string();
+
     default:
       NOTREACHED()
           << "Unsupported substitution for this function: "
@@ -502,7 +534,7 @@ std::string SubstitutionWriter::GetCompilerSubstitution(
 
   // Fall-through to the source ones.
   return GetSourceSubstitution(
-      target->settings(), source, type, OUTPUT_RELATIVE,
+      target, target->settings(), source, type, OUTPUT_RELATIVE,
       target->settings()->build_settings()->build_dir());
 }
 

tools/gn/substitution_writer.h

@@ -46,6 +46,10 @@ extern const char kSourceExpansion_Help[];
 // The compiler and linker specific substitutions do NOT include the various
 // cflags, ldflags, libraries, etc. These are written by the ninja target
 // writer since they depend on traversing the dependency tree.
+//
+// The methods which take a target as an argument can accept null target
+// pointer if there is no target context, in which case the substitutions
+// requiring target context will not work.
 class SubstitutionWriter {
  public:
   enum OutputStyle {
@@ -81,15 +85,20 @@ class SubstitutionWriter {
   // expected to be a SourceFile or an OutputFile, this will CHECK if the
   // result isn't in the correct directory. The caller should validate this
   // first (see for example IsFileInOuputDir).
+  //
+  // The target can be null (see class comment above).
   static SourceFile ApplyPatternToSource(
+      const Target* target,
       const Settings* settings,
       const SubstitutionPattern& pattern,
       const SourceFile& source);
   static std::string ApplyPatternToSourceAsString(
+      const Target* target,
       const Settings* settings,
       const SubstitutionPattern& pattern,
       const SourceFile& source);
   static OutputFile ApplyPatternToSourceAsOutputFile(
+      const Target* target,
       const Settings* settings,
       const SubstitutionPattern& pattern,
       const SourceFile& source);
@@ -98,17 +107,22 @@ class SubstitutionWriter {
   // given output vector. It works this way so one can call multiple times to
   // apply to multiple files and create a list. The result can either be
   // SourceFiles or OutputFiles.
+  //
+  // The target can be null (see class comment above).
   static void ApplyListToSource(
+      const Target* target,
       const Settings* settings,
       const SubstitutionList& list,
       const SourceFile& source,
       std::vector<SourceFile>* output);
   static void ApplyListToSourceAsString(
+      const Target* target,
       const Settings* settings,
       const SubstitutionList& list,
       const SourceFile& source,
       std::vector<std::string>* output);
   static void ApplyListToSourceAsOutputFile(
+      const Target* target,
       const Settings* settings,
       const SubstitutionList& list,
       const SourceFile& source,
@@ -116,17 +130,22 @@ class SubstitutionWriter {
 
   // Like ApplyListToSource but applies the list to all sources and replaces
   // rather than appends the output (this produces the complete output).
+  //
+  // The target can be null (see class comment above).
   static void ApplyListToSources(
+      const Target* target,
       const Settings* settings,
       const SubstitutionList& list,
       const std::vector<SourceFile>& sources,
       std::vector<SourceFile>* output);
   static void ApplyListToSourcesAsString(
+      const Target* target,
       const Settings* settings,
       const SubstitutionList& list,
       const std::vector<SourceFile>& sources,
       std::vector<std::string>* output);
   static void ApplyListToSourcesAsOutputFile(
+      const Target* target,
       const Settings* settings,
       const SubstitutionList& list,
       const std::vector<SourceFile>& sources,
@@ -138,7 +157,10 @@ class SubstitutionWriter {
   // Ninja files, paths will be relative to the build dir, and no definition
   // for {{source}} will be written since that maps to Ninja's implicit $in
   // variable.
+  //
+  // The target can be null (see class comment above).
   static void WriteNinjaVariablesForSource(
+      const Target* target,
       const Settings* settings,
       const SourceFile& source,
       const std::vector<SubstitutionType>& types,
@@ -149,7 +171,10 @@ class SubstitutionWriter {
   // given source file. If output_style is OUTPUT_RELATIVE, relative_to
   // indicates the directory that the relative directories should be relative
   // to, otherwise it is ignored.
+  //
+  // The target can be null (see class comment above).
   static std::string GetSourceSubstitution(
+      const Target* target,
       const Settings* settings,
       const SourceFile& source,
       SubstitutionType type,

tools/gn/substitution_writer_unittest.cc

@@ -43,7 +43,7 @@ TEST(SubstitutionWriter, ApplyPatternToSource) {
                             nullptr, &err));
 
   SourceFile result = SubstitutionWriter::ApplyPatternToSource(
-      setup.settings(), pattern, SourceFile("//foo/bar/myfile.txt"));
+      nullptr, setup.settings(), pattern, SourceFile("//foo/bar/myfile.txt"));
   ASSERT_EQ("//out/Debug/gen/foo/bar/myfile.tmp", result.value());
 }
 
@@ -56,7 +56,7 @@ TEST(SubstitutionWriter, ApplyPatternToSourceAsOutputFile) {
                             nullptr, &err));
 
   OutputFile result = SubstitutionWriter::ApplyPatternToSourceAsOutputFile(
-      setup.settings(), pattern, SourceFile("//foo/bar/myfile.txt"));
+      nullptr, setup.settings(), pattern, SourceFile("//foo/bar/myfile.txt"));
   ASSERT_EQ("gen/foo/bar/myfile.tmp", result.value());
 }
 
@@ -73,7 +73,8 @@ TEST(SubstitutionWriter, WriteNinjaVariablesForSource) {
 
   std::ostringstream out;
   SubstitutionWriter::WriteNinjaVariablesForSource(
-      setup.settings(), SourceFile("//foo/bar/baz.txt"), types, options, out);
+      nullptr, setup.settings(), SourceFile("//foo/bar/baz.txt"), types,
+      options, out);
 
   // The "source" should be skipped since that will expand to $in which is
   // implicit.
@@ -103,10 +104,17 @@ TEST(SubstitutionWriter, WriteWithNinjaVariables) {
 
 TEST(SubstitutionWriter, SourceSubstitutions) {
   TestWithScope setup;
+  Err err;
+
+  Target target(setup.settings(), Label(SourceDir("//foo/bar/"), "baz"));
+  target.set_output_type(Target::STATIC_LIBRARY);
+  target.SetToolchain(setup.toolchain());
+  ASSERT_TRUE(target.OnResolved(&err));
 
   // Call to get substitutions relative to the build dir.
   #define GetRelSubst(str, what) \
       SubstitutionWriter::GetSourceSubstitution( \
+          &target, \
           setup.settings(), \
           SourceFile(str), \
           what, \
@@ -116,6 +124,7 @@ TEST(SubstitutionWriter, SourceSubstitutions) {
   // Call to get absolute directory substitutions.
   #define GetAbsSubst(str, what) \
       SubstitutionWriter::GetSourceSubstitution( \
+          &target, \
           setup.settings(), \
           SourceFile(str), \
           what, \
@@ -175,6 +184,11 @@ TEST(SubstitutionWriter, SourceSubstitutions) {
   EXPECT_EQ(".",
             GetRelSubst("//baz.txt", SUBSTITUTION_SOURCE_ROOT_RELATIVE_DIR));
 
+  EXPECT_EQ("baz.txt",
+      GetRelSubst("//foo/bar/baz.txt", SUBSTITUTION_SOURCE_TARGET_RELATIVE));
+  EXPECT_EQ("baz.txt",
+      GetAbsSubst("//foo/bar/baz.txt", SUBSTITUTION_SOURCE_TARGET_RELATIVE));
+
   #undef GetAbsSubst
   #undef GetRelSubst
 }