Hook up .d files and outputs to grit

Pull grit 171 to get the new deps fixes required to implement this.

This removes the call to grit for querying inputs completely, and enables depfile writing to enable automatic rebuilds when things are out-of-date.

Adds a new outputs variable that bypasses querying grit for outputs. This will assert that the given files exist in grit. This version allows outputs to be empty in which case it will revert to the old dynamic behavior. I'm going to land this with just the worst cases fixed to make landing easier. Then followup with converting the rest of the grit targets and removing the dynamic option.

R=viettrungluu@chromium.org

Review URL: https://codereview.chromium.org/407653003

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@284705 0039d316-1c4b-4281-b951-d872f2087c98

DEPS

@@ -157,7 +157,7 @@ deps = {
     (Var("googlecode_url") % "snappy") + "/trunk@80",
 
   "src/tools/grit":
-    (Var("googlecode_url") % "grit-i18n") + "/trunk@170",
+    (Var("googlecode_url") % "grit-i18n") + "/trunk@171",
 
   "src/tools/gyp":
     (Var("googlecode_url") % "gyp") + "/trunk@1953",

build/secondary/tools/grit/BUILD.gn

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This target creates a stamp file that depends on all the sources in the grit
+# directory. By depending on this, a target can force itself to be rebuilt if
+# grit itself changes.
+action("grit_sources") {
+  depfile = "$target_out_dir/grit_sources.d"
+  script = "//build/secondary/tools/grit/stamp_grit_sources.py"
+
+  inputs = [ "grit.py" ]
+
+  # Note that we can't call this "grit_sources.stamp" because that file is
+  # implicitly created by GN for script actions.
+  outputs = [ "$target_out_dir/grit_sources.script.stamp" ]
+
+  args = [
+    rebase_path("//tools/grit", root_build_dir),
+    rebase_path(outputs[0], root_build_dir),
+    rebase_path(depfile, root_build_dir)
+  ]
+}

build/secondary/tools/grit/grit_rule.gni

@@ -10,6 +10,23 @@
 #   source
 #       Path to .grd file.
 #
+#   outputs (optional)
+#       List of outputs from grit, relative to the target_gen_dir. If supplied,
+#       a call to Grit to compute the outputs can be skipped which will make
+#       GN run faster. Grit will verify at build time that this list is correct
+#       and will fail if there is a mismatch between the outputs specified by
+#       the .grd file and the outputs list here.
+#
+#       To get this list, you can look in the .grd file for
+#       <output filename="..." and put those filename here. The base directory
+#       of the list in Grit and the output list specified in the GN grit target
+#       are the same (the target_gen_dir) so you can generally copy the names
+#       exactly.
+#
+#       To get the list of outputs programatically, run:
+#           python tools/grit/grit_info.py --outputs . path/to/your.grd
+#       And strip the leading "./" from the output files.
+#
 #   grit_flags (optional)
 #       List of strings containing extra command-line flags to pass to Grit.
 #
@@ -165,9 +182,6 @@ grit_info_script = "//tools/grit/grit_info.py"
 template("grit") {
   assert(defined(invoker.source),
          "\"source\" must be defined for the grit template $target_name")
-  assert(!defined(invoker.sources) && !defined(invoker.outputs),
-         "Neither \"sources\" nor \"outputs\" can be defined for the grit " +
-         "template $target_name")
 
   if (defined(invoker.resource_ids)) {
     resource_ids = invoker.resource_ids
@@ -195,25 +209,37 @@ template("grit") {
     grit_flags = []  # These are optional so default to empty list.
   }
 
-  grit_inputs_build_rel = exec_script(grit_info_script,
-    [ "--inputs", source_path, "-f", resource_ids] + grit_flags, "list lines")
-  # The inputs are relative to the current (build) directory, rebase to
-  # the current one.
-  grit_inputs = rebase_path(grit_inputs_build_rel, ".", root_build_dir) + [
-    grit_resource_id_file,
-  ]
-
-  grit_outputs_build_rel = exec_script(grit_info_script,
-    [ "--outputs", "$rebased_output_dir", source_path, "-f", resource_ids ] +
-    grit_flags,
-    "list lines")
-
-  # The names returned by grit are relative to the current (build) directory,
-  # but references to files in this template are expected to be relative to the
-  # invoking BUILD.gn file's directory. Make it absolute so there's no
-  # ambiguity.
-  grit_outputs = get_path_info(
-      rebase_path(grit_outputs_build_rel, ".", root_build_dir), "abspath")
+  grit_inputs = [ invoker.source ]
+
+  assert_files_flags = []
+  if (defined(invoker.outputs)) {
+    # If the declaration specified outputs, we want to make sure that they
+    # actually match what Grit is writing. We write the list to a file (some
+    # of the output lists are long enough to not fit on a Windows command line)
+    # and ask Grit to verify those are the actual outputs at runtime.
+    asserted_list_file = "$target_out_dir/${target_name}_expected_outputs.txt"
+    write_file(asserted_list_file,
+               rebase_path(invoker.outputs, root_build_dir, target_gen_dir))
+    assert_files_flags += [
+      "--assert-file-list=" + rebase_path(asserted_list_file, root_build_dir),
+    ]
+    grit_outputs = get_path_info(
+        rebase_path(invoker.outputs, ".", target_gen_dir),
+        "abspath")
+  } else {
+    # Ask Grit for the output list.
+    grit_outputs_build_rel = exec_script(grit_info_script,
+      [ "--outputs", "$rebased_output_dir", source_path, "-f", resource_ids ] +
+      grit_flags,
+      "list lines")
+
+    # The names returned by grit are relative to the current (build) directory,
+    # but references to files in this template are expected to be relative to
+    # the invoking BUILD.gn file's directory. Make it absolute so there's no
+    # ambiguity.
+    grit_outputs = get_path_info(
+        rebase_path(grit_outputs_build_rel, ".", root_build_dir), "abspath")
+  }
 
   # The config and the action below get this visibility son only the generated
   # source set can depend on them. The variable "target_name" will get
@@ -235,17 +261,21 @@ template("grit") {
     script = "//tools/grit/grit.py"
     inputs = grit_inputs
     outputs = grit_outputs
+    depfile = "$target_out_dir/${target_name}.d"
 
     args = [
       "-i", source_path, "build",
       "-f", resource_ids,
       "-o", rebased_output_dir,
-    ] + grit_defines + grit_flags
+      "--depdir", ".",
+      "--depfile", rebase_path(depfile, root_build_dir),
+    ] + grit_defines + grit_flags + assert_files_flags
 
     visibility = target_visibility
 
+    deps = [ "//tools/grit:grit_sources" ]
     if (defined(invoker.deps)) {
-      deps = invoker.deps
+      deps += invoker.deps
     }
   }
 

build/secondary/tools/grit/stamp_grit_sources.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This script enumerates the files in the given directory, writing an empty
+# stamp file and a .d file listing the inputs required to make the stamp. This
+# allows us to dynamically depend on the grit sources without enumerating the
+# grit directory for every invocation of grit (which is what adding the source
+# files to every .grd file's .d file would entail) or shelling out to grit
+# synchronously during GN execution to get the list (which would be slow).
+#
+# Usage:
+#    stamp_grit_sources.py <directory> <stamp-file> <.d-file>
+
+import os
+import sys
+
+def GritSourceFiles(grit_root_dir):
+  files = []
+  for root, _, filenames in os.walk(grit_root_dir):
+    grit_src = [os.path.join(root, f) for f in filenames
+                if f.endswith('.py') and not f.endswith('_unittest.py')]
+    files.extend(grit_src)
+  files = [f.replace('\\', '/') for f in files]
+  return sorted(files)
+
+
+def WriteDepFile(dep_file, stamp_file, source_files):
+  with open(dep_file, "w") as f:
+    f.write(stamp_file)
+    f.write(": ")
+    f.write(' '.join(source_files))
+
+
+def WriteStampFile(stamp_file):
+  with open(stamp_file, "w"):
+    pass
+
+
+def main(argv):
+  if len(argv) != 4:
+    print "Error: expecting 3 args."
+    return 1
+
+  grit_root_dir = sys.argv[1]
+  stamp_file = sys.argv[2]
+  dep_file = sys.argv[3]
+
+  WriteStampFile(stamp_file)
+  WriteDepFile(dep_file, stamp_file, GritSourceFiles(grit_root_dir))
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))

chrome/browser/BUILD.gn

@@ -612,6 +612,10 @@ static_library("browser") {
 # (generate_browser_resources action)
 grit("resources") {
   source = "browser_resources.grd"
+  outputs = [
+    "grit/browser_resources.h",
+    "browser_resources.pak",
+  ]
 
   omnibox_mojom_file = "$root_gen_dir/chrome/browser/ui/webui/omnibox/omnibox.mojom.js"
 

chrome/browser/resources/BUILD.gn

@@ -6,30 +6,58 @@ import("//tools/grit/grit_rule.gni")
 
 grit("memory_internals_resources") {
   source = "memory_internals_resources.grd"
+  outputs = [
+    "grit/memory_internals_resources.h",
+    "memory_internals_resources.pak",
+  ]
 }
 
 grit("net_internals_resources") {
   source = "net_internals_resources.grd"
+  outputs = [
+    "grit/net_internals_resources.h",
+    "net_internals_resources.pak",
+  ]
 }
 
 grit("invalidations_resources") {
   source = "invalidations_resources.grd"
+  outputs = [
+    "grit/invalidations_resources.h",
+    "invalidations_resources.pak",
+  ]
 }
 
 grit("password_manager_internals_resources") {
   source = "password_manager_internals_resources.grd"
+  outputs = [
+    "grit/password_manager_internals_resources.h",
+    "password_manager_internals_resources.pak",
+  ]
 }
 
 grit("signin_internals_resources") {
   source = "signin_internals_resources.grd"
+  outputs = [
+    "grit/signin_internals_resources.h",
+    "signin_internals_resources.pak",
+  ]
 }
 
 grit("sync_internals_resources") {
   source = "sync_internals_resources.grd"
+  outputs = [
+    "grit/sync_internals_resources.h",
+    "sync_internals_resources.pak",
+  ]
 }
 
 grit("translate_internals_resources") {
   source = "translate_internals_resources.grd"
+  outputs = [
+    "grit/translate_internals_resources.h",
+    "translate_internals_resources.pak",
+  ]
 }
 
 # GYP version: copy command of chrome_extra_resources
@@ -41,17 +69,35 @@ copy("extension_resource_demo") {
 if (!is_ios) {
   grit("component_extension_resources") {
     source = "component_extension_resources.grd"
+    outputs = [
+      "grit/component_extension_resources.h",
+      "grit/component_extension_resources_map.cc",
+      "grit/component_extension_resources_map.h",
+      "component_extension_resources.pak",
+    ]
   }
 
   grit("options_resources") {
     source = "options_resources.grd"
+    outputs = [
+      "grit/options_resources.h",
+      "options_resources.pak",
+    ]
   }
 
   grit("quota_internals_resources") {
     source = "quota_internals_resources.grd"
+    outputs = [
+      "grit/quota_internals_resources.h",
+      "quota_internals_resources.pak",
+    ]
   }
 
   grit("sync_file_system_internals_resources") {
     source = "sync_file_system_internals_resources.grd"
+    outputs = [
+      "grit/sync_file_system_internals_resources.h",
+      "sync_file_system_internals_resources.pak",
+    ]
   }
 }