Reduce the number of actions related to mojom generator using response file

Spawning many python processes can be bottleneck of build time.
This patch is to improve the time by replacing action_foreach to action for mojom related actions.

Especially, windows buildbot sometimes uses a few minutes for mojom generator.
https://chromium-build-stats.appspot.com/ninja_log/2018/03/22/win1357-c4/ninja_log.win1357-c4.chrome-bot.20180322-005012.7392.gz/trace.html from https://ci.chromium.org/buildbot/tryserver.chromium.win/win7_chromium_rel_ng/127904
The trace shows there are many mojom related actions around 70s~110s and 190s~260s (Those are emphasized by typing mojom to search box.)

I took build stats after touching mojom_bindings_generator.py
On Z840 Windows 10, time to build content/common:mojo_bindings improved from 16.755s to 7.646s and the number of build target is reduced from 2342 to 600.
On Z840 Linux, time to build content/common:mojo_bindings improved from 7.501s to 4.279s and the number of build target is reduced from 2340 to 611.


Change-Id: I56168a233839e1386f31c0b769b263b43dc865d8
Reviewed-on: https://chromium-review.googlesource.com/974982Reviewed-by: Ken Rockot <rockot@chromium.org>
Commit-Queue: Takuto Ikuta <tikuta@chromium.org>
Cr-Commit-Position: refs/heads/master@{#546048}

mojo/public/tools/bindings/mojom.gni

@@ -398,16 +398,25 @@ template("mojom") {
     } else if (is_win) {
       enabled_features += [ "is_win" ]
     }
-    action_foreach(parser_target_name) {
+
+    action(parser_target_name) {
       script = mojom_generator_script
       inputs = mojom_generator_sources + jinja2_sources
       sources = invoker.sources
-      outputs = [
-        "{{source_gen_dir}}/{{source_name_part}}.p",
-      ]
+      outputs = []
+      filelist = []
+      foreach(source, invoker.sources) {
+        filename = get_path_info("$source", "name")
+        dirname = get_path_info("$source", "gen_dir")
+        outputs += [ "$dirname/$filename.p" ]
+        filelist += [ rebase_path("$source", root_build_dir) ]
+      }
+
+      response_file_contents = filelist
+
       args = [
         "parse",
-        "{{source}}",
+        "--filelist={{response_file_name}}",
         "-o",
         rebase_path(root_gen_dir, root_build_dir),
         "-d",
@@ -443,20 +452,28 @@ template("mojom") {
       verify_deps_target_name = "${target_name}__verify_deps"
       verify_deps_target_names += [ ":$verify_deps_target_name" ]
       source_file_name = target_name
-      action_foreach(verify_deps_target_name) {
+
+      action(verify_deps_target_name) {
         script = mojom_generator_script
         inputs = mojom_generator_sources + jinja2_sources
         sources = invoker.sources
         deps = [
           ":$parsed_target_name",
         ]
-        outputs = [
-          "{{source_gen_dir}}/{{source_name_part}}.v",
-        ]
+        outputs = []
+        filelist = []
+        foreach(source, invoker.sources) {
+          filename = get_path_info("$source", "name")
+          dirname = get_path_info("$source", "gen_dir")
+          outputs += [ "$dirname/$filename.v" ]
+          filelist += [ rebase_path("$source", root_build_dir) ]
+        }
+
+        response_file_contents = filelist
 
         args = [
           "verify",
-          "{{source}}",
+          "--filelist={{response_file_name}}",
           "-f",
           rebase_path("$target_gen_dir/$source_file_name.deps_sources_list",
                       root_build_dir),
@@ -476,7 +493,6 @@ template("mojom") {
     common_generator_args = [
       "--use_bundled_pylibs",
       "generate",
-      "{{source}}",
       "-d",
       rebase_path("//", root_build_dir),
       "-I",
@@ -536,7 +552,7 @@ template("mojom") {
       }
     }
 
-    action_foreach(generator_cpp_message_ids_target_name) {
+    action(generator_cpp_message_ids_target_name) {
       script = mojom_generator_script
       inputs = mojom_generator_sources + jinja2_sources
       sources = invoker.sources
@@ -544,11 +560,18 @@ template("mojom") {
         ":$parsed_target_name",
         "//mojo/public/tools/bindings:precompile_templates",
       ]
-      outputs = [
-        "{{source_gen_dir}}/{{source_name_part}}.mojom-shared-message-ids.h",
-      ]
+      outputs = []
       args = common_generator_args
+      filelist = []
+      foreach(source, invoker.sources) {
+        outputs += [ "$target_gen_dir/$source-shared-message-ids.h" ]
+        filelist += [ rebase_path("$source", root_build_dir) ]
+      }
+
+      response_file_contents = filelist
+
       args += [
+        "--filelist={{response_file_name}}",
         "--generate_non_variant_code",
         "--generate_message_ids",
         "-g",
@@ -568,7 +591,7 @@ template("mojom") {
       "{{source_gen_dir}}/{{source_name_part}}.mojom-shared.h",
     ]
     generator_shared_target_name = "${target_name}_shared__generator"
-    action_foreach(generator_shared_target_name) {
+    action(generator_shared_target_name) {
       script = mojom_generator_script
       inputs = mojom_generator_sources + jinja2_sources
       sources = invoker.sources
@@ -577,9 +600,22 @@ template("mojom") {
                "//mojo/public/tools/bindings:precompile_templates",
              ] + verify_deps_target_names
 
-      outputs = generator_shared_cpp_outputs
+      outputs = []
       args = common_generator_args
+      filelist = []
+      foreach(source, invoker.sources) {
+        filelist += [ rebase_path("$source", root_build_dir) ]
+        outputs += [
+          "$target_gen_dir/$source-shared-internal.h",
+          "$target_gen_dir/$source-shared.cc",
+          "$target_gen_dir/$source-shared.h",
+        ]
+      }
+
+      response_file_contents = filelist
+
       args += [
+        "--filelist={{response_file_name}}",
         "--generate_non_variant_code",
         "-g",
         "c++",
@@ -742,7 +778,7 @@ template("mojom") {
       }
 
       generator_target_name = "${target_name}${variant_suffix}__generator"
-      action_foreach(generator_target_name) {
+      action(generator_target_name) {
         script = mojom_generator_script
         inputs = mojom_generator_sources + jinja2_sources
         sources = invoker.sources
@@ -751,9 +787,21 @@ template("mojom") {
                  ":$type_mappings_target_name",
                  "//mojo/public/tools/bindings:precompile_templates",
                ] + verify_deps_target_names
-        outputs = generator_cpp_outputs
+        outputs = []
         args = common_generator_args
+        filelist = []
+        foreach(source, invoker.sources) {
+          filelist += [ rebase_path("$source", root_build_dir) ]
+          outputs += [
+            "$target_gen_dir/${source}${variant_dash_suffix}.cc",
+            "$target_gen_dir/${source}${variant_dash_suffix}.h",
+          ]
+        }
+
+        response_file_contents = filelist
+
         args += [
+          "--filelist={{response_file_name}}",
           "-g",
           "c++",
         ]
@@ -994,7 +1042,7 @@ template("mojom") {
       if (enabled_sources != []) {
         generator_java_outputs =
             [ "{{source_gen_dir}}/{{source_name_part}}.mojom.srcjar" ]
-        action_foreach(java_generator_target_name) {
+        action(java_generator_target_name) {
           script = mojom_generator_script
           inputs = mojom_generator_sources + jinja2_sources
           sources = enabled_sources
@@ -1003,9 +1051,18 @@ template("mojom") {
                    ":$type_mappings_target_name",
                    "//mojo/public/tools/bindings:precompile_templates",
                  ] + verify_deps_target_names
-          outputs = generator_java_outputs
+          outputs = []
           args = common_generator_args
+          filelist = []
+          foreach(source, invoker.sources) {
+            filelist += [ rebase_path("$source", root_build_dir) ]
+            outputs += [ "$target_gen_dir/$source.srcjar" ]
+          }
+
+          response_file_contents = filelist
+
           args += [
+            "--filelist={{response_file_name}}",
             "-g",
             "java",
           ]
@@ -1078,7 +1135,7 @@ template("mojom") {
         "{{source_gen_dir}}/{{source_name_part}}.mojom.js",
         "{{source_gen_dir}}/{{source_name_part}}.mojom.externs.js",
       ]
-      action_foreach(generator_js_target_name) {
+      action(generator_js_target_name) {
         script = mojom_generator_script
         inputs = mojom_generator_sources + jinja2_sources
         sources = []
@@ -1089,9 +1146,21 @@ template("mojom") {
                  ":$parsed_target_name",
                  "//mojo/public/tools/bindings:precompile_templates",
                ] + verify_deps_target_names
-        outputs = generator_js_outputs
+        outputs = []
         args = common_generator_args
+        filelist = []
+        foreach(source, invoker.sources) {
+          filelist += [ rebase_path("$source", root_build_dir) ]
+          outputs += [
+            "$target_gen_dir/$source.js",
+            "$target_gen_dir/$source.externs.js",
+          ]
+        }
+
+        response_file_contents = filelist
+
         args += [
+          "--filelist={{response_file_name}}",
           "-g",
           "javascript",
         ]

mojo/public/tools/bindings/mojom_bindings_generator.py

@@ -243,6 +243,11 @@ def _Generate(args, remaining_args):
 
   processor = MojomProcessor(lambda filename: filename in args.filename)
   processor.LoadTypemaps(set(args.typemaps))
+
+  if args.filelist:
+    with open(args.filelist) as f:
+      args.filename.extend(f.read().split())
+
   for filename in args.filename:
     processor._GenerateModule(args, remaining_args, generator_modules,
                               RelativePath(filename, args.depth), [])
@@ -293,6 +298,11 @@ def _ParseFile(args, rel_filename):
 
 def _Parse(args, _):
   fileutil.EnsureDirectoryExists(args.output_dir)
+
+  if args.filelist:
+    with open(args.filelist) as f:
+      args.filename.extend(f.read().split())
+
   for filename in args.filename:
     _ParseFile(args, RelativePath(filename, args.depth))
   return 0
@@ -307,6 +317,10 @@ def _Precompile(args, _):
 def _VerifyImportDeps(args, __):
   fileutil.EnsureDirectoryExists(args.gen_dir)
 
+  if args.filelist:
+    with open(args.filelist) as f:
+      args.filename.extend(f.read().split())
+
   for filename in args.filename:
     rel_path = RelativePath(filename, args.depth)
     tree = _UnpickleAST(_GetPicklePath(rel_path, args.gen_dir))
@@ -352,7 +366,8 @@ def main():
   parse_parser = subparsers.add_parser(
       "parse", description="Parse mojom to AST and remove disabled definitions."
                            " Pickle pruned AST into output_dir.")
-  parse_parser.add_argument("filename", nargs="+", help="mojom input file")
+  parse_parser.add_argument("filename", nargs="*", help="mojom input file")
+  parse_parser.add_argument("--filelist", help="mojom input file list")
   parse_parser.add_argument(
       "-o",
       "--output_dir",
@@ -373,8 +388,9 @@ def main():
 
   generate_parser = subparsers.add_parser(
       "generate", description="Generate bindings from mojom files.")
-  generate_parser.add_argument("filename", nargs="+",
+  generate_parser.add_argument("filename", nargs="*",
                                help="mojom input file")
+  generate_parser.add_argument("--filelist", help="mojom input file list")
   generate_parser.add_argument("-d", "--depth", dest="depth", default=".",
                                help="depth from source root")
   generate_parser.add_argument("-o", "--output_dir", dest="output_dir",
@@ -461,8 +477,9 @@ def main():
 
   verify_parser = subparsers.add_parser("verify", description="Checks "
       "the set of imports against the set of dependencies.")
-  verify_parser.add_argument("filename", nargs="+",
+  verify_parser.add_argument("filename", nargs="*",
       help="mojom input file")
+  verify_parser.add_argument("--filelist", help="mojom input file list")
   verify_parser.add_argument("-f", "--file", dest="deps_file",
       help="file containing paths to the sources files for "
       "dependencies")