Web UI: Support adding input_files for generate_grd, remove gen paths

Add input_files and input_files_base_dir arguments for generate_grd, to
allow inclusion of files that are not preprocessed and therefore
included in the manifest in the grd file. Additional updates to
generate_grd:
- Rename rollup files that aren't shared.rollup.js
- Set file type in the grd as "chrome_html" for HTML/JS files
- Adding tests

Started from:
https://chromium-review.googlesource.com/c/chromium/src/+/2443211
by dpapad@

Also removing some unnecessary kGeneratedPath constants from various
Web UI controllers that have been updated to use autogenerated grd
files. These autogenerated files contain resource_path properties for
each <include>, so stripping the generated portion of the path in
webui_util.cc is no longer necessary.

Bug: 1132403
Change-Id: Ia707d48fe348b168dce6bf806099ffd209385bed
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2463333
Commit-Queue: Rebekah Potter <rbpotter@chromium.org>
Reviewed-by: dpapad <dpapad@chromium.org>
Cr-Commit-Position: refs/heads/master@{#815746}

chrome/browser/ui/webui/extensions/extensions_ui.cc

@@ -59,11 +59,6 @@ constexpr char kInDevModeKey[] = "inDevMode";
 constexpr char kShowActivityLogKey[] = "showActivityLog";
 constexpr char kLoadTimeClassesKey[] = "loadTimeClasses";
 
-#if !BUILDFLAG(OPTIMIZE_WEBUI)
-constexpr char kGeneratedPath[] =
-    "@out_folder@/gen/chrome/browser/resources/extensions/preprocessed/";
-#endif
-
 std::string GetLoadTimeClasses(bool in_dev_mode) {
   return in_dev_mode ? "in-dev-mode" : std::string();
 }
@@ -82,7 +77,7 @@ content::WebUIDataSource* CreateMdExtensionsSource(Profile* profile,
 #else
   webui::SetupWebUIDataSource(
       source, base::make_span(kExtensionsResources, kExtensionsResourcesSize),
-      kGeneratedPath, IDR_EXTENSIONS_EXTENSIONS_HTML);
+      "", IDR_EXTENSIONS_EXTENSIONS_HTML);
 #endif
 
   static constexpr webui::LocalizedString kLocalizedStrings[] = {

chrome/browser/ui/webui/print_preview/print_preview_ui.cc

@@ -88,11 +88,6 @@ const char kBasicPrintShortcut[] = "\x28\xE2\x8c\xA5\xE2\x8C\x98\x50\x29";
 const char kBasicPrintShortcut[] = "(Ctrl+Shift+P)";
 #endif
 
-#if !BUILDFLAG(OPTIMIZE_WEBUI)
-constexpr char kGeneratedPath[] =
-    "@out_folder@/gen/chrome/browser/resources/print_preview/preprocessed/";
-#endif
-
 PrintPreviewUI::TestDelegate* g_test_delegate = nullptr;
 
 void StopWorker(int document_cookie) {
@@ -478,8 +473,8 @@ content::WebUIDataSource* CreatePrintPreviewUISource(Profile* profile) {
 #else
   webui::SetupWebUIDataSource(
       source,
-      base::make_span(kPrintPreviewResources, kPrintPreviewResourcesSize),
-      kGeneratedPath, IDR_PRINT_PREVIEW_PRINT_PREVIEW_HTML);
+      base::make_span(kPrintPreviewResources, kPrintPreviewResourcesSize), "",
+      IDR_PRINT_PREVIEW_PRINT_PREVIEW_HTML);
 #endif
   AddPrintPreviewStrings(source);
   SetupPrintPreviewPlugin(source);

chrome/browser/ui/webui/settings/settings_ui.cc

@@ -136,11 +136,6 @@
 
 namespace settings {
 
-#if !BUILDFLAG(OPTIMIZE_WEBUI)
-constexpr char kGeneratedPath[] =
-    "@out_folder@/gen/chrome/browser/resources/settings/preprocessed/";
-#endif
-
 // static
 void SettingsUI::RegisterProfilePrefs(
     user_prefs::PrefRegistrySyncable* registry) {
@@ -373,7 +368,7 @@ SettingsUI::SettingsUI(content::WebUI* web_ui)
 #else
   webui::SetupWebUIDataSource(
       html_source, base::make_span(kSettingsResources, kSettingsResourcesSize),
-      kGeneratedPath, IDR_SETTINGS_SETTINGS_V3_HTML);
+      "", IDR_SETTINGS_SETTINGS_V3_HTML);
 #endif
 
   AddLocalizedStrings(html_source, profile, web_ui->GetWebContents());

chrome/browser/ui/webui/webui_util.cc

@@ -43,9 +43,12 @@ void SetupWebUIDataSource(content::WebUIDataSource* source,
                           const std::string& generated_path,
                           int default_resource) {
   SetupPolymer3Defaults(source);
+  // TODO (crbug.com/1132403): Replace usages of |generated_path| with the new
+  // |resource_path| GRD property, and remove from here.
+  bool has_gen_path = !generated_path.empty();
   for (const GritResourceMap& resource : resources) {
     std::string path = resource.name;
-    if (path.rfind(generated_path, 0) == 0) {
+    if (has_gen_path && path.rfind(generated_path, 0) == 0) {
       path = path.substr(generated_path.size());
     }
 

ui/webui/resources/tools/generate_grd.gni

@@ -11,6 +11,10 @@ template("generate_grd") {
     }
 
     inputs = invoker.manifest_files
+    if (defined(invoker.input_files)) {
+      inputs += invoker.input_files
+    }
+
     outputs = [ invoker.out_grd ]
 
     args = [
@@ -22,5 +26,13 @@ template("generate_grd") {
              rebase_path(root_gen_dir, root_build_dir),
              "--manifest-files",
            ] + rebase_path(invoker.manifest_files, root_build_dir)
+
+    if (defined(invoker.input_files)) {
+      args += [
+        "--input-files-base-dir",
+        invoker.input_files_base_dir,
+      ]
+      args += [ "--input-files" ] + invoker.input_files
+    }
   }
 }

ui/webui/resources/tools/generate_grd.py

@@ -29,6 +29,14 @@
 #     from the root generated directory for setting file paths, as grd files
 #     with generated file paths must specify these paths as
 #     "${root_gen_dir}/<path_to_file>"
+#
+#   input_files:
+#     List of file paths (from |input_files_base_dir|) that are not included in
+#     any manifest files, but should be added to the grd.
+#
+#   input_files_base_dir:
+#     The base directory for the paths in |input_files|. |input_files| and
+#     |input_files_base_dir| must either both be provided or both be omitted.
 
 import argparse
 import json
@@ -56,38 +64,65 @@ GRD_BEGIN_TEMPLATE = '<?xml version="1.0" encoding="UTF-8"?>\n'\
                      '    <includes>\n'
 
 GRD_INCLUDE_TEMPLATE = '      <include name="{name}" ' \
-                       'file="${{root_gen_dir}}/{path_from_gen}" ' \
-                       'resource_path="{path}" use_base_dir="false" ' \
-                       'type="BINDATA" />\n'
+                       'file="{file}" resource_path="{path}" ' \
+                       'use_base_dir="false" type="{type}" />\n'
 
 GRD_END_TEMPLATE = '    </includes>\n'\
                    '  </release>\n'\
                    '</grit>\n'
 
+
+# Generates an <include .... /> row for the given file.
+def _generate_include_row(grd_prefix, filename, pathname):
+  name_suffix = filename.upper().replace('/', '_').replace('.', '_')
+  name = 'IDR_%s_%s' % (grd_prefix.upper(), name_suffix)
+  extension = os.path.splitext(filename)[1]
+  type = 'chrome_html' if extension == '.html' or extension == '.js' \
+          else 'BINDATA'
+  resource_path = filename
+  # Remove 'rollup' from *.rollup.js paths, except for shared.rollup.js.
+  # Possibly pass such replacements from the gni file, if this ends up not being
+  # sufficient for all cases.
+  if ('rollup' in resource_path and 'shared' not in resource_path):
+    resource_path = resource_path.replace('rollup.', '')
+  return GRD_INCLUDE_TEMPLATE.format(
+      file=pathname,
+      path=resource_path,
+      name=name,
+      type=type)
+
+
 def main(argv):
   parser = argparse.ArgumentParser()
   parser.add_argument('--manifest-files', required=True, nargs="*")
   parser.add_argument('--out-grd', required=True)
   parser.add_argument('--grd-prefix', required=True)
   parser.add_argument('--root-gen-dir', required=True)
+  parser.add_argument('--input-files', nargs="*")
+  parser.add_argument('--input-files-base-dir')
   args = parser.parse_args(argv)
 
   grd_file = open(os.path.normpath(os.path.join(_CWD, args.out_grd)), 'w')
   grd_file.write(GRD_BEGIN_TEMPLATE.format(prefix=args.grd_prefix))
 
+  if args.input_files != None:
+    assert(args.input_files_base_dir)
+    for filename in args.input_files:
+      filepath = os.path.join(
+          args.input_files_base_dir, filename).replace('\\', '/')
+      grd_file.write(_generate_include_row(
+          args.grd_prefix, filename, '${root_src_dir}/' + filepath))
+
   for manifest_file in args.manifest_files:
     manifest_path = os.path.normpath(os.path.join(_CWD, manifest_file))
     with open(manifest_path, 'r') as f:
       data = json.load(f)
       base_dir= os.path.normpath(os.path.join(_CWD, data['base_dir']))
       for filename in data['files']:
-        name_suffix = filename.upper().replace('/', '_').replace('.', '_')
-        name = 'IDR_%s_%s' % (args.grd_prefix.upper(), name_suffix)
         filepath = os.path.join(base_dir, filename).replace('\\', '/')
         rebased_path = os.path.relpath(filepath, args.root_gen_dir)
-        grd_file.write(GRD_INCLUDE_TEMPLATE.format(name=name,
-                                                   path=filename,
-                                                   path_from_gen=rebased_path))
+        grd_file.write(_generate_include_row(
+            args.grd_prefix, filename, '${root_gen_dir}/' + rebased_path))
 
   grd_file.write(GRD_END_TEMPLATE)
   return

ui/webui/resources/tools/generate_grd_test.py

@@ -26,7 +26,8 @@ class GenerateGrdTest(unittest.TestCase):
     assert self._out_folder
     return open(os.path.join(self._out_folder, file_name), 'rb').read()
 
-  def _run_test_(self, manifest_files, grd_expected):
+  def _run_test_(self, grd_expected, manifest_files, input_files=None,
+                 input_files_base_dir=None):
     assert not self._out_folder
     self._out_folder = tempfile.mkdtemp(dir=_HERE_DIR)
     args = [
@@ -36,6 +37,14 @@ class GenerateGrdTest(unittest.TestCase):
       '--manifest-files',
     ] + manifest_files
 
+    if (input_files_base_dir):
+      args += [
+        '--input-files-base-dir',
+        input_files_base_dir,
+        '--input-files',
+      ]
+      args += input_files
+
     generate_grd.main(args)
 
     actual_grd = self._read_out_file('test_resources.grd')
@@ -44,10 +53,23 @@ class GenerateGrdTest(unittest.TestCase):
     self.assertEquals(expected_grd, actual_grd)
 
   def testSuccess(self):
-    self._run_test_([
+    self._run_test_(
+      'expected_grd.grd',
+      [
         os.path.join(pathToHere, 'tests', 'test_manifest_1.json'),
         os.path.join(pathToHere, 'tests', 'test_manifest_2.json'),
-      ], 'expected_grd.grd')
+      ])
+
+  def testSuccessWithInputFiles(self):
+    self._run_test_(
+      'expected_grd_with_input_files.grd',
+      [
+        os.path.join(pathToHere, 'tests', 'test_manifest_1.json'),
+        os.path.join(pathToHere, 'tests', 'test_manifest_2.json'),
+      ],
+      [ 'images/test_svg.svg', 'test_html_in_src.html' ],
+      'test_src_dir')
+
 
 if __name__ == '__main__':
   unittest.main()

ui/webui/resources/tools/tests/expected_grd.grd

@@ -12,12 +12,13 @@
   </outputs>
   <release seq="1">
     <includes>
-      <include name="IDR_TEST_TEST_HTML" file="${root_gen_dir}/preprocessed/test.html" resource_path="test.html" use_base_dir="false" type="BINDATA" />
-      <include name="IDR_TEST_TEST_JS" file="${root_gen_dir}/preprocessed/test.js" resource_path="test.js" use_base_dir="false" type="BINDATA" />
-      <include name="IDR_TEST_DIR_ELEMENT_IN_DIR_JS" file="${root_gen_dir}/preprocessed/dir/element_in_dir.js" resource_path="dir/element_in_dir.js" use_base_dir="false" type="BINDATA" />
-      <include name="IDR_TEST_TEST_UI_JS" file="${root_gen_dir}/preprocessed/test_ui.js" resource_path="test_ui.js" use_base_dir="false" type="BINDATA" />
-      <include name="IDR_TEST_TEST_PROXY_JS" file="${root_gen_dir}/preprocessed/test_proxy.js" resource_path="test_proxy.js" use_base_dir="false" type="BINDATA" />
-      <include name="IDR_TEST_DIR_ANOTHER_ELEMENT_IN_DIR_JS" file="${root_gen_dir}/preprocessed/dir/another_element_in_dir.js" resource_path="dir/another_element_in_dir.js" use_base_dir="false" type="BINDATA" />
+      <include name="IDR_TEST_TEST_HTML" file="${root_gen_dir}/preprocessed/test.html" resource_path="test.html" use_base_dir="false" type="chrome_html" />
+      <include name="IDR_TEST_TEST_ROLLUP_JS" file="${root_gen_dir}/preprocessed/test.rollup.js" resource_path="test.js" use_base_dir="false" type="chrome_html" />
+      <include name="IDR_TEST_DIR_ELEMENT_IN_DIR_JS" file="${root_gen_dir}/preprocessed/dir/element_in_dir.js" resource_path="dir/element_in_dir.js" use_base_dir="false" type="chrome_html" />
+      <include name="IDR_TEST_DIR_TEST_SVG_SVG" file="${root_gen_dir}/preprocessed/dir/test_svg.svg" resource_path="dir/test_svg.svg" use_base_dir="false" type="BINDATA" />
+      <include name="IDR_TEST_TEST_UI_JS" file="${root_gen_dir}/preprocessed/test_ui.js" resource_path="test_ui.js" use_base_dir="false" type="chrome_html" />
+      <include name="IDR_TEST_TEST_PROXY_JS" file="${root_gen_dir}/preprocessed/test_proxy.js" resource_path="test_proxy.js" use_base_dir="false" type="chrome_html" />
+      <include name="IDR_TEST_DIR_ANOTHER_ELEMENT_IN_DIR_JS" file="${root_gen_dir}/preprocessed/dir/another_element_in_dir.js" resource_path="dir/another_element_in_dir.js" use_base_dir="false" type="chrome_html" />
     </includes>
   </release>
 </grit>

ui/webui/resources/tools/tests/expected_grd_with_input_files.grd

+<?xml version="1.0" encoding="UTF-8"?>
+<grit latest_public_release="0" current_release="1" output_all_resource_defines="false">
+  <outputs>
+    <output filename="grit/test_resources.h" type="rc_header">
+      <emit emit_type='prepend'></emit>
+    </output>
+    <output filename="grit/test_resources_map.cc"
+            type="resource_file_map_source" />
+    <output filename="grit/test_resources_map.h"
+            type="resource_map_header" />
+    <output filename="test_resources.pak" type="data_package" />
+  </outputs>
+  <release seq="1">
+    <includes>
+      <include name="IDR_TEST_IMAGES_TEST_SVG_SVG" file="${root_src_dir}/test_src_dir/images/test_svg.svg" resource_path="images/test_svg.svg" use_base_dir="false" type="BINDATA" />
+      <include name="IDR_TEST_TEST_HTML_IN_SRC_HTML" file="${root_src_dir}/test_src_dir/test_html_in_src.html" resource_path="test_html_in_src.html" use_base_dir="false" type="chrome_html" />
+      <include name="IDR_TEST_TEST_HTML" file="${root_gen_dir}/preprocessed/test.html" resource_path="test.html" use_base_dir="false" type="chrome_html" />
+      <include name="IDR_TEST_TEST_ROLLUP_JS" file="${root_gen_dir}/preprocessed/test.rollup.js" resource_path="test.js" use_base_dir="false" type="chrome_html" />
+      <include name="IDR_TEST_DIR_ELEMENT_IN_DIR_JS" file="${root_gen_dir}/preprocessed/dir/element_in_dir.js" resource_path="dir/element_in_dir.js" use_base_dir="false" type="chrome_html" />
+      <include name="IDR_TEST_DIR_TEST_SVG_SVG" file="${root_gen_dir}/preprocessed/dir/test_svg.svg" resource_path="dir/test_svg.svg" use_base_dir="false" type="BINDATA" />
+      <include name="IDR_TEST_TEST_UI_JS" file="${root_gen_dir}/preprocessed/test_ui.js" resource_path="test_ui.js" use_base_dir="false" type="chrome_html" />
+      <include name="IDR_TEST_TEST_PROXY_JS" file="${root_gen_dir}/preprocessed/test_proxy.js" resource_path="test_proxy.js" use_base_dir="false" type="chrome_html" />
+      <include name="IDR_TEST_DIR_ANOTHER_ELEMENT_IN_DIR_JS" file="${root_gen_dir}/preprocessed/dir/another_element_in_dir.js" resource_path="dir/another_element_in_dir.js" use_base_dir="false" type="chrome_html" />
+    </includes>
+  </release>
+</grit>

ui/webui/resources/tools/tests/test_manifest_1.json

-{"files": ["test.html", "test.js", "dir/element_in_dir.js"], "base_dir": "tools/tests/preprocessed" }
+{"files": ["test.html", "test.rollup.js", "dir/element_in_dir.js", "dir/test_svg.svg"], "base_dir": "tools/tests/preprocessed" }