Support compressed .json grit files, compress chromevox msgs (Take 2)

This provides support for type="chrome_messages_json_gzip" in
grit/build.py, which compresses the output files (which should have
.gz appended to the name, e.g. messages.json.gz).

This makes it fairly simple to convert an extension (e.g. chromevox)
to use compressed l10n files.

Original CL:
https://chromium-review.googlesource.com/c/chromium/src/+/1913610

For changes reviewed in the original CL:
TBR=agrieve@chromium.org

Bug: 1023568
Change-Id: Ibf8dfa374f5e759b3b459d007fde2423f83873a5
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1931097
Commit-Queue: Steven Bennetts <stevenjb@chromium.org>
Reviewed-by: David Tseng <dtseng@chromium.org>
Reviewed-by: Devlin <rdevlin.cronin@chromium.org>
Cr-Commit-Position: refs/heads/master@{#720686}

chrome/browser/resources/chromeos/accessibility/chromevox/BUILD.gn

@@ -398,7 +398,7 @@ if (is_chromeos) {
 action("chromevox_test_messages_js") {
   script = "tools/generate_test_messages.py"
   sources = [
-    "$chromevox_out_dir/_locales/en/messages.json",
+    "$chromevox_out_dir/_locales/en/messages.json.gz",
   ]
   output_file = "$root_out_dir/test_data/chrome/browser/resources/chromeos/accessibility/chromevox/host/testing/test_messages.js"
   outputs = [

chrome/browser/resources/chromeos/accessibility/chromevox/background/language_switching_test.js

@@ -28,6 +28,7 @@ ChromeVoxLanguageSwitchingTest.prototype = {
 #include "base/callback.h"
 #include "chrome/browser/chromeos/accessibility/accessibility_manager.h"
 #include "chrome/common/extensions/extension_constants.h"
+#include "extensions/common/extension_l10n_util.h"
 
 // The following includes are necessary for this test file.
 #include "base/command_line.h"

chrome/browser/resources/chromeos/accessibility/chromevox/strings/chromevox_strings.gni

@@ -10,59 +10,59 @@ template("chromevox_strings") {
     source = "//chrome/browser/resources/chromeos/accessibility/chromevox/strings/chromevox_strings.grd"
     defines = chrome_grit_defines
     outputs = [
-      "_locales/am/messages.json",
-      "_locales/ar/messages.json",
-      "_locales/bg/messages.json",
-      "_locales/bn/messages.json",
-      "_locales/ca/messages.json",
-      "_locales/cs/messages.json",
-      "_locales/da/messages.json",
-      "_locales/de/messages.json",
-      "_locales/el/messages.json",
-      "_locales/en_GB/messages.json",
-      "_locales/en/messages.json",
-      "_locales/es/messages.json",
-      "_locales/es_419/messages.json",
-      "_locales/et/messages.json",
-      "_locales/fa/messages.json",
-      "_locales/fi/messages.json",
-      "_locales/fil/messages.json",
-      "_locales/fr/messages.json",
-      "_locales/gu/messages.json",
-      "_locales/he/messages.json",
-      "_locales/hi/messages.json",
-      "_locales/hr/messages.json",
-      "_locales/hu/messages.json",
-      "_locales/id/messages.json",
-      "_locales/it/messages.json",
-      "_locales/ja/messages.json",
-      "_locales/kn/messages.json",
-      "_locales/ko/messages.json",
-      "_locales/lt/messages.json",
-      "_locales/lv/messages.json",
-      "_locales/ml/messages.json",
-      "_locales/mr/messages.json",
-      "_locales/ms/messages.json",
-      "_locales/nl/messages.json",
-      "_locales/nb/messages.json",
-      "_locales/pl/messages.json",
-      "_locales/pt_BR/messages.json",
-      "_locales/pt_PT/messages.json",
-      "_locales/ro/messages.json",
-      "_locales/ru/messages.json",
-      "_locales/sk/messages.json",
-      "_locales/sl/messages.json",
-      "_locales/sr/messages.json",
-      "_locales/sv/messages.json",
-      "_locales/sw/messages.json",
-      "_locales/ta/messages.json",
-      "_locales/te/messages.json",
-      "_locales/th/messages.json",
-      "_locales/tr/messages.json",
-      "_locales/uk/messages.json",
-      "_locales/vi/messages.json",
-      "_locales/zh_CN/messages.json",
-      "_locales/zh_TW/messages.json",
+      "_locales/am/messages.json.gz",
+      "_locales/ar/messages.json.gz",
+      "_locales/bg/messages.json.gz",
+      "_locales/bn/messages.json.gz",
+      "_locales/ca/messages.json.gz",
+      "_locales/cs/messages.json.gz",
+      "_locales/da/messages.json.gz",
+      "_locales/de/messages.json.gz",
+      "_locales/el/messages.json.gz",
+      "_locales/en_GB/messages.json.gz",
+      "_locales/en/messages.json.gz",
+      "_locales/es/messages.json.gz",
+      "_locales/es_419/messages.json.gz",
+      "_locales/et/messages.json.gz",
+      "_locales/fa/messages.json.gz",
+      "_locales/fi/messages.json.gz",
+      "_locales/fil/messages.json.gz",
+      "_locales/fr/messages.json.gz",
+      "_locales/gu/messages.json.gz",
+      "_locales/he/messages.json.gz",
+      "_locales/hi/messages.json.gz",
+      "_locales/hr/messages.json.gz",
+      "_locales/hu/messages.json.gz",
+      "_locales/id/messages.json.gz",
+      "_locales/it/messages.json.gz",
+      "_locales/ja/messages.json.gz",
+      "_locales/kn/messages.json.gz",
+      "_locales/ko/messages.json.gz",
+      "_locales/lt/messages.json.gz",
+      "_locales/lv/messages.json.gz",
+      "_locales/ml/messages.json.gz",
+      "_locales/mr/messages.json.gz",
+      "_locales/ms/messages.json.gz",
+      "_locales/nl/messages.json.gz",
+      "_locales/nb/messages.json.gz",
+      "_locales/pl/messages.json.gz",
+      "_locales/pt_BR/messages.json.gz",
+      "_locales/pt_PT/messages.json.gz",
+      "_locales/ro/messages.json.gz",
+      "_locales/ru/messages.json.gz",
+      "_locales/sk/messages.json.gz",
+      "_locales/sl/messages.json.gz",
+      "_locales/sr/messages.json.gz",
+      "_locales/sv/messages.json.gz",
+      "_locales/sw/messages.json.gz",
+      "_locales/ta/messages.json.gz",
+      "_locales/te/messages.json.gz",
+      "_locales/th/messages.json.gz",
+      "_locales/tr/messages.json.gz",
+      "_locales/uk/messages.json.gz",
+      "_locales/vi/messages.json.gz",
+      "_locales/zh_CN/messages.json.gz",
+      "_locales/zh_TW/messages.json.gz",
     ]
     output_dir = invoker.out_dir
 

chrome/browser/resources/chromeos/accessibility/chromevox/testing/chromevox_e2e_test_base.js

@@ -39,12 +39,14 @@ ChromeVoxE2ETest.prototype = {
 #include "base/callback.h"
 #include "chrome/browser/chromeos/accessibility/accessibility_manager.h"
 #include "chrome/common/extensions/extension_constants.h"
+#include "extensions/common/extension_l10n_util.h"
     `);
   },
 
   /** @override */
   testGenPreamble: function() {
     GEN(`
+  auto allow = extension_l10n_util::AllowGzippedMessagesAllowedForTest();
   base::Closure load_cb =
       base::Bind(&chromeos::AccessibilityManager::EnableSpokenFeedback,
           base::Unretained(chromeos::AccessibilityManager::Get()),

chrome/browser/resources/chromeos/accessibility/chromevox/tools/generate_test_messages.py

@@ -5,6 +5,7 @@
 # found in the LICENSE file.
 '''Generates test_messages.js from an extension message json file.'''
 
+import gzip
 import optparse
 import sys
 
@@ -38,7 +39,11 @@ def main():
   if len(args) != 1:
     Die('Exactly one input file must be specified')
   in_file_name = args[0]
-  with open(in_file_name) as in_file:
+  def _OpenFile(filename):
+    if filename.endswith('.gz'):
+      return gzip.open(filename)
+    return open(filename)
+  with _OpenFile(in_file_name) as in_file:
     json = in_file.read().strip()
   with open(options.output_file, 'w') as out_file:
     out_file.write(_JS_TEMPLATE % {'in_file': in_file_name, 'json': json})

extensions/common/extension_l10n_util.cc

@@ -36,6 +36,8 @@ namespace keys = extensions::manifest_keys;
 
 namespace {
 
+bool g_allow_gzipped_messages_for_test = false;
+
 // Loads contents of the messages file for given locale. If file is not found,
 // or there was parsing error we return null and set |error|. If
 // |gzip_permission| is kAllowForTrustedSource, this will also look for a .gz
@@ -54,7 +56,8 @@ std::unique_ptr<base::DictionaryValue> LoadMessageFile(
     dictionary = base::DictionaryValue::From(
         messages_deserializer.Deserialize(nullptr, error));
   } else if (gzip_permission == extension_l10n_util::GzippedMessagesPermission::
-                                    kAllowForTrustedSource) {
+                                    kAllowForTrustedSource ||
+             g_allow_gzipped_messages_for_test) {
     // If a compressed version of the file exists, load that.
     base::FilePath compressed_file_path =
         file_path.AddExtension(FILE_PATH_LITERAL(".gz"));
@@ -75,6 +78,8 @@ std::unique_ptr<base::DictionaryValue> LoadMessageFile(
       dictionary = base::DictionaryValue::From(
           messages_deserializer.Deserialize(nullptr, error));
     }
+  } else {
+    LOG(ERROR) << "Unable to load message file: " << locale_path.AsUTF8Unsafe();
   }
 
   if (!dictionary) {
@@ -154,6 +159,10 @@ std::string LocaleForLocalization() {
 
 namespace extension_l10n_util {
 
+base::AutoReset<bool> AllowGzippedMessagesAllowedForTest() {
+  return base::AutoReset<bool>(&g_allow_gzipped_messages_for_test, true);
+}
+
 void SetProcessLocale(const std::string& locale) {
   GetProcessLocale() = locale;
 }

extensions/common/extension_l10n_util.h

@@ -11,6 +11,7 @@
 #include <string>
 #include <vector>
 
+#include "base/auto_reset.h"
 #include "base/strings/string_piece.h"
 
 namespace base {
@@ -32,6 +33,10 @@ enum class GzippedMessagesPermission {
   kAllowForTrustedSource,
 };
 
+// Called from tests to temporarily allow loading gzipped messages for non
+// component test extensions.
+base::AutoReset<bool> AllowGzippedMessagesAllowedForTest();
+
 // Set the locale for this process to a fixed value, rather than using the
 // normal file-based lookup mechanisms. This is used to set the locale inside
 // the sandboxed utility process, where file reading is not allowed.

tools/grit/grit/tool/build.py

@@ -10,6 +10,7 @@ from __future__ import print_function
 import codecs
 import filecmp
 import getopt
+import gzip
 import os
 import shutil
 import sys
@@ -34,6 +35,7 @@ _format_modules = {
   'android': 'android_xml',
   'c_format': 'c_format',
   'chrome_messages_json': 'chrome_messages_json',
+  'chrome_messages_json_gzip': 'chrome_messages_json',
   'data_package': 'data_pack',
   'policy_templates': 'policy_templates_json',
   'rc_all': 'rc',
@@ -359,7 +361,7 @@ are exported to translation interchange files (e.g. XMB files), etc.
       return 'cp1252'
     if output_type in ('android', 'c_format',  'plist', 'plist_strings', 'doc',
                        'json', 'android_policy', 'chrome_messages_json',
-                       'policy_templates'):
+                       'chrome_messages_json_gzip', 'policy_templates'):
       return 'utf_8'
     # TODO(gfeher) modify here to set utf-8 encoding for admx/adml
     return 'utf_16'
@@ -392,24 +394,34 @@ are exported to translation interchange files (e.g. XMB files), etc.
 
       # Write the results to a temporary file and only overwrite the original
       # if the file changed.  This avoids unnecessary rebuilds.
-      outfile = self.fo_create(output.GetOutputFilename() + '.tmp', 'wb')
+      out_filename = output.GetOutputFilename()
+      tmp_filename = out_filename + '.tmp'
+      tmpfile = self.fo_create(tmp_filename, 'wb')
 
-      if output.GetType() != 'data_package':
-        encoding = self._EncodingForOutputType(output.GetType())
-        outfile = util.WrapOutputStream(outfile, encoding)
+      output_type = output.GetType()
+      if output_type != 'data_package':
+        encoding = self._EncodingForOutputType(output_type)
+        tmpfile = util.WrapOutputStream(tmpfile, encoding)
 
       # Iterate in-order through entire resource tree, calling formatters on
       # the entry into a node and on exit out of it.
-      with outfile:
-        self.ProcessNode(self.res, output, outfile)
+      with tmpfile:
+        self.ProcessNode(self.res, output, tmpfile)
+
+      if output_type == 'chrome_messages_json_gzip':
+        gz_filename = tmp_filename + '.gz'
+        with open(tmp_filename, 'rb') as tmpfile, open(gz_filename, 'wb') as f:
+          with gzip.GzipFile(filename='', mode='wb', fileobj=f, mtime=0) as fgz:
+            shutil.copyfileobj(tmpfile, fgz)
+        os.remove(tmp_filename)
+        tmp_filename = gz_filename
 
       # Now copy from the temp file back to the real output, but on Windows,
       # only if the real output doesn't exist or the contents of the file
       # changed.  This prevents identical headers from being written and .cc
       # files from recompiling (which is painful on Windows).
-      if not os.path.exists(output.GetOutputFilename()):
-        os.rename(output.GetOutputFilename() + '.tmp',
-                  output.GetOutputFilename())
+      if not os.path.exists(out_filename):
+        os.rename(tmp_filename, out_filename)
       else:
         # CHROMIUM SPECIFIC CHANGE.
         # This clashes with gyp + vstudio, which expect the output timestamp
@@ -418,13 +430,11 @@ are exported to translation interchange files (e.g. XMB files), etc.
         if not self.write_only_new:
           write_file = True
         else:
-          files_match = filecmp.cmp(output.GetOutputFilename(),
-              output.GetOutputFilename() + '.tmp')
+          files_match = filecmp.cmp(out_filename, tmp_filename)
           write_file = not files_match
         if write_file:
-          shutil.copy2(output.GetOutputFilename() + '.tmp',
-                       output.GetOutputFilename())
-        os.remove(output.GetOutputFilename() + '.tmp')
+          shutil.copy2(tmp_filename, out_filename)
+        os.remove(tmp_filename)
 
       self.VerboseOut(' done.\n')
 

tools/grit/grit_rule.gni

@@ -480,7 +480,8 @@ template("grit") {
     sources = []
     foreach(output, grit_outputs) {
       extension = get_path_info(output, "extension")
-      if (extension != "json" && extension != "pak" && extension != "xml") {
+      if (extension != "json" && extension != "gz" && extension != "pak" &&
+          extension != "xml") {
         sources += [ output ]
       }
     }

ui/accessibility/extensions/chromevoxclassic/BUILD.gn

@@ -459,7 +459,7 @@ chromevox_strings("chromevox_strings") {
 action("chromevox_test_messages_js") {
   script = "//chrome/browser/resources/chromeos/accessibility/chromevox/tools/generate_test_messages.py"
   sources = [
-    "$chromevox_out_dir/_locales/en/messages.json",
+    "$chromevox_out_dir/_locales/en/messages.json.gz",
   ]
   output_file = "$root_out_dir/test_data/ui/accessibility/extensions/chromevoxclassic/host/testing/test_messages.js"
   outputs = [