ResourceBundale and DataPack: Support gzip pak files

This adds support to DataPack to load gzipped .pak.gz files into memory,
instead of using mmap to map memory to an uncompressed pak file.

The advantage is that .pak files compress by as much as 70% on disk.

The disadvantage is that mmapped memory (which is initially resident)
can be trivially swapped out when not in use (since it is already
backed by a file), whereas allocated memory uses regular swap storage.

For Chrome OS this is likely to be a worthwhile tradeoff. Testing shows
that fairly minimal use of the Chrome OS UI swaps in most of the file
after it is swapped out. See the issue for details.

Note: This CL does not implelent compression of .pak files, that will be
handled in a follow-up CL.

This also removes the unused |test_file_exists| parameter from
ResourceBundle::GetLocaleFilePath (it was always set to true).

Bug: 1017864
Change-Id: I16eaaf8d294199d8310ec06fedc79dabfb0d27a0
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1882759
Commit-Queue: Steven Bennetts <stevenjb@chromium.org>
Reviewed-by: Scott Violet <sky@chromium.org>
Cr-Commit-Position: refs/heads/master@{#713124}

ui/base/BUILD.gn

@@ -46,6 +46,7 @@ component("ui_data_pack") {
   deps = [
     "//base",
     "//net",
+    "//third_party/zlib/google:compression_utils",
   ]
 
   defines = [ "UI_DATA_PACK_IMPLEMENTATION" ]
@@ -919,6 +920,7 @@ test("ui_base_unittests") {
     "//testing/gtest",
     "//third_party/brotli:dec",
     "//third_party/icu",
+    "//third_party/zlib/google:compression_utils",
     "//ui/base",
     "//ui/base:test_support",
     "//ui/base:ui_data_pack",

ui/base/resource/data_pack.cc

@@ -21,6 +21,7 @@
 #include "base/synchronization/lock.h"
 #include "base/sys_byteorder.h"
 #include "build/build_config.h"
+#include "third_party/zlib/google/compression_utils.h"
 
 // For details of the file layout, see
 // http://dev.chromium.org/developers/design-documents/linuxresourcesandlocalizedstrings
@@ -47,6 +48,7 @@ enum LoadErrors {
   HEADER_TRUNCATED,
   WRONG_ENCODING,
   INIT_FAILED_FROM_FILE,
+  UNZIP_FAILED,
 
   LOAD_ERRORS_COUNT,
 };
@@ -200,7 +202,6 @@ class DataPack::MemoryMappedDataSource : public DataPack::DataSource {
 
   // DataPack::DataSource:
   size_t GetLength() const override { return mmap_->length(); }
-
   const uint8_t* GetData() const override { return mmap_->data(); }
 
  private:
@@ -209,6 +210,25 @@ class DataPack::MemoryMappedDataSource : public DataPack::DataSource {
   DISALLOW_COPY_AND_ASSIGN(MemoryMappedDataSource);
 };
 
+// Takes ownership of a string of uncompressed pack data.
+class DataPack::StringDataSource : public DataPack::DataSource {
+ public:
+  explicit StringDataSource(std::string&& data) : data_(std::move(data)) {}
+
+  ~StringDataSource() override {}
+
+  // DataPack::DataSource:
+  size_t GetLength() const override { return data_.size(); }
+  const uint8_t* GetData() const override {
+    return reinterpret_cast<const uint8_t*>(data_.c_str());
+  }
+
+ private:
+  const std::string data_;
+
+  DISALLOW_COPY_AND_ASSIGN(StringDataSource);
+};
+
 class DataPack::BufferDataSource : public DataPack::DataSource {
  public:
   explicit BufferDataSource(base::StringPiece buffer) : buffer_(buffer) {}
@@ -217,7 +237,6 @@ class DataPack::BufferDataSource : public DataPack::DataSource {
 
   // DataPack::DataSource:
   size_t GetLength() const override { return buffer_.length(); }
-
   const uint8_t* GetData() const override {
     return reinterpret_cast<const uint8_t*>(buffer_.data());
   }
@@ -249,9 +268,19 @@ bool DataPack::LoadFromPath(const base::FilePath& path) {
   if (!mmap->Initialize(path)) {
     DLOG(ERROR) << "Failed to mmap datapack";
     LogDataPackError(INIT_FAILED);
-    mmap.reset();
     return false;
   }
+  if (path.FinalExtension() == FILE_PATH_LITERAL(".gz")) {
+    base::StringPiece compressed(reinterpret_cast<char*>(mmap->data()),
+                                 mmap->length());
+    std::string data;
+    if (!compression::GzipUncompress(compressed, &data)) {
+      LOG(ERROR) << "Failed to unzip compressed datapack: " << path;
+      LogDataPackError(UNZIP_FAILED);
+      return false;
+    }
+    return LoadImpl(std::make_unique<StringDataSource>(std::move(data)));
+  }
   return LoadImpl(std::make_unique<MemoryMappedDataSource>(std::move(mmap)));
 }
 

ui/base/resource/data_pack.h

@@ -36,13 +36,19 @@ class UI_DATA_PACK_EXPORT DataPack : public ResourceHandle {
   explicit DataPack(ui::ScaleFactor scale_factor);
   ~DataPack() override;
 
-  // Load a pack file from |path|, returning false on error.
+  // Load a pack file from |path|, returning false on error. If the final
+  // extension of |path| is .gz, the file will be uncompressed and stored in
+  // memory owned by |data_source_|. Otherwise the file will be mapped to
+  // memory, with the mapping owned by |data_source_|.
   bool LoadFromPath(const base::FilePath& path);
 
-  // Loads a pack file from |file|, returning false on error.
+  // Invokes LoadFromFileRegion with the entire contents of |file|. Compressed
+  // files are not supported.
   bool LoadFromFile(base::File file);
 
   // Loads a pack file from |region| of |file|, returning false on error.
+  // The file region will be mapped to memory with the mapping owned by
+  // |data_source_|.
   bool LoadFromFileRegion(base::File file,
                           const base::MemoryMappedFile::Region& region);
 
@@ -87,6 +93,7 @@ class UI_DATA_PACK_EXPORT DataPack : public ResourceHandle {
   class DataSource;
   class BufferDataSource;
   class MemoryMappedDataSource;
+  class StringDataSource;
 
   // Does the actual loading of a pack file.
   // Called by Load and LoadFromFile and LoadFromBuffer.

ui/base/resource/data_pack_unittest.cc

@@ -17,6 +17,7 @@
 #include "base/strings/string_piece.h"
 #include "build/build_config.h"
 #include "testing/gtest/include/gtest/gtest.h"
+#include "third_party/zlib/google/compression_utils.h"
 #include "ui/base/resource/data_pack_literal.h"
 #include "ui/base/ui_base_paths.h"
 
@@ -61,6 +62,42 @@ TEST(DataPackTest, LoadFromPath) {
   ASSERT_FALSE(pack.GetStringPiece(140, &data));
 }
 
+TEST(DataPackTest, LoadFromPathCompressed) {
+  base::ScopedTempDir dir;
+  ASSERT_TRUE(dir.CreateUniqueTempDir());
+  base::FilePath data_path =
+      dir.GetPath().Append(FILE_PATH_LITERAL("sample.pak.gz"));
+
+  // Dump contents into a compressed pak file.
+  std::string compressed;
+  ASSERT_TRUE(compression::GzipCompress(
+      base::StringPiece(kSamplePakContentsV4, kSamplePakSizeV4), &compressed));
+  ASSERT_EQ(base::WriteFile(data_path, compressed.c_str(), compressed.length()),
+            static_cast<int>(compressed.length()));
+
+  // Load the file through the data pack API.
+  DataPack pack(SCALE_FACTOR_100P);
+  ASSERT_TRUE(pack.LoadFromPath(data_path));
+
+  base::StringPiece data;
+  ASSERT_TRUE(pack.HasResource(4));
+  ASSERT_TRUE(pack.GetStringPiece(4, &data));
+  EXPECT_EQ("this is id 4", data);
+  ASSERT_TRUE(pack.HasResource(6));
+  ASSERT_TRUE(pack.GetStringPiece(6, &data));
+  EXPECT_EQ("this is id 6", data);
+
+  // Try reading zero-length data blobs, just in case.
+  ASSERT_TRUE(pack.GetStringPiece(1, &data));
+  EXPECT_EQ(0U, data.length());
+  ASSERT_TRUE(pack.GetStringPiece(10, &data));
+  EXPECT_EQ(0U, data.length());
+
+  // Try looking up an invalid key.
+  ASSERT_FALSE(pack.HasResource(140));
+  ASSERT_FALSE(pack.GetStringPiece(140, &data));
+}
+
 TEST(DataPackTest, LoadFromFile) {
   base::ScopedTempDir dir;
   ASSERT_TRUE(dir.CreateUniqueTempDir());

ui/base/resource/resource_bundle.cc

@@ -318,7 +318,7 @@ void ResourceBundle::LoadSecondaryLocaleDataWithPakFileRegion(
 #if !defined(OS_ANDROID)
 // static
 bool ResourceBundle::LocaleDataPakExists(const std::string& locale) {
-  return !GetLocaleFilePath(locale, true).empty();
+  return !GetLocaleFilePath(locale).empty();
 }
 #endif  // !defined(OS_ANDROID)
 
@@ -364,8 +364,8 @@ void ResourceBundle::AddDataPackFromFileRegion(
 
 #if !defined(OS_MACOSX)
 // static
-base::FilePath ResourceBundle::GetLocaleFilePath(const std::string& app_locale,
-                                                 bool test_file_exists) {
+base::FilePath ResourceBundle::GetLocaleFilePath(
+    const std::string& app_locale) {
   if (app_locale.empty())
     return base::FilePath();
 
@@ -406,10 +406,16 @@ base::FilePath ResourceBundle::GetLocaleFilePath(const std::string& app_locale,
   if (locale_file_path.empty() || !locale_file_path.IsAbsolute())
     return base::FilePath();
 
-  if (test_file_exists && !base::PathExists(locale_file_path))
-    return base::FilePath();
+  if (base::PathExists(locale_file_path))
+    return locale_file_path;
+
+  // If a compressed version of the file exists, load that.
+  base::FilePath compressed_locale_file_path =
+      locale_file_path.AddExtension(FILE_PATH_LITERAL(".gz"));
+  if (base::PathExists(compressed_locale_file_path))
+    return compressed_locale_file_path;
 
-  return locale_file_path;
+  return base::FilePath();
 }
 #endif
 
@@ -420,7 +426,7 @@ std::string ResourceBundle::LoadLocaleResources(
   std::string app_locale = l10n_util::GetApplicationLocale(pref_locale);
   base::FilePath locale_file_path = GetOverriddenPakPath();
   if (locale_file_path.empty())
-    locale_file_path = GetLocaleFilePath(app_locale, true);
+    locale_file_path = GetLocaleFilePath(app_locale);
 
   if (locale_file_path.empty()) {
     // It's possible that there is no locale.pak.

ui/base/resource/resource_bundle.h

@@ -317,12 +317,12 @@ class UI_BASE_EXPORT ResourceBundle {
   void OverrideLocaleStringResource(int resource_id,
                                     const base::string16& string);
 
-  // Returns the full pathname of the locale file to load.  May return an empty
-  // string if no locale data files are found and |test_file_exists| is true.
+  // Returns the full pathname of the locale file to load, which may be a
+  // compressed locale file ending in .gz. Returns an empty string if no locale
+  // data files are found.
   // Used on Android to load the local file in the browser process and pass it
   // to the sandboxed renderer process.
-  static base::FilePath GetLocaleFilePath(const std::string& app_locale,
-                                          bool test_file_exists);
+  static base::FilePath GetLocaleFilePath(const std::string& app_locale);
 
   // Returns the maximum scale factor currently loaded.
   // Returns SCALE_FACTOR_100P if no resource is loaded.

ui/base/resource/resource_bundle_android.cc

@@ -97,7 +97,7 @@ bool ResourceBundle::LocaleDataPakExists(const std::string& locale) {
   if (g_locale_paks_in_apk) {
     return !GetPathForAndroidLocalePakWithinApk(locale).empty();
   }
-  return !GetLocaleFilePath(locale, true).empty();
+  return !GetLocaleFilePath(locale).empty();
 }
 
 std::string ResourceBundle::LoadLocaleResources(
@@ -117,7 +117,7 @@ std::string ResourceBundle::LoadLocaleResources(
   } else {
     base::FilePath locale_file_path = GetOverriddenPakPath();
     if (locale_file_path.empty())
-      locale_file_path = GetLocaleFilePath(app_locale, true);
+      locale_file_path = GetLocaleFilePath(app_locale);
 
     if (locale_file_path.empty()) {
       // It's possible that there is no locale.pak.

ui/base/resource/resource_bundle_ios.mm

@@ -60,8 +60,8 @@ void ResourceBundle::LoadCommonResources() {
 }
 
 // static
-base::FilePath ResourceBundle::GetLocaleFilePath(const std::string& app_locale,
-                                                 bool test_file_exists) {
+base::FilePath ResourceBundle::GetLocaleFilePath(
+    const std::string& app_locale) {
   NSString* mac_locale = base::SysUTF8ToNSString(app_locale);
 
   // iOS uses "_" instead of "-", so swap to get a iOS-style value.
@@ -84,7 +84,7 @@ base::FilePath ResourceBundle::GetLocaleFilePath(const std::string& app_locale,
   if (locale_file_path.empty() || !locale_file_path.IsAbsolute())
     return base::FilePath();
 
-  if (test_file_exists && !base::PathExists(locale_file_path))
+  if (!base::PathExists(locale_file_path))
     return base::FilePath();
 
   return locale_file_path;

ui/base/resource/resource_bundle_mac.mm

@@ -60,8 +60,8 @@ void ResourceBundle::LoadCommonResources() {
 }
 
 // static
-base::FilePath ResourceBundle::GetLocaleFilePath(const std::string& app_locale,
-                                                 bool test_file_exists) {
+base::FilePath ResourceBundle::GetLocaleFilePath(
+    const std::string& app_locale) {
   NSString* mac_locale = base::SysUTF8ToNSString(app_locale);
 
   // Mac OS X uses "_" instead of "-", so swap to get a Mac-style value.
@@ -84,7 +84,7 @@ base::FilePath ResourceBundle::GetLocaleFilePath(const std::string& app_locale,
   if (locale_file_path.empty() || !locale_file_path.IsAbsolute())
     return base::FilePath();
 
-  if (test_file_exists && !base::PathExists(locale_file_path))
+  if (!base::PathExists(locale_file_path))
     return base::FilePath();
 
   return locale_file_path;