Add method to rewrite a LevelDB instance

Add the infrastructure to remove expired data by rewriting LevelDB
instances.
The integration with actual storage implementation will be added in
followup Cls.

Bug: 823071
Change-Id: Iff1b93481468423d292266f16a45f6ef4ac6e7f2
Reviewed-on: https://chromium-review.googlesource.com/1152917
Commit-Queue: Christian Dullweber <dullweber@chromium.org>
Reviewed-by: Victor Costan <pwnall@chromium.org>
Cr-Commit-Position: refs/heads/master@{#587980}

third_party/leveldatabase/BUILD.gn

@@ -32,6 +32,8 @@ config("leveldb_config") {
 leveldb_sources = [
   "env_chromium.cc",
   "env_chromium.h",
+  "leveldb_features.h",
+  "leveldb_features.cc",
   "leveldb_chrome.cc",
   "leveldb_chrome.h",
   "port/port_chromium.cc",

third_party/leveldatabase/env_chromium.cc

@@ -37,6 +37,7 @@
 #include "build/build_config.h"
 #include "third_party/leveldatabase/chromium_logger.h"
 #include "third_party/leveldatabase/leveldb_chrome.h"
+#include "third_party/leveldatabase/leveldb_features.h"
 #include "third_party/leveldatabase/src/include/leveldb/options.h"
 #include "third_party/re2/src/re2/re2.h"
 
@@ -63,6 +64,10 @@ const FilePath::CharType table_extension[] = FILE_PATH_LITERAL(".ldb");
 static const FilePath::CharType kLevelDBTestDirectoryPrefix[] =
     FILE_PATH_LITERAL("leveldb-test-");
 
+// This name should not be changed or users involved in a crash might not be
+// able to recover data.
+static const char kDatabaseNameSuffixForRebuildDB[] = "__tmp_for_rebuild";
+
 // Making direct platform in lieu of using base::FileEnumerator because the
 // latter can fail quietly without return an error result.
 static base::File::Error GetDirectoryEntries(const FilePath& dir_param,
@@ -737,6 +742,10 @@ bool IndicatesDiskFull(const leveldb::Status& status) {
               base::File::FILE_ERROR_NO_SPACE);
 }
 
+std::string DatabaseNameForRewriteDB(const std::string& original_name) {
+  return original_name + kDatabaseNameSuffixForRebuildDB;
+}
+
 // Given the size of the disk, identified by |disk_size| in bytes, determine the
 // appropriate write_buffer_size. Ignoring snapshots, if the current set of
 // tables in a database contains a set of key/value pairs identified by {A}, and
@@ -1611,13 +1620,84 @@ leveldb::Status OpenDB(const leveldb_env::Options& options,
     mem_options.write_buffer_size = 0;  // minimum size.
     s = DBTracker::GetInstance()->OpenDatabase(mem_options, name, &tracked_db);
   } else {
+    std::string tmp_name = DatabaseNameForRewriteDB(name);
+    // If Chrome crashes during rewrite, there might be a temporary db but
+    // no actual db.
+    if (options.env->FileExists(tmp_name) &&
+        !options.env->FileExists(name + "/CURRENT")) {
+      s = leveldb::DestroyDB(name, options);
+      if (!s.ok())
+        return s;
+      s = options.env->RenameFile(tmp_name, name);
+      if (!s.ok())
+        return s;
+    }
     s = DBTracker::GetInstance()->OpenDatabase(options, name, &tracked_db);
+    // It is possible that the database was partially deleted during a
+    // rewrite and can't be opened anymore.
+    if (!s.ok() && options.env->FileExists(tmp_name)) {
+      s = leveldb::DestroyDB(name, options);
+      if (!s.ok())
+        return s;
+      s = options.env->RenameFile(tmp_name, name);
+      if (!s.ok())
+        return s;
+      s = DBTracker::GetInstance()->OpenDatabase(options, name, &tracked_db);
+    }
+    // There might be a temporary database that needs to be cleaned up.
+    if (options.env->FileExists(tmp_name)) {
+      leveldb::DestroyDB(tmp_name, options);
+    }
   }
   if (s.ok())
     dbptr->reset(tracked_db);
   return s;
 }
 
+leveldb::Status RewriteDB(const leveldb_env::Options& options,
+                          const std::string& name,
+                          std::unique_ptr<leveldb::DB>* dbptr) {
+  if (!base::FeatureList::IsEnabled(leveldb::kLevelDBRewriteFeature))
+    return Status::OK();
+  if (leveldb_chrome::IsMemEnv(options.env))
+    return Status::OK();
+  TRACE_EVENT0("leveldb", "ChromiumEnv::RewriteDB");
+  leveldb::Status s;
+  std::string tmp_name = DatabaseNameForRewriteDB(name);
+  if (options.env->FileExists(tmp_name)) {
+    s = leveldb::DestroyDB(tmp_name, options);
+    if (!s.ok())
+      return s;
+  }
+  // Copy all data from *dbptr to a temporary db.
+  std::unique_ptr<leveldb::DB> tmp_db;
+  s = leveldb_env::OpenDB(options, tmp_name, &tmp_db);
+  if (!s.ok())
+    return s;
+  std::unique_ptr<leveldb::Iterator> it(
+      (*dbptr)->NewIterator(leveldb::ReadOptions()));
+  for (it->SeekToFirst(); it->Valid(); it->Next()) {
+    s = tmp_db->Put(leveldb::WriteOptions(), it->key(), it->value());
+    if (!s.ok())
+      break;
+  }
+  it.reset();
+  tmp_db.reset();
+  if (!s.ok()) {
+    leveldb::DestroyDB(tmp_name, options);
+    return s;
+  }
+  // Replace the old database with tmp_db.
+  (*dbptr).reset();
+  s = leveldb::DestroyDB(name, options);
+  if (!s.ok())
+    return s;
+  s = options.env->RenameFile(tmp_name, name);
+  if (!s.ok())
+    return s;
+  return leveldb_env::OpenDB(options, name, dbptr);
+}
+
 base::StringPiece MakeStringPiece(const leveldb::Slice& s) {
   return base::StringPiece(s.data(), s.size());
 }

third_party/leveldatabase/env_chromium.h

@@ -109,6 +109,10 @@ LEVELDB_EXPORT int GetNumCorruptionCodes();
 LEVELDB_EXPORT std::string GetCorruptionMessage(const leveldb::Status& status);
 LEVELDB_EXPORT bool IndicatesDiskFull(const leveldb::Status& status);
 
+// Returns the name for a temporary database copy during RewriteDB().
+LEVELDB_EXPORT std::string DatabaseNameForRewriteDB(
+    const std::string& original_name);
+
 // Determine the appropriate leveldb write buffer size to use. The default size
 // (4MB) may result in a log file too large to be compacted given the available
 // storage space. This function will return smaller values for smaller disks,
@@ -350,6 +354,16 @@ LEVELDB_EXPORT leveldb::Status OpenDB(const leveldb_env::Options& options,
                                       const std::string& name,
                                       std::unique_ptr<leveldb::DB>* dbptr);
 
+// Copies the content of |dbptr| into a fresh database to remove traces of
+// deleted data. |options| and |name| of the old database are required to create
+// an identical copy. |dbptr| will be replaced with the new database on success.
+// If the rewrite fails e.g. because we can't write to the temporary location,
+// the old db is returned if possible, otherwise |*dbptr| can become NULL.
+// The rewrite will only be performed if |kLevelDBRewriteFeature| is enabled.
+LEVELDB_EXPORT leveldb::Status RewriteDB(const leveldb_env::Options& options,
+                                         const std::string& name,
+                                         std::unique_ptr<leveldb::DB>* dbptr);
+
 LEVELDB_EXPORT base::StringPiece MakeStringPiece(const leveldb::Slice& s);
 LEVELDB_EXPORT leveldb::Slice MakeSlice(const base::StringPiece& s);
 

third_party/leveldatabase/env_chromium_unittest.cc

@@ -20,6 +20,7 @@
 #include "testing/gtest/include/gtest/gtest.h"
 #include "third_party/leveldatabase/env_chromium.h"
 #include "third_party/leveldatabase/leveldb_chrome.h"
+#include "third_party/leveldatabase/leveldb_features.h"
 #include "third_party/leveldatabase/src/include/leveldb/cache.h"
 #include "third_party/leveldatabase/src/include/leveldb/db.h"
 
@@ -649,6 +650,143 @@ TEST(ChromiumLevelDB, DeleteInMemoryDB) {
   EXPECT_TRUE(leveldb_chrome::PossiblyValidDB(db_path, on_disk_options.env));
 }
 
+class ChromiumLevelDBRebuildTest : public ::testing::Test {
+ protected:
+  ChromiumLevelDBRebuildTest() {
+    feature_list_.InitAndEnableFeature(leveldb::kLevelDBRewriteFeature);
+  }
+
+  void SetUp() override {
+    testing::Test::SetUp();
+    ASSERT_TRUE(scoped_temp_dir_.CreateUniqueTempDir());
+  }
+
+  const base::FilePath& temp_path() const { return scoped_temp_dir_.GetPath(); }
+
+ private:
+  base::test::ScopedFeatureList feature_list_;
+  base::ScopedAllowBlockingForTesting allow_blocking_;
+  base::ScopedTempDir scoped_temp_dir_;
+};
+
+TEST_F(ChromiumLevelDBRebuildTest, RebuildDb) {
+  std::unique_ptr<leveldb::DB> db;
+  base::FilePath db_path = temp_path().AppendASCII("db");
+  leveldb_env::Options options;
+  options.create_if_missing = true;
+
+  auto s = leveldb_env::OpenDB(options, db_path.AsUTF8Unsafe(), &db);
+  ASSERT_TRUE(s.ok());
+  db->Put(leveldb::WriteOptions(), "key1", "value1");
+  db->Put(leveldb::WriteOptions(), "key2", "value2");
+  db->Delete(leveldb::WriteOptions(), "key1");
+
+  leveldb::DB* old_db_ptr = db.get();
+  s = leveldb_env::RewriteDB(options, db_path.AsUTF8Unsafe(), &db);
+  EXPECT_TRUE(s.ok());
+  EXPECT_NE(old_db_ptr, db.get());
+  EXPECT_TRUE(db);
+
+  std::string value;
+  s = db->Get(leveldb::ReadOptions(), "key1", &value);
+  EXPECT_TRUE(s.IsNotFound());
+  s = db->Get(leveldb::ReadOptions(), "key2", &value);
+  EXPECT_TRUE(s.ok());
+  EXPECT_EQ("value2", value);
+}
+
+TEST_F(ChromiumLevelDBRebuildTest, RecoverMissingDB) {
+  std::unique_ptr<leveldb::DB> db;
+  base::FilePath db_path = temp_path().AppendASCII("db");
+  base::FilePath tmp_path =
+      temp_path().AppendASCII(leveldb_env::DatabaseNameForRewriteDB("db"));
+  leveldb_env::Options options;
+  options.create_if_missing = true;
+
+  // Write a temporary db to simulate a failed rewrite attempt where only the
+  // temporary db exists.
+  auto s = leveldb_env::OpenDB(options, tmp_path.AsUTF8Unsafe(), &db);
+  ASSERT_TRUE(s.ok());
+  db->Put(leveldb::WriteOptions(), "key", "value");
+  db.reset();
+
+  EXPECT_FALSE(base::DirectoryExists(db_path));
+  EXPECT_TRUE(base::DirectoryExists(tmp_path));
+
+  // Open the regular db and check if the temporary one is recovered.
+  s = leveldb_env::OpenDB(options, db_path.AsUTF8Unsafe(), &db);
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  std::string value;
+  s = db->Get(leveldb::ReadOptions(), "key", &value);
+  EXPECT_TRUE(s.ok()) << s.ToString();
+  EXPECT_EQ("value", value);
+  EXPECT_TRUE(base::DirectoryExists(db_path));
+  EXPECT_FALSE(base::DirectoryExists(tmp_path));
+}
+
+TEST_F(ChromiumLevelDBRebuildTest, RecoverCorruptDB) {
+  std::unique_ptr<leveldb::DB> db;
+  base::FilePath db_path = temp_path().AppendASCII("db");
+  base::FilePath tmp_path =
+      temp_path().AppendASCII(leveldb_env::DatabaseNameForRewriteDB("db"));
+  leveldb_env::Options options;
+  options.create_if_missing = true;
+
+  // Create a corrupt db.
+  ASSERT_TRUE(base::CreateDirectory(db_path));
+  ASSERT_TRUE(leveldb_chrome::CorruptClosedDBForTesting(db_path));
+
+  // Write a temporary db to simulate a failed rewrite attempt.
+  auto s = leveldb_env::OpenDB(options, tmp_path.AsUTF8Unsafe(), &db);
+  ASSERT_TRUE(s.ok());
+  db->Put(leveldb::WriteOptions(), "key", "value");
+  db.reset();
+
+  // Open the regular db and check if the temporary one is recovered.
+  s = leveldb_env::OpenDB(options, db_path.AsUTF8Unsafe(), &db);
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  std::string value;
+  s = db->Get(leveldb::ReadOptions(), "key", &value);
+  EXPECT_TRUE(s.ok()) << s.ToString();
+  EXPECT_EQ("value", value);
+  EXPECT_TRUE(base::DirectoryExists(db_path));
+  EXPECT_FALSE(base::DirectoryExists(tmp_path));
+}
+
+TEST_F(ChromiumLevelDBRebuildTest, FinishCleanup) {
+  std::unique_ptr<leveldb::DB> db;
+  base::FilePath db_path = temp_path().AppendASCII("db");
+  base::FilePath tmp_path =
+      temp_path().AppendASCII(leveldb_env::DatabaseNameForRewriteDB("db"));
+  leveldb_env::Options options;
+  options.create_if_missing = true;
+
+  // Write a regular and a temporary db to simulate a rewrite attempt that
+  // crashed before finishing.
+  auto s = leveldb_env::OpenDB(options, db_path.AsUTF8Unsafe(), &db);
+  ASSERT_TRUE(s.ok());
+  db->Put(leveldb::WriteOptions(), "key", "regular");
+  db.reset();
+
+  s = leveldb_env::OpenDB(options, tmp_path.AsUTF8Unsafe(), &db);
+  ASSERT_TRUE(s.ok());
+  db->Put(leveldb::WriteOptions(), "key", "temp");
+  db.reset();
+
+  EXPECT_TRUE(base::DirectoryExists(db_path));
+  EXPECT_TRUE(base::DirectoryExists(tmp_path));
+
+  // Open the regular db and check that the temporary one was cleaned up.
+  s = leveldb_env::OpenDB(options, db_path.AsUTF8Unsafe(), &db);
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  std::string value;
+  s = db->Get(leveldb::ReadOptions(), "key", &value);
+  EXPECT_TRUE(s.ok()) << s.ToString();
+  EXPECT_EQ("regular", value);
+  EXPECT_TRUE(base::DirectoryExists(db_path));
+  EXPECT_FALSE(base::DirectoryExists(tmp_path));
+}
+
 }  // namespace leveldb_env
 
 int main(int argc, char** argv) { return base::TestSuite(argc, argv).Run(); }

third_party/leveldatabase/leveldb_features.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "third_party/leveldatabase/leveldb_features.h"
+
+namespace leveldb {
+
+const base::Feature kLevelDBRewriteFeature{"LevelDBPerformRewrite",
+                                           base::FEATURE_DISABLED_BY_DEFAULT};
+}  // namespace leveldb

third_party/leveldatabase/leveldb_features.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifndef THIRD_PARTY_LEVELDATABASE_LEVELDB_FEATURES_H_
+#define THIRD_PARTY_LEVELDATABASE_LEVELDB_FEATURES_H_
+
+#include "base/feature_list.h"
+#include "leveldb/export.h"
+
+namespace leveldb {
+LEVELDB_EXPORT extern const base::Feature kLevelDBRewriteFeature;
+}  // namespace leveldb
+
+#endif  // THIRD_PARTY_LEVELDATABASE_LEVELDB_FEATURES_H_

tools/metrics/histograms/enums.xml

@@ -29580,6 +29580,7 @@ from previous Chrome versions.
   <int value="1000706989" label="AutomaticTabDiscarding:disabled"/>
   <int value="1002585107" label="emphasize-titles-in-omnibox-dropdown"/>
   <int value="1003002105" label="MaterialDesignBookmarks:disabled"/>
+  <int value="1004593833" label="LevelDBPerformRewrite:enabled"/>
   <int value="1004909189"
       label="ContentSuggestionsThumbnailDominantColor:disabled"/>
   <int value="1005684777" label="PictureInPicture:disabled"/>
@@ -29791,6 +29792,7 @@ from previous Chrome versions.
   <int value="1377056573" label="browser-side-navigation:enabled"/>
   <int value="1378310092" label="disable-suggestions-service"/>
   <int value="1381746642" label="enable-automatic-password-saving"/>
+  <int value="1382107019" label="LevelDBPerformRewrite:disabled"/>
   <int value="1382500494" label="disable-drive-apps-in-app-list"/>
   <int value="1383591631" label="enable-gesture-typing"/>
   <int value="1384614036" label="disable-unified-media-pipeline"/>