indexeddb: mark databases corrupted if missing blob file

This patch uses a schema bump to do a one-time check of all blob files
on disk.  http://crbug.com/1131151 is a bug where under some rare
circumstances blob files may be written and then secretly deleted,
leaving the database with knowledge of an external file that doesn't
exist.

During this one-time check, if any blob files are missing, then the
database is marked as corrupted and will be deleted.  We use a schema
bump to only do this once.

Bug: 1143975
Change-Id: Ibb78e2bfaa4a1ba1becd77b88b5e07b8e0f3fe6e
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2508645
Commit-Queue: enne <enne@chromium.org>
Reviewed-by: Marijn Kruisselbrink <mek@chromium.org>
Cr-Commit-Position: refs/heads/master@{#826411}

content/browser/indexed_db/indexed_db_backing_store.cc

@@ -728,6 +728,9 @@ leveldb::Status IndexedDBBackingStore::Initialize(bool clean_active_journal) {
     if (s.ok() && db_schema_version < 4) {
       s = MigrateToV4(write_batch.get());
     }
+    if (s.ok() && db_schema_version < 5) {
+      s = MigrateToV5(write_batch.get());
+    }
     db_schema_version = indexed_db::kLatestKnownSchemaVersion;
   }
 
@@ -935,6 +938,83 @@ Status IndexedDBBackingStore::UpgradeBlobEntriesToV4(
   return Status::OK();
 }
 
+Status IndexedDBBackingStore::ValidateBlobFiles(
+    TransactionalLevelDBDatabase* db) {
+  DCHECK(filesystem_proxy_);
+
+  Status status = leveldb::Status::OK();
+  std::vector<base::string16> names;
+  IndexedDBMetadataCoding metadata_coding;
+  status = metadata_coding.ReadDatabaseNames(db, origin_identifier_, &names);
+  if (!status.ok())
+    return status;
+
+  for (const auto& name : names) {
+    IndexedDBDatabaseMetadata metadata;
+    bool found = false;
+    status = metadata_coding.ReadMetadataForDatabaseName(
+        db, origin_identifier_, name, &metadata, &found);
+    if (!found)
+      return Status::NotFound("Metadata not found for \"%s\".",
+                              base::UTF16ToUTF8(name));
+    for (const auto& store_id_metadata_pair : metadata.object_stores) {
+      leveldb::ReadOptions options;
+      // Since this is a scan, don't fill up the cache, as it's not likely these
+      // blocks will be reloaded.
+      options.fill_cache = false;
+      options.verify_checksums = true;
+      std::unique_ptr<TransactionalLevelDBIterator> iterator =
+          db->CreateIterator(options);
+      std::string min_key = BlobEntryKey::EncodeMinKeyForObjectStore(
+          metadata.id, store_id_metadata_pair.first);
+      std::string max_key = BlobEntryKey::EncodeStopKeyForObjectStore(
+          metadata.id, store_id_metadata_pair.first);
+      status = iterator->Seek(base::StringPiece(min_key));
+      if (status.IsNotFound()) {
+        status = Status::OK();
+        continue;
+      }
+      if (!status.ok())
+        return status;
+      // Loop through all blob entries in for the given object store.
+      for (; status.ok() && iterator->IsValid() &&
+             db->leveldb_state()->comparator()->Compare(
+                 leveldb_env::MakeSlice(iterator->Key()), max_key) < 0;
+           status = iterator->Next()) {
+        std::vector<IndexedDBExternalObject> temp_external_objects;
+        DecodeExternalObjects(iterator->Value().as_string(),
+                              &temp_external_objects);
+        for (auto& object : temp_external_objects) {
+          if (object.object_type() !=
+              IndexedDBExternalObject::ObjectType::kFile) {
+            continue;
+          }
+          // Empty blobs are not written to disk.
+          if (!object.size())
+            continue;
+
+          base::FilePath path =
+              GetBlobFileName(metadata.id, object.blob_number());
+          base::Optional<base::File::Info> info =
+              filesystem_proxy_->GetFileInfo(path);
+          if (!info.has_value()) {
+            return leveldb::Status::Corruption(
+                "Unable to upgrade to database version 5.", "");
+          }
+        }
+      }
+      if (status.IsNotFound())
+        status = leveldb::Status::OK();
+      if (!status.ok())
+        return status;
+    }
+
+    if (!status.ok())
+      return status;
+  }
+  return Status::OK();
+}
+
 Status IndexedDBBackingStore::RevertSchemaToV2() {
 #if DCHECK_IS_ON()
   DCHECK_CALLED_ON_VALID_SEQUENCE(idb_sequence_checker_);
@@ -3034,6 +3114,27 @@ Status IndexedDBBackingStore::MigrateToV4(LevelDBWriteBatch* write_batch) {
   return s;
 }
 
+Status IndexedDBBackingStore::MigrateToV5(LevelDBWriteBatch* write_batch) {
+  // Some blob files were not written to disk due to a bug.
+  // Validate that all blob files in the db exist on disk,
+  // and return InternalInconsistencyStatus if any do not.
+  // See http://crbug.com/1131151 for more details.
+  const int64_t db_schema_version = 5;
+  const std::string schema_version_key = SchemaVersionKey::Encode();
+  Status s;
+
+  if (origin_.host() != "docs.google.com") {
+    s = ValidateBlobFiles(db_.get());
+    if (!s.ok()) {
+      INTERNAL_CONSISTENCY_ERROR(SET_UP_METADATA);
+      return InternalInconsistencyStatus();
+    }
+  }
+  ignore_result(PutInt(write_batch, schema_version_key, db_schema_version));
+
+  return s;
+}
+
 Status IndexedDBBackingStore::Transaction::HandleBlobPreTransaction() {
   DCHECK_CALLED_ON_VALID_SEQUENCE(idb_sequence_checker_);
   DCHECK(backing_store_);

content/browser/indexed_db/indexed_db_backing_store.h

@@ -538,10 +538,18 @@ class CONTENT_EXPORT IndexedDBBackingStore {
       TransactionalLevelDBDatabase* database,
       bool* blobs_exist);
 
+  // A helper function for V4 schema migration.
+  // It iterates through all blob files.  It will add to the db entry both the
+  // size and modified date for the blob based on the written file.  If any blob
+  // file in the db is missing on disk, it will return an inconsistency status.
   leveldb::Status UpgradeBlobEntriesToV4(
       TransactionalLevelDBDatabase* database,
       LevelDBWriteBatch* write_batch,
       std::vector<base::FilePath>* empty_blobs_to_delete);
+  // A helper function for V5 schema miration.
+  // Iterates through all blob files on disk and validates they exist,
+  // returning an internal inconsistency corruption error if any are missing.
+  leveldb::Status ValidateBlobFiles(TransactionalLevelDBDatabase* database);
 
   // TODO(dmurph): Move this completely to IndexedDBMetadataFactory.
   leveldb::Status GetCompleteMetadata(
@@ -568,6 +576,7 @@ class CONTENT_EXPORT IndexedDBBackingStore {
   leveldb::Status MigrateToV2(LevelDBWriteBatch* write_batch);
   leveldb::Status MigrateToV3(LevelDBWriteBatch* write_batch);
   leveldb::Status MigrateToV4(LevelDBWriteBatch* write_batch);
+  leveldb::Status MigrateToV5(LevelDBWriteBatch* write_batch);
 
   leveldb::Status FindKeyInIndex(
       IndexedDBBackingStore::Transaction* transaction,

content/browser/indexed_db/indexed_db_backing_store_unittest.cc

@@ -203,6 +203,13 @@ class MockBlobStorageContext : public ::storage::mojom::BlobStorageContext {
                        base::Optional<base::Time> last_modified,
                        WriteBlobToFileCallback callback) override {
     writes_.emplace_back(std::move(blob), path);
+
+    if (write_files_to_disk_) {
+      auto filesystem_proxy = std::make_unique<storage::FilesystemProxy>(
+          storage::FilesystemProxy::UNRESTRICTED, base::FilePath());
+      filesystem_proxy->WriteFileAtomically(path, "fake contents");
+    }
+
     base::SequencedTaskRunnerHandle::Get()->PostTask(
         FROM_HERE,
         base::BindOnce(std::move(callback),
@@ -212,8 +219,13 @@ class MockBlobStorageContext : public ::storage::mojom::BlobStorageContext {
   const std::vector<BlobWrite>& writes() { return writes_; }
   void ClearWrites() { writes_.clear(); }
 
+  // If true, writes a fake file for each blob file to disk.
+  // The contents are bogus, but the files will exist.
+  void SetWriteFilesToDisk(bool write) { write_files_to_disk_ = write; }
+
  private:
   std::vector<BlobWrite> writes_;
+  bool write_files_to_disk_ = false;
 };
 
 class FakeNativeFileSystemTransferToken
@@ -636,14 +648,25 @@ class IndexedDBBackingStoreTestWithExternalObjects
   bool CheckBlobWrites() {
     DCHECK(idb_context_->IDBTaskRunner()->RunsTasksInCurrentSequence());
 
-    if (blob_context_->writes().size() +
-            native_file_system_context_->writes().size() !=
-        external_objects_.size()) {
+    size_t num_empty_blobs = 0;
+    for (const auto& info : external_objects_) {
+      if (info.object_type() == IndexedDBExternalObject::ObjectType::kFile &&
+          !info.size()) {
+        num_empty_blobs++;
+      }
+    }
+
+    size_t num_written = blob_context_->writes().size() +
+                         native_file_system_context_->writes().size();
+    if (num_written != external_objects_.size() - num_empty_blobs) {
       return false;
     }
     for (size_t i = 0; i < blob_context_->writes().size(); ++i) {
       const BlobWrite& desc = blob_context_->writes()[i];
       const IndexedDBExternalObject& info = external_objects_[i];
+      if (!info.size())
+        continue;
+
       base::RunLoop uuid_loop;
       std::string uuid_out;
       DCHECK(desc.blob.is_bound());
@@ -1778,7 +1801,7 @@ TEST_F(IndexedDBBackingStoreTest, SchemaUpgradeWithoutBlobsSurvives) {
   ASSERT_TRUE(success);
 
   EXPECT_TRUE(found);
-  EXPECT_EQ(4, found_int);
+  EXPECT_EQ(indexed_db::kLatestKnownSchemaVersion, found_int);
   task_environment_.RunUntilIdle();
 }
 
@@ -2053,6 +2076,140 @@ TEST_F(IndexedDBBackingStoreTestWithBlobs, SchemaUpgradeV3ToV4) {
   EXPECT_TRUE(CheckBlobInfoMatches(result_value.external_objects));
 }
 
+TEST_F(IndexedDBBackingStoreTestWithBlobs, SchemaUpgradeV4ToV5) {
+  int64_t database_id;
+  const int64_t object_store_id = 99;
+
+  const base::string16 database_name(ASCIIToUTF16("db1"));
+  const int64_t version = 9;
+
+  const base::string16 object_store_name(ASCIIToUTF16("object_store1"));
+  const bool auto_increment = true;
+  const IndexedDBKeyPath object_store_key_path(
+      ASCIIToUTF16("object_store_key"));
+
+  // Add an empty blob here to test with.  Empty blobs are not written
+  // to disk, so it's important to verify that a database with empty blobs
+  // should be considered still valid.
+  external_objects().push_back(CreateBlobInfo(base::UTF8ToUTF16("empty blob"),
+                                              base::UTF8ToUTF16("file type"),
+                                              base::Time::Now(), 0u));
+  // The V5 migration checks files on disk, so make sure our fake blob
+  // context writes something there to check.
+  blob_context_->SetWriteFilesToDisk(true);
+
+  IndexedDBMetadataCoding metadata_coding;
+
+  {
+    IndexedDBDatabaseMetadata database;
+    leveldb::Status s = metadata_coding.CreateDatabase(
+        backing_store()->db(), backing_store()->origin_identifier(),
+        database_name, version, &database);
+    EXPECT_TRUE(s.ok());
+    EXPECT_GT(database.id, 0);
+    database_id = database.id;
+
+    IndexedDBBackingStore::Transaction transaction(
+        backing_store()->AsWeakPtr(),
+        blink::mojom::IDBTransactionDurability::Relaxed,
+        blink::mojom::IDBTransactionMode::ReadWrite);
+    transaction.Begin(CreateDummyLock());
+
+    IndexedDBObjectStoreMetadata object_store;
+    s = metadata_coding.CreateObjectStore(
+        transaction.transaction(), database.id, object_store_id,
+        object_store_name, object_store_key_path, auto_increment,
+        &object_store);
+    EXPECT_TRUE(s.ok());
+
+    bool succeeded = false;
+    EXPECT_TRUE(
+        transaction.CommitPhaseOne(CreateBlobWriteCallback(&succeeded)).ok());
+    EXPECT_TRUE(succeeded);
+    EXPECT_TRUE(transaction.CommitPhaseTwo().ok());
+  }
+  task_environment_.RunUntilIdle();
+
+  // Initiate transaction - writing blobs.
+  std::unique_ptr<IndexedDBBackingStore::Transaction> transaction =
+      std::make_unique<IndexedDBBackingStore::Transaction>(
+          backing_store()->AsWeakPtr(),
+          blink::mojom::IDBTransactionDurability::Relaxed,
+          blink::mojom::IDBTransactionMode::ReadWrite);
+  transaction->Begin(CreateDummyLock());
+  IndexedDBBackingStore::RecordIdentifier record;
+
+  IndexedDBKey key = IndexedDBKey(ASCIIToUTF16("key"));
+  IndexedDBValue value = IndexedDBValue("value3", external_objects());
+
+  EXPECT_TRUE(backing_store()
+                  ->PutRecord(transaction.get(), database_id, object_store_id,
+                              key, &value, &record)
+                  .ok());
+  bool succeeded = false;
+  base::RunLoop write_blobs_loop;
+  EXPECT_TRUE(transaction
+                  ->CommitPhaseOne(CreateBlobWriteCallback(
+                      &succeeded, write_blobs_loop.QuitClosure()))
+                  .ok());
+  write_blobs_loop.Run();
+  task_environment_.RunUntilIdle();
+
+  // Finish up transaction, verifying blob writes.
+  EXPECT_TRUE(succeeded);
+  EXPECT_TRUE(CheckBlobWrites());
+  ASSERT_TRUE(transaction->CommitPhaseTwo().ok());
+  transaction.reset();
+
+  task_environment_.RunUntilIdle();
+  ASSERT_EQ(blob_context_->writes().size(), 3u);
+
+  // Verify V4 to V5 conversion with all blobs intact has no data loss.
+  {
+    // Change the schema to be v4.
+    const int64_t old_version = 4;
+    std::unique_ptr<LevelDBWriteBatch> write_batch =
+        LevelDBWriteBatch::Create();
+    const std::string schema_version_key = SchemaVersionKey::Encode();
+    ASSERT_TRUE(
+        indexed_db::PutInt(write_batch.get(), schema_version_key, old_version)
+            .ok());
+    ASSERT_TRUE(backing_store()->db()->Write(write_batch.get()).ok());
+
+    DestroyFactoryAndBackingStore();
+    CreateFactoryAndBackingStore();
+
+    // There should be no corruption here.
+    ASSERT_EQ(data_loss_info_.status, blink::mojom::IDBDataLoss::None);
+  }
+
+  // Verify V4 to V5 conversion with missing blobs has data loss.
+  {
+    // Change the schema to be v4.
+    const int64_t old_version = 4;
+    std::unique_ptr<LevelDBWriteBatch> write_batch =
+        LevelDBWriteBatch::Create();
+    const std::string schema_version_key = SchemaVersionKey::Encode();
+    ASSERT_TRUE(
+        indexed_db::PutInt(write_batch.get(), schema_version_key, old_version)
+            .ok());
+    ASSERT_TRUE(backing_store()->db()->Write(write_batch.get()).ok());
+
+    // Pick a blob we wrote arbitrarily and delete it.
+    auto path = blob_context_->writes()[1].path;
+    auto filesystem_proxy = std::make_unique<storage::FilesystemProxy>(
+        storage::FilesystemProxy::UNRESTRICTED, base::FilePath());
+    filesystem_proxy->DeleteFile(path);
+
+    DestroyFactoryAndBackingStore();
+    CreateFactoryAndBackingStore();
+
+    // This should be corrupted.
+    ASSERT_NE(data_loss_info_.status, blink::mojom::IDBDataLoss::None);
+    DestroyFactoryAndBackingStore();
+  }
+}
+
 // This tests that external objects are deleted when ClearObjectStore is called.
 // See: http://crbug.com/488851
 // TODO(enne): we could use more comprehensive testing for ClearObjectStore.

content/browser/indexed_db/indexed_db_leveldb_coding.h

@@ -30,7 +30,8 @@ namespace indexed_db {
 // 2 - Adds DataVersion to to global metadata.
 // 3 - Adds metadata needed for blob support.
 // 4 - Adds size & last_modified to 'file' blob_info encodings.
-const constexpr int64_t kLatestKnownSchemaVersion = 4;
+// 5 - One time verification that blob files exist on disk.
+const constexpr int64_t kLatestKnownSchemaVersion = 5;
 }  // namespace indexed_db
 
 CONTENT_EXPORT extern const unsigned char kMinimumIndexId;