Database support for GDataDirectoryService.

* Add methods InitFromDB and SaveToDB to GDataDirectoryService, along with other helper methods to save/load the directory service from level db instead of from a proto file.
* Add a wrapper class GDataDirectoryServiceDB to hold the level db, with methods Create, Save, Read, Truncate, etc. This object lives on the blocking thread.
* Add CreateDBParams to facilitate creation of GDataDirectoryServiceDB on the blocking thread.
* Add GDataDirectoryService::FromProtoString to create a GDataEntry from a string saved in the db.
* Add unit tests for the db methods.
* Move LoadRootFeedParams to gdata_params. Move typedefs before struct definitions.
* Add a timer to measure the time to restore the filesystem from db or proto. For hugefileman, it's 3000 msec for db, and 2500 for proto, with debug code on a z600. Measurements on device TBD.

BUG=127856
TEST=unit tests, manual tests with hugefileman.
TBR=sky@chromium.org

Review URL: https://chromiumcodereview.appspot.com/10800092
Patch from Achuith Bhandarkar <achuith@chromium.org>.

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@150022 0039d316-1c4b-4281-b951-d872f2087c98

chrome/browser/chromeos/gdata/gdata_file_system.cc

@@ -8,6 +8,7 @@
 #include <utility>
 
 #include "base/bind.h"
+#include "base/command_line.h"
 #include "base/file_util.h"
 #include "base/json/json_file_value_serializer.h"
 #include "base/json/json_reader.h"
@@ -27,6 +28,7 @@
 #include "chrome/browser/chromeos/gdata/gdata_util.h"
 #include "chrome/browser/prefs/pref_service.h"
 #include "chrome/browser/profiles/profile.h"
+#include "chrome/common/chrome_switches.h"
 #include "chrome/common/chrome_notification_types.h"
 #include "chrome/common/pref_names.h"
 #include "content/public/browser/browser_thread.h"
@@ -45,6 +47,8 @@ const FilePath::CharType kAccountMetadataFile[] =
     FILE_PATH_LITERAL("account_metadata.json");
 const FilePath::CharType kFilesystemProtoFile[] =
     FILE_PATH_LITERAL("file_system.pb");
+const FilePath::CharType kResourceMetadataDBFile[] =
+    FILE_PATH_LITERAL("resource_metadata.db");
 
 const char kEmptyFilePath[] = "/dev/null";
 
@@ -78,28 +82,6 @@ SerializationTimetable kSerializeTimetable[] = {
 #endif
 };
 
-// Defines set of parameters sent to callback OnProtoLoaded().
-struct LoadRootFeedParams {
-  LoadRootFeedParams(
-        FilePath search_file_path,
-        bool should_load_from_server,
-        const FindEntryCallback& callback)
-        : search_file_path(search_file_path),
-          should_load_from_server(should_load_from_server),
-          load_error(GDATA_FILE_OK),
-          callback(callback) {
-    }
-  ~LoadRootFeedParams() {
-  }
-
-  FilePath search_file_path;
-  bool should_load_from_server;
-  std::string proto;
-  GDataFileError load_error;
-  base::Time last_modified;
-  const FindEntryCallback callback;
-};
-
 // Returns true if file system is due to be serialized on disk based on it
 // |serialized_size| and |last_serialized| timestamp.
 bool ShouldSerializeFileSystemNow(size_t serialized_size,
@@ -273,6 +255,11 @@ void SaveFeedOnBlockingPoolForDebugging(
   }
 }
 
+bool UseLevelDB() {
+  return CommandLine::ForCurrentProcess()->HasSwitch(
+      switches::kUseLevelDBForGData);
+}
+
 // Gets the file size of |local_file|.
 void GetLocalFileSizeOnBlockingPool(const FilePath& local_file,
                                     GDataFileError* error,
@@ -995,6 +982,9 @@ void GDataWapiFeedLoader::ReloadFromServerIfNeeded(
     const FindEntryCallback& callback) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 
+  DVLOG(1) << "ReloadFeedFromServerIfNeeded local_changestamp="
+           << local_changestamp << ", initial_origin=" << initial_origin;
+
   // First fetch the latest changestamp to see if there were any new changes
   // there at all.
   documents_service_->GetAccountMetadata(
@@ -1162,7 +1152,7 @@ void GDataWapiFeedLoader::OnFeedFromServerLoaded(GetDocumentsParams* params,
   }
 
   // Save file system metadata to disk.
-  SaveFileSystemAsProto();
+  SaveFileSystem();
 
   // If we had someone to report this too, then this retrieval was done in a
   // context of search... so continue search.
@@ -2996,17 +2986,25 @@ void GDataWapiFeedLoader::LoadFromCache(
     const FindEntryCallback& callback) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 
-  const FilePath path =
-      cache_->GetCacheDirectoryPath(GDataCache::CACHE_TYPE_META).Append(
-          kFilesystemProtoFile);
   LoadRootFeedParams* params = new LoadRootFeedParams(search_file_path,
                                                       should_load_from_server,
                                                       callback);
+  FilePath path = cache_->GetCacheDirectoryPath(GDataCache::CACHE_TYPE_META);
+  if (UseLevelDB()) {
+    path = path.Append(kResourceMetadataDBFile);
+    directory_service_->InitFromDB(path, blocking_task_runner_,
+        base::Bind(
+            &GDataWapiFeedLoader::ContinueWithInitializedDirectoryService,
+            weak_ptr_factory_.GetWeakPtr(),
+            base::Owned(params)));
+  } else {
+    path = path.Append(kFilesystemProtoFile);
     BrowserThread::GetBlockingPool()->PostTaskAndReply(FROM_HERE,
         base::Bind(&LoadProtoOnBlockingPool, path, params),
         base::Bind(&GDataWapiFeedLoader::OnProtoLoaded,
                    weak_ptr_factory_.GetWeakPtr(),
                    base::Owned(params)));
+  }
 }
 
 void GDataFileSystem::OnDirectoryChanged(const FilePath& directory_path) {
@@ -3058,7 +3056,6 @@ void GDataWapiFeedLoader::OnProtoLoaded(LoadRootFeedParams* params) {
   if (directory_service_->origin() == FROM_SERVER)
     return;
 
-  int local_changestamp = 0;
   // Update directory structure only if everything is OK and we haven't yet
   // received the feed from the server yet.
   if (params->load_error == GDATA_FILE_OK) {
@@ -3066,16 +3063,27 @@ void GDataWapiFeedLoader::OnProtoLoaded(LoadRootFeedParams* params) {
     if (directory_service_->ParseFromString(params->proto)) {
       directory_service_->set_last_serialized(params->last_modified);
       directory_service_->set_serialized_size(params->proto.size());
-      local_changestamp = directory_service_->largest_changestamp();
     } else {
       params->load_error = GDATA_FILE_ERROR_FAILED;
       LOG(WARNING) << "Parse of cached proto file failed";
     }
   }
 
+  ContinueWithInitializedDirectoryService(params, params->load_error);
+}
+
+void GDataWapiFeedLoader::ContinueWithInitializedDirectoryService(
+    LoadRootFeedParams* params,
+    GDataFileError error) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
+
+  DVLOG(1) << "Time elapsed to load directory service from disk="
+           << (base::Time::Now() - params->load_start_time).InMilliseconds()
+           << " milliseconds";
+
   FindEntryCallback callback = params->callback;
   // If we got feed content from cache, try search over it.
-  if (params->load_error == GDATA_FILE_OK && !callback.is_null()) {
+  if (error == GDATA_FILE_OK && !callback.is_null()) {
     // Continue file content search operation if the delegate hasn't terminated
     // this search branch already.
     directory_service_->FindEntryByPathAndRunSync(params->search_file_path,
@@ -3103,21 +3111,22 @@ void GDataWapiFeedLoader::OnProtoLoaded(LoadRootFeedParams* params) {
   // |reported| to the original callback, then we just need to refresh the
   // content without continuing search upon operation completion.
   ReloadFromServerIfNeeded(initial_origin,
-                           local_changestamp,
+                           directory_service_->largest_changestamp(),
                            params->search_file_path,
                            callback);
 }
 
-void GDataWapiFeedLoader::SaveFileSystemAsProto() {
+void GDataWapiFeedLoader::SaveFileSystem() {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 
-  DVLOG(1) << "SaveFileSystemAsProto";
-
   if (!ShouldSerializeFileSystemNow(directory_service_->serialized_size(),
                                     directory_service_->last_serialized())) {
     return;
   }
 
+  if (UseLevelDB()) {
+    directory_service_->SaveToDB();
+  } else {
     const FilePath path =
         cache_->GetCacheDirectoryPath(GDataCache::CACHE_TYPE_META).Append(
             kFilesystemProtoFile);
@@ -3130,6 +3139,7 @@ void GDataWapiFeedLoader::SaveFileSystemAsProto() {
         blocking_task_runner_,
         base::Bind(&SaveProtoOnBlockingPool, path,
                    base::Passed(serialized_proto.Pass())));
+  }
 }
 
 void GDataFileSystem::OnFilePathUpdated(const FileOperationCallback& callback,
@@ -3524,10 +3534,11 @@ void GDataFileSystem::RunAndNotifyInitialLoadFinished(
     GDataEntry* entry) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 
-  DVLOG(1) << "Initial load finished";
   if (!callback.is_null())
     callback.Run(error, entry);
 
+  DVLOG(1) << "RunAndNotifyInitialLoadFinished";
+
   // Notify the observers that root directory has been initialized.
   FOR_EACH_OBSERVER(GDataFileSystemInterface::Observer, observers_,
                     OnInitialLoadFinished());

chrome/browser/chromeos/gdata/gdata_file_system.h

@@ -38,10 +38,6 @@ struct GetDocumentsParams;
 struct GetDocumentsUiState;
 struct UploadFileInfo;
 
-namespace {
-struct LoadRootFeedParams;
-}  // namespace
-
 // Callback run as a response to LoadFromServer.
 //
 // TODO(satorux): Move this to a new file: crbug.com/138268
@@ -145,6 +141,11 @@ class GDataWapiFeedLoader {
   // Callback for handling root directory refresh from the cache.
   void OnProtoLoaded(LoadRootFeedParams* params);
 
+  // Continues handling root directory refresh after the directory service
+  // is fully loaded.
+  void ContinueWithInitializedDirectoryService(LoadRootFeedParams* params,
+                                               GDataFileError error);
+
   // Helper callback for handling results of metadata retrieval initiated from
   // ReloadFeedFromServerIfNeeded(). This method makes a decision about fetching
   // the content of the root feed during the root directory refresh process.
@@ -174,13 +175,13 @@ class GDataWapiFeedLoader {
       GDataErrorCode status,
       scoped_ptr<base::Value> data);
 
+  // Save filesystem to disk.
+  void SaveFileSystem();
+
   // Callback for handling UI updates caused by document fetching.
   void OnNotifyDocumentFeedFetched(
       base::WeakPtr<GetDocumentsUiState> ui_state);
 
-  // Save filesystem as proto file.
-  void SaveFileSystemAsProto();
-
   GDataDirectoryService* directory_service_;  // Not owned.
   DocumentsServiceInterface* documents_service_;  // Not owned.
   DriveWebAppsRegistryInterface* webapps_registry_;  // Not owned.

chrome/browser/chromeos/gdata/gdata_files.cc

@@ -4,25 +4,37 @@
 
 #include "chrome/browser/chromeos/gdata/gdata_files.h"
 
+#include <leveldb/db.h>
 #include <vector>
 
 #include "base/message_loop_proxy.h"
 #include "base/platform_file.h"
+#include "base/string_number_conversions.h"
 #include "base/string_util.h"
 #include "base/stringprintf.h"
+#include "base/sequenced_task_runner.h"
 #include "base/tracked_objects.h"
 #include "base/utf_string_conversions.h"
 #include "chrome/browser/chromeos/gdata/gdata.pb.h"
 #include "chrome/browser/chromeos/gdata/gdata_util.h"
 #include "chrome/browser/chromeos/gdata/gdata_wapi_parser.h"
+#include "content/public/browser/browser_thread.h"
 #include "net/base/escape.h"
 
+using content::BrowserThread;
+
 namespace gdata {
 namespace {
 
 const char kSlash[] = "/";
 const char kEscapedSlash[] = "\xE2\x88\x95";
 
+// m: prefix for filesystem metadata db keys, version and largest_changestamp.
+// r: prefix for resource id db keys.
+const char kDBKeyLargestChangestamp[] = "m:largest_changestamp";
+const char kDBKeyVersion[] = "m:version";
+const char kDBKeyResourceIdPrefix[] = "r:";
+
 // Extracts resource_id out of edit url.
 std::string ExtractResourceId(const GURL& url) {
   return net::UnescapeURLComponent(url.ExtractFileName(),
@@ -423,12 +435,131 @@ void GDataDirectory::RemoveChildDirectories() {
   child_directories_.clear();
 }
 
+// ResourceMetadataDB implementation.
+
+// Params for GDatadirectoryServiceDB::Create.
+struct CreateDBParams {
+  CreateDBParams(const FilePath& db_path,
+                 base::SequencedTaskRunner* blocking_task_runner)
+                 : db_path(db_path),
+                   blocking_task_runner(blocking_task_runner) {
+  }
+
+  FilePath db_path;
+  scoped_refptr<base::SequencedTaskRunner> blocking_task_runner;
+  scoped_ptr<ResourceMetadataDB> db;
+  GDataDirectoryService::SerializedMap serialized_resources;
+};
+
+// Wrapper for level db. All methods must be called on blocking thread.
+class ResourceMetadataDB {
+ public:
+  ResourceMetadataDB(const FilePath& db_path,
+                          base::SequencedTaskRunner* blocking_task_runner);
+
+  // Initializes the database.
+  void Init();
+
+  // Reads the database into |serialized_resources|.
+  void Read(GDataDirectoryService::SerializedMap* serialized_resources);
+
+  // Saves |serialized_resources| to the database.
+  void Save(const GDataDirectoryService::SerializedMap& serialized_resources);
+
+ private:
+  // Clears the database.
+  void Clear();
+
+  scoped_refptr<base::SequencedTaskRunner> blocking_task_runner_;
+  scoped_ptr<leveldb::DB> level_db_;
+  FilePath db_path_;
+};
+
+ResourceMetadataDB::ResourceMetadataDB(const FilePath& db_path,
+    base::SequencedTaskRunner* blocking_task_runner)
+  : blocking_task_runner_(blocking_task_runner),
+    db_path_(db_path) {
+  DCHECK(blocking_task_runner_->RunsTasksOnCurrentThread());
+}
+
+// Creates, initializes and reads from the database.
+// This must be defined after ResourceMetadataDB and CreateDBParams.
+static void CreateResourceMetadataDBOnBlockingPool(
+    CreateDBParams* params) {
+  DCHECK(params->blocking_task_runner->RunsTasksOnCurrentThread());
+  DCHECK(!params->db_path.empty());
+
+  params->db.reset(new ResourceMetadataDB(params->db_path,
+                                               params->blocking_task_runner));
+  params->db->Init();
+  params->db->Read(&params->serialized_resources);
+}
+
+void ResourceMetadataDB::Init() {
+  DCHECK(blocking_task_runner_->RunsTasksOnCurrentThread());
+  DCHECK(!db_path_.empty());
+
+  DVLOG(1) << "Init " << db_path_.value();
+
+  leveldb::DB* level_db = NULL;
+  leveldb::Options options;
+  options.create_if_missing = true;
+  leveldb::Status db_status = leveldb::DB::Open(options, db_path_.value(),
+                                                &level_db);
+  DCHECK(level_db);
+  DCHECK(db_status.ok());
+  level_db_.reset(level_db);
+}
+
+void ResourceMetadataDB::Read(
+  GDataDirectoryService::SerializedMap* serialized_resources) {
+  DCHECK(blocking_task_runner_->RunsTasksOnCurrentThread());
+  DCHECK(serialized_resources);
+  DVLOG(1) << "Read " << db_path_.value();
+
+  scoped_ptr<leveldb::Iterator> iter(level_db_->NewIterator(
+        leveldb::ReadOptions()));
+  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+    DVLOG(1) << "Read, resource " << iter->key().ToString();
+    serialized_resources->insert(std::make_pair(iter->key().ToString(),
+                                                iter->value().ToString()));
+  }
+}
+
+void ResourceMetadataDB::Save(
+    const GDataDirectoryService::SerializedMap& serialized_resources) {
+  DCHECK(blocking_task_runner_->RunsTasksOnCurrentThread());
+
+  Clear();
+  for (GDataDirectoryService::SerializedMap::const_iterator iter =
+      serialized_resources.begin();
+      iter != serialized_resources.end(); ++iter) {
+    DVLOG(1) << "Saving resource " << iter->first << " to db";
+    leveldb::Status status = level_db_->Put(leveldb::WriteOptions(),
+                                            leveldb::Slice(iter->first),
+                                            leveldb::Slice(iter->second));
+    if (!status.ok()) {
+      LOG(ERROR) << "leveldb Put failed of " << iter->first
+                 << ", with " << status.ToString();
+      NOTREACHED();
+    }
+  }
+}
+
+void ResourceMetadataDB::Clear() {
+  level_db_.reset();
+  leveldb::DestroyDB(db_path_.value(), leveldb::Options());
+  Init();
+}
+
 // GDataDirectoryService class implementation.
 
 GDataDirectoryService::GDataDirectoryService()
-    : serialized_size_(0),
+    : blocking_task_runner_(NULL),
+      serialized_size_(0),
       largest_changestamp_(0),
-      origin_(UNINITIALIZED) {
+      origin_(UNINITIALIZED),
+      weak_ptr_factory_(ALLOW_THIS_IN_INITIALIZER_LIST(this)) {
   root_.reset(new GDataDirectory(NULL, this));
   root_->set_title(kGDataRootDirectory);
   root_->SetBaseNameFromTitle();
@@ -437,12 +568,22 @@ GDataDirectoryService::GDataDirectoryService()
 }
 
 GDataDirectoryService::~GDataDirectoryService() {
+  ClearRoot();
+
+  // Ensure db is closed on the blocking pool.
+  if (blocking_task_runner_ && directory_service_db_.get())
+    blocking_task_runner_->DeleteSoon(FROM_HERE,
+                                      directory_service_db_.release());
+}
+
+void GDataDirectoryService::ClearRoot() {
   // Note that children have a reference to root_,
   // so we need to delete them here.
   root_->RemoveChildren();
   RemoveEntryFromResourceMap(root_.get());
   DCHECK(resource_map_.empty());
   resource_map_.clear();
+  root_.reset();
 }
 
 void GDataDirectoryService::AddEntryToDirectory(
@@ -561,6 +702,173 @@ void GDataDirectoryService::RefreshFileInternal(
   }
 }
 
+void GDataDirectoryService::InitFromDB(
+    const FilePath& db_path,
+    base::SequencedTaskRunner* blocking_task_runner,
+    const FileOperationCallback& callback) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
+  DCHECK(!db_path.empty());
+  DCHECK(blocking_task_runner);
+
+  if (directory_service_db_.get()) {
+    if (!callback.is_null())
+      callback.Run(GDATA_FILE_ERROR_FAILED);
+    return;
+  }
+
+  blocking_task_runner_ = blocking_task_runner;
+
+  DVLOG(1) << "InitFromDB " << db_path.value();
+
+  CreateDBParams* create_params =
+      new CreateDBParams(db_path, blocking_task_runner);
+  blocking_task_runner_->PostTaskAndReply(
+      FROM_HERE,
+      base::Bind(&CreateResourceMetadataDBOnBlockingPool,
+                 create_params),
+      base::Bind(&GDataDirectoryService::InitResourceMap,
+                 weak_ptr_factory_.GetWeakPtr(),
+                 base::Owned(create_params),
+                 callback));
+}
+
+void GDataDirectoryService::InitResourceMap(
+    CreateDBParams* create_params,
+    const FileOperationCallback& callback) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
+  DCHECK(create_params);
+  DCHECK(!directory_service_db_.get());
+
+  SerializedMap* serialized_resources = &create_params->serialized_resources;
+  directory_service_db_ = create_params->db.Pass();
+  if (serialized_resources->empty()) {
+    origin_ = INITIALIZING;
+    if (!callback.is_null())
+      callback.Run(GDATA_FILE_ERROR_NOT_FOUND);
+    return;
+  }
+
+  ClearRoot();
+
+  // Version check.
+  int32 version = 0;
+  SerializedMap::iterator iter = serialized_resources->find(kDBKeyVersion);
+  if (iter == serialized_resources->end() ||
+      !base::StringToInt(iter->second, &version) ||
+      version != kProtoVersion) {
+    if (!callback.is_null())
+      callback.Run(GDATA_FILE_ERROR_FAILED);
+    return;
+  }
+  serialized_resources->erase(iter);
+
+  // Get the largest changestamp.
+  iter = serialized_resources->find(kDBKeyLargestChangestamp);
+  if (iter == serialized_resources->end() ||
+      !base::StringToInt(iter->second, &largest_changestamp_)) {
+    NOTREACHED() << "Could not find/parse largest_changestamp";
+    if (!callback.is_null())
+      callback.Run(GDATA_FILE_ERROR_FAILED);
+    return;
+  } else {
+    DVLOG(1) << "InitResourceMap largest_changestamp_" << largest_changestamp_;
+    serialized_resources->erase(iter);
+  }
+
+  ResourceMap resource_map;
+  for (SerializedMap::const_iterator iter = serialized_resources->begin();
+      iter != serialized_resources->end(); ++iter) {
+    if (iter->first.find(kDBKeyResourceIdPrefix) != 0) {
+      NOTREACHED() << "Incorrect prefix for db key " << iter->first;
+      continue;
+    }
+
+    const std::string resource_id =
+        iter->first.substr(strlen(kDBKeyResourceIdPrefix));
+    scoped_ptr<GDataEntry> entry = FromProtoString(iter->second);
+    if (entry.get()) {
+      DVLOG(1) << "Inserting resource " << resource_id
+               << " into resource_map";
+      resource_map.insert(std::make_pair(resource_id, entry.release()));
+    } else {
+      NOTREACHED() << "Failed to parse GDataEntry for resource " << resource_id;
+    }
+  }
+
+  // Fix up parent-child relations.
+  for (ResourceMap::iterator iter = resource_map.begin();
+      iter != resource_map.end(); ++iter) {
+    GDataEntry* entry = iter->second;
+    ResourceMap::iterator parent_it =
+        resource_map.find(entry->parent_resource_id());
+    if (parent_it != resource_map.end()) {
+      GDataDirectory* parent = parent_it->second->AsGDataDirectory();
+      if (parent) {
+        DVLOG(1) << "Adding " << entry->resource_id()
+                 << " as a child of " << parent->resource_id();
+        parent->AddEntry(entry);
+      } else {
+        NOTREACHED() << "Parent is not a directory " << parent->resource_id();
+      }
+    } else if (entry->resource_id() == kGDataRootDirectoryResourceId) {
+      root_.reset(entry->AsGDataDirectory());
+      DCHECK(root_.get());
+      AddEntryToResourceMap(root_.get());
+    } else {
+      NOTREACHED() << "Missing parent id " << entry->parent_resource_id()
+                   << " for resource " << entry->resource_id();
+    }
+  }
+
+  DCHECK(root_.get());
+  DCHECK_EQ(resource_map.size(), resource_map_.size());
+  DCHECK_EQ(resource_map.size(), serialized_resources->size());
+
+  origin_ = FROM_CACHE;
+
+  if (!callback.is_null())
+    callback.Run(GDATA_FILE_OK);
+}
+
+void GDataDirectoryService::SaveToDB() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
+
+  if (!blocking_task_runner_ || !directory_service_db_.get()) {
+    NOTREACHED();
+    return;
+  }
+
+  size_t serialized_size = 0;
+  SerializedMap serialized_resources;
+  for (ResourceMap::const_iterator iter = resource_map_.begin();
+      iter != resource_map_.end(); ++iter) {
+    GDataEntryProto proto;
+    iter->second->ToProtoFull(&proto);
+    std::string serialized_string;
+    const bool ok = proto.SerializeToString(&serialized_string);
+    DCHECK(ok);
+    if (ok) {
+      serialized_resources.insert(
+          std::make_pair(std::string(kDBKeyResourceIdPrefix) + iter->first,
+                         serialized_string));
+      serialized_size += serialized_string.size();
+    }
+  }
+
+  serialized_resources.insert(std::make_pair(kDBKeyVersion,
+      base::IntToString(kProtoVersion)));
+  serialized_resources.insert(std::make_pair(kDBKeyLargestChangestamp,
+      base::IntToString(largest_changestamp_)));
+  set_last_serialized(base::Time::Now());
+  set_serialized_size(serialized_size);
+
+  blocking_task_runner_->PostTask(
+      FROM_HERE,
+      base::Bind(&ResourceMetadataDB::Save,
+                 base::Unretained(directory_service_db_.get()),
+                 serialized_resources));
+}
+
 // Convert to/from proto.
 
 // static
@@ -768,4 +1076,31 @@ bool GDataDirectoryService::ParseFromString(
   return true;
 }
 
+scoped_ptr<GDataEntry> GDataDirectoryService::FromProtoString(
+    const std::string& serialized_proto) {
+  GDataEntryProto entry_proto;
+  if (!entry_proto.ParseFromString(serialized_proto))
+    return scoped_ptr<GDataEntry>();
+
+  scoped_ptr<GDataEntry> entry;
+  if (entry_proto.file_info().is_directory()) {
+    entry.reset(new GDataDirectory(NULL, this));
+    // Call GDataEntry::FromProto instead of GDataDirectory::FromProto because
+    // the proto does not include children.
+    if (!entry->FromProto(entry_proto)) {
+      NOTREACHED() << "FromProto (directory) failed";
+      entry.reset();
+    }
+  } else {
+    scoped_ptr<GDataFile> file(new GDataFile(NULL, this));
+    // Call GDataFile::FromProto.
+    if (file->FromProto(entry_proto)) {
+      entry.reset(file.release());
+    } else {
+      NOTREACHED() << "FromProto (file) failed";
+    }
+  }
+  return entry.Pass();
+}
+
 }  // namespace gdata

chrome/browser/chromeos/gdata/gdata_files.h

@@ -21,11 +21,17 @@
 #include "chrome/browser/profiles/profile_keyed_service.h"
 #include "chrome/browser/profiles/profile_keyed_service_factory.h"
 
+namespace base {
+class SequencedTaskRunner;
+}
+
 namespace gdata {
 
+struct CreateDBParams;
 class GDataFile;
 class GDataDirectory;
 class GDataDirectoryService;
+class ResourceMetadataDB;
 
 class GDataEntryProto;
 class GDataDirectoryProto;
@@ -102,7 +108,7 @@ class GDataEntry {
       PlatformFileInfoProto* proto);
 
   // Converts to/from proto. Only handles the common part (i.e. does not
-  // touch |file_specific_info| and |directory_specific_info|.
+  // touch |file_specific_info|).
   bool FromProto(const GDataEntryProto& proto) WARN_UNUSED_RESULT;
   void ToProto(GDataEntryProto* proto) const;
 
@@ -332,12 +338,17 @@ class GDataDirectory : public GDataEntry {
   DISALLOW_COPY_AND_ASSIGN(GDataDirectory);
 };
 
+// TODO(achuith,hashimoto,satorux): Move this to a separate file.
+// crbug.com/140317.
 // Class to handle GDataEntry* lookups, add/remove GDataEntry*.
 class GDataDirectoryService {
  public:
   // Callback for GetEntryByResourceIdAsync.
   typedef base::Callback<void(GDataEntry* entry)> GetEntryByResourceIdCallback;
 
+  // Map of resource id and serialized GDataEntry.
+  typedef std::map<std::string, std::string> SerializedMap;
+
   GDataDirectoryService();
   ~GDataDirectoryService();
 
@@ -401,18 +412,45 @@ class GDataDirectoryService {
   void SerializeToString(std::string* serialized_proto) const;
   bool ParseFromString(const std::string& serialized_proto);
 
+  // Restores from and saves to database.
+  void InitFromDB(const FilePath& db_path,
+                  base::SequencedTaskRunner* blocking_task_runner,
+                  const FileOperationCallback& callback);
+  void SaveToDB();
+
  private:
   // A map table of file's resource string to its GDataFile* entry.
   typedef std::map<std::string, GDataEntry*> ResourceMap;
 
-  scoped_ptr<GDataDirectory> root_;  // Stored in the serialized proto.
+  // Initializes the resource map using serialized_resources fetched from the
+  // database.
+  void InitResourceMap(CreateDBParams* create_params,
+                       const FileOperationCallback& callback);
+
+  // Clears root_ and the resource map.
+  void ClearRoot();
+
+  // Creates GDataEntry from serialized string.
+  scoped_ptr<GDataEntry> FromProtoString(
+      const std::string& serialized_proto);
+
+  // Private data members.
+  scoped_refptr<base::SequencedTaskRunner> blocking_task_runner_;
+  scoped_ptr<ResourceMetadataDB> directory_service_db_;
+
   ResourceMap resource_map_;
 
+  scoped_ptr<GDataDirectory> root_;  // Stored in the serialized proto.
+
   base::Time last_serialized_;
   size_t serialized_size_;
   int largest_changestamp_;  // Stored in the serialized proto.
   ContentOrigin origin_;
 
+  // This should remain the last member so it'll be destroyed first and
+  // invalidate its weak pointers before other members are destroyed.
+  base::WeakPtrFactory<GDataDirectoryService> weak_ptr_factory_;
+
   DISALLOW_COPY_AND_ASSIGN(GDataDirectoryService);
 };
 

chrome/browser/chromeos/gdata/gdata_files_unittest.cc

@@ -7,19 +7,179 @@
 #include <string>
 #include <utility>
 #include <vector>
+
+#include "base/sequenced_task_runner.h"
+#include "base/string_number_conversions.h"
+#include "base/threading/sequenced_worker_pool.h"
 #include "base/message_loop.h"
 #include "chrome/browser/chromeos/gdata/gdata.pb.h"
+#include "chrome/browser/chromeos/gdata/gdata_cache.h"
 #include "chrome/browser/chromeos/gdata/gdata_test_util.h"
+#include "chrome/test/base/testing_profile.h"
+#include "content/public/test/test_browser_thread.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 namespace gdata {
-
 namespace {
 
 // See gdata.proto for the difference between the two URLs.
 const char kResumableEditMediaUrl[] = "http://resumable-edit-media/";
 const char kResumableCreateMediaUrl[] = "http://resumable-create-media/";
 
+// Add a directory to |parent| and return that directory. The name and
+// resource_id are determined by the incrementing counter |sequence_id|.
+GDataDirectory* AddDirectory(GDataDirectory* parent,
+                             GDataDirectoryService* directory_service,
+                             int sequence_id) {
+  GDataDirectory* dir = new GDataDirectory(parent, directory_service);
+  const std::string dir_name = "dir" + base::IntToString(sequence_id);
+  const std::string resource_id = std::string("dir_resource_id:") +
+                                  dir_name;
+  dir->set_title(dir_name);
+  dir->set_resource_id(resource_id);
+  GDataFileError error = GDATA_FILE_ERROR_FAILED;
+  directory_service->AddEntryToDirectory(
+      parent->GetFilePath(),
+      dir,
+      base::Bind(&test_util::CopyErrorCodeFromFileOperationCallback, &error));
+  test_util::RunBlockingPoolTask();
+  EXPECT_EQ(GDATA_FILE_OK, error);
+  return dir;
+}
+
+// Add a file to |parent| and return that file. The name and
+// resource_id are determined by the incrementing counter |sequence_id|.
+GDataFile* AddFile(GDataDirectory* parent,
+                   GDataDirectoryService* directory_service,
+                   int sequence_id) {
+  GDataFile* file = new GDataFile(parent, directory_service);
+  const std::string title = "file" + base::IntToString(sequence_id);
+  const std::string resource_id = std::string("file_resource_id:") +
+                                  title;
+  file->set_title(title);
+  file->set_resource_id(resource_id);
+  file->set_file_md5(std::string("file_md5:") + title);
+  GDataFileError error = GDATA_FILE_ERROR_FAILED;
+  directory_service->AddEntryToDirectory(
+      parent->GetFilePath(),
+      file,
+      base::Bind(&test_util::CopyErrorCodeFromFileOperationCallback, &error));
+  test_util::RunBlockingPoolTask();
+  EXPECT_EQ(GDATA_FILE_OK, error);
+  return file;
+}
+
+// Creates the following files/directories
+// drive/dir1/
+// drive/dir2/
+// drive/dir1/dir3/
+// drive/dir1/file4
+// drive/dir1/file5
+// drive/dir2/file6
+// drive/dir2/file7
+// drive/dir2/file8
+// drive/dir1/dir3/file9
+// drive/dir1/dir3/file10
+void InitDirectoryService(GDataDirectoryService* directory_service) {
+  int sequence_id = 1;
+  GDataDirectory* dir1 = AddDirectory(directory_service->root(),
+      directory_service, sequence_id++);
+  GDataDirectory* dir2 = AddDirectory(directory_service->root(),
+      directory_service, sequence_id++);
+  GDataDirectory* dir3 = AddDirectory(dir1, directory_service, sequence_id++);
+
+  AddFile(dir1, directory_service, sequence_id++);
+  AddFile(dir1, directory_service, sequence_id++);
+
+  AddFile(dir2, directory_service, sequence_id++);
+  AddFile(dir2, directory_service, sequence_id++);
+  AddFile(dir2, directory_service, sequence_id++);
+
+  AddFile(dir3, directory_service, sequence_id++);
+  AddFile(dir3, directory_service, sequence_id++);
+}
+
+// Find directory by path.
+GDataDirectory* FindDirectory(GDataDirectoryService* directory_service,
+                              const char* path) {
+  return directory_service->FindEntryByPathSync(
+      FilePath(path))->AsGDataDirectory();
+}
+
+// Find file by path.
+GDataFile* FindFile(GDataDirectoryService* directory_service,
+                    const char* path) {
+  return directory_service->FindEntryByPathSync(FilePath(path))->AsGDataFile();
+}
+
+// Verify that the recreated directory service matches what we created in
+// InitDirectoryService.
+void VerifyDirectoryService(GDataDirectoryService* directory_service) {
+  ASSERT_TRUE(directory_service->root());
+
+  GDataDirectory* dir1 = FindDirectory(directory_service, "drive/dir1");
+  ASSERT_TRUE(dir1);
+  GDataDirectory* dir2 = FindDirectory(directory_service, "drive/dir2");
+  ASSERT_TRUE(dir2);
+  GDataDirectory* dir3 = FindDirectory(directory_service, "drive/dir1/dir3");
+  ASSERT_TRUE(dir3);
+
+  GDataFile* file4 = FindFile(directory_service, "drive/dir1/file4");
+  ASSERT_TRUE(file4);
+  EXPECT_EQ(file4->parent(), dir1);
+
+  GDataFile* file5 = FindFile(directory_service, "drive/dir1/file5");
+  ASSERT_TRUE(file5);
+  EXPECT_EQ(file5->parent(), dir1);
+
+  GDataFile* file6 = FindFile(directory_service, "drive/dir2/file6");
+  ASSERT_TRUE(file6);
+  EXPECT_EQ(file6->parent(), dir2);
+
+  GDataFile* file7 = FindFile(directory_service, "drive/dir2/file7");
+  ASSERT_TRUE(file7);
+  EXPECT_EQ(file7->parent(), dir2);
+
+  GDataFile* file8 = FindFile(directory_service, "drive/dir2/file8");
+  ASSERT_TRUE(file8);
+  EXPECT_EQ(file8->parent(), dir2);
+
+  GDataFile* file9 = FindFile(directory_service, "drive/dir1/dir3/file9");
+  ASSERT_TRUE(file9);
+  EXPECT_EQ(file9->parent(), dir3);
+
+  GDataFile* file10 = FindFile(directory_service, "drive/dir1/dir3/file10");
+  ASSERT_TRUE(file10);
+  EXPECT_EQ(file10->parent(), dir3);
+
+  EXPECT_EQ(dir1, directory_service->GetEntryByResourceId(
+      "dir_resource_id:dir1"));
+  EXPECT_EQ(dir2, directory_service->GetEntryByResourceId(
+      "dir_resource_id:dir2"));
+  EXPECT_EQ(dir3, directory_service->GetEntryByResourceId(
+      "dir_resource_id:dir3"));
+  EXPECT_EQ(file4, directory_service->GetEntryByResourceId(
+      "file_resource_id:file4"));
+  EXPECT_EQ(file5, directory_service->GetEntryByResourceId(
+      "file_resource_id:file5"));
+  EXPECT_EQ(file6, directory_service->GetEntryByResourceId(
+      "file_resource_id:file6"));
+  EXPECT_EQ(file7, directory_service->GetEntryByResourceId(
+      "file_resource_id:file7"));
+  EXPECT_EQ(file8, directory_service->GetEntryByResourceId(
+      "file_resource_id:file8"));
+  EXPECT_EQ(file9, directory_service->GetEntryByResourceId(
+      "file_resource_id:file9"));
+  EXPECT_EQ(file10, directory_service->GetEntryByResourceId(
+      "file_resource_id:file10"));
+}
+
+// Callback for GDataDirectoryService::InitFromDB.
+void InitFromDBCallback(GDataFileError expected_error,
+                        GDataFileError actual_error) {
+  EXPECT_EQ(expected_error, actual_error);
+}
+
 }  // namespace
 
 TEST(GDataEntryTest, FromProto_DetectBadUploadUrl) {
@@ -276,4 +436,38 @@ TEST(GDataRootDirectoryTest, GetEntryByResourceId_RootDirectory) {
   EXPECT_EQ(kGDataRootDirectoryResourceId, entry->resource_id());
 }
 
+TEST(GDataRootDirectoryTest, DBTest) {
+  MessageLoopForUI message_loop;
+  content::TestBrowserThread ui_thread(content::BrowserThread::UI,
+                                       &message_loop);
+
+  scoped_ptr<TestingProfile> profile(new TestingProfile);
+  scoped_refptr<base::SequencedWorkerPool> pool =
+      content::BrowserThread::GetBlockingPool();
+  scoped_refptr<base::SequencedTaskRunner> blocking_task_runner =
+      pool->GetSequencedTaskRunner(pool->GetSequenceToken());
+
+  GDataDirectoryService directory_service;
+  FilePath db_path(GDataCache::GetCacheRootPath(profile.get()).
+      AppendASCII("meta").AppendASCII("resource_metadata.db"));
+  // InitFromDB should fail with GDATA_FILE_ERROR_NOT_FOUND since the db
+  // doesn't exist.
+  directory_service.InitFromDB(db_path, blocking_task_runner,
+      base::Bind(&InitFromDBCallback, GDATA_FILE_ERROR_NOT_FOUND));
+  test_util::RunBlockingPoolTask();
+  InitDirectoryService(&directory_service);
+
+  // Write the filesystem to db.
+  directory_service.SaveToDB();
+  test_util::RunBlockingPoolTask();
+
+  GDataDirectoryService directory_service2;
+  // InitFromDB should succeed with GDATA_FILE_OK as the db now exists.
+  directory_service2.InitFromDB(db_path, blocking_task_runner,
+      base::Bind(&InitFromDBCallback, GDATA_FILE_OK));
+  test_util::RunBlockingPoolTask();
+
+  VerifyDirectoryService(&directory_service2);
+}
+
 }  // namespace gdata

chrome/browser/chromeos/gdata/gdata_params.cc

@@ -56,4 +56,18 @@ ResumeUploadParams::ResumeUploadParams(
 ResumeUploadParams::~ResumeUploadParams() {
 }
 
+LoadRootFeedParams::LoadRootFeedParams(
+    FilePath search_file_path,
+    bool should_load_from_server,
+    const FindEntryCallback& callback)
+    : search_file_path(search_file_path),
+      should_load_from_server(should_load_from_server),
+      load_error(GDATA_FILE_OK),
+      load_start_time(base::Time::Now()),
+      callback(callback) {
+}
+
+LoadRootFeedParams::~LoadRootFeedParams() {
+}
+
 }  // namespace gdata

chrome/browser/chromeos/gdata/gdata_params.h

@@ -22,6 +22,50 @@
 namespace gdata {
 
 class GDataEntry;
+struct ResumeUploadResponse;
+
+// Different callback types for various functionalities in DocumentsService.
+
+// Callback type for authentication related DocumentService calls.
+typedef base::Callback<void(GDataErrorCode error,
+                            const std::string& token)> AuthStatusCallback;
+
+// Callback type for DocumentServiceInterface::GetDocuments.
+// Note: feed_data argument should be passed using base::Passed(&feed_data), not
+// feed_data.Pass().
+typedef base::Callback<void(GDataErrorCode error,
+                            scoped_ptr<base::Value> feed_data)> GetDataCallback;
+
+// Callback type for Delete/Move DocumentServiceInterface calls.
+typedef base::Callback<void(GDataErrorCode error,
+                            const GURL& document_url)> EntryActionCallback;
+
+// Callback type for DownloadDocument/DownloadFile DocumentServiceInterface
+// calls.
+typedef base::Callback<void(GDataErrorCode error,
+                            const GURL& content_url,
+                            const FilePath& temp_file)> DownloadActionCallback;
+
+// Callback type for getting download data from DownloadFile
+// DocumentServiceInterface calls.
+typedef base::Callback<void(
+    GDataErrorCode error,
+    scoped_ptr<std::string> download_data)> GetDownloadDataCallback;
+
+// Callback type for DocumentServiceInterface::InitiateUpload.
+typedef base::Callback<void(GDataErrorCode error,
+                            const GURL& upload_url)> InitiateUploadCallback;
+
+// Callback type for DocumentServiceInterface::ResumeUpload.
+typedef base::Callback<void(
+    const ResumeUploadResponse& response,
+    scoped_ptr<gdata::DocumentEntry> new_entry)> ResumeUploadCallback;
+
+// Callback type used to get result of file search.
+// If |error| is not PLATFORM_FILE_OK, |entry| is set to NULL.
+typedef base::Callback<void(GDataFileError error, GDataEntry* entry)>
+    FindEntryCallback;
+
 
 // Struct for response to ResumeUpload.
 struct ResumeUploadResponse {
@@ -88,47 +132,23 @@ struct InitiateUploadParams {
   const FilePath& virtual_path;
 };
 
-// Different callback types for various functionalities in DocumentsService.
-
-// Callback type for authentication related DocumentService calls.
-typedef base::Callback<void(GDataErrorCode error,
-                            const std::string& token)> AuthStatusCallback;
-
-// Callback type for DocumentServiceInterface::GetDocuments.
-// Note: feed_data argument should be passed using base::Passed(&feed_data), not
-// feed_data.Pass().
-typedef base::Callback<void(GDataErrorCode error,
-                            scoped_ptr<base::Value> feed_data)> GetDataCallback;
-
-// Callback type for Delete/Move DocumentServiceInterface calls.
-typedef base::Callback<void(GDataErrorCode error,
-                            const GURL& document_url)> EntryActionCallback;
-
-// Callback type for DownloadDocument/DownloadFile DocumentServiceInterface
-// calls.
-typedef base::Callback<void(GDataErrorCode error,
-                            const GURL& content_url,
-                            const FilePath& temp_file)> DownloadActionCallback;
-
-// Callback type for getting download data from DownloadFile
-// DocumentServiceInterface calls.
-typedef base::Callback<void(
-    GDataErrorCode error,
-    scoped_ptr<std::string> download_data)> GetDownloadDataCallback;
-
-// Callback type for DocumentServiceInterface::InitiateUpload.
-typedef base::Callback<void(GDataErrorCode error,
-                            const GURL& upload_url)> InitiateUploadCallback;
-
-// Callback type for DocumentServiceInterface::ResumeUpload.
-typedef base::Callback<void(
-    const ResumeUploadResponse& response,
-    scoped_ptr<gdata::DocumentEntry> new_entry)> ResumeUploadCallback;
-
-// Callback type used to get result of file search.
-// If |error| is not PLATFORM_FILE_OK, |entry| is set to NULL.
-typedef base::Callback<void(GDataFileError error, GDataEntry* entry)>
-    FindEntryCallback;
+// Defines set of parameters sent to callback OnProtoLoaded().
+struct LoadRootFeedParams {
+  LoadRootFeedParams(
+        FilePath search_file_path,
+        bool should_load_from_server,
+        const FindEntryCallback& callback);
+  ~LoadRootFeedParams();
+
+  FilePath search_file_path;
+  bool should_load_from_server;
+  std::string proto;
+  GDataFileError load_error;
+  base::Time last_modified;
+  // Time when filesystem began to be loaded from disk.
+  base::Time load_start_time;
+  const FindEntryCallback callback;
+};
 
 }  // namespace gdata
 

chrome/common/chrome_switches.cc

@@ -1369,6 +1369,9 @@ const char kEnableDevicePolicy[]            = "enable-device-policy";
 // Enables Drive v2 API instead of Google Documents List API.
 const char kEnableDriveV2Api[]              = "enable-drive-v2-api";
 
+// Use level db for drive metadata storage.
+const char kUseLevelDBForGData[]            = "use-leveldb-for-gdata";
+
 // Enables the redirection of viewable document requests to the Google Document
 // Viewer.
 const char kEnableGView[]                   = "enable-gview";

chrome/common/chrome_switches.h

@@ -373,6 +373,7 @@ extern const char kEnableTouchpadThreeFingerClick[];
 extern const char kSkipOAuthLogin[];
 extern const char kEnableDevicePolicy[];
 extern const char kEnableDriveV2Api[];
+extern const char kUseLevelDBForGData[];
 extern const char kEnableGView[];
 extern const char kEnableKioskMode[];
 extern const char kEnableONCPolicy[];