Recover unsynced Drive files to Downloads when switching to DriveFS.

When running with DriveFS enabled for the first time, recover any files
in the local cache marked as dirty to a folder in Downloads, similar to
the existing process for recovering from database errors, and clean up
any old Drive client metadata. The existing Drive client code does not
expect dirty files to be removed from its cache so clearing its metadata
is necessary to limit confusion when switching back from DriveFS.

Reset the DriveFS migration pref when running without DriveFS; each
first DriveFS after running without DriveFS should migrate pinned files
and recover any unsynced files. This should avoid losing data if a user
toggles between DriveFS and not multiple times with unsynced files.

Move Drive and DriveFS data to separate subdirectories in files app
browsertests to avoid interference, in particular in DriveFS migration
tests.

Bug: 883242
Tbr: slangley@chromium.org
Change-Id: I20199ffd04ad7f63cb77ac9a53e49ba1bf830502
Reviewed-on: https://chromium-review.googlesource.com/1242783
Commit-Queue: Sam McNally <sammc@chromium.org>
Reviewed-by: Noel Gordon <noel@chromium.org>
Reviewed-by: Sergei Datsenko <dats@chromium.org>
Cr-Commit-Position: refs/heads/master@{#594585}

chrome/browser/chromeos/drive/drive_integration_service.cc

@@ -129,6 +129,22 @@ void DeleteDirectoryContents(const base::FilePath& dir) {
   }
 }
 
+base::FilePath FindUniquePath(const base::FilePath& base_name) {
+  auto target = base_name;
+  for (int uniquifier = 1; base::PathExists(target); ++uniquifier) {
+    target = base_name.InsertBeforeExtensionASCII(
+        base::StringPrintf(" (%d)", uniquifier));
+  }
+  return target;
+}
+
+base::FilePath GetRecoveredFilesPath(
+    const base::FilePath& downloads_directory) {
+  const std::string& dest_directory_name = l10n_util::GetStringUTF8(
+      IDS_FILE_BROWSER_RECOVERED_FILES_FROM_GOOGLE_DRIVE_DIRECTORY_NAME);
+  return FindUniquePath(downloads_directory.Append(dest_directory_name));
+}
+
 // Initializes FileCache and ResourceMetadata.
 // Must be run on the same task runner used by |cache| and |resource_metadata|.
 FileError InitializeMetadata(
@@ -170,6 +186,13 @@ FileError InitializeMetadata(
       base::FILE_PERMISSION_EXECUTE_BY_GROUP |
       base::FILE_PERMISSION_EXECUTE_BY_OTHERS);
 
+  // If attempting to migrate to DriveFS without previous Drive sync data
+  // present, skip the migration.
+  if (base::IsDirectoryEmpty(cache_root_directory.Append(kMetadataDirectory)) &&
+      !cache) {
+    return FILE_ERROR_FAILED;
+  }
+
   internal::ResourceMetadataStorage::UpgradeOldDB(
       metadata_storage->directory_path());
 
@@ -190,15 +213,7 @@ FileError InitializeMetadata(
 
   if (metadata_storage->cache_file_scan_is_needed()) {
     // Generate unique directory name.
-    const std::string& dest_directory_name = l10n_util::GetStringUTF8(
-        IDS_FILE_BROWSER_RECOVERED_FILES_FROM_GOOGLE_DRIVE_DIRECTORY_NAME);
-    base::FilePath dest_directory = downloads_directory.Append(
-        base::FilePath::FromUTF8Unsafe(dest_directory_name));
-    for (int uniquifier = 1; base::PathExists(dest_directory); ++uniquifier) {
-      dest_directory = downloads_directory.Append(
-          base::FilePath::FromUTF8Unsafe(dest_directory_name))
-          .InsertBeforeExtensionASCII(base::StringPrintf(" (%d)", uniquifier));
-    }
+    auto dest_directory = GetRecoveredFilesPath(downloads_directory);
 
     internal::ResourceMetadataStorage::RecoveredCacheInfoMap
         recovered_cache_info;
@@ -257,21 +272,74 @@ base::FilePath GetFullPath(internal::ResourceMetadataStorage* metadata_storage,
   return path;
 }
 
+// Recover any dirty files in GCache/v1 to a recovered files directory in
+// Downloads. This imitates the behavior of recovering cache files when database
+// corruption occurs; however, in this case, we have an intact database so can
+// use the exact file names, potentially with uniquifiers added since the
+// directory structure is discarded.
+void RecoverDirtyFiles(
+    const base::FilePath& cache_directory,
+    const base::FilePath& downloads_directory,
+    const std::vector<std::pair<base::FilePath, std::string>>& dirty_files) {
+  if (dirty_files.empty()) {
+    return;
+  }
+  auto recovery_directory = GetRecoveredFilesPath(downloads_directory);
+  if (!base::CreateDirectory(recovery_directory)) {
+    return;
+  }
+  for (auto& dirty_file : dirty_files) {
+    auto target_path =
+        FindUniquePath(recovery_directory.Append(dirty_file.first.BaseName()));
+    base::Move(cache_directory.Append(dirty_file.second), target_path);
+  }
+}
+
+// Remove the data used by the old Drive client, first moving any dirty files
+// into the user's Downloads.
+void CleanupGCacheV1(
+    const base::FilePath& cache_directory,
+    const base::FilePath& downloads_directory,
+    std::vector<std::pair<base::FilePath, std::string>> dirty_files) {
+  RecoverDirtyFiles(cache_directory.Append(kCacheFileDirectory),
+                    downloads_directory, dirty_files);
+  DeleteDirectoryContents(cache_directory);
+}
+
 std::vector<base::FilePath> GetPinnedFiles(
-    internal::ResourceMetadataStorage* metadata_storage) {
+    std::unique_ptr<internal::ResourceMetadataStorage, util::DestroyHelper>
+        metadata_storage,
+    base::FilePath cache_directory,
+    base::FilePath downloads_directory) {
   std::vector<base::FilePath> pinned_files;
+  std::vector<std::pair<base::FilePath, std::string>> dirty_files;
   for (auto it = metadata_storage->GetIterator(); !it->IsAtEnd();
        it->Advance()) {
     const auto& value = it->GetValue();
-    if (!value.has_file_specific_info() ||
-        !value.file_specific_info().cache_state().is_pinned()) {
+    if (!value.has_file_specific_info()) {
       continue;
     }
-    auto path = GetFullPath(metadata_storage, value);
-    if (!path.empty()) {
-      pinned_files.push_back(std::move(path));
+    const auto& info = value.file_specific_info();
+    if (info.cache_state().is_pinned()) {
+      auto path = GetFullPath(metadata_storage.get(), value);
+      if (!path.empty()) {
+        pinned_files.push_back(std::move(path));
+      }
+    }
+    if (info.cache_state().is_dirty()) {
+      dirty_files.push_back(std::make_pair(
+          GetFullPath(metadata_storage.get(), value), value.local_id()));
     }
   }
+  // Destructing |metadata_storage| requires a posted task to run, so defer
+  // deleting its data until after it's been destructed. This also returns the
+  // list of files to pin to the UI thread without waiting for the remaining
+  // data to be cleared.
+  metadata_storage.reset();
+  base::SequencedTaskRunnerHandle::Get()->PostTask(
+      FROM_HERE,
+      base::BindOnce(&CleanupGCacheV1, std::move(cache_directory),
+                     std::move(downloads_directory), std::move(dirty_files)));
   return pinned_files;
 }
 
@@ -908,6 +976,12 @@ void DriveIntegrationService::InitializeAfterMetadataInitialized(
     if (error != FILE_ERROR_OK) {
       profile_->GetPrefs()->SetBoolean(prefs::kDriveFsPinnedMigrated, true);
       metadata_storage_.reset();
+      blocking_task_runner_->PostTask(
+          FROM_HERE,
+          base::BindOnce(
+              &CleanupGCacheV1, cache_root_directory_, base::FilePath(),
+              std::vector<std::pair<base::FilePath, std::string>>()));
+      metadata_storage_.reset();
     }
     state_ = INITIALIZED;
     if (enabled_)
@@ -931,6 +1005,11 @@ void DriveIntegrationService::InitializeAfterMetadataInitialized(
     return;
   }
 
+  // Reset the pref so any migration to DriveFS is performed again the next time
+  // DriveFS is enabled. This is necessary to ensure any newly-pinned files are
+  // migrated and any dirty files are recovered whenever switching to DriveFS.
+  profile_->GetPrefs()->ClearPref(prefs::kDriveFsPinnedMigrated);
+
   // Initialize Download Handler for hooking downloads to the Drive folder.
   content::DownloadManager* download_manager =
       g_browser_process->download_status_updater()
@@ -1004,7 +1083,9 @@ void DriveIntegrationService::MigratePinnedFiles() {
 
   base::PostTaskAndReplyWithResult(
       blocking_task_runner_.get(), FROM_HERE,
-      base::BindOnce(&GetPinnedFiles, metadata_storage_.get()),
+      base::BindOnce(
+          &GetPinnedFiles, std::move(metadata_storage_), cache_root_directory_,
+          file_manager::util::GetDownloadsFolderForProfile(profile_)),
       base::BindOnce(&DriveIntegrationService::PinFiles,
                      weak_ptr_factory_.GetWeakPtr()));
 }
@@ -1018,7 +1099,6 @@ void DriveIntegrationService::PinFiles(
     GetDriveFsInterface()->SetPinned(path, true, base::DoNothing());
   }
   profile_->GetPrefs()->SetBoolean(prefs::kDriveFsPinnedMigrated, true);
-  metadata_storage_.reset();
 }
 
 //===================== DriveIntegrationServiceFactory =======================

chrome/browser/chromeos/file_manager/file_manager_browsertest.cc

@@ -803,6 +803,10 @@ class DriveFsFilesAppBrowserTest : public FileManagerBrowserTestBase {
                 .starts_with("PRE");
   }
 
+  base::FilePath GetDriveDataDirectory() {
+    return profile()->GetPath().Append("drive/v1");
+  }
+
  private:
   std::string test_case_name_;
 
@@ -817,6 +821,28 @@ IN_PROC_BROWSER_TEST_F(DriveFsFilesAppBrowserTest, PRE_MigratePinnedFiles) {
 IN_PROC_BROWSER_TEST_F(DriveFsFilesAppBrowserTest, MigratePinnedFiles) {
   set_test_case_name("driveMigratePinnedFile");
   StartTest();
+
+  EXPECT_TRUE(base::IsDirectoryEmpty(GetDriveDataDirectory()));
+}
+
+IN_PROC_BROWSER_TEST_F(DriveFsFilesAppBrowserTest, PRE_RecoverDirtyFiles) {
+  set_test_case_name("PRE_driveRecoverDirtyFiles");
+  StartTest();
+
+  // Create a non-dirty file in the cache.
+  base::WriteFile(GetDriveDataDirectory().Append("files/foo"), "data", 4);
+}
+
+IN_PROC_BROWSER_TEST_F(DriveFsFilesAppBrowserTest, RecoverDirtyFiles) {
+  set_test_case_name("driveRecoverDirtyFiles");
+  StartTest();
+
+  EXPECT_TRUE(base::IsDirectoryEmpty(GetDriveDataDirectory()));
+}
+
+IN_PROC_BROWSER_TEST_F(DriveFsFilesAppBrowserTest, LaunchWithoutOldDriveData) {
+  // After starting up, GCache/v1 should still be empty.
+  EXPECT_TRUE(base::IsDirectoryEmpty(GetDriveDataDirectory()));
 }
 
 }  // namespace file_manager

chrome/browser/chromeos/file_manager/file_manager_browsertest_base.cc

@@ -804,8 +804,8 @@ class DriveTestVolume : public TestVolume {
 
     EXPECT_FALSE(integration_service_);
     integration_service_ = new drive::DriveIntegrationService(
-        profile, nullptr, fake_drive_service_, std::string(), root_path(),
-        nullptr, CreateDriveFsConnectionDelegate());
+        profile, nullptr, fake_drive_service_, std::string(),
+        root_path().Append("v1"), nullptr, CreateDriveFsConnectionDelegate());
 
     return integration_service_;
   }
@@ -881,7 +881,7 @@ class DriveFsTestVolume : public DriveTestVolume {
   }
 
   void InitializeFakeDriveFs() {
-    fake_drivefs_ = std::make_unique<drivefs::FakeDriveFs>(root_path());
+    fake_drivefs_ = std::make_unique<drivefs::FakeDriveFs>(mount_path());
     fake_drivefs_->RegisterMountingForAccountId(base::BindRepeating(
         [](Profile* profile) {
           auto* user =
@@ -933,14 +933,16 @@ class DriveFsTestVolume : public DriveTestVolume {
       const AddEntriesMessage::TestEntryInfo& entry) {
     const base::FilePath target_path = GetTargetPathForTestEntry(entry);
     base::FilePath drive_path("/");
-    CHECK(root_path().AppendRelativePath(target_path, &drive_path));
+    CHECK(mount_path().AppendRelativePath(target_path, &drive_path));
     return drive_path;
   }
 
-  base::FilePath GetMyDrivePath() { return root_path().Append("root"); }
+  base::FilePath mount_path() { return root_path().Append("v2"); }
+
+  base::FilePath GetMyDrivePath() { return mount_path().Append("root"); }
 
   base::FilePath GetTeamDriveGrandRoot() {
-    return root_path().Append("team_drives");
+    return mount_path().Append("team_drives");
   }
 
   base::FilePath GetTeamDrivePath(const std::string& team_drive_name) {
@@ -1372,6 +1374,12 @@ FileManagerBrowserTestBase::CreateDriveIntegrationService(Profile* profile) {
                                    kPredefinedProfileSalt);
     drive_volumes_[profile->GetOriginalProfile()] =
         std::make_unique<DriveFsTestVolume>(profile->GetOriginalProfile());
+    if (!IsIncognitoModeTest()) {
+      base::ThreadTaskRunnerHandle::Get()->PostTask(
+          FROM_HERE,
+          base::BindOnce(base::IgnoreResult(&LocalTestVolume::Mount),
+                         base::Unretained(local_volume_.get()), profile));
+    }
   } else {
     drive_volumes_[profile->GetOriginalProfile()] =
         std::make_unique<DriveTestVolume>();

chromeos/components/drivefs/fake_drivefs.cc

@@ -125,13 +125,6 @@ class FakeDriveFs::SearchQuery : public mojom::SearchQuery {
       auto item = drivefs::mojom::QueryItem::New();
       item->path = base::FilePath("/");
       CHECK(mount_path.AppendRelativePath(file, &item->path));
-      std::vector<std::string> components;
-      item->path.GetComponents(&components);
-      // During tests, metadata for the other drive sync implementation can
-      // end up in |mount_path| so filter it out.
-      if (components.size() < 2u || components[1] == "meta") {
-        continue;
-      }
       results.push_back(std::move(item));
     }
     return results;

components/drive/service/fake_drive_service.cc

@@ -1099,6 +1099,10 @@ CancelCallback FakeDriveService::InitiateUploadNewFile(
                     "",  // etag
                     title);
 
+  if (title == "never-sync.txt") {
+    return CancelCallback();
+  }
+
   base::ThreadTaskRunnerHandle::Get()->PostTask(
       FROM_HERE, base::BindOnce(callback, HTTP_SUCCESS, session_url));
   return CancelCallback();

ui/file_manager/integration_tests/file_manager/drive_specific.js

@@ -344,8 +344,10 @@ testcase.drivePressCtrlAFromSearch = function() {
   StepsRunner.run(steps);
 };
 
+/**
+ * Pin hello.txt in the old Drive client.
+ */
 testcase.PRE_driveMigratePinnedFile = function() {
-  // Pin a file.
   testPromise(
       setupAndWaitUntilReady(null, RootPath.DRIVE).then(function(results) {
         var windowId = results.windowId;
@@ -388,6 +390,9 @@ testcase.PRE_driveMigratePinnedFile = function() {
       }));
 };
 
+/**
+ * Verify hello.txt is still pinned after migrating to DriveFS.
+ */
 testcase.driveMigratePinnedFile = function() {
   // After enabling DriveFS, ensure the file is still pinned.
   testPromise(
@@ -430,6 +435,10 @@ function formatDate(date) {
   return `${year}-${month}-${day}`;
 }
 
+/**
+ * Test that a images within a DCIM directory on removable media is backed up to
+ * Drive, in the Chrome OS Cloud backup/<current date> directory.
+ */
 testcase.driveBackupPhotos = function() {
   let appId;
 
@@ -499,3 +508,143 @@ testcase.driveBackupPhotos = function() {
     }
   ]);
 };
+
+/**
+ * Create some dirty files in Drive.
+ *
+ * Create /root/never-sync.txt and /root/A/never-sync.txt. These files will
+ * never complete syncing to the fake drive service so will remain dirty
+ * forever.
+ */
+testcase.PRE_driveRecoverDirtyFiles = function() {
+  let appId;
+  StepsRunner.run([
+    function() {
+      setupAndWaitUntilReady(
+          null, RootPath.DOWNLOADS, this.next, [ENTRIES.neverSync],
+          [ENTRIES.directoryA]);
+    },
+    // Select never-sync.txt.
+    function(results) {
+      appId = results.windowId;
+
+      remoteCall.callRemoteTestUtil('selectFile', appId, ['never-sync.txt'])
+          .then(this.next);
+    },
+    // Copy it.
+    function(result) {
+      chrome.test.assertTrue(result, 'selectFile failed');
+
+      return remoteCall
+          .callRemoteTestUtil(
+              'fakeKeyDown', appId, ['#file-list', 'c', true, false, false])
+          .then(this.next);
+    },
+    // Navigate to My Drive.
+    function(result) {
+      chrome.test.assertTrue(result, 'copy failed');
+
+      return remoteCall
+          .navigateWithDirectoryTree(appId, '/root', 'My Drive', 'drive')
+          .then(this.next);
+    },
+    // Paste.
+    function() {
+      return remoteCall
+          .callRemoteTestUtil(
+              'fakeKeyDown', appId, ['#file-list', 'v', true, false, false])
+          .then(this.next);
+    },
+    // Wait for the paste to complete.
+    function(result) {
+      chrome.test.assertTrue(result, 'paste failed');
+      const expectedEntryRows = [
+        ENTRIES.neverSync.getExpectedRow(),
+        ENTRIES.directoryA.getExpectedRow(),
+      ];
+      remoteCall
+          .waitForFiles(
+              appId, expectedEntryRows, {ignoreLastModifiedTime: true})
+          .then(this.next);
+    },
+    // Navigate to My Drive/A.
+    function() {
+      return remoteCall
+          .navigateWithDirectoryTree(appId, '/root/A', 'My Drive', 'drive')
+          .then(this.next);
+    },
+    // Paste.
+    function() {
+      return remoteCall
+          .callRemoteTestUtil(
+              'fakeKeyDown', appId, ['#file-list', 'v', true, false, false])
+          .then(this.next);
+    },
+    // Wait for the paste to complete.
+    function(result) {
+      chrome.test.assertTrue(result, 'paste failed');
+      const expectedEntryRows = [ENTRIES.neverSync.getExpectedRow()];
+      remoteCall
+          .waitForFiles(
+              appId, expectedEntryRows, {ignoreLastModifiedTime: true})
+          .then(this.next);
+    },
+    function() {
+      checkIfNoErrorsOccured(this.next);
+    },
+  ]);
+};
+
+/**
+ * Verify that when enabling DriveFS, the dirty files are recovered to
+ * Downloads/Recovered files from Google Drive. The directory structure should
+ * be flattened with uniquified names:
+ * - never-sync.txt
+ * - never-sync (1).txt
+ */
+testcase.driveRecoverDirtyFiles = function() {
+  let appId;
+  // After enabling DriveFS, ensure the dirty files have been recovered into
+  // Downloads.
+  StepsRunner.run([
+    function() {
+      setupAndWaitUntilReady(null, RootPath.DOWNLOADS, this.next, [], []);
+    },
+    // Wait for the Recovered files directory to be in Downloads.
+    function(results) {
+      appId = results.windowId;
+
+      const expectedEntryRows = [
+        ENTRIES.neverSync.getExpectedRow(),
+        ['Recovered files from Google Drive', '--', 'Folder'],
+      ];
+      remoteCall
+          .waitForFiles(
+              appId, expectedEntryRows, {ignoreLastModifiedTime: true})
+          .then(this.next);
+    },
+    // Navigate to the recovered files directory.
+    function() {
+      return remoteCall
+          .navigateWithDirectoryTree(
+              appId, '/Recovered files from Google Drive', 'Downloads')
+          .then(this.next);
+    },
+    // Ensure it contains never-sync.txt and never-sync (1).txt.
+    function() {
+      var uniquifiedNeverSync = ENTRIES.neverSync.getExpectedRow();
+      uniquifiedNeverSync[0] = 'never-sync (1).txt';
+      const expectedEntryRows = [
+        ENTRIES.neverSync.getExpectedRow(),
+        uniquifiedNeverSync,
+      ];
+      remoteCall
+          .waitForFiles(
+              appId, expectedEntryRows, {ignoreLastModifiedTime: true})
+          .then(this.next);
+    },
+    function() {
+      checkIfNoErrorsOccured(this.next);
+    },
+  ]);
+};

ui/file_manager/integration_tests/remote_call.js

@@ -555,7 +555,7 @@ RemoteCallFilesApp.prototype.navigateWithDirectoryTree = function(
       .then(() => {
         // Entries within Drive starts with /root/ but it isn't displayed in the
         // breadcrubms used by waitUntilCurrentDirectoryIsChanged.
-        path = path.replace(/^\/root\//, '/').replace(/^\/team_drives/, '');
+        path = path.replace(/^\/root/, '').replace(/^\/team_drives/, '');
 
         // Wait until the Files app is navigated to the path.
         return this.waitUntilCurrentDirectoryIsChanged(

ui/file_manager/integration_tests/test_util.js

@@ -796,4 +796,15 @@ var ENTRIES = {
       canShare: true
     },
   }),
+
+  neverSync: new TestEntryInfo({
+    type: EntryType.FILE,
+    sourceFileName: 'text.txt',
+    targetPath: 'never-sync.txt',
+    mimeType: 'text/plain',
+    lastModifiedTime: 'Sep 4, 1998, 12:34 PM',
+    nameText: 'never-sync.txt',
+    sizeText: '51 bytes',
+    typeText: 'Plain text'
+  }),
 };