Files.app: Add a deduplication step to avoid importing duplicate media.

Add a DuplicateFinder and introduce it into the cloud import pipeline.  Note: Deduplication takes place in-place on the source files, which could be slow for MTP devices.  Profiling will be necessary to test this.  The duplicate check returns a simple yes/no answer.  Non-duplicate media is then imported as before.

Destination is now promoted to a top-level concept.  MediaImportHandler has a new function which enables clients to register new import destinations.  Each destination requires a DestinationFactory and an optional DuplicateFinder.  The import destination is selected at import time by its ID.

The Observer pattern on the ImportTask class was tweaked to support auxiliary data being passed with each observer call.  This is to allow richer updates (e.g. an entry-changed update that informs observers when a file was copied, and sends along the source and new destination).

A new importer.TestLogger had to be created and inserted into some tests in order to circumvent the running of the regular RunTimeLogger, which uses functionality that doesn't exist at test time (i.e. chrome.syncFileSystem.onFileStatusChanged)

MockEntry et al. were  augmented to do more "realistic" moves and copies, needed to test the new two-stage import.

Unit tests were cleaned up.

BUG=420680
TEST=browser_test: FileManagerJsTest.MediaImportHandlerTest

Review URL: https://codereview.chromium.org/881463003

Cr-Commit-Position: refs/heads/master@{#313503}

ui/file_manager/file_manager/background/js/background.js

@@ -90,7 +90,8 @@ function FileBrowserBackground() {
   this.mediaImportHandler =
       new importer.MediaImportHandler(
           this.progressCenter,
-          this.historyLoader);
+          this.historyLoader,
+          new importer.DriveDuplicateFinder());
 
   /**
    * Promise of string data.

ui/file_manager/file_manager/background/js/compiled_resources.gyp

@@ -29,6 +29,7 @@
           'app_window_wrapper.js',
           'device_handler.js',
           'drive_sync_handler.js',
+          'duplicate_finder.js',
           'file_operation_handler.js',
           'file_operation_manager.js',
           'file_operation_util.js',

ui/file_manager/file_manager/background/js/duplicate_finder.js

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Namespace
+var importer = importer || {};
+
+/**
+ * Interface for import deduplicators.  A duplicate finder is linked to an
+ * import destination, and will check whether files already exist in that import
+ * destination.
+ * @interface
+ */
+importer.DuplicateFinder = function() {};
+
+/**
+ * Checks whether the given file already exists in the import destination.
+ * @param {!FileEntry} entry The file entry to check.
+ * @return {!Promise<boolean>}
+ */
+importer.DuplicateFinder.prototype.checkDuplicate;
+
+/**
+ * A duplicate finder for Google Drive.
+ * @constructor
+ * @implements {importer.DuplicateFinder}
+ * @struct
+ */
+importer.DriveDuplicateFinder = function() {};
+
+/** @override */
+importer.DriveDuplicateFinder.prototype.checkDuplicate = function(entry) {
+  // TODO(kenobi): Implement content hash deduplication.
+  return Promise.resolve(false);
+};
+
+/**
+ * A duplicate finder for testing.  Allows the return value to be set.
+ * @constructor
+ * @implements {importer.DuplicateFinder}
+ * @struct
+ */
+importer.TestDuplicateFinder = function() {
+  /** @type {boolean} */
+  this.returnValue = false;
+};
+
+/** @override */
+importer.TestDuplicateFinder.prototype.checkDuplicate = function(entry) {
+  return Promise.resolve(this.returnValue);
+};

ui/file_manager/file_manager/background/js/media_import_handler.js

@@ -12,13 +12,17 @@ var importer = importer || {};
  */
 importer.ImportRunner = function() {};
 
+/**
+ * @typedef {function():(!Promise<!DirectoryEntry>)}
+ */
+importer.ImportRunner.DestinationFactory;
+
 /**
  * Imports all media identified by scanResult.
  *
  * @param {!importer.ScanResult} scanResult
- * @param {!importer.MediaImportHandler.DestinationFactory=} opt_destination A
+ * @param {!importer.ImportRunner.DestinationFactory=} opt_destination A
  *     function that returns the directory into which media will be imported.
- *     The function will be executed only when the import task actually runs.
  *
  * @return {!importer.MediaImportHandler.ImportTask} The resulting import task.
  */
@@ -33,8 +37,10 @@ importer.ImportRunner.prototype.importFromScanResult;
  *
  * @param {!ProgressCenter} progressCenter
  * @param {!importer.HistoryLoader} historyLoader
+ * @param {!importer.DuplicateFinder} duplicateFinder
  */
-importer.MediaImportHandler = function(progressCenter, historyLoader) {
+importer.MediaImportHandler =
+    function(progressCenter, historyLoader, duplicateFinder) {
   /** @private {!ProgressCenter} */
   this.progressCenter_ = progressCenter;
 
@@ -44,19 +50,16 @@ importer.MediaImportHandler = function(progressCenter, historyLoader) {
   /** @private {!importer.TaskQueue} */
   this.queue_ = new importer.TaskQueue();
 
+  /** @private {!importer.DuplicateFinder} */
+  this.duplicateFinder_ = duplicateFinder;
+
   /** @private {number} */
   this.nextTaskId_ = 0;
 };
 
-/**
- * @typedef {function():(!Promise<!DirectoryEntry>)}
- */
-importer.MediaImportHandler.DestinationFactory;
-
 /** @override */
 importer.MediaImportHandler.prototype.importFromScanResult =
     function(scanResult, opt_destination) {
-
   var destination = opt_destination ||
       importer.MediaImportHandler.defaultDestination.getImportDestination;
 
@@ -64,9 +67,10 @@ importer.MediaImportHandler.prototype.importFromScanResult =
       this.generateTaskId_(),
       this.historyLoader_,
       scanResult,
-      destination);
+      destination,
+      this.duplicateFinder_);
 
-  task.addObserver(this.onTaskProgress_.bind(this));
+  task.addObserver(this.onTaskProgress_.bind(this, task));
 
   this.queue_.queueTask(task);
 
@@ -84,12 +88,12 @@ importer.MediaImportHandler.prototype.generateTaskId_ = function() {
 /**
  * Sends updates to the ProgressCenter when an import is happening.
  *
- * @param {!importer.TaskQueue.UpdateType} updateType
  * @param {!importer.TaskQueue.Task} task
+ * @param {string} updateType
  * @private
  */
 importer.MediaImportHandler.prototype.onTaskProgress_ =
-    function(updateType, task) {
+    function(task, updateType) {
   var UpdateType = importer.TaskQueue.UpdateType;
 
   var item = this.progressCenter_.getItemById(task.taskId);
@@ -143,22 +147,28 @@ importer.MediaImportHandler.prototype.onTaskProgress_ =
  * @param {string} taskId
  * @param {!importer.HistoryLoader} historyLoader
  * @param {!importer.ScanResult} scanResult
- * @param {!importer.MediaImportHandler.DestinationFactory} destinationFactory A
+ * @param {!importer.ImportRunner.DestinationFactory} destinationFactory A
  *     function that returns the directory into which media will be imported.
+  * @param {!importer.DuplicateFinder} duplicateFinder A duplicate-finder linked
+  *     to the import destination, that will be used to deduplicate imports.
  */
 importer.MediaImportHandler.ImportTask = function(
     taskId,
     historyLoader,
     scanResult,
-    destinationFactory) {
+    destinationFactory,
+    duplicateFinder) {
 
   importer.TaskQueue.BaseTask.call(this, taskId);
   /** @private {string} */
   this.taskId_ = taskId;
 
-  /** @private {!importer.MediaImportHandler.DestinationFactory} */
+  /** @private {!importer.ImportRunner.DestinationFactory} */
   this.destinationFactory_ = destinationFactory;
 
+  /** @private {!importer.DuplicateFinder} */
+  this.deduplicator_ = duplicateFinder;
+
   /** @private {!importer.ScanResult} */
   this.scanResult_ = scanResult;
 
@@ -184,6 +194,24 @@ importer.MediaImportHandler.ImportTask = function(
   this.canceled_ = false;
 };
 
+/**
+ * Update types that are specific to ImportTask.  Clients can add Observers to
+ * ImportTask to listen for these kinds of updates.
+ * @enum {string}
+ */
+importer.MediaImportHandler.ImportTask.UpdateType = {
+  ENTRY_CHANGED: 'ENTRY_CHANGED'
+};
+
+/**
+ * Auxilliary info for ENTRY_CHANGED notifications.
+ * @typedef {{
+ *   sourceUrl: string,
+ *   destination: !Entry
+ * }}
+ */
+importer.MediaImportHandler.ImportTask.EntryChangedInfo;
+
 /** @struct */
 importer.MediaImportHandler.ImportTask.prototype = {
   /** @return {number} Number of imported bytes */
@@ -267,19 +295,47 @@ importer.MediaImportHandler.ImportTask.prototype.importTo_ =
  * @param {function()} completionCallback Called after this operation is
  *     complete.
  * @param {!FileEntry} entry The entry to import.
- * @param {number} index The entry's index in the scan results.
  * @private
  */
 importer.MediaImportHandler.ImportTask.prototype.importOne_ =
-    function(destination, completionCallback, entry, index) {
+    function(destination, completionCallback, entry) {
   if (this.canceled_) {
     this.notify(importer.TaskQueue.UpdateType.CANCELED);
     return;
   }
 
+  this.deduplicator_.checkDuplicate(entry)
+      .then(
+          /** @param {boolean} isDuplicate */
+          function(isDuplicate) {
+            if (isDuplicate) {
+              // If the given file is a duplicate, don't import it again.  Just
+              // update the progress indicator.
+              // TODO(kenobi): Update import history to mark the dupe as sync'd.
+              this.processedBytes_ += entry.size;
+              this.notify(importer.TaskQueue.UpdateType.PROGRESS);
+              return Promise.resolve();
+            } else {
+              return this.copy_(entry, destination);
+            }
+          }.bind(this))
+      .then(completionCallback);
+};
+
+/**
+ * @param {!FileEntry} entry The file to copy.
+ * @param {!DirectoryEntry} destination The destination directory.
+ * @return {!Promise<!FileEntry>} Resolves to the destination file when the copy
+ *     is complete.
+ * @private
+ */
+importer.MediaImportHandler.ImportTask.prototype.copy_ =
+    function(entry, destination) {
   // A count of the current number of processed bytes for this entry.
   var currentBytes = 0;
 
+  var resolver = new importer.Resolver();
+
   /**
    * Updates the task when the copy code reports progress.
    * @param {string} sourceUrl
@@ -297,28 +353,38 @@ importer.MediaImportHandler.ImportTask.prototype.importOne_ =
   /**
    * Updates the task when the new file has been created.
    * @param {string} sourceUrl
-   * @param {Entry} destEntry
+   * @param {Entry} destinationEntry
    * @this {importer.MediaImportHandler.ImportTask}
    */
-  var onEntryChanged = function(sourceUrl, destEntry) {
+  var onEntryChanged = function(sourceUrl, destinationEntry) {
     this.processedBytes_ -= currentBytes;
     this.processedBytes_ += entry.size;
-    this.onEntryChanged_(sourceUrl, destEntry);
+    destinationEntry.size = entry.size;
+    this.notify(
+        importer.MediaImportHandler.ImportTask.UpdateType.ENTRY_CHANGED,
+        {
+          sourceUrl: sourceUrl,
+          destination: destinationEntry
+        });
     this.notify(importer.TaskQueue.UpdateType.PROGRESS);
   };
 
-  /** @this {importer.MediaImportHandler.ImportTask} */
-  var onComplete = function() {
+  /**
+   * @param {Entry} destinationEntry The new destination entry.
+   * @this {importer.MediaImportHandler.ImportTask}
+   */
+  var onComplete = function(destinationEntry) {
     this.cancelCallback_ = null;
     this.markAsCopied_(entry, destination);
     this.notify(importer.TaskQueue.UpdateType.PROGRESS);
-    completionCallback();
+    resolver.resolve(destinationEntry);
   };
 
   /** @this {importer.MediaImportHandler.ImportTask} */
   var onError = function(error) {
     this.cancelCallback_ = null;
     this.onError_(error);
+    resolver.reject(error);
   };
 
   this.cancelCallback_ = fileOperationUtil.copyTo(
@@ -329,6 +395,8 @@ importer.MediaImportHandler.ImportTask.prototype.importOne_ =
       onProgress.bind(this),
       onComplete.bind(this),
       onError.bind(this));
+
+  return resolver.promise;
 };
 
 /**
@@ -374,7 +442,7 @@ importer.MediaImportHandler.ImportTask.prototype.onError_ = function(error) {
 
 /**
  * Namespace for a default import destination factory. The
- * defaultDestionation.getImportDestination function creates and returns the
+ * defaultDestination.getImportDestination function creates and returns the
  * directory /photos/YYYY-MM-DD in the user's Google Drive.  YYYY-MM-DD is the
  * current date.
  */

ui/file_manager/file_manager/background/js/media_import_handler_unittest.html

@@ -18,6 +18,7 @@
   <script src="../../common/js/util.js"></script>
   <script src="../../common/js/volume_manager_common.js"></script>
   <script src="../../common/js/importer_common.js"></script>
+  <script src="../../common/js/test_importer_common.js"></script>
   <script src="../../common/js/progress_center_common.js"></script>
 
   <script src="test_import_history.js"></script>
@@ -26,6 +27,8 @@
   <script src="mock_volume_manager.js"></script>
   <script src="task_queue.js"></script>
   <script src="media_import_handler.js"></script>
+  <script src="duplicate_finder.js"></script>
+  <script src="test_duplicate_finder.js"></script>
   <script src="volume_manager.js"></script>
   <script src="file_operation_util.js"></script>
 

ui/file_manager/file_manager/background/js/media_import_handler_unittest.js

@@ -17,12 +17,15 @@ var importHistory;
 /** @type {!VolumeInfo} */
 var drive;
 
-/** @type {!MockFileSystem} */
-var fileSystem;
-
 /** @type {!MockCopyTo} */
 var mockCopier;
 
+/** @type {!MockFileSystem} */
+var destinationFileSystem;
+
+/** @type {!importer.DuplicateFinder} */
+var duplicateFinder;
+
 // Set up string assets.
 loadTimeData.data = {
   CLOUD_IMPORT_ITEMS_REMAINING: '',
@@ -31,6 +34,8 @@ loadTimeData.data = {
 };
 
 function setUp() {
+  importer.setupTestLogger();
+
   progressCenter = new MockProgressCenter();
 
   // Replaces fileOperationUtil.copyTo with test function.
@@ -49,9 +54,11 @@ function setUp() {
 
   importHistory = new importer.TestImportHistory();
   mediaScanner = new TestMediaScanner();
+  destinationFileSystem = new MockFileSystem(destinationFactory);
+  duplicateFinder = new importer.TestDuplicateFinder();
+
   mediaImporter = new importer.MediaImportHandler(
-      progressCenter,
-      importHistory);
+      progressCenter, importHistory, duplicateFinder);
 }
 
 function testImportMedia(callback) {
@@ -59,16 +66,14 @@ function testImportMedia(callback) {
     '/DCIM/photos0/IMG00001.jpg',
     '/DCIM/photos0/IMG00002.jpg',
     '/DCIM/photos0/IMG00003.jpg',
-    '/DCIM/photos1/IMG00001.jpg',
-    '/DCIM/photos1/IMG00002.jpg',
-    '/DCIM/photos1/IMG00003.jpg'
+    '/DCIM/photos1/IMG00004.jpg',
+    '/DCIM/photos1/IMG00005.jpg',
+    '/DCIM/photos1/IMG00006.jpg'
   ]);
 
-  var destinationFileSystem = new MockFileSystem('fake-destination');
-  var destination = function() { return destinationFileSystem.root; };
-
   var scanResult = new TestScanResult(media);
-  var importTask = mediaImporter.importFromScanResult(scanResult, destination);
+  var importTask =
+      mediaImporter.importFromScanResult(scanResult, destinationFactory);
   var whenImportDone = new Promise(
       function(resolve, reject) {
         importTask.addObserver(
@@ -91,15 +96,8 @@ function testImportMedia(callback) {
   reportPromise(
       whenImportDone.then(
         function() {
-          assertEquals(media.length, mockCopier.copiedFiles.length);
-          mockCopier.copiedFiles.forEach(
-            /** @param {!MockCopyTo.CopyInfo} copy */
-            function(copy) {
-              // Verify the copied file is one of the expected files.
-              assertTrue(media.indexOf(copy.source) >= 0);
-              // Verify that the files are being copied to the right locations.
-              assertEquals(destination(), copy.destination);
-            });
+          var copiedEntries = destinationFileSystem.root.getAllChildren();
+          assertEquals(media.length, copiedEntries.length);
         }),
       callback);
 
@@ -112,11 +110,9 @@ function testUpdatesHistoryAfterImport(callback) {
     '/DCIM/photos1/IMG00003.jpg'
   ]);
 
-  var destinationFileSystem = new MockFileSystem('fake-destination');
-  var destination = function() { return destinationFileSystem.root; };
-
   var scanResult = new TestScanResult(entries);
-  var importTask = mediaImporter.importFromScanResult(scanResult, destination);
+  var importTask =
+      mediaImporter.importFromScanResult(scanResult, destinationFactory);
   var whenImportDone = new Promise(
       function(resolve, reject) {
         importTask.addObserver(
@@ -157,19 +153,17 @@ function testImportCancellation(callback) {
     '/DCIM/photos0/IMG00001.jpg',
     '/DCIM/photos0/IMG00002.jpg',
     '/DCIM/photos0/IMG00003.jpg',
-    '/DCIM/photos1/IMG00001.jpg',
-    '/DCIM/photos1/IMG00002.jpg',
-    '/DCIM/photos1/IMG00003.jpg'
+    '/DCIM/photos1/IMG00004.jpg',
+    '/DCIM/photos1/IMG00005.jpg',
+    '/DCIM/photos1/IMG00006.jpg'
   ]);
 
   /** @const {number} */
   var EXPECTED_COPY_COUNT = 3;
 
-  var destinationFileSystem = new MockFileSystem('fake-destination');
-  var destination = function() { return destinationFileSystem.root; };
-
   var scanResult = new TestScanResult(media);
-  var importTask = mediaImporter.importFromScanResult(scanResult, destination);
+  var importTask =
+      mediaImporter.importFromScanResult(scanResult, destinationFactory);
   var whenImportCancelled = new Promise(
       function(resolve, reject) {
         importTask.addObserver(
@@ -184,32 +178,26 @@ function testImportCancellation(callback) {
             });
       });
 
+  // Simulate cancellation after the expected number of copies is done.
+  var copyCount = 0;
+  importTask.addObserver(function(updateType) {
+    if (updateType ===
+        importer.MediaImportHandler.ImportTask.UpdateType.ENTRY_CHANGED) {
+      copyCount++;
+      if (copyCount === EXPECTED_COPY_COUNT) {
+        importTask.requestCancel();
+      }
+    }
+  });
+
   reportPromise(
       whenImportCancelled.then(
         function() {
-          assertEquals(EXPECTED_COPY_COUNT, mockCopier.copiedFiles.length);
-          mockCopier.copiedFiles.forEach(
-            /** @param {!MockCopyTo.CopyInfo} copy */
-            function(copy) {
-              // Verify the copied file is one of the expected files.
-              assertTrue(media.indexOf(copy.source) >= 0);
-              // Verify that the files are being copied to the right locations.
-              assertEquals(destination(), copy.destination);
-            });
+          var copiedEntries = destinationFileSystem.root.getAllChildren();
+          assertEquals(EXPECTED_COPY_COUNT, copiedEntries.length);
         }),
       callback);
 
-  // Simulate cancellation after the expected number of copies is done.
-  var copyCount = 0;
-  mockCopier.onCopy(
-      /** @param {!MockCopyTo.CopyInfo} copy */
-      function(copy) {
-        mockCopier.doCopy(copy);
-        if (++copyCount === EXPECTED_COPY_COUNT) {
-          importTask.requestCancel();
-        }
-      });
-
   scanResult.finalize();
 }
 
@@ -219,7 +207,7 @@ function testImportCancellation(callback) {
  */
 function setupFileSystem(fileNames) {
   // Set up a filesystem with some files.
-  fileSystem = new MockFileSystem('fake-media-volume');
+  var fileSystem = new MockFileSystem('fake-media-volume');
   fileSystem.populate(fileNames);
 
   return fileNames.map(
@@ -228,6 +216,10 @@ function setupFileSystem(fileNames) {
       });
 }
 
+/** @return {!DirectoryEntry} The destination root, for testing. */
+function destinationFactory() {
+  return destinationFileSystem.root;
+}
 
 /**
  * Replaces fileOperationUtil.copyTo with some mock functionality for testing.
@@ -244,9 +236,6 @@ function MockCopyTo() {
   this.progressCallback_ = null;
   this.successCallback_ = null;
   this.errorCallback_ = null;
-
-  // Default copy callback just does the copy.
-  this.copyCallback_ = this.doCopy.bind(this);
 }
 
 /**
@@ -276,31 +265,18 @@ MockCopyTo.prototype.copyTo_ = function(source, parent, newName,
   this.successCallback_ = successCallback;
   this.errorCallback_ = errorCallback;
 
-  this.copyCallback_({
+  // Log the copy, then copy the file.
+  this.copiedFiles.push({
     source: source,
     destination: parent,
     newName: newName
   });
-};
-
-/**
- * Set a callback to be called whenever #copyTo_ is called.  This can be used to
- * simulate errors, etc, during copying.  The default copy callback just calls
- * #doCopy.
- * @param {!function(!MockCopyTo.CopyInfo)} copyCallback
- */
-MockCopyTo.prototype.onCopy = function(copyCallback) {
-  this.copyCallback_ = copyCallback;
-};
-
-/**
- * Completes the given copy.  Call this in the callback passed to #onCopy, to
- * simulate the current copy operation completing successfully.
- * @param {!MockCopyTo.CopyInfo} copy
- */
-MockCopyTo.prototype.doCopy = function(copy) {
-  this.copiedFiles.push(copy);
-  this.entryChangedCallback_(copy.source.toURL(),
-      copy.destination);
-  this.successCallback_();
+  source.copyTo(
+      parent,
+      newName,
+      function(newEntry) {
+        this.entryChangedCallback_(source.toURL(), parent);
+        this.successCallback_(newEntry);
+      }.bind(this),
+      this.errorCallback_.bind(this));
 };

ui/file_manager/file_manager/background/js/task_queue.js

@@ -52,7 +52,7 @@ importer.TaskQueue.prototype.queueTask = function(task) {
   // The Tasks that are pushed onto the queue aren't required to be inherently
   // asynchronous.  This code force task execution to occur asynchronously.
   Promise.resolve().then(function() {
-    task.addObserver(this.onTaskUpdate_.bind(this));
+    task.addObserver(this.onTaskUpdate_.bind(this, task));
     this.tasks_.push(task);
     // If more than one task is queued, then the queue is already running.
     if (this.tasks_.length === 1) {
@@ -91,11 +91,11 @@ importer.TaskQueue.prototype.setIdleCallback = function(callback) {
 /**
  * Sends out notifications when a task updates.  This is meant to be called by
  * the running tasks owned by this queue.
- * @param {!importer.TaskQueue.UpdateType} updateType
  * @param {!importer.TaskQueue.Task} task
+ * @param {!importer.TaskQueue.UpdateType} updateType
  * @private
  */
-importer.TaskQueue.prototype.onTaskUpdate_ = function(updateType, task) {
+importer.TaskQueue.prototype.onTaskUpdate_ = function(task, updateType) {
   // Send a task update to clients.
   this.updateCallbacks_.forEach(function(callback) {
     callback.call(null, updateType, task);
@@ -146,7 +146,13 @@ importer.TaskQueue.prototype.runPending_ = function() {
 importer.TaskQueue.Task = function() {};
 
 /**
- * @typedef {function(!importer.TaskQueue.UpdateType, !importer.TaskQueue.Task)}
+ * A callback that is triggered whenever an update is reported on the observed
+ * task.  The first argument is a string specifying the type of the update.
+ * Standard values used by all tasks are enumerated in
+ * importer.TaskQueue.UpdateType, but child classes may add supplementary update
+ * types of their own.  The second argument is an Object containing
+ * supplementary information pertaining to the update.
+ * @typedef {function(string, Object=)}
  */
 importer.TaskQueue.Task.Observer;
 
@@ -196,13 +202,14 @@ importer.TaskQueue.BaseTask.prototype.addObserver = function(observer) {
 importer.TaskQueue.BaseTask.prototype.run = function() {};
 
 /**
- * @param {!importer.TaskQueue.UpdateType} updateType
+ * @param {string} updateType
+ * @param {Object=} opt_data
  * @protected
  */
-importer.TaskQueue.BaseTask.prototype.notify = function(updateType) {
+importer.TaskQueue.BaseTask.prototype.notify = function(updateType, opt_data) {
   this.observers_.forEach(
       /** @param {!importer.TaskQueue.Task.Observer} callback */
       function(callback) {
-        callback.call(null, updateType, this);
+        callback.call(null, updateType, opt_data);
       }.bind(this));
 };

ui/file_manager/file_manager/background/js/test_duplicate_finder.js

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Namespace
+var importer = importer || {};
+
+/**
+ * A duplicate finder for testing.  Allows the return value to be set.
+ * @constructor
+ * @implements {importer.DuplicateFinder}
+ * @struct
+ */
+importer.TestDuplicateFinder = function() {
+  /** @type {boolean} */
+  this.returnValue = false;
+};
+
+/** @override */
+importer.TestDuplicateFinder.prototype.checkDuplicate = function(entry) {
+  return Promise.resolve(this.returnValue);
+};

ui/file_manager/file_manager/common/js/importer_common.js

@@ -578,7 +578,7 @@ importer.RuntimeLogger.prototype.writeLine_ = function(type, line, writer) {
       }.bind(this));
 };
 
-/** @type {importer.Logger} */
+/** @private {importer.Logger} */
 importer.logger_ = null;
 
 /**

ui/file_manager/file_manager/common/js/mock_entry.js

@@ -83,11 +83,12 @@ MockFileSystem.prototype.findChildren_ = function(directory) {
 /**
  * Base class of mock entries.
  *
- * @param {TestFileSystem} filesystem File system where the entry is localed.
+ * @param {MockFileSystem} filesystem File system where the entry is localed.
  * @param {string} fullPath Full path of the entry.
  * @constructor
  */
 function MockEntry(filesystem, fullPath) {
+  filesystem.entries[fullPath] = this;
   this.filesystem = filesystem;
   this.fullPath = fullPath;
 }
@@ -139,10 +140,27 @@ MockEntry.prototype.getParent = function(
 MockEntry.prototype.moveTo = function(parent, opt_newName, onSuccess, onError) {
   Promise.resolve().then(function() {
     this.filesystem.entries[this.fullPath] = null;
-    return this.clone(joinPath(parent.fullPath, opt_newName || this.name));
+    return this.clone(
+        joinPath(parent.fullPath, opt_newName || this.name),
+        parent.filesystem);
   }.bind(this)).then(onSuccess, onError);
 };
 
+/**
+ * @param {MockDirectoryEntry} parent
+ * @param {string=} opt_newName
+ * @param {function(!MockEntry)} successCallback
+ * @param {function} errorCallback
+ */
+MockEntry.prototype.copyTo =
+    function(parent, opt_newName, successCallback, errorCallback) {
+  Promise.resolve().then(function() {
+    return this.clone(
+        joinPath(parent.fullPath, opt_newName || this.name),
+        parent.filesystem);
+  }.bind(this)).then(successCallback, errorCallback);
+};
+
 /**
  * Removes the entry.
  *
@@ -159,9 +177,10 @@ MockEntry.prototype.remove = function(onSuccess, onError) {
  * Clones the entry with the new fullpath.
  *
  * @param {string} fullpath New fullpath.
+ * @param {FileSystem} opt_filesystem New file system
  * @return {MockEntry} Cloned entry.
  */
-MockEntry.prototype.clone = function(fullpath) {
+MockEntry.prototype.clone = function(fullpath, opt_filesystem) {
   throw new Error('Not implemented.');
 };
 
@@ -203,8 +222,9 @@ MockFileEntry.prototype.getMetadata = function(onSuccess, onError) {
 /**
  * @override
  */
-MockFileEntry.prototype.clone = function(path) {
-  return new MockFileEntry(this.filesystem, path, this.metadata);
+MockFileEntry.prototype.clone = function(path, opt_filesystem) {
+  return new MockFileEntry(
+      opt_filesystem || this.filesystem, path, this.metadata);
 };
 
 /**
@@ -228,11 +248,19 @@ MockDirectoryEntry.prototype = {
 /**
  * @override
  */
-MockDirectoryEntry.prototype.clone = function(path) {
-  return new MockDirectoryEntry(this.filesystem, path);
+MockDirectoryEntry.prototype.clone = function(path, opt_filesystem) {
+  return new MockDirectoryEntry(opt_filesystem || this.filesystem, path);
 };
 
 /**
+ * Returns all children of the supplied directoryEntry.
+ * @return {!Array.<!Entry>}
+ */
+MockDirectoryEntry.prototype.getAllChildren = function() {
+  return this.filesystem.findChildren_(this);
+};
+
+  /**
  * Returns a file under the directory.
  *
  * @param {string} path Path.

ui/file_manager/file_manager/common/js/test_importer_common.js

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Shared cloud importer namespace
+var importer = importer || {};
+
+/**
+ * Sets up a logger for use in unit tests.  The test logger doesn't attempt to
+ * access chrome's sync file system.  Call this during setUp.
+ */
+importer.setupTestLogger = function() {
+  importer.logger_ = new importer.TestLogger();
+};
+
+/**
+ * A {@code importer.Logger} for testing.  Just sends output to the console.
+ *
+ * @constructor
+ * @implements {importer.Logger}
+ * @struct
+ * @final
+ */
+importer.TestLogger = function() {};
+
+/** @override */
+importer.TestLogger.prototype.info = function(content) {
+  console.log(content);
+};
+
+/** @override */
+importer.TestLogger.prototype.error = function(content) {
+  console.error(content);
+  console.error(new Error('Error stack').stack);
+};
+
+/** @override */
+importer.TestLogger.prototype.catcher = function(context) {
+  return function(error) {
+    this.error('Caught promise error. Context: ' + context +
+        ' Error: ' + error.message);
+    console.error(error.stack);
+  }.bind(this);
+};

ui/file_manager/file_manager/foreground/js/compiled_resources.gyp

@@ -52,6 +52,7 @@
           '../../background/js/file_operation_handler.js',
           '../../background/js/device_handler.js',
           '../../background/js/drive_sync_handler.js',
+          '../../background/js/duplicate_finder.js',
           '../../background/js/volume_manager.js',
           '../../background/js/progress_center.js',
           '../../background/js/app_window_wrapper.js',

ui/file_manager/file_manager/foreground/js/import_controller_unittest.js

@@ -23,25 +23,22 @@ var destinationVolume;
 /** @type {!TestCallRecorder} */
 var commandUpdateRecorder;
 
-function setUp() {
-  // Set up string assets.
-  loadTimeData.data = {
-    CLOUD_IMPORT_BUTTON_LABEL: 'Import it!',
-    CLOUD_IMPORT_INSUFFICIENT_SPACE_BUTTON_LABEL: 'Not enough space!',
-    CLOUD_IMPORT_SCANNING_BUTTON_LABEL: 'Scanning... ...!'
-  };
+// Set up string assets.
+loadTimeData.data = {
+  CLOUD_IMPORT_BUTTON_LABEL: 'Import it!',
+  CLOUD_IMPORT_EMPTY_SCAN_BUTTON_LABEL: 'No new media',
+  CLOUD_IMPORT_INSUFFICIENT_SPACE_BUTTON_LABEL: 'Not enough space!',
+  CLOUD_IMPORT_SCANNING_BUTTON_LABEL: 'Scanning... ...!',
+  DRIVE_DIRECTORY_LABEL: 'My Drive',
+  DOWNLOADS_DIRECTORY_LABEL: 'Downloads'
+};
 
+function setUp() {
   // Stub out metrics support.
   metrics = {
     recordEnum: function() {}
   };
 
-  // Set up string assets.
-  loadTimeData.data = {
-    DRIVE_DIRECTORY_LABEL: 'My Drive',
-    DOWNLOADS_DIRECTORY_LABEL: 'Downloads'
-  };
-
   commandUpdateRecorder = new TestCallRecorder();
 
   volumeManager = new MockVolumeManager();
@@ -175,8 +172,9 @@ function testGetCommandUpdate_CanExecuteAfterScanIsFinalized() {
       ],
       '/DCIM');
 
+  var fileSystem = new MockFileSystem('testFs');
   mediaScanner.fileEntries.push(
-      new MockFileEntry(null, '/DCIM/photos0/IMG00001.jpg', {size: 0}));
+      new MockFileEntry(fileSystem, '/DCIM/photos0/IMG00001.jpg', {size: 0}));
   controller.getCommandUpdate();
   mediaScanner.finalizeScans();
 

ui/file_manager/file_manager/foreground/js/metadata/metadata_cache_unittest.html

@@ -5,6 +5,7 @@
   -->
 <script src="../../../../../../ui/webui/resources/js/assert.js"></script>
 <script src="../../../common/js/mock_entry.js"></script>
+<script src="../../../common/js/unittest_util.js"></script>
 <script src="metadata_cache.js"></script>
 
 <script src="metadata_cache_unittest.js"></script>

ui/file_manager/file_manager/foreground/js/metadata/metadata_cache_unittest.js

@@ -2,6 +2,13 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+/** @type {!MockFileSystem} */
+var fileSystem;
+
+function setUp() {
+  fileSystem = new MockFileSystem('volumeId');
+}
+
 /**
  * Mock of MetadataProvider.
  *
@@ -59,22 +66,6 @@ function getLatest(metadataCache, entries, type) {
       metadataCache, entries, type));
 }
 
-/**
- * Invokes a callback function depending on the result of promise.
- *
- * @param {Promise} promise Promise.
- * @param {function(boolean)} calllback Callback function. True is passed if the
- * test failed.
- */
-function reportPromise(promise, callback) {
-  promise.then(function() {
-    callback(/* error */ false);
-  }, function(error) {
-    console.error(error.stack || error);
-    callback(/* error */ true);
-  });
-}
-
 /**
  * Confirms metadata is cached for the same entry.
  *
@@ -84,7 +75,7 @@ function reportPromise(promise, callback) {
 function testCache(callback) {
   var provider = new MockProvider('instrument');
   var metadataCache = new MetadataCache([provider]);
-  var entry = new MockFileEntry('volumeId', '/music.txt');
+  var entry = new MockFileEntry(fileSystem, '/music.txt');
 
   var metadataFromProviderPromise =
       getMetadata(metadataCache, [entry], 'instrument');
@@ -119,8 +110,8 @@ function testNoCacheForDifferentEntries(callback) {
   var provider = new MockProvider('instrument');
   var metadataCache = new MetadataCache([provider]);
 
-  var entry1 = new MockFileEntry('volumeId', '/music1.txt');
-  var entry2 = new MockFileEntry('volumeId', '/music2.txt');
+  var entry1 = new MockFileEntry(fileSystem, '/music1.txt');
+  var entry2 = new MockFileEntry(fileSystem, '/music2.txt');
 
   var entry1MetadataPromise =
       getMetadata(metadataCache, [entry1], 'instrument');
@@ -155,7 +146,7 @@ function testNoCacheForDifferentTypes(callback) {
   ];
   var metadataCache = new MetadataCache(providers);
 
-  var entry = new MockFileEntry('volumeId', '/music.txt');
+  var entry = new MockFileEntry(fileSystem, '/music.txt');
   var instrumentMedatataPromise =
       getMetadata(metadataCache, [entry], 'instrument');
   var beatMetadataPromise = getMetadata(metadataCache, [entry], 'beat');
@@ -192,7 +183,7 @@ function testGetDiffrentTypesInVeriousOrders(callback) {
     });
   };
 
-  var entry1 = new MockFileEntry('volumeId', '/music1.txt');
+  var entry1 = new MockFileEntry(fileSystem, '/music1.txt');
   var promise1 = Promise.all([
     getAndCheckMetadata(entry1, 'instrument', {name: 'banjo'}),
     getAndCheckMetadata(entry1, 'beat', {number: 2}),
@@ -204,7 +195,7 @@ function testGetDiffrentTypesInVeriousOrders(callback) {
   assertEquals(1, providers[0].callbackPool.length);
   assertEquals(1, providers[1].callbackPool.length);
 
-  var entry2 = new MockFileEntry('volumeId', '/music2.txt');
+  var entry2 = new MockFileEntry(fileSystem, '/music2.txt');
   var promise2 = Promise.all([
     getAndCheckMetadata(entry2, 'instrument', {name: 'banjo'}),
     getAndCheckMetadata(
@@ -216,7 +207,7 @@ function testGetDiffrentTypesInVeriousOrders(callback) {
   assertEquals(2, providers[0].callbackPool.length);
   assertEquals(2, providers[1].callbackPool.length);
 
-  var entry3 = new MockFileEntry('volumeId', '/music3.txt');
+  var entry3 = new MockFileEntry(fileSystem, '/music3.txt');
   var promise3 = Promise.all([
     getAndCheckMetadata(
         entry3,
@@ -250,7 +241,7 @@ function testGetCached(callback) {
   var provider = new MockProvider('instrument');
   var metadataCache = new MetadataCache([provider]);
 
-  var entry = new MockFileEntry('volumeId', '/music.txt');
+  var entry = new MockFileEntry(fileSystem, '/music.txt');
 
   // Check the cache does exist before calling getMetadata.
   assertEquals(null, metadataCache.getCached(entry, 'instrument'));
@@ -278,7 +269,7 @@ function testGetCached(callback) {
 function testGetLatest(callback) {
   var provider = new MockProvider('instrument');
   var metadataCache = new MetadataCache([provider]);
-  var entry = new MockFileEntry('volumeId', '/music.txt');
+  var entry = new MockFileEntry(fileSystem, '/music.txt');
 
   var promise = getLatest(metadataCache, [entry], 'instrument');
   assertEquals(1, provider.callbackPool.length);
@@ -298,7 +289,7 @@ function testGetLatest(callback) {
 function testGetLatestToIgnoreCache(callback) {
   var provider = new MockProvider('instrument');
   var metadataCache = new MetadataCache([provider]);
-  var entry = new MockFileEntry('volumeId', '/music.txt');
+  var entry = new MockFileEntry(fileSystem, '/music.txt');
 
   var promise1 = getMetadata(metadataCache, [entry], 'instrument');
   assertEquals(1, provider.callbackPool.length);
@@ -329,7 +320,7 @@ function testGetLatestToIgnoreCache(callback) {
 function testGetLatestAndPreviousCall(callback) {
   var provider = new MockProvider('instrument');
   var metadataCache = new MetadataCache([provider]);
-  var entry = new MockFileEntry('volumeId', '/music.txt');
+  var entry = new MockFileEntry(fileSystem, '/music.txt');
 
   var promise1 = getMetadata(metadataCache, [entry], 'instrument');
   assertEquals(1, provider.callbackPool.length);
@@ -359,7 +350,7 @@ function testClear(callback) {
     new MockProvider('beat')
   ];
   var metadataCache = new MetadataCache(providers);
-  var entry = new MockFileEntry('volumeId', '/music.txt');
+  var entry = new MockFileEntry(fileSystem, '/music.txt');
 
   var promise1 = getMetadata(metadataCache, [entry], 'instrument');
   var promise2 = getMetadata(metadataCache, [entry], 'beat');
@@ -401,7 +392,7 @@ function testAddObserver() {
   var metadataCache = new MetadataCache(providers);
 
   var directoryEntry = new MockFileEntry(
-      'volumeId',
+      fileSystem,
       '/mu\\^$.*.+?|&{}[si]()<>cs');
   var observerCalls = [];
   var observerCallback = function(entries, properties) {
@@ -411,7 +402,7 @@ function testAddObserver() {
   metadataCache.addObserver(directoryEntry, MetadataCache.CHILDREN,
       'filesystem', observerCallback);
 
-  var fileEntry1 = new MockFileEntry('volumeId',
+  var fileEntry1 = new MockFileEntry(fileSystem,
       '/mu\\^$.*.+?|&{}[si]()<>cs/foo.mp3');
   var fileEntry1URL = fileEntry1.toURL();
   metadataCache.set(fileEntry1, 'filesystem', 'test1');
@@ -419,7 +410,7 @@ function testAddObserver() {
   assertArrayEquals([fileEntry1], observerCalls[0].entries);
   assertEquals('test1', observerCalls[0].properties[fileEntry1URL]);
 
-  var fileEntry2 = new MockFileEntry('volumeId',
+  var fileEntry2 = new MockFileEntry(fileSystem,
       '/mu\\^$.*.+?|&{}[si]()<>cs/f.[o]o.mp3');
   var fileEntry2URL = fileEntry2.toURL();
   metadataCache.set(fileEntry2, 'filesystem', 'test2');
@@ -428,14 +419,14 @@ function testAddObserver() {
   assertEquals('test2', observerCalls[1].properties[fileEntry2URL]);
 
   // Descendant case does not invoke the observer.
-  var fileEntry3 = new MockFileEntry('volumeId',
+  var fileEntry3 = new MockFileEntry(fileSystem,
       '/mu\\^$.*.+?|&{}[si]()<>cs/foo/bar.mp3');
   metadataCache.set(fileEntry3, 'filesystem', 'test3');
   assertEquals(2, observerCalls.length);
 
   // This case does not invoke the observer.
   // (This is a case which matches when regexp special chars are not escaped).
-  var fileEntry4 = new MockFileEntry('volumeId', '/&{}i<>cs/foo.mp3');
+  var fileEntry4 = new MockFileEntry(fileSystem, '/&{}i<>cs/foo.mp3');
   metadataCache.set(fileEntry4);
   assertEquals(2, observerCalls.length);
 }
@@ -444,7 +435,6 @@ function testAddObserver() {
  * Tests content provider.
  */
 function testContentProvider(callback) {
-  var fileSystem = new MockFileSystem('volumeId');
   var entry = new MockFileEntry(fileSystem, '/sample.txt');
   var metadataCache = new MetadataCache([new ContentProvider({
     start: function() {},

ui/file_manager/file_manager/foreground/js/navigation_list_model_unittest.js

@@ -2,21 +2,31 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-function setUp() {
-  // Set up string assets.
-  loadTimeData.data = {
-    DRIVE_DIRECTORY_LABEL: 'My Drive',
-    DOWNLOADS_DIRECTORY_LABEL: 'Downloads'
-  };
+/** @type {!MockFileSystem} Simulate the drive file system. */
+var drive;
+
+/** @type {!MockFileSystem} Simulate a removable volume. */
+var hoge;
+
+// Set up string assets.
+loadTimeData.data = {
+  DRIVE_DIRECTORY_LABEL: 'My Drive',
+  DOWNLOADS_DIRECTORY_LABEL: 'Downloads'
+};
 
+function setUp() {
   // Override VolumeInfo.prototype.resolveDisplayRoot.
   VolumeInfo.prototype.resolveDisplayRoot = function() {};
+
+  drive = new MockFileSystem('drive');
+  hoge = new MockFileSystem('removable:hoge');
+
 }
 
 function testModel() {
   var volumeManager = new MockVolumeManagerWrapper();
   var shortcutListModel = new MockFolderShortcutDataModel(
-      [new MockFileEntry('drive', '/root/shortcut')]);
+      [new MockFileEntry(drive, '/root/shortcut')]);
   var model = new NavigationListModel(volumeManager, shortcutListModel);
 
   assertEquals(3, model.length);
@@ -28,18 +38,18 @@ function testModel() {
 function testAddAndRemoveShortcuts() {
   var volumeManager = new MockVolumeManagerWrapper();
   var shortcutListModel = new MockFolderShortcutDataModel(
-      [new MockFileEntry('drive', '/root/shortcut')]);
+      [new MockFileEntry(drive, '/root/shortcut')]);
   var model = new NavigationListModel(volumeManager, shortcutListModel);
 
   assertEquals(3, model.length);
 
   // Add a shortcut at the tail.
-  shortcutListModel.splice(1, 0, new MockFileEntry('drive', '/root/shortcut2'));
+  shortcutListModel.splice(1, 0, new MockFileEntry(drive, '/root/shortcut2'));
   assertEquals(4, model.length);
   assertEquals('/root/shortcut2', model.item(3).entry.fullPath);
 
   // Add a shortcut at the head.
-  shortcutListModel.splice(0, 0, new MockFileEntry('drive', '/root/hoge'));
+  shortcutListModel.splice(0, 0, new MockFileEntry(drive, '/root/hoge'));
   assertEquals(5, model.length);
   assertEquals('/root/hoge', model.item(2).entry.fullPath);
   assertEquals('/root/shortcut', model.item(3).entry.fullPath);
@@ -60,7 +70,7 @@ function testAddAndRemoveShortcuts() {
 function testAddAndRemoveVolumes() {
   var volumeManager = new MockVolumeManagerWrapper();
   var shortcutListModel = new MockFolderShortcutDataModel(
-      [new MockFileEntry('drive', '/root/shortcut')]);
+      [new MockFileEntry(drive, '/root/shortcut')]);
   var model = new NavigationListModel(volumeManager, shortcutListModel);
 
   assertEquals(3, model.length);
@@ -88,7 +98,7 @@ function testAddAndRemoveVolumes() {
 
   // A shortcut is created on the 'hoge' volume.
   shortcutListModel.splice(
-      1, 0, new MockFileEntry('removable:hoge', '/shortcut2'));
+      1, 0, new MockFileEntry(hoge, '/shortcut2'));
   assertEquals(6, model.length);
   assertEquals('drive', model.item(0).volumeInfo.volumeId);
   assertEquals('downloads', model.item(1).volumeInfo.volumeId);

ui/file_manager/file_manager/foreground/js/ui/banners_unittest.js

@@ -47,7 +47,7 @@ function setUp() {
   mtpDcimEntry = new MockDirectoryEntry(mtpVolumeInfo.fileSystem, '/DCIM');
   downloadsEntry = new MockDirectoryEntry(
       volumeManager.getCurrentProfileVolumeInfo(
-          VolumeManagerCommon.VolumeType.DOWNLOADS),
+          VolumeManagerCommon.VolumeType.DOWNLOADS).fileSystem,
       '/hello-world');
 
   cloudImportBanner = new CloudImportBanner(directoryModel, volumeManager);

ui/file_manager/file_manager/manifest.json

@@ -179,6 +179,7 @@
         "background/js/background_base.js",
         "background/js/device_handler.js",
         "background/js/drive_sync_handler.js",
+        "background/js/duplicate_finder.js",
         "background/js/file_operation_handler.js",
         "background/js/file_operation_manager.js",
         "background/js/file_operation_util.js",

ui/file_manager/file_manager_resources.grd

@@ -30,6 +30,7 @@
       <include name="IDR_FILE_MANAGER_DEVICE_BACKGROUND_BASE_JS" file="file_manager/background/js/background_base.js" flattenhtml="false" type="BINDATA" />
       <include name="IDR_FILE_MANAGER_DEVICE_HANDLER_JS" file="file_manager/background/js/device_handler.js" flattenhtml="false" type="BINDATA" />
       <include name="IDR_FILE_MANAGER_DRIVE_SYNC_HANDLER_JS" file="file_manager/background/js/drive_sync_handler.js" flattenhtml="false" type="BINDATA" />
+      <include name="IDR_FILE_MANAGER_DUPLICATE_FINDER_JS" file="file_manager/background/js/duplicate_finder.js" flattenhtml="false" type="BINDATA" />
       <include name="IDR_FILE_MANAGER_FILE_OPERATION_HANDLER_JS" file="file_manager/background/js/file_operation_handler.js" flattenhtml="false" type="BINDATA" />
       <include name="IDR_FILE_MANAGER_FILE_OPERATION_MANAGER_JS" file="file_manager/background/js/file_operation_manager.js" flattenhtml="false" type="BINDATA" />
       <include name="IDR_FILE_MANAGER_FILE_OPERATION_UTIL_JS" file="file_manager/background/js/file_operation_util.js" flattenhtml="false" type="BINDATA" />