DevTools: do not index excluded folders

This patch starts skipping indexing for excluded folders.

R=pfeldman, dgozman, caseq

Change-Id: I862b45b56afe8798d9b54772014ffe8002f12007
Reviewed-on: https://chromium-review.googlesource.com/972579
Commit-Queue: Andrey Lushnikov <lushnikov@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Pavel Feldman <pfeldman@chromium.org>
Cr-Commit-Position: refs/heads/master@{#544794}

chrome/browser/devtools/devtools_embedder_message_dispatcher.h

@@ -53,7 +53,8 @@ class DevToolsEmbedderMessageDispatcher {
     virtual void UpgradeDraggedFileSystemPermissions(
         const std::string& file_system_url) = 0;
     virtual void IndexPath(int index_request_id,
-                           const std::string& file_system_path) = 0;
+                           const std::string& file_system_path,
+                           const std::string& excluded_folders) = 0;
     virtual void StopIndexing(int index_request_id) = 0;
     virtual void LoadNetworkResource(const DispatchCallback& callback,
                                      const std::string& url,

chrome/browser/devtools/devtools_file_system_indexer.cc

@@ -267,10 +267,12 @@ typedef Callback<void(bool, const vector<bool>&)> IndexerCallback;
 
 DevToolsFileSystemIndexer::FileSystemIndexingJob::FileSystemIndexingJob(
     const FilePath& file_system_path,
+    const std::vector<base::FilePath>& excluded_folders,
     const TotalWorkCallback& total_work_callback,
     const WorkedCallback& worked_callback,
     const DoneCallback& done_callback)
     : file_system_path_(file_system_path),
+      excluded_folders_(excluded_folders),
       total_work_callback_(total_work_callback),
       worked_callback_(worked_callback),
       done_callback_(done_callback),
@@ -278,6 +280,7 @@ DevToolsFileSystemIndexer::FileSystemIndexingJob::FileSystemIndexingJob(
       stopped_(false) {
   current_trigrams_set_.resize(kTrigramCount);
   current_trigrams_.reserve(kTrigramCount);
+  pending_folders_.push_back(file_system_path);
 }
 
 DevToolsFileSystemIndexer::FileSystemIndexingJob::~FileSystemIndexingJob() {}
@@ -303,10 +306,22 @@ void DevToolsFileSystemIndexer::FileSystemIndexingJob::CollectFilesToIndex() {
   if (stopped_)
     return;
   if (!file_enumerator_) {
-    file_enumerator_.reset(
-        new FileEnumerator(file_system_path_, true, FileEnumerator::FILES));
+    file_enumerator_.reset(new FileEnumerator(
+        pending_folders_.back(), false,
+        FileEnumerator::FILES | FileEnumerator::DIRECTORIES));
+    pending_folders_.pop_back();
   }
   FilePath file_path = file_enumerator_->Next();
+  if (file_path.empty() && !pending_folders_.empty()) {
+    file_enumerator_.reset(new FileEnumerator(
+        pending_folders_.back(), false,
+        FileEnumerator::FILES | FileEnumerator::DIRECTORIES));
+    pending_folders_.pop_back();
+    impl_task_runner()->PostTask(
+        FROM_HERE, BindOnce(&FileSystemIndexingJob::CollectFilesToIndex, this));
+    return;
+  }
+
   if (file_path.empty()) {
     BrowserThread::PostTask(
         BrowserThread::UI, FROM_HERE,
@@ -315,6 +330,20 @@ void DevToolsFileSystemIndexer::FileSystemIndexingJob::CollectFilesToIndex() {
     IndexFiles();
     return;
   }
+  if (file_enumerator_->GetInfo().IsDirectory()) {
+    bool excluded = false;
+    for (const FilePath& excluded_folder : excluded_folders_) {
+      excluded = excluded_folder.IsParent(file_path);
+      if (excluded)
+        break;
+    }
+    if (!excluded)
+      pending_folders_.push_back(file_path);
+    impl_task_runner()->PostTask(
+        FROM_HERE, BindOnce(&FileSystemIndexingJob::CollectFilesToIndex, this));
+    return;
+  }
+
   Time saved_last_modified_time =
       g_trigram_index.Get().LastModifiedTimeForFile(file_path);
   FileEnumerator::FileInfo file_info = file_enumerator_->GetInfo();
@@ -447,15 +476,18 @@ DevToolsFileSystemIndexer::~DevToolsFileSystemIndexer() {
 scoped_refptr<DevToolsFileSystemIndexer::FileSystemIndexingJob>
 DevToolsFileSystemIndexer::IndexPath(
     const string& file_system_path,
+    const vector<string>& excluded_folders,
     const TotalWorkCallback& total_work_callback,
     const WorkedCallback& worked_callback,
     const DoneCallback& done_callback) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
-  scoped_refptr<FileSystemIndexingJob> indexing_job =
-      new FileSystemIndexingJob(FilePath::FromUTF8Unsafe(file_system_path),
-                                total_work_callback,
-                                worked_callback,
-                                done_callback);
+  vector<base::FilePath> paths;
+  for (const string& path : excluded_folders) {
+    paths.push_back(FilePath::FromUTF8Unsafe(path));
+  }
+  scoped_refptr<FileSystemIndexingJob> indexing_job = new FileSystemIndexingJob(
+      FilePath::FromUTF8Unsafe(file_system_path), paths, total_work_callback,
+      worked_callback, done_callback);
   indexing_job->Start();
   return indexing_job;
 }

chrome/browser/devtools/devtools_file_system_indexer.h

@@ -41,6 +41,7 @@ class DevToolsFileSystemIndexer
     friend class base::RefCountedThreadSafe<FileSystemIndexingJob>;
     friend class DevToolsFileSystemIndexer;
     FileSystemIndexingJob(const base::FilePath& file_system_path,
+                          const std::vector<base::FilePath>& excluded_folders,
                           const TotalWorkCallback& total_work_callback,
                           const WorkedCallback& worked_callback,
                           const DoneCallback& done_callback);
@@ -59,6 +60,9 @@ class DevToolsFileSystemIndexer
     void ReportWorked();
 
     base::FilePath file_system_path_;
+    std::vector<base::FilePath> excluded_folders_;
+
+    std::vector<base::FilePath> pending_folders_;
     TotalWorkCallback total_work_callback_;
     WorkedCallback worked_callback_;
     DoneCallback done_callback_;
@@ -83,6 +87,7 @@ class DevToolsFileSystemIndexer
   // progress callbacks.
   scoped_refptr<FileSystemIndexingJob> IndexPath(
       const std::string& file_system_path,
+      const std::vector<std::string>& excluded_folders,
       const TotalWorkCallback& total_work_callback,
       const WorkedCallback& worked_callback,
       const DoneCallback& done_callback);

chrome/browser/devtools/devtools_file_system_indexer_unittest.cc

@@ -49,9 +49,10 @@ TEST_F(DevToolsFileSystemIndexerTest, BasicUsage) {
       base_test_path.Append(FILE_PATH_LITERAL("devtools"))
           .Append(FILE_PATH_LITERAL("indexer"));
 
+  std::vector<std::string> excluded_folders;
   scoped_refptr<DevToolsFileSystemIndexer::FileSystemIndexingJob> job =
-      indexer_->IndexPath(index_path.AsUTF8Unsafe(), base::DoNothing(),
-                          base::DoNothing(),
+      indexer_->IndexPath(index_path.AsUTF8Unsafe(), excluded_folders,
+                          base::DoNothing(), base::DoNothing(),
                           base::Bind(&DevToolsFileSystemIndexerTest::SetDone,
                                      base::Unretained(this)));
 

chrome/browser/devtools/devtools_ui_bindings.cc

@@ -755,8 +755,10 @@ void DevToolsUIBindings::UpgradeDraggedFileSystemPermissions(
                                   weak_factory_.GetWeakPtr()));
 }
 
-void DevToolsUIBindings::IndexPath(int index_request_id,
-                                   const std::string& file_system_path) {
+void DevToolsUIBindings::IndexPath(
+    int index_request_id,
+    const std::string& file_system_path,
+    const std::string& excluded_folders_message) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
   CHECK(IsValidFrontendURL(web_contents_->GetURL()) && frontend_host_);
   if (!file_helper_->IsFileSystemAdded(file_system_path)) {
@@ -765,21 +767,30 @@ void DevToolsUIBindings::IndexPath(int index_request_id,
   }
   if (indexing_jobs_.count(index_request_id) != 0)
     return;
+  std::vector<std::string> excluded_folders;
+  std::unique_ptr<base::Value> parsed_excluded_folders =
+      base::JSONReader::Read(excluded_folders_message);
+  if (parsed_excluded_folders && parsed_excluded_folders->is_list()) {
+    const std::vector<base::Value>& folder_paths =
+        parsed_excluded_folders->GetList();
+    for (const base::Value& folder_path : folder_paths) {
+      if (folder_path.is_string())
+        excluded_folders.push_back(folder_path.GetString());
+    }
+  }
+
   indexing_jobs_[index_request_id] =
       scoped_refptr<DevToolsFileSystemIndexer::FileSystemIndexingJob>(
           file_system_indexer_->IndexPath(
-              file_system_path,
+              file_system_path, excluded_folders,
               Bind(&DevToolsUIBindings::IndexingTotalWorkCalculated,
-                   weak_factory_.GetWeakPtr(),
-                   index_request_id,
+                   weak_factory_.GetWeakPtr(), index_request_id,
                    file_system_path),
               Bind(&DevToolsUIBindings::IndexingWorked,
-                   weak_factory_.GetWeakPtr(),
-                   index_request_id,
+                   weak_factory_.GetWeakPtr(), index_request_id,
                    file_system_path),
               Bind(&DevToolsUIBindings::IndexingDone,
-                   weak_factory_.GetWeakPtr(),
-                   index_request_id,
+                   weak_factory_.GetWeakPtr(), index_request_id,
                    file_system_path)));
 }
 

chrome/browser/devtools/devtools_ui_bindings.h

@@ -122,7 +122,8 @@ class DevToolsUIBindings : public DevToolsEmbedderMessageDispatcher::Delegate,
   void UpgradeDraggedFileSystemPermissions(
       const std::string& file_system_url) override;
   void IndexPath(int index_request_id,
-                 const std::string& file_system_path) override;
+                 const std::string& file_system_path,
+                 const std::string& excluded_folders) override;
   void StopIndexing(int index_request_id) override;
   void SearchInPath(int search_request_id,
                     const std::string& file_system_path,

third_party/WebKit/Source/devtools/front_end/devtools_compatibility.js

@@ -581,9 +581,13 @@
      * @override
      * @param {number} requestId
      * @param {string} fileSystemPath
+     * @param {string} excludedFolders
      */
-    indexPath(requestId, fileSystemPath) {
-      DevToolsAPI.sendMessageToEmbedder('indexPath', [requestId, fileSystemPath], null);
+    indexPath(requestId, fileSystemPath, excludedFolders) {
+      // |excludedFolders| added in M67. For backward compatibility,
+      // pass empty array.
+      excludedFolders = excludedFolders || '[]';
+      DevToolsAPI.sendMessageToEmbedder('indexPath', [requestId, fileSystemPath, excludedFolders], null);
     }
 
     /**

third_party/WebKit/Source/devtools/front_end/host/InspectorFrontendHost.js

@@ -309,8 +309,9 @@ Host.InspectorFrontendHostStub = class {
    * @override
    * @param {number} requestId
    * @param {string} fileSystemPath
+   * @param {string} excludedFolders
    */
-  indexPath(requestId, fileSystemPath) {
+  indexPath(requestId, fileSystemPath, excludedFolders) {
   }
 
   /**

third_party/WebKit/Source/devtools/front_end/host/InspectorFrontendHostAPI.js

@@ -111,8 +111,9 @@ InspectorFrontendHostAPI.prototype = {
   /**
    * @param {number} requestId
    * @param {string} fileSystemPath
+   * @param {string} excludedFolders
    */
-  indexPath(requestId, fileSystemPath) {},
+  indexPath(requestId, fileSystemPath, excludedFolders) {},
 
   /**
    * @return {string}

third_party/WebKit/Source/devtools/front_end/persistence/IsolatedFileSystem.js

@@ -48,6 +48,8 @@ Persistence.IsolatedFileSystem = class {
     this._excludedFoldersSetting = Common.settings.createLocalSetting('workspaceExcludedFolders', {});
     /** @type {!Set<string>} */
     this._excludedFolders = new Set(this._excludedFoldersSetting.get()[path] || []);
+    /** @type {!Array<string>} */
+    this._excludedEmbedderFolders = [];
 
     /** @type {!Set<string>} */
     this._initialFilePaths = new Set();
@@ -173,8 +175,11 @@ Persistence.IsolatedFileSystem = class {
             const parentFolder = entry.fullPath.substring(1, lastSlash);
             this._initialGitFolders.add(parentFolder);
           }
-          if (this.isFileExcluded(entry.fullPath + '/'))
+          if (this.isFileExcluded(entry.fullPath + '/')) {
+            this._excludedEmbedderFolders.push(
+                Common.ParsedURL.urlToPlatformPath(this._path + entry.fullPath, Host.isWin()));
             continue;
+          }
           ++pendingRequests;
           this._requestEntries(entry.fullPath, boundInnerCallback);
         }
@@ -604,7 +609,7 @@ Persistence.IsolatedFileSystem = class {
   indexContent(progress) {
     progress.setTotalWork(1);
     const requestId = this._manager.registerProgress(progress);
-    InspectorFrontendHost.indexPath(requestId, this._embedderPath);
+    InspectorFrontendHost.indexPath(requestId, this._embedderPath, JSON.stringify(this._excludedEmbedderFolders));
   }
 };