Add CachingFileSystem on top of CloudStorageFileSystem

to improve performance of the docserver, avoiding hit Cloud Storage
on every request.

BUG=343315
NOTRY=true

Review URL: https://codereview.chromium.org/162043003

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@250901 0039d316-1c4b-4281-b951-d872f2087c98

chrome/common/extensions/docs/server2/app.yaml

 application: chrome-apps-doc
-version: 3-7-0
+version: 3-7-1
 runtime: python27
 api_version: 1
 threadsafe: false

chrome/common/extensions/docs/server2/cron.yaml

@@ -2,4 +2,4 @@ cron:
 - description: Repopulates all cached data.
   url: /_cron
   schedule: every 5 minutes
-  target: 3-7-0
+  target: 3-7-1

chrome/common/extensions/docs/server2/gcs_file_system.py

@@ -13,6 +13,10 @@ from future import Gettable, Future
 import logging
 import traceback
 
+# Name of the file containing the Git hash of the latest commit sync'ed
+# to Cloud Storage. This file is generated by the Github->GCS sync script
+LAST_COMMIT_HASH_FILENAME='.__lastcommit.txt'
+
 '''See gcs_file_system_provider.py for documentation on using Google Cloud
 Storage as a filesystem.
 '''
@@ -35,30 +39,25 @@ def _ListDir(dir_name):
 def _CreateStatInfo(bucket, path):
   bucket = '/%s' % bucket
   full_path = '/'.join( (bucket, path.lstrip('/')) )
+  last_commit_file = '%s/%s' % (bucket, LAST_COMMIT_HASH_FILENAME)
   try:
+    last_commit = _ReadFile(last_commit_file)
     if full_path.endswith('/'):
       child_versions = dict()
-      version = 0
       # Fetching stats for all files under full_path, recursively. The
       # listbucket method uses a prefix approach to simulate hierarchy,
       # but calling it without the "delimiter" argument searches for prefix,
       # which means, for directories, everything beneath it.
       for _file in cloudstorage_api.listbucket(full_path):
-        if not _file.is_dir:
-          # GCS doesn't have metadata for dirs
-          child_stat = cloudstorage_api.stat('%s' % _file.filename).st_ctime
-          filename = _file.filename[len(bucket)+1:]
-          child_versions[filename] = child_stat
-          version = max(version, child_stat)
+        filename = _file.filename[len(full_path):]
+        child_versions[filename] = last_commit
     else:
       child_versions = None
-      version = cloudstorage_api.stat(full_path).st_ctime
-    return StatInfo(version, child_versions)
+    return StatInfo(last_commit, child_versions)
   except (TypeError, errors.Error):
     raise FileNotFoundError('cloudstorage.stat failed for %s: %s' % (path,
                             traceback.format_exc()))
 
-
 class CloudStorageFileSystem(FileSystem):
   '''FileSystem implementation which fetches resources from Google Cloud
   Storage.

chrome/common/extensions/docs/server2/gcs_file_system_provider.py

@@ -5,6 +5,7 @@
 import os
 import environment
 
+from caching_file_system import CachingFileSystem
 from empty_dir_file_system import EmptyDirFileSystem
 from extensions_paths import LOCAL_GCS_DIR, LOCAL_GCS_DEBUG_CONF
 from local_file_system import LocalFileSystem
@@ -81,9 +82,9 @@ class CloudStorageFileSystemProvider(object):
     # gcs_file_system has strong dependencies on runtime appengine APIs,
     # so we only import it when we are sure we are not on preview.py or tests.
     from gcs_file_system import CloudStorageFileSystem
-    return CloudStorageFileSystem(
-        bucket, debug_access_token, debug_bucket_prefix)
-
+    return CachingFileSystem(CloudStorageFileSystem(bucket,
+        debug_access_token, debug_bucket_prefix),
+        self._object_store_creator)
 
   @staticmethod
   def ForEmpty():