Commit eac4a32f authored by mangini@chromium.org's avatar mangini@chromium.org

Add CachingFileSystem on top of CloudStorageFileSystem

to improve performance of the docserver, avoiding hit Cloud Storage
on every request.

BUG=343315
NOTRY=true

Review URL: https://codereview.chromium.org/162043003

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@250901 0039d316-1c4b-4281-b951-d872f2087c98
parent 652174c8
application: chrome-apps-doc
version: 3-7-0
version: 3-7-1
runtime: python27
api_version: 1
threadsafe: false
......
......@@ -2,4 +2,4 @@ cron:
- description: Repopulates all cached data.
url: /_cron
schedule: every 5 minutes
target: 3-7-0
target: 3-7-1
......@@ -13,6 +13,10 @@ from future import Gettable, Future
import logging
import traceback
# Name of the file containing the Git hash of the latest commit sync'ed
# to Cloud Storage. This file is generated by the Github->GCS sync script
LAST_COMMIT_HASH_FILENAME='.__lastcommit.txt'
'''See gcs_file_system_provider.py for documentation on using Google Cloud
Storage as a filesystem.
'''
......@@ -35,30 +39,25 @@ def _ListDir(dir_name):
def _CreateStatInfo(bucket, path):
bucket = '/%s' % bucket
full_path = '/'.join( (bucket, path.lstrip('/')) )
last_commit_file = '%s/%s' % (bucket, LAST_COMMIT_HASH_FILENAME)
try:
last_commit = _ReadFile(last_commit_file)
if full_path.endswith('/'):
child_versions = dict()
version = 0
# Fetching stats for all files under full_path, recursively. The
# listbucket method uses a prefix approach to simulate hierarchy,
# but calling it without the "delimiter" argument searches for prefix,
# which means, for directories, everything beneath it.
for _file in cloudstorage_api.listbucket(full_path):
if not _file.is_dir:
# GCS doesn't have metadata for dirs
child_stat = cloudstorage_api.stat('%s' % _file.filename).st_ctime
filename = _file.filename[len(bucket)+1:]
child_versions[filename] = child_stat
version = max(version, child_stat)
filename = _file.filename[len(full_path):]
child_versions[filename] = last_commit
else:
child_versions = None
version = cloudstorage_api.stat(full_path).st_ctime
return StatInfo(version, child_versions)
return StatInfo(last_commit, child_versions)
except (TypeError, errors.Error):
raise FileNotFoundError('cloudstorage.stat failed for %s: %s' % (path,
traceback.format_exc()))
class CloudStorageFileSystem(FileSystem):
'''FileSystem implementation which fetches resources from Google Cloud
Storage.
......
......@@ -5,6 +5,7 @@
import os
import environment
from caching_file_system import CachingFileSystem
from empty_dir_file_system import EmptyDirFileSystem
from extensions_paths import LOCAL_GCS_DIR, LOCAL_GCS_DEBUG_CONF
from local_file_system import LocalFileSystem
......@@ -81,9 +82,9 @@ class CloudStorageFileSystemProvider(object):
# gcs_file_system has strong dependencies on runtime appengine APIs,
# so we only import it when we are sure we are not on preview.py or tests.
from gcs_file_system import CloudStorageFileSystem
return CloudStorageFileSystem(
bucket, debug_access_token, debug_bucket_prefix)
return CachingFileSystem(CloudStorageFileSystem(bucket,
debug_access_token, debug_bucket_prefix),
self._object_store_creator)
@staticmethod
def ForEmpty():
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment