Devserver: only populate data during cron jobs, meaning all data from instances

are served out of caches. Add a PersistentObjectStore to make that work.

BUG=226625

Review URL: https://codereview.chromium.org/14218004

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@194924 0039d316-1c4b-4281-b951-d872f2087c98

chrome/common/extensions/docs/server2/api_data_source.py

@@ -20,12 +20,9 @@ def _RemoveNoDocs(item):
   if json_parse.IsDict(item):
     if item.get('nodoc', False):
       return True
-    to_remove = []
     for key, value in item.items():
       if _RemoveNoDocs(value):
-        to_remove.append(key)
-    for k in to_remove:
-      del item[k]
+        del item[key]
   elif type(item) == list:
     to_remove = []
     for i in item:

chrome/common/extensions/docs/server2/api_data_source_test.py

@@ -14,7 +14,6 @@ from api_data_source import (APIDataSource,
                              _RemoveNoDocs)
 from compiled_file_system import CompiledFileSystem
 from file_system import FileNotFoundError
-from in_memory_object_store import InMemoryObjectStore
 from local_file_system import LocalFileSystem
 from object_store_creator import ObjectStoreCreator
 from reference_resolver import ReferenceResolver
@@ -62,29 +61,6 @@ class APIDataSourceTest(unittest.TestCase):
                                      data_source,
                                      ObjectStoreCreator.Factory()).Create()
 
-  def DISABLED_testSimple(self):
-    compiled_fs_factory = CompiledFileSystem.Factory(
-        LocalFileSystem(self._base_path),
-        InMemoryObjectStore('fake_branch'))
-    data_source_factory = APIDataSource.Factory(compiled_fs_factory,
-                                                '.')
-    data_source_factory.SetSamplesDataSourceFactory(FakeSamplesDataSource())
-    data_source = data_source_factory.Create({}, disable_refs=True)
-
-    # Take the dict out of the list.
-    expected = json.loads(self._ReadLocalFile('expected_test_file.json'))
-    expected['permissions'] = None
-    test1 = data_source.get('test_file')
-    test1.pop('samples')
-    self.assertEqual(expected, test1)
-    test2 = data_source.get('testFile')
-    test2.pop('samples')
-    self.assertEqual(expected, test2)
-    test3 = data_source.get('testFile.html')
-    test3.pop('samples')
-    self.assertEqual(expected, test3)
-    self.assertRaises(FileNotFoundError, data_source.get, 'junk')
-
   def _LoadJSON(self, filename):
     return json.loads(self._ReadLocalFile(filename))
 

chrome/common/extensions/docs/server2/app.yaml

 application: chrome-apps-doc
-version: 2-0-20
+version: 2-0-21
 runtime: python27
 api_version: 1
 threadsafe: false

chrome/common/extensions/docs/server2/appengine_wrappers.py

@@ -2,6 +2,11 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import os
+
+def IsDevServer():
+  return os.environ.get('SERVER_SOFTWARE', '').find('Development') == 0
+
 # This will attempt to import the actual App Engine modules, and if it fails,
 # they will be replaced with fake modules. This is useful during testing.
 try:
@@ -12,11 +17,7 @@ try:
   import google.appengine.api.files as files
   import google.appengine.api.memcache as memcache
   import google.appengine.api.urlfetch as urlfetch
-  # Default to a 5 minute cache timeout.
-  CACHE_TIMEOUT = 300
 except ImportError:
-  # Cache for one second because zero means cache forever.
-  CACHE_TIMEOUT = 1
   import re
   from StringIO import StringIO
 
@@ -48,6 +49,9 @@ except ImportError:
     def get_result(self):
       return self.result
 
+    def wait(self):
+      pass
+
   class FakeUrlFetch(object):
     """A fake urlfetch module that uses the current
     |FAKE_URL_FETCHER_CONFIGURATION| to map urls to fake fetchers.
@@ -130,23 +134,37 @@ except ImportError:
   files = FakeFiles()
 
   class InMemoryMemcache(object):
-    """A fake memcache that does nothing.
+    """An in-memory memcache implementation.
     """
+    def __init__(self):
+      self._namespaces = {}
+
     class Client(object):
       def set_multi_async(self, mapping, namespace='', time=0):
-        return
+        for k, v in mapping.iteritems():
+          memcache.set(k, v, namespace=namespace, time=time)
 
       def get_multi_async(self, keys, namespace='', time=0):
-        return _RPC(result=dict((k, None) for k in keys))
+        return _RPC(result=dict(
+          (k, memcache.get(k, namespace=namespace, time=time)) for k in keys))
 
     def set(self, key, value, namespace='', time=0):
-      return
+      self._GetNamespace(namespace)[key] = value
 
     def get(self, key, namespace='', time=0):
-      return None
+      return self._GetNamespace(namespace).get(key)
+
+    def delete(self, key, namespace=''):
+      self._GetNamespace(namespace).pop(key, None)
 
-    def delete(self, key, namespace):
-      return
+    def delete_multi(self, keys, namespace=''):
+      for k in keys:
+        self.delete(k, namespace=namespace)
+
+    def _GetNamespace(self, namespace):
+      if namespace not in self._namespaces:
+        self._namespaces[namespace] = {}
+      return self._namespaces[namespace]
 
   memcache = InMemoryMemcache()
 
@@ -176,20 +194,60 @@ except ImportError:
 
   class db(object):
     _store = {}
+
     class StringProperty(object):
       pass
 
+    class BlobProperty(object):
+      pass
+
+    class Key(object):
+      def __init__(self, key):
+        self._key = key
+
+      @staticmethod
+      def from_path(model_name, path):
+        return db.Key('%s/%s' % (model_name, path))
+
+      def __eq__(self, obj):
+        return self.__class__ == obj.__class__ and self._key == obj._key
+
+      def __hash__(self):
+        return hash(self._key)
+
+      def __str__(self):
+        return str(self._key)
+
     class Model(object):
-      def __init__(self, key_='', value=''):
-        self._key = key_
-        self._value = value
+      key = None
+
+      def __init__(self, **optargs):
+        cls = self.__class__
+        for k, v in optargs.iteritems():
+          assert hasattr(cls, k), '%s does not define property %s' % (
+              cls.__name__, k)
+          setattr(self, k, v)
 
       @staticmethod
       def gql(query, key):
-        return _Db_Result(db._store.get(key, None))
+        return _Db_Result(db._store.get(key))
 
       def put(self):
-        db._store[self._key] = self._value
+        db._store[self.key_] = self.value
+
+    @staticmethod
+    def get_async(key):
+      return _RPC(result=db._store.get(key))
+
+    @staticmethod
+    def delete_async(key):
+      db._store.pop(key, None)
+      return _RPC()
+
+    @staticmethod
+    def put_async(value):
+      db._store[value.key] = value
+      return _RPC()
 
   class BlobReferenceProperty(object):
     pass

chrome/common/extensions/docs/server2/branch_utility_test.py

@@ -7,7 +7,6 @@ import os
 import sys
 import unittest
 
-from in_memory_object_store import InMemoryObjectStore
 from branch_utility import BranchUtility
 from fake_url_fetcher import FakeUrlFetcher
 from test_object_store import TestObjectStore

chrome/common/extensions/docs/server2/cache_chain_object_store.py

+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from future import Future
+from object_store import ObjectStore
+
+class _GetMultiFuture(object):
+  '''A Future for GetMulti.
+
+  Params:
+  - |toplevel_cache| CacheChainObjectStore's cache.
+  - |object_store_futures| a list of (object store, future) pairs, where future
+    is the result of calling GetMulti on the missing keys for the object store.
+  - |cached_items| a mapping of cache items already in memory.
+  - |missing_keys| the keys that were missing from the GetMulti call
+  '''
+  def __init__(self,
+               toplevel_cache,
+               object_store_futures,
+               cached_items,
+               missing_keys):
+    self._toplevel_cache = toplevel_cache
+    self._object_store_futures = object_store_futures
+    self._results_so_far = cached_items
+    self._missing_keys = missing_keys
+
+  def Get(self):
+    # Approach:
+    #
+    # Try each object store in order, until there are no more missing keys.
+    # Don't realise the Future value of an object store that we don't need to;
+    # this is important e.g. to avoid querying data store constantly.
+    #
+    # When a value is found, cache it in all object stores further up the
+    # chain, including the object-based cache on CacheChainObjectStore.
+    object_store_updates = []
+    for object_store, object_store_future in self._object_store_futures:
+      if len(self._missing_keys) == 0:
+        break
+      result = object_store_future.Get()
+      for k, v in result.items():  # use items(); changes during iteration
+        if v is None or k not in self._missing_keys:
+          del result[k]
+          continue
+        self._toplevel_cache[k] = v
+        self._results_so_far[k] = v
+        self._missing_keys.remove(k)
+      for _, updates in object_store_updates:
+        updates.update(result)
+      object_store_updates.append((object_store, {}))
+    # Update the caches of all object stores that need it.
+    for object_store, updates in object_store_updates:
+      if updates:
+        object_store.SetMulti(updates)
+    return self._results_so_far
+
+class CacheChainObjectStore(ObjectStore):
+  '''Maintains an in-memory cache along with a chain of other object stores to
+  try for the same keys. This is useful for implementing a multi-layered cache.
+  The in-memory cache is inbuilt since it's synchronous, but the object store
+  interface is asynchronous.
+  The rules for the object store chain are:
+    - When setting (or deleting) items, all object stores in the hierarcy will
+      have that item set.
+    - When getting items, each object store is tried in order. The first object
+      store to find the item will trickle back up, setting it on all object
+      stores higher in the hierarchy.
+  '''
+  def __init__(self, object_stores):
+    self._object_stores = object_stores
+    self._cache = {}
+
+  def SetMulti(self, mapping):
+    self._cache.update(mapping)
+    for object_store in self._object_stores:
+      object_store.SetMulti(mapping)
+
+  def GetMulti(self, keys):
+    missing_keys = list(keys)
+    cached_items = {}
+    for key in keys:
+      if key in self._cache:
+        cached_items[key] = self._cache.get(key)
+        missing_keys.remove(key)
+    if len(missing_keys) == 0:
+      return Future(value=cached_items)
+    object_store_futures = [(object_store, object_store.GetMulti(missing_keys))
+                            for object_store in self._object_stores]
+    return Future(delegate=_GetMultiFuture(
+        self._cache, object_store_futures, cached_items, missing_keys))
+
+  def DelMulti(self, keys):
+    for k in keys:
+      self._cache.pop(k, None)
+    for object_store in self._object_stores:
+      object_store.DelMulti(keys)

chrome/common/extensions/docs/server2/cache_chain_object_store_test.py

+#!/usr/bin/env python
+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from cache_chain_object_store import CacheChainObjectStore
+from test_object_store import TestObjectStore
+import unittest
+
+class CacheChainObjectStoreTest(unittest.TestCase):
+  def setUp(self):
+    self._first = TestObjectStore('first', init={
+      'storage.html': 'storage',
+    })
+    self._second = TestObjectStore('second', init={
+      'runtime.html': 'runtime',
+      'storage.html': 'storage',
+    })
+    self._third = TestObjectStore('third', init={
+      'commands.html': 'commands',
+      'runtime.html': 'runtime',
+      'storage.html': 'storage',
+    })
+    self._store = CacheChainObjectStore(
+        (self._first, self._second, self._third))
+
+  def testGetFromFirstLayer(self):
+    self.assertEqual('storage', self._store.Get('storage.html').Get())
+    self.assertTrue(*self._first.CheckAndReset(get_count=1))
+    # Found in first layer, stop.
+    self.assertTrue(*self._second.CheckAndReset())
+    self.assertTrue(*self._third.CheckAndReset())
+    # Cached in memory, won't re-query.
+    self.assertEqual('storage', self._store.Get('storage.html').Get())
+    self.assertTrue(*self._first.CheckAndReset())
+    self.assertTrue(*self._second.CheckAndReset())
+    self.assertTrue(*self._third.CheckAndReset())
+
+  def testGetFromSecondLayer(self):
+    self.assertEqual('runtime', self._store.Get('runtime.html').Get())
+    # Not found in first layer but found in second.
+    self.assertTrue(*self._first.CheckAndReset(get_count=1, set_count=1))
+    self.assertTrue(*self._second.CheckAndReset(get_count=1))
+    self.assertTrue(*self._third.CheckAndReset())
+    # First will now have it cached.
+    self.assertEqual('runtime', self._first.Get('runtime.html').Get())
+    self._first.Reset()
+    # Cached in memory, won't re-query.
+    self.assertEqual('runtime', self._store.Get('runtime.html').Get())
+    self.assertTrue(*self._first.CheckAndReset())
+    self.assertTrue(*self._second.CheckAndReset())
+    self.assertTrue(*self._third.CheckAndReset())
+
+  def testGetFromThirdLayer(self):
+    self.assertEqual('commands', self._store.Get('commands.html').Get())
+    # As above but for third.
+    self.assertTrue(*self._first.CheckAndReset(get_count=1, set_count=1))
+    self.assertTrue(*self._second.CheckAndReset(get_count=1, set_count=1))
+    self.assertTrue(*self._third.CheckAndReset(get_count=1))
+    # First and second will now have it cached.
+    self.assertEqual('commands', self._first.Get('commands.html').Get())
+    self.assertEqual('commands', self._second.Get('commands.html').Get())
+    self._first.Reset()
+    self._second.Reset()
+    # Cached in memory, won't re-query.
+    self.assertEqual('commands', self._store.Get('commands.html').Get())
+    self.assertTrue(*self._first.CheckAndReset())
+    self.assertTrue(*self._second.CheckAndReset())
+    self.assertTrue(*self._third.CheckAndReset())
+
+  def testGetFromAllLayers(self):
+    self.assertEqual({
+      'commands.html': 'commands',
+      'runtime.html': 'runtime',
+      'storage.html': 'storage',
+    }, self._store.GetMulti(('commands.html',
+                             'runtime.html',
+                             'storage.html')).Get())
+    self.assertTrue(*self._first.CheckAndReset(get_count=1, set_count=1))
+    self.assertTrue(*self._second.CheckAndReset(get_count=1, set_count=1))
+    self.assertTrue(*self._third.CheckAndReset(get_count=1))
+    # First and second will have it all cached.
+    self.assertEqual('runtime', self._first.Get('runtime.html').Get())
+    self.assertEqual('commands', self._first.Get('commands.html').Get())
+    self.assertEqual('commands', self._second.Get('commands.html').Get())
+    self._first.Reset()
+    self._second.Reset()
+    # Cached in memory.
+    self.assertEqual({
+      'commands.html': 'commands',
+      'runtime.html': 'runtime',
+      'storage.html': 'storage',
+    }, self._store.GetMulti(('commands.html',
+                             'runtime.html',
+                             'storage.html')).Get())
+    self.assertTrue(*self._first.CheckAndReset())
+    self.assertTrue(*self._second.CheckAndReset())
+    self.assertTrue(*self._third.CheckAndReset())
+
+  def testPartiallyCachedInMemory(self):
+    self.assertEqual({
+      'commands.html': 'commands',
+      'storage.html': 'storage',
+    }, self._store.GetMulti(('commands.html', 'storage.html')).Get())
+    self.assertTrue(*self._first.CheckAndReset(get_count=1, set_count=1))
+    self.assertTrue(*self._second.CheckAndReset(get_count=1, set_count=1))
+    self.assertTrue(*self._third.CheckAndReset(get_count=1))
+    # runtime wasn't cached in memory, so stores should still be queried.
+    self.assertEqual({
+      'commands.html': 'commands',
+      'runtime.html': 'runtime',
+    }, self._store.GetMulti(('commands.html', 'runtime.html')).Get())
+    self.assertTrue(*self._first.CheckAndReset(get_count=1, set_count=1))
+    self.assertTrue(*self._second.CheckAndReset(get_count=1))
+    self.assertTrue(*self._third.CheckAndReset())
+
+  def testNotFound(self):
+    self.assertEqual(None, self._store.Get('notfound.html').Get())
+    self.assertTrue(*self._first.CheckAndReset(get_count=1))
+    self.assertTrue(*self._second.CheckAndReset(get_count=1))
+    self.assertTrue(*self._third.CheckAndReset(get_count=1))
+    # Not-foundedness shouldn't be cached.
+    self.assertEqual(None, self._store.Get('notfound.html').Get())
+    self.assertTrue(*self._first.CheckAndReset(get_count=1))
+    self.assertTrue(*self._second.CheckAndReset(get_count=1))
+    self.assertTrue(*self._third.CheckAndReset(get_count=1))
+    # Test some things not found, some things found.
+    self.assertEqual({
+      'runtime.html': 'runtime',
+    }, self._store.GetMulti(('runtime.html', 'notfound.html')).Get())
+    self.assertTrue(*self._first.CheckAndReset(get_count=1, set_count=1))
+    self.assertTrue(*self._second.CheckAndReset(get_count=1))
+    self.assertTrue(*self._third.CheckAndReset(get_count=1))
+
+  def testSet(self):
+    self._store.Set('hello.html', 'hello')
+    self.assertTrue(*self._first.CheckAndReset(set_count=1))
+    self.assertTrue(*self._second.CheckAndReset(set_count=1))
+    self.assertTrue(*self._third.CheckAndReset(set_count=1))
+    # Should have cached it.
+    self.assertEqual('hello', self._store.Get('hello.html').Get())
+    self.assertTrue(*self._first.CheckAndReset())
+    self.assertTrue(*self._second.CheckAndReset())
+    self.assertTrue(*self._third.CheckAndReset())
+
+  def testDel(self):
+    # Cache it.
+    self.assertEqual('storage', self._store.Get('storage.html').Get())
+    self.assertTrue(*self._first.CheckAndReset(get_count=1))
+    # Delete it.
+    self._store.Del('storage.html')
+    self.assertTrue(*self._first.CheckAndReset(del_count=1))
+    self.assertTrue(*self._second.CheckAndReset(del_count=1))
+    self.assertTrue(*self._third.CheckAndReset(del_count=1))
+    # Not cached anymore.
+    self.assertEqual(None, self._store.Get('storage.html').Get())
+    self.assertTrue(*self._first.CheckAndReset(get_count=1))
+    self.assertTrue(*self._second.CheckAndReset(get_count=1))
+    self.assertTrue(*self._third.CheckAndReset(get_count=1))
+
+if __name__ == '__main__':
+  unittest.main()

chrome/common/extensions/docs/server2/caching_file_system.py

@@ -23,7 +23,7 @@ class _AsyncUncachedFuture(object):
       version = self._file_system.Stat(item).version
       mapping[item] = (new_items[item], version)
       self._current_result[item] = new_items[item]
-    self._object_store.SetMulti(mapping, time=0)
+    self._object_store.SetMulti(mapping)
     return self._current_result
 
 class CachingFileSystem(FileSystem):
@@ -33,7 +33,8 @@ class CachingFileSystem(FileSystem):
     self._file_system = file_system
     def create_object_store(category):
       return (object_store_creator_factory.Create(CachingFileSystem)
-          .Create(category=category, version=file_system.GetVersion()))
+          .Create(category='%s/%s' % (file_system.GetName(), category),
+                  version=file_system.GetVersion()))
     self._stat_object_store = create_object_store('stat')
     self._read_object_store = create_object_store('read')
     self._read_binary_object_store = create_object_store('read-binary')
@@ -73,31 +74,30 @@ class CachingFileSystem(FileSystem):
     """Reads a list of files. If a file is in memcache and it is not out of
     date, it is returned. Otherwise, the file is retrieved from the file system.
     """
-    result = {}
-    uncached = []
     read_object_store = (self._read_binary_object_store if binary else
                          self._read_object_store)
-    results = read_object_store.GetMulti(paths).Get()
-    result_values = [x[1] for x in sorted(results.iteritems())]
-    stats = self._stat_object_store.GetMulti(paths).Get()
-    stat_values = [x[1] for x in sorted(stats.iteritems())]
-    for path, cached_result, stat in zip(sorted(paths),
-                                         result_values,
-                                         stat_values):
-      if cached_result is None:
+    read_values = read_object_store.GetMulti(paths).Get()
+    stat_values = self._stat_object_store.GetMulti(paths).Get()
+    result = {}
+    uncached = []
+    for path in paths:
+      read_value = read_values.get(path)
+      stat_value = stat_values.get(path)
+      if read_value is None:
         uncached.append(path)
         continue
-      data, version = cached_result
+      data, version = read_value
       # TODO(cduvall): Make this use a multi stat.
-      if stat is None:
-        stat = self.Stat(path).version
-      if stat != version:
+      if stat_value is None:
+        stat_value = self.Stat(path).version
+      if stat_value != version:
         uncached.append(path)
         continue
       result[path] = data
 
     if not uncached:
       return Future(value=result)
+
     return Future(delegate=_AsyncUncachedFuture(
         self._file_system.Read(uncached, binary=binary),
         result,

chrome/common/extensions/docs/server2/caching_file_system_test.py

@@ -10,7 +10,6 @@ import unittest
 from caching_file_system import CachingFileSystem
 from file_system import FileSystem, StatInfo
 from future import Future
-from in_memory_object_store import InMemoryObjectStore
 from local_file_system import LocalFileSystem
 from object_store_creator import ObjectStoreCreator
 from test_file_system import TestFileSystem
@@ -66,20 +65,20 @@ class CachingFileSystemTest(unittest.TestCase):
     self.assertTrue(fake_fs.CheckAndReset())
 
     # Test if the Stat version is the same the resource is not re-fetched.
-    file_system._stat_object_store.Delete('bob/bob0')
+    file_system._stat_object_store.Del('bob/bob0')
     self.assertEqual('bob/bob0 contents', file_system.ReadSingle('bob/bob0'))
     self.assertTrue(fake_fs.CheckAndReset(stat_count=1))
 
     # Test if there is a newer version, the resource is re-fetched.
-    file_system._stat_object_store.Delete('bob/bob0')
+    file_system._stat_object_store.Del('bob/bob0')
     fake_fs.IncrementStat();
     self.assertEqual('bob/bob0 contents', file_system.ReadSingle('bob/bob0'))
     self.assertTrue(fake_fs.CheckAndReset(read_count=1, stat_count=1))
 
     # Test directory and subdirectory stats are cached.
-    file_system._stat_object_store.Delete('bob/bob0')
-    file_system._read_object_store.Delete('bob/bob0')
-    file_system._stat_object_store.Delete('bob/bob1')
+    file_system._stat_object_store.Del('bob/bob0')
+    file_system._read_object_store.Del('bob/bob0')
+    file_system._stat_object_store.Del('bob/bob1')
     fake_fs.IncrementStat();
     self.assertEqual('bob/bob1 contents', file_system.ReadSingle('bob/bob1'))
     self.assertEqual('bob/bob0 contents', file_system.ReadSingle('bob/bob0'))
@@ -88,8 +87,8 @@ class CachingFileSystemTest(unittest.TestCase):
     self.assertTrue(fake_fs.CheckAndReset())
 
     # Test a more recent parent directory doesn't force a refetch of children.
-    file_system._read_object_store.Delete('bob/bob0')
-    file_system._read_object_store.Delete('bob/bob1')
+    file_system._read_object_store.Del('bob/bob0')
+    file_system._read_object_store.Del('bob/bob1')
     self.assertEqual('bob/bob1 contents', file_system.ReadSingle('bob/bob1'))
     self.assertEqual('bob/bob2 contents', file_system.ReadSingle('bob/bob2'))
     self.assertEqual('bob/bob3 contents', file_system.ReadSingle('bob/bob3'))
@@ -100,7 +99,7 @@ class CachingFileSystemTest(unittest.TestCase):
     self.assertEqual('bob/bob3 contents', file_system.ReadSingle('bob/bob3'))
     self.assertTrue(fake_fs.CheckAndReset())
 
-    file_system._stat_object_store.Delete('bob/bob0')
+    file_system._stat_object_store.Del('bob/bob0')
     self.assertEqual('bob/bob0 contents', file_system.ReadSingle('bob/bob0'))
     self.assertTrue(fake_fs.CheckAndReset(read_count=1, stat_count=1))
     self.assertEqual('bob/bob0 contents', file_system.ReadSingle('bob/bob0'))

chrome/common/extensions/docs/server2/compiled_file_system.py

@@ -89,13 +89,13 @@ class CompiledFileSystem(object):
     will be ignored.
     """
     version = self._file_system.Stat(path).version
-    cache_entry = self._file_object_store.Get(path, time=0).Get()
+    cache_entry = self._file_object_store.Get(path).Get()
     if (cache_entry is not None) and (version == cache_entry.version):
       return cache_entry._cache_data
     cache_data = self._populate_function(
         path,
         self._file_system.ReadSingle(path, binary=binary))
-    self._file_object_store.Set(path, _CacheEntry(cache_data, version), time=0)
+    self._file_object_store.Set(path, _CacheEntry(cache_data, version))
     return cache_data
 
   def GetFromFileListing(self, path):
@@ -105,9 +105,9 @@ class CompiledFileSystem(object):
     if not path.endswith('/'):
       path += '/'
     version = self._file_system.Stat(path).version
-    cache_entry = self._list_object_store.Get(path, time=0).Get()
+    cache_entry = self._list_object_store.Get(path).Get()
     if (cache_entry is not None) and (version == cache_entry.version):
         return cache_entry._cache_data
     cache_data = self._populate_function(path, self._RecursiveList(path))
-    self._list_object_store.Set(path, _CacheEntry(cache_data, version), time=0)
+    self._list_object_store.Set(path, _CacheEntry(cache_data, version))
     return cache_data

chrome/common/extensions/docs/server2/datastore_models.py

+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from appengine_wrappers import db
+import cPickle
+
+# A collection of the data store models used throughout the server.
+# These values are global within datastore.
+
+class PersistentObjectStoreItem(db.Model):
+  pickled_value = db.BlobProperty()
+
+  @classmethod
+  def CreateKey(cls, namespace, key):
+    return db.Key.from_path(cls.__name__, '%s/%s' % (namespace, key))
+
+  @classmethod
+  def CreateItem(cls, namespace, key, value):
+    return PersistentObjectStoreItem(key=cls.CreateKey(namespace, key),
+                                     pickled_value=cPickle.dumps(value))
+
+  def GetValue(self):
+    return cPickle.loads(self.pickled_value)

chrome/common/extensions/docs/server2/file_system.py

@@ -63,8 +63,16 @@ class FileSystem(object):
     '''
     raise NotImplementedError()
 
-  def GetVersion(self):
+  @classmethod
+  def GetName(cls):
+    '''The type of the file system, exposed for caching classes to namespace
+    their caches. It is unlikely that this needs to be overridden.
+    '''
+    return cls.__name__
+
+  @classmethod
+  def GetVersion(cls):
     '''The version of the file system, exposed for caching classes backed to
-    file systems. This is unlikely to change.
+    file systems. It is unlikely that this needs to be overridden.
     '''
     return None

chrome/common/extensions/docs/server2/github_file_system.py

@@ -131,8 +131,11 @@ class GithubFileSystem(FileSystem):
 
   def _DefaultStat(self, path):
     version = 0
-    # Cache for a minute so we don't try to keep fetching bad data.
-    self._stat_object_store.Set(path, version, time=60)
+    # TODO(kalman): we should replace all of this by wrapping the
+    # GithubFileSystem in a CachingFileSystem. A lot of work has been put into
+    # CFS to be robust, and GFS is missing out.
+    # For example: the following line is wrong, but it could be moot.
+    self._stat_object_store.Set(path, version)
     return StatInfo(version)
 
   def Stat(self, path):

chrome/common/extensions/docs/server2/github_file_system_test.py

@@ -13,7 +13,6 @@ from appengine_url_fetcher import AppEngineUrlFetcher
 from appengine_wrappers import files
 from fake_fetchers import ConfigureFakeFetchers
 from github_file_system import GithubFileSystem
-from in_memory_object_store import InMemoryObjectStore
 import url_constants
 
 class GithubFileSystemTest(unittest.TestCase):

chrome/common/extensions/docs/server2/handler.py

@@ -18,6 +18,24 @@ import time
 _DEFAULT_CHANNEL = 'stable'
 
 class Handler(webapp.RequestHandler):
+  # AppEngine instances should never need to call out to SVN. That should only
+  # ever be done by the cronjobs, which then write the result into DataStore,
+  # which is as far as instances look.
+  #
+  # Why? SVN is slow and a bit flaky. Cronjobs failing is annoying but
+  # temporary. Instances failing affects users, and is really bad.
+  #
+  # Anyway - to enforce this, we actually don't give instances access to SVN.
+  # If anything is missing from datastore, it'll be a 404. If the cronjobs
+  # don't manage to catch everything - uhoh. On the other hand, we'll figure it
+  # out pretty soon, and it also means that legitimate 404s are caught before a
+  # round trip to SVN.
+  #
+  # However, we can't expect users of preview.py to run a cronjob first, so,
+  # this is a hack allow that to be online all of the time.
+  # TODO(kalman): achieve this via proper dependency injection.
+  ALWAYS_ONLINE = False
+
   def __init__(self, request, response):
     super(Handler, self).__init__(request, response)
 
@@ -38,16 +56,17 @@ class Handler(webapp.RequestHandler):
     if real_path.strip('/') == 'extensions':
       real_path = 'extensions/index.html'
 
-    server_instance = ServerInstance.GetOrCreate(channel_name)
+    constructor = (
+        ServerInstance.GetOrCreateOnline if Handler.ALWAYS_ONLINE else
+        ServerInstance.GetOrCreateOffline)
+    server_instance = constructor(channel_name)
 
     canonical_path = server_instance.path_canonicalizer.Canonicalize(real_path)
     if real_path != canonical_path:
       self.redirect(canonical_path)
       return
 
-    ServerInstance.GetOrCreate(channel_name).Get(real_path,
-                                                 self.request,
-                                                 self.response)
+    server_instance.Get(real_path, self.request, self.response)
 
   def _HandleCron(self, path):
     # Cron strategy:
@@ -75,26 +94,33 @@ class Handler(webapp.RequestHandler):
     channel = path.split('/')[-1]
     logging.info('cron/%s: starting' % channel)
 
-    server_instance = ServerInstance.GetOrCreate(channel)
+    server_instance = ServerInstance.GetOrCreateOnline(channel)
 
-    def run_cron_for_dir(d):
+    def run_cron_for_dir(d, path_prefix=''):
       error = None
       start_time = time.time()
       files = [f for f in server_instance.content_cache.GetFromFileListing(d)
                if not f.endswith('/')]
       for f in files:
+        path = '%s%s' % (path_prefix, f)
         try:
-          server_instance.Get(f, MockRequest(f), MockResponse())
+          response = MockResponse()
+          server_instance.Get(path, MockRequest(path), response)
+          if response.status != 200:
+            error = 'Got %s response' % response.status
         except error:
-          logging.error('cron/%s: error rendering %s/%s: %s' % (
-              channel, d, f, error))
-      logging.info('cron/%s: rendering %s files in %s took %s seconds' % (
-          channel, len(files), d, time.time() - start_time))
+          pass
+        if error:
+          logging.error('cron/%s: error rendering %s: %s' % (
+              channel, path, error))
+      logging.info('cron/%s: rendering %s files took %s seconds' % (
+          channel, len(files), time.time() - start_time))
       return error
 
     # Don't use "or" since we want to evaluate everything no matter what.
-    was_error = any((run_cron_for_dir(svn_constants.PUBLIC_TEMPLATE_PATH),
-                     run_cron_for_dir(svn_constants.STATIC_PATH)))
+    was_error = any((
+        run_cron_for_dir(svn_constants.PUBLIC_TEMPLATE_PATH),
+        run_cron_for_dir(svn_constants.STATIC_PATH, path_prefix='static/')))
 
     if was_error:
       self.response.status = 500

chrome/common/extensions/docs/server2/in_memory_object_store.py

-# Copyright (c) 2012 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import time
-
-from appengine_wrappers import CACHE_TIMEOUT
-from future import Future
-from object_store import ObjectStore
-
-class _CacheEntry(object):
-  def __init__(self, value, expire_time):
-    self.value = value
-    self._never_expires = (expire_time == 0)
-    self._expiry = time.time() + expire_time
-
-  def HasExpired(self):
-    if self._never_expires:
-      return False
-    return time.time() > self._expiry
-
-class _AsyncGetFuture(object):
-  """A future for memcache gets.
-
-  Properties:
-  - |cache| the in-memory cache used by InMemoryObjectStore
-  - |time| the cache timeout
-  - |future| the |Future| from the backing |ObjectStore|
-  - |initial_mapping| a mapping of cache items already in memory
-  """
-  def __init__(self, cache, time, future, initial_mapping):
-    self._cache = cache
-    self._time = time
-    self._future = future
-    self._mapping = initial_mapping
-
-  def Get(self):
-    if self._future is not None:
-      result = self._future.Get()
-      self._cache.update(
-          dict((k, _CacheEntry(v, self._time)) for k, v in result.iteritems()))
-      self._mapping.update(result)
-    return self._mapping
-
-class InMemoryObjectStore(ObjectStore):
-  def __init__(self, object_store):
-    self._object_store = object_store
-    self._cache = {}
-
-  def SetMulti(self, mapping, time=CACHE_TIMEOUT):
-    for k, v in mapping.iteritems():
-      self._cache[k] = _CacheEntry(v, time)
-      # TODO(cduvall): Use a batch set? App Engine kept throwing:
-      # ValueError: Values may not be more than 1000000 bytes in length
-      # for the batch set.
-      self._object_store.Set(k, v, time=time)
-
-  def GetMulti(self, keys, time=CACHE_TIMEOUT):
-    keys = keys[:]
-    mapping = {}
-    for key in keys:
-      cache_entry = self._cache.get(key, None)
-      if cache_entry is None or cache_entry.HasExpired():
-        mapping[key] = None
-      else:
-        mapping[key] = cache_entry.value
-        keys.remove(key)
-    future = self._object_store.GetMulti(keys, time=time)
-    return Future(delegate=_AsyncGetFuture(self._cache,
-                                           time,
-                                           future,
-                                           mapping))
-
-  def Delete(self, key):
-    if key in self._cache:
-      self._cache.pop(key)
-    self._object_store.Delete(key)

chrome/common/extensions/docs/server2/integration_test.py

 #!/usr/bin/env python
-# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import logging
+from handler import Handler
+from local_renderer import LocalRenderer
 import optparse
 import os
 import sys
-from StringIO import StringIO
-import re
+import time
 import unittest
 
 # Run build_server so that files needed by tests are copied to the local
@@ -16,150 +16,94 @@ import unittest
 import build_server
 build_server.main()
 
-BASE_PATH = None
-EXPLICIT_TEST_FILES = None
-
-from fake_fetchers import ConfigureFakeFetchers
-ConfigureFakeFetchers(os.path.join(sys.path[0], os.pardir))
-
-# Import Handler later because it immediately makes a request to github. We need
-# the fake urlfetch to be in place first.
-from handler import Handler
-
-class _MockResponse(object):
-  def __init__(self):
-    self.status = 200
-    self.out = StringIO()
-    self.headers = {}
-
-  def set_status(self, status):
-    self.status = status
+# Arguments set up if __main__ specifies them.
+_BASE_PATH = os.path.join(
+    os.path.abspath(os.path.dirname(__file__)), os.pardir, os.pardir)
+_EXPLICIT_TEST_FILES = None
 
-class _MockRequest(object):
-  def __init__(self, path):
-    self.headers = {}
-    self.path = path
-    self.url = 'http://localhost' + path
+def _GetPublicFiles():
+  '''Gets all public files mapped to their contents.
+  '''
+  public_path = os.path.join(_BASE_PATH, 'docs', 'templates', 'public', '')
+  public_files = {}
+  for path, dirs, files in os.walk(public_path):
+    relative_path = path[len(public_path):]
+    for filename in files:
+      with open(os.path.join(path, filename), 'r') as f:
+        public_files[os.path.join(relative_path, filename)] = f.read()
+  return public_files
 
 class IntegrationTest(unittest.TestCase):
-  def _TestSamplesLocales(self, sample_path, failures):
-    # Use US English, Spanish, and Arabic.
-    for lang in ['en-US', 'es', 'ar']:
-      request = _MockRequest(sample_path)
-      request.headers['Accept-Language'] = lang + ';q=0.8'
-      response = _MockResponse()
-      try:
-        Handler(request, response).get()
-        if 200 != response.status:
-          failures.append(
-              'Samples page with language %s does not have 200 status.'
-              ' Status was %d.' %  (lang, response.status))
-        if not response.out.getvalue():
-          failures.append(
-              'Rendering samples page with language %s produced no output.' %
-                  lang)
-      except Exception as e:
-        failures.append('Error rendering samples page with language %s: %s' %
-            (lang, e))
+  def setUp(self):
+    self._renderer = LocalRenderer(_BASE_PATH)
 
-  def _RunPublicTemplatesTest(self):
-    base_path = os.path.join(BASE_PATH, 'docs', 'templates', 'public')
-    if EXPLICIT_TEST_FILES is None:
-      test_files = []
-      for path, dirs, files in os.walk(base_path):
-        for dir_ in dirs:
-          if dir_.startswith('.'):
-            dirs.remove(dir_)
-        for name in files:
-          if name.startswith('.') or name == '404.html':
-            continue
-          test_files.append(os.path.join(path, name)[len(base_path + os.sep):])
-    else:
-      test_files = EXPLICIT_TEST_FILES
-    test_files = [f.replace(os.sep, '/') for f in test_files]
-    failures = []
-    for filename in test_files:
-      request = _MockRequest(filename)
-      response = _MockResponse()
-      try:
-        Handler(request, response).get()
-        if 200 != response.status:
-          failures.append('%s does not have 200 status. Status was %d.' %
-              (filename, response.status))
-        if not response.out.getvalue():
-          failures.append('Rendering %s produced no output.' % filename)
-        if filename.endswith('samples.html'):
-          self._TestSamplesLocales(filename, failures)
-      except Exception as e:
-        failures.append('Error rendering %s: %s' % (filename, e))
-    if failures:
-      self.fail('\n'.join(failures))
+  def testCronAndPublicFiles(self):
+    '''Runs cron then requests every public file. Cron needs to be run first
+    because the public file requests are offline.
+    '''
+    if _EXPLICIT_TEST_FILES is not None:
+      return
 
-  def testAllPublicTemplates(self):
-    logging.getLogger().setLevel(logging.ERROR)
-    logging_error = logging.error
+    print('Running cron...')
+    start_time = time.time()
     try:
-      logging.error = self.fail
-      self._RunPublicTemplatesTest()
+      render_content, render_status, _ = self._renderer.Render('/cron/stable')
+      self.assertEqual(200, render_status)
+      self.assertEqual('Success', render_content)
     finally:
-      logging.error = logging_error
+      print('Took %s seconds' % (time.time() - start_time))
 
-  def testNonexistentFile(self):
-    logging.getLogger().setLevel(logging.CRITICAL)
-    request = _MockRequest('extensions/junk.html')
-    bad_response = _MockResponse()
-    Handler(request, bad_response).get()
-    self.assertEqual(404, bad_response.status)
-    request_404 = _MockRequest('404.html')
-    response_404 = _MockResponse()
-    Handler(request_404, response_404).get()
-    self.assertEqual(200, response_404.status)
-    self.assertEqual(response_404.out.getvalue(), bad_response.out.getvalue())
+    public_files = _GetPublicFiles()
 
-  def testCron(self):
-    if EXPLICIT_TEST_FILES is not None:
-      return
-    logging_error = logging.error
+    print('Rendering %s public files...' % len(public_files.keys()))
+    start_time = time.time()
     try:
-      logging.error = self.fail
-      request = _MockRequest('/cron/trunk')
-      response = _MockResponse()
-      Handler(request, response).get()
-      self.assertEqual(200, response.status)
-      self.assertEqual('Success', response.out.getvalue())
+      for path, content in _GetPublicFiles().iteritems():
+        def check_result(render_content, render_status, _):
+          self.assertEqual(200, render_status)
+          # This is reaaaaally rough since usually these will be tiny templates
+          # that render large files. At least it'll catch zero-length responses.
+          self.assertTrue(len(render_content) >= len(content))
+        check_result(*self._renderer.Render(path))
+        # Samples are internationalized, test some locales.
+        if path.endswith('/samples.html'):
+          for lang in ['en-US', 'es', 'ar']:
+            check_result(*self._renderer.Render(
+                path, headers={'Accept-Language': '%s;q=0.8' % lang}))
     finally:
-      logging.error = logging_error
+      print('Took %s seconds' % (time.time() - start_time))
+
+  def testExplicitFiles(self):
+    '''Tests just the files in _EXPLICIT_TEST_FILES.
+    '''
+    if _EXPLICIT_TEST_FILES is None:
+      return
+    print('Rendering %s explicit files...' % len(_EXPLICIT_TEST_FILES))
+    for filename in _EXPLICIT_TEST_FILES:
+      print('Rendering %s...' % filename)
+      start_time = time.time()
+      try:
+        render_content, render_status, _ = self._renderer.Render(
+            filename, always_online=True)
+        self.assertEqual(200, render_status)
+        self.assertTrue(render_content != '')
+      finally:
+        print('Took %s seconds' % (time.time() - start_time))
+
+  def testFileNotFound(self):
+    render_content, render_status, _ = self._renderer.Render(
+        '/extensions/notfound.html')
+    self.assertEqual(404, render_status)
 
 if __name__ == '__main__':
   parser = optparse.OptionParser()
-  parser.add_option('-p',
-                    '--path',
-                    default=os.path.join(
-                        os.path.abspath(os.path.dirname(__file__)),
-                        os.pardir,
-                        os.pardir))
-  parser.add_option('-a',
-                    '--all',
-                    action='store_true',
-                    default=False)
+  parser.add_option('-p', '--path', default=None)
+  parser.add_option('-a', '--all', action='store_true', default=False)
   (opts, args) = parser.parse_args()
-
   if not opts.all:
-    EXPLICIT_TEST_FILES = args
-  BASE_PATH = opts.path
-  suite = unittest.TestSuite(tests=[
-    IntegrationTest('testNonexistentFile'),
-    IntegrationTest('testCron'),
-    IntegrationTest('testAllPublicTemplates')
-  ])
-  result = unittest.TestResult()
-  suite.run(result)
-  if result.failures:
-    print('*----------------------------------*')
-    print('| integration_test.py has failures |')
-    print('*----------------------------------*')
-    for test, failure in result.failures:
-      print(test)
-      print(failure)
-    exit(1)
-  exit(0)
+    _EXPLICIT_TEST_FILES = args
+  if opts.path is not None:
+    _BASE_PATH = opts.path
+  # Kill sys.argv because we have our own flags.
+  sys.argv = [sys.argv[0]]
+  unittest.main()

chrome/common/extensions/docs/server2/local_renderer.py

+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from handler import Handler
+from fake_fetchers import ConfigureFakeFetchers
+import os
+from StringIO import StringIO
+import urlparse
+
+class _Request(object):
+  def __init__(self, path, headers):
+    self.path = path
+    self.url = 'http://localhost/%s' % path
+    self.headers = headers
+
+class _Response(object):
+  def __init__(self):
+    self.status = 200
+    self.out = StringIO()
+    self.headers = {}
+
+  def set_status(self, status):
+    self.status = status
+
+class LocalRenderer(object):
+  '''Renders pages fetched from the local file system.
+  '''
+  def __init__(self, base_dir):
+    self._base_dir = base_dir.rstrip(os.path.sep)
+
+  def Render(self, path, headers={}, always_online=False):
+    '''Renders |path|, returning a tuple of (status, contents, headers).
+    '''
+    # TODO(kalman): do this via a LocalFileSystem not this fake AppEngine stuff.
+    ConfigureFakeFetchers(os.path.join(self._base_dir, 'docs'))
+    handler_was_always_online = Handler.ALWAYS_ONLINE
+    Handler.ALWAYS_ONLINE = always_online
+    try:
+      response = _Response()
+      Handler(_Request(urlparse.urlparse(path).path, headers), response).get()
+      content = response.out.getvalue()
+      if isinstance(content, unicode):
+        content = content.encode('utf-8', 'replace')
+      return (content, response.status, response.headers)
+    finally:
+      Handler.ALWAYS_ONLINE = handler_was_always_online

chrome/common/extensions/docs/server2/memcache_object_store.py

@@ -3,7 +3,7 @@
 # found in the LICENSE file.
 
 from appengine_wrappers import memcache
-from object_store import ObjectStore, CACHE_TIMEOUT
+from object_store import ObjectStore
 
 class _AsyncMemcacheGetFuture(object):
   def __init__(self, rpc):
@@ -16,14 +16,12 @@ class MemcacheObjectStore(ObjectStore):
   def __init__(self, namespace):
     self._namespace = namespace
 
-  def SetMulti(self, mapping, time=CACHE_TIMEOUT):
-    memcache.Client().set_multi_async(mapping,
-                                      namespace=self._namespace,
-                                      time=time)
+  def SetMulti(self, mapping):
+    memcache.Client().set_multi_async(mapping, namespace=self._namespace)
 
-  def GetMulti(self, keys, time=CACHE_TIMEOUT):
+  def GetMulti(self, keys):
     rpc = memcache.Client().get_multi_async(keys, namespace=self._namespace)
     return _AsyncMemcacheGetFuture(rpc)
 
-  def Delete(self, key):
-    memcache.delete(key, namespace=self._namespace)
+  def DelMulti(self, keys):
+    memcache.delete_multi(keys, namespace=self._namespace)

chrome/common/extensions/docs/server2/object_store.py

@@ -2,7 +2,6 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-from appengine_wrappers import CACHE_TIMEOUT
 from future import Future
 
 class _SingleGetFuture(object):
@@ -14,33 +13,36 @@ class _SingleGetFuture(object):
     return self._future.Get().get(self._key)
 
 class ObjectStore(object):
-  """A class for caching picklable objects.
-  """
-  def Set(self, key, value, time=CACHE_TIMEOUT):
-    """Sets key -> value in the object store, with the specified timeout.
-    """
-    self.SetMulti({ key: value }, time=time)
-
-  def SetMulti(self, mapping, time=CACHE_TIMEOUT):
-    """Sets the mapping of keys to values in the object store with the specified
-    timeout.
-    """
+  '''A class for caching picklable objects.
+  '''
+  def Get(self, key):
+    '''Gets a |Future| with the value of |key| in the object store, or None
+    if |key| is not in the object store.
+    '''
+    return Future(delegate=_SingleGetFuture(self.GetMulti([key]), key))
+
+  def GetMulti(self, keys):
+    '''Gets a |Future| with values mapped to |keys| from the object store, with
+    any keys not in the object store omitted.
+    '''
     raise NotImplementedError()
 
-  def Get(self, key, time=CACHE_TIMEOUT):
-    """Gets a |Future| with the value of |key| in the object store, or None
-    if |key| is not in the object store.
-    """
-    return Future(delegate=_SingleGetFuture(self.GetMulti([key], time=time),
-                                            key))
-
-  def GetMulti(self, keys, time=CACHE_TIMEOUT):
-    """Gets a |Future| with values mapped to |keys| from the object store, with
-    any keys not in the object store mapped to None.
-    """
+  def Set(self, key, value):
+    '''Sets key -> value in the object store.
+    '''
+    self.SetMulti({ key: value })
+
+  def SetMulti(self, items):
+    '''Atomically sets the mapping of keys to values in the object store.
+    '''
     raise NotImplementedError()
 
-  def Delete(self, key):
-    """Deletes a key from the object store.
-    """
+  def Del(self, key):
+    '''Deletes a key from the object store.
+    '''
+    self.DelMulti([key])
+
+  def DelMulti(self, keys):
+    '''Deletes |keys| from the object store.
+    '''
     raise NotImplementedError()

chrome/common/extensions/docs/server2/object_store_creator.py

@@ -2,8 +2,9 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-from in_memory_object_store import InMemoryObjectStore
+from cache_chain_object_store import CacheChainObjectStore
 from memcache_object_store import MemcacheObjectStore
+from persistent_object_store import PersistentObjectStore
 
 class ObjectStoreCreator(object):
   class Factory(object):
@@ -45,4 +46,5 @@ class ObjectStoreCreator(object):
       namespace = '%s/%s' % (namespace, version)
     if self._store_type is not None:
       return self._store_type(namespace)
-    return InMemoryObjectStore(MemcacheObjectStore(namespace))
+    return CacheChainObjectStore((MemcacheObjectStore(namespace),
+                                  PersistentObjectStore(namespace)))

chrome/common/extensions/docs/server2/offline_file_system.py

+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from file_system import FileSystem, FileNotFoundError
+
+class OfflineFileSystem(FileSystem):
+  '''An offline FileSystem which masquerades as another file system. It throws
+  FileNotFound error for all operations, and overrides GetName and GetVersion.
+  '''
+  def __init__(self, cls):
+    self._cls = cls
+
+  def Read(self, paths, binary=False):
+    raise FileNotFoundError(paths)
+
+  def Stat(self, path):
+    raise FileNotFoundError(path)
+
+  # HACK: despite GetName/GetVersion being @classmethods, these need to be
+  # instance methods so that we can grab the name and version from the class
+  # given on construction.
+
+  def GetName(self):
+    return self._cls.GetName()
+
+  def GetVersion(self):
+    return self._cls.GetVersion()

chrome/common/extensions/docs/server2/persistent_object_store.py

+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from appengine_wrappers import db, IsDevServer
+from datastore_models import PersistentObjectStoreItem
+from future import Future
+import logging
+from object_store import ObjectStore
+
+class _AsyncGetFuture(object):
+  def __init__(self, object_store, keys):
+    self._futures = dict(
+        (k, db.get_async(
+            PersistentObjectStoreItem.CreateKey(object_store._namespace, k)))
+         for k in keys)
+
+  def Get(self):
+    return dict((key, future.get_result().GetValue())
+                for key, future in self._futures.iteritems()
+                if future.get_result() is not None)
+
+class PersistentObjectStore(ObjectStore):
+  '''Stores data persistently using the AppEngine Datastore API.
+  '''
+  def __init__(self, namespace):
+    self._namespace = namespace
+
+  def SetMulti(self, mapping):
+    futures = []
+    for key, value in mapping.items():
+      futures.append(db.put_async(
+          PersistentObjectStoreItem.CreateItem(self._namespace, key, value)))
+    # If running the dev server, the futures don't complete until the server is
+    # *quitting*. This is annoying. Flush now.
+    if IsDevServer():
+      [future.wait() for future in futures]
+
+  def GetMulti(self, keys):
+    return Future(delegate=_AsyncGetFuture(self, keys))
+
+  def DelMulti(self, keys):
+    futures = []
+    for key in keys:
+      futures.append(db.delete_async(
+        PersistentObjectStoreItem.CreateKey(self._namespace, key)))
+    # If running the dev server, the futures don't complete until the server is
+    # *quitting*. This is annoying. Flush now.
+    if IsDevServer():
+      [future.wait() for future in futures]

chrome/common/extensions/docs/server2/persistent_object_store_test.py

+#!/usr/bin/env python
+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from persistent_object_store import PersistentObjectStore
+import unittest
+
+class PersistentObjectStoreTest(unittest.TestCase):
+  '''Tests for PersistentObjectStore. These are all a bit contrived because
+  ultimately it comes down to our use of the appengine datastore API, and we
+  mock it out for tests anyway. Who knows whether it's correct.
+  '''
+  def testPersistence(self):
+    # First object store.
+    object_store = PersistentObjectStore('test')
+    object_store.Set('key', 'value')
+    self.assertEqual('value', object_store.Get('key').Get())
+    # Other object store should have it too.
+    another_object_store = PersistentObjectStore('test')
+    self.assertEqual('value', another_object_store.Get('key').Get())
+    # Setting in the other store should set in both.
+    mapping = {'key2': 'value2', 'key3': 'value3'}
+    another_object_store.SetMulti(mapping)
+    self.assertEqual(mapping, object_store.GetMulti(mapping.keys()).Get())
+    self.assertEqual(mapping,
+                     another_object_store.GetMulti(mapping.keys()).Get())
+    # And delete.
+    object_store.DelMulti(mapping.keys())
+    self.assertEqual({}, object_store.GetMulti(mapping.keys()).Get())
+    self.assertEqual({}, another_object_store.GetMulti(mapping.keys()).Get())
+
+  def testNamespaceIsolation(self):
+    object_store = PersistentObjectStore('test')
+    another_object_store = PersistentObjectStore('another')
+    object_store.Set('key', 'value')
+    self.assertEqual(None, another_object_store.Get('key').Get())
+
+if __name__ == '__main__':
+  unittest.main()

chrome/common/extensions/docs/server2/preview.py

@@ -28,55 +28,48 @@
 # relative paths (e.g. static/css/site.css) for convenient sandboxing.
 
 from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+from local_renderer import LocalRenderer
 import logging
 import optparse
 import os
-import shutil
-from StringIO import StringIO
 import sys
 import time
-import urlparse
 
 import build_server
 # Copy all the files necessary to run the server. These are cleaned up when the
 # server quits.
 build_server.main()
 
-from fake_fetchers import ConfigureFakeFetchers
-
-class _Response(object):
-  def __init__(self):
-    self.status = 200
-    self.out = StringIO()
-    self.headers = {}
-
-  def set_status(self, status):
-    self.status = status
-
-class _Request(object):
-  def __init__(self, path):
-    self.headers = {}
-    self.path = path
-    self.url = 'http://localhost' + path
-
-def _Render(path):
-  response = _Response()
-  Handler(_Request(urlparse.urlparse(path).path), response).get()
-  content = response.out.getvalue()
-  if isinstance(content, unicode):
-    content = content.encode('utf-8', 'replace')
-  return (content, response.status, response.headers)
-
 def _GetLocalPath():
   if os.sep in sys.argv[0]:
     return os.path.join(sys.argv[0].rsplit(os.sep, 1)[0], os.pardir, os.pardir)
   return os.path.join(os.pardir, os.pardir)
 
+def _Render(base_dir, path):
+  renderer = LocalRenderer(base_dir)
+  content, status, headers = renderer.Render(path, always_online=True)
+  while status in [301, 302]:
+    redirect = headers['Location'].lstrip('/')
+    sys.stderr.write('<!-- Redirected %s to %s -->\n' % (path, redirect))
+    content, status, headers = renderer.Render(redirect, always_online=True)
+  return (content, status, headers)
+
 class RequestHandler(BaseHTTPRequestHandler):
+  class Factory(object):
+    def __init__(self, base_dir):
+      self._base_dir = base_dir
+
+    def Create(self, *args):
+      return RequestHandler(self._base_dir, *args)
+
+  def __init__(self, base_dir, *args):
+    self._base_dir = base_dir
+    BaseHTTPRequestHandler.__init__(self, *args)
+
   """A HTTPRequestHandler that outputs the docs page generated by Handler.
   """
   def do_GET(self):
-    content, status, headers = _Render(self.path)
+    content, status, headers = _Render(self._base_dir, self.path)
     self.send_response(status)
     for k, v in headers.iteritems():
       self.send_header(k, v)
@@ -109,9 +102,6 @@ if __name__ == '__main__':
           'docs.')
     exit()
 
-  ConfigureFakeFetchers(os.path.join(opts.directory, 'docs'))
-  from handler import Handler
-
   if opts.render:
     if opts.render.find('#') >= 0:
       (path, iterations) = opts.render.rsplit('#', 1)
@@ -123,18 +113,13 @@ if __name__ == '__main__':
     if opts.time:
       start_time = time.time()
 
-    content, status, headers = _Render(path)
-    if status in [301, 302]:
-      # Handle a single level of redirection.
-      redirect_path = headers['Location'].lstrip('/')
-      sys.stderr.write('<!-- Redirected %s to %s -->\n' % (path, redirect_path))
-      content, status, headers = _Render(redirect_path)
+    content, status, headers = _Render(opts.directory, path)
     if status != 200:
       print('Error status: %s' % status)
       exit(1)
 
     for _ in range(extra_iterations):
-      _Render(path)
+      _Render(opts.directory, path)
 
     if opts.time:
       print('Took %s seconds' % (time.time() - start_time))
@@ -156,7 +141,8 @@ if __name__ == '__main__':
   print('')
 
   logging.getLogger().setLevel(logging.INFO)
-  server = HTTPServer(('', int(opts.port)), RequestHandler)
+  server = HTTPServer(('', int(opts.port)),
+                      RequestHandler.Factory(opts.directory).Create)
   try:
     server.serve_forever()
   finally:

chrome/common/extensions/docs/server2/reference_resolver_test.py

@@ -9,7 +9,6 @@ import sys
 import unittest
 
 from file_system import FileNotFoundError
-from in_memory_object_store import InMemoryObjectStore
 from reference_resolver import ReferenceResolver
 from test_object_store import TestObjectStore
 

chrome/common/extensions/docs/server2/server_instance.py

@@ -11,15 +11,15 @@ from api_list_data_source import APIListDataSource
 from appengine_blobstore import AppEngineBlobstore
 from appengine_url_fetcher import AppEngineUrlFetcher
 from branch_utility import BranchUtility
+from caching_file_system import CachingFileSystem
 from compiled_file_system import CompiledFileSystem
 from example_zipper import ExampleZipper
 from file_system import FileNotFoundError
 from github_file_system import GithubFileSystem
-from in_memory_object_store import InMemoryObjectStore
 from intro_data_source import IntroDataSource
 from local_file_system import LocalFileSystem
-from caching_file_system import CachingFileSystem
 from object_store_creator import ObjectStoreCreator
+from offline_file_system import OfflineFileSystem
 from path_canonicalizer import PathCanonicalizer
 from reference_resolver import ReferenceResolver
 from samples_data_source import SamplesDataSource
@@ -27,6 +27,7 @@ from sidenav_data_source import SidenavDataSource
 from subversion_file_system import SubversionFileSystem
 import svn_constants
 from template_data_source import TemplateDataSource
+from third_party.json_schema_compiler.memoize import memoize
 from third_party.json_schema_compiler.model import UnixName
 import url_constants
 
@@ -35,32 +36,43 @@ def _IsBinaryMimetype(mimetype):
              for prefix in ['audio', 'image', 'video'])
 
 class ServerInstance(object):
-  '''Per-instance per-branch state.
-  '''
-  _instances = {}
-
+  # Lazily create so we don't create github file systems unnecessarily in
+  # tests.
   branch_utility = None
   github_file_system = None
 
   @staticmethod
-  def GetOrCreate(channel):
-    # Lazily create so that we don't do unnecessary work in tests.
-    if ServerInstance.branch_utility is None:
-      ServerInstance.branch_utility = BranchUtility(
-          url_constants.OMAHA_PROXY_URL, AppEngineUrlFetcher())
-    branch = ServerInstance.branch_utility.GetBranchNumberForChannelName(
-        channel)
-
-    # Use the branch as the key to |_instances| since the branch data is
-    # predictable while the channel data (channels can swich branches) isn't.
-    instance = ServerInstance._instances.get(branch)
-    if instance is None:
-      instance = ServerInstance._CreateForProduction(channel, branch)
-      ServerInstance._instances[branch] = instance
-    return instance
+  @memoize
+  def GetOrCreateOffline(channel):
+    '''Gets/creates a local ServerInstance, meaning that only resources local to
+    the server - memcache, object store, etc, are queried. This amounts to not
+    setting up the subversion nor github file systems.
+    '''
+    branch_utility = ServerInstance._GetOrCreateBranchUtility()
+    branch = branch_utility.GetBranchNumberForChannelName(channel)
+    object_store_creator_factory = ObjectStoreCreator.Factory(branch)
+    # No svn nor github file systems. Rely on the crons to fill the caches, and
+    # for the caches to exist.
+    return ServerInstance(
+        channel,
+        object_store_creator_factory,
+        CachingFileSystem(OfflineFileSystem(SubversionFileSystem),
+                          object_store_creator_factory),
+        # TODO(kalman): convert GithubFileSystem to be wrappable in a
+        # CachingFileSystem so that it can be replaced with an
+        # OfflineFileSystem. Currently GFS doesn't set the child versions of
+        # stat requests so it doesn't.
+        ServerInstance._GetOrCreateGithubFileSystem())
 
   @staticmethod
-  def _CreateForProduction(channel, branch):
+  @memoize
+  def GetOrCreateOnline(channel):
+    '''Creates/creates an online server instance, meaning that both local and
+    subversion/github resources are queried.
+    '''
+    branch_utility = ServerInstance._GetOrCreateBranchUtility()
+    branch = branch_utility.GetBranchNumberForChannelName(channel)
+
     if branch == 'trunk':
       svn_url = '/'.join((url_constants.SVN_TRUNK_URL,
                           'src',
@@ -81,17 +93,10 @@ class ServerInstance(object):
                              AppEngineUrlFetcher(viewvc_url)),
         object_store_creator_factory)
 
-    # Lazily create so we don't create github file systems unnecessarily in
-    # tests.
-    if ServerInstance.github_file_system is None:
-      ServerInstance.github_file_system = GithubFileSystem(
-          AppEngineUrlFetcher(url_constants.GITHUB_URL),
-          AppEngineBlobstore())
-
     return ServerInstance(channel,
                           object_store_creator_factory,
                           svn_file_system,
-                          ServerInstance.github_file_system)
+                          ServerInstance._GetOrCreateGithubFileSystem())
 
   @staticmethod
   def CreateForTest(file_system):
@@ -100,6 +105,22 @@ class ServerInstance(object):
                           file_system,
                           None)
 
+  @staticmethod
+  def _GetOrCreateBranchUtility():
+    if ServerInstance.branch_utility is None:
+      ServerInstance.branch_utility = BranchUtility(
+          url_constants.OMAHA_PROXY_URL,
+          AppEngineUrlFetcher())
+    return ServerInstance.branch_utility
+
+  @staticmethod
+  def _GetOrCreateGithubFileSystem():
+    if ServerInstance.github_file_system is None:
+      ServerInstance.github_file_system = GithubFileSystem(
+          AppEngineUrlFetcher(url_constants.GITHUB_URL),
+          AppEngineBlobstore())
+    return ServerInstance.github_file_system
+
   def __init__(self,
                channel,
                object_store_creator_factory,

chrome/common/extensions/docs/server2/sidenav_data_source_test.py

@@ -8,7 +8,6 @@ import sys
 import unittest
 
 from compiled_file_system import CompiledFileSystem
-from in_memory_object_store import InMemoryObjectStore
 from local_file_system import LocalFileSystem
 from object_store_creator import ObjectStoreCreator
 from sidenav_data_source import SidenavDataSource

chrome/common/extensions/docs/server2/subversion_file_system.py

@@ -134,5 +134,6 @@ class SubversionFileSystem(FileSystem):
       stat_info.version = stat_info.child_versions[filename]
     return stat_info
 
-  def GetVersion(self):
+  @classmethod
+  def GetVersion(cls):
     return _VERSION

chrome/common/extensions/docs/server2/template_data_source.py

@@ -147,5 +147,6 @@ class TemplateDataSource(object):
     try:
       return self._cache.GetFromFile(base_path + '/' + real_path)
     except FileNotFoundError as e:
-      logging.info(e)
+      logging.warning('Template %s in %s not found: %s' % (
+          template_name, base_path, e))
       return None

chrome/common/extensions/docs/server2/test_file_system.py

@@ -103,7 +103,7 @@ class TestFileSystem(FileSystem):
     return self._StatImpl(path)
 
   def _StatImpl(self, path):
-    read_result = self._ReadImpl([path]).Get()[path]
+    read_result = self._ReadImpl([path]).Get().get(path)
     stat_result = StatInfo(self._SinglePathStat(path))
     if isinstance(read_result, list):
       stat_result.child_versions = dict(

chrome/common/extensions/docs/server2/test_object_store.py

@@ -6,19 +6,58 @@ from future import Future
 from object_store import ObjectStore
 
 class TestObjectStore(ObjectStore):
-  '''An object store which records its namespace and behaves like a dict, for
-  testing.
+  '''An object store which records its namespace and behaves like a dict.
+  Specify |init| with an initial object for the object store.
+  Use CheckAndReset to assert how many times Get/Set/Del have been called. Get
+  is a special case; it is only incremented once the future has had Get called.
   '''
-  def __init__(self, namespace):
+  def __init__(self, namespace, init=None):
     self.namespace = namespace
-    self._store = {}
+    self._store = init or {}
+    self._get_count = 0
+    self._set_count = 0
+    self._del_count = 0
 
-  def SetMulti(self, mapping, **optarg):
+  #
+  # ObjectStore implementation.
+  #
+
+  def GetMulti(self, keys):
+    class FutureImpl(object):
+      def Get(self2):
+        self._get_count += 1
+        return dict((k, self._store.get(k)) for k in keys if k in self._store)
+    return Future(delegate=FutureImpl())
+
+  def SetMulti(self, mapping):
+    self._set_count += 1
     self._store.update(mapping)
 
-  def GetMulti(self, keys, **optargs):
-    return Future(value=dict((k, v) for k, v in self._store.items()
-                                    if k in keys))
+  def DelMulti(self, keys):
+    self._del_count += 1
+    for k in keys:
+      self._store.pop(k, None)
+
+  #
+  # Testing methods.
+  #
+
+  def CheckAndReset(self, get_count=0, set_count=0, del_count=0):
+    '''Returns a tuple (success, error). Use in tests like:
+    self.assertTrue(*object_store.CheckAndReset(...))
+    '''
+    errors = []
+    for desc, expected, actual in (('get_count', get_count, self._get_count),
+                                   ('set_count', set_count, self._set_count),
+                                   ('del_count', del_count, self._del_count)):
+      if actual != expected:
+        errors.append('%s: expected %s got %s' % (desc, expected, actual))
+    try:
+      return (len(errors) == 0, ','.join(errors))
+    finally:
+      self.Reset()
 
-  def Delete(self, key):
-    del self._store[key]
+  def Reset(self):
+    self._get_count = 0
+    self._set_count = 0
+    self._del_count = 0

chrome/common/extensions/docs/server2/test_object_store_test.py

@@ -20,9 +20,24 @@ class TestObjectStoreTest(unittest.TestCase):
     store.Set('hi', 'blah')
     self.assertEqual('blah', store.Get('hi').Get())
     self.assertEqual({'hi': 'blah'}, store.GetMulti(['hi', 'lo']).Get())
-    store.Delete('hi')
+    store.Del('hi')
     self.assertEqual(None, store.Get('hi').Get())
     self.assertEqual({}, store.GetMulti(['hi', 'lo']).Get())
 
+  def testCheckAndReset(self):
+    store = TestObjectStore('namespace')
+    store.Set('x', 'y')
+    self.assertTrue(store.CheckAndReset(set_count=1))
+    store.Set('x', 'y')
+    store.Set('x', 'y')
+    self.assertTrue(store.CheckAndReset(set_count=2))
+    store.Set('x', 'y')
+    store.Set('x', 'y')
+    store.Get('x')
+    store.Get('x')
+    store.Get('x')
+    store.Del('x')
+    self.assertTrue(store.CheckAndReset(get_count=3, set_count=2, del_count=1))
+
 if __name__ == '__main__':
   unittest.main()