Docserver: Add commit history and _reset_commit servlet

This adds commit history to named commits in the CommitTracker. The history can be viewed by requesting /_query_commit/<commit-name>.

This also adds a _reset_commit servlet which will reset the current commit ID of a named commit. This servlet will only obey the request if the given commit ID is present in the named commit's history.

BUG=None
R=kalman@chromium.org
NOTRY=true

Review URL: https://codereview.chromium.org/736773002

Cr-Commit-Position: refs/heads/master@{#314395}

chrome/common/extensions/docs/server2/admin_servlets.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from appengine_wrappers import taskqueue
+from commit_tracker import CommitTracker
+from future import All
+from object_store_creator import ObjectStoreCreator
+from refresh_tracker import RefreshTracker
+from servlet import Servlet, Response
+
+
+class EnqueueServlet(Servlet):
+  '''This Servlet can be used to manually enqueue tasks on the default
+  taskqueue. Useful for when an admin wants to manually force a specific
+  DataSource refresh, but the refresh operation takes longer than the 60 sec
+  timeout of a non-taskqueue request. For example, you might query
+
+  /_enqueue/_refresh/content_providers/cr-native-client?commit=123ff65468dcafff0
+
+  which will enqueue a task (/_refresh/content_providers/cr-native-client) to
+  refresh the NaCl documentation cache for commit 123ff65468dcafff0.
+
+  Access to this servlet should always be restricted to administrative users.
+  '''
+  def __init__(self, request):
+    Servlet.__init__(self, request)
+
+  def Get(self):
+    queue = taskqueue.Queue()
+    queue.add(taskqueue.Task(url='/%s' % self._request.path,
+                             params=self._request.arguments))
+    return Response.Ok('Task enqueued.')
+
+
+class QueryCommitServlet(Servlet):
+  '''Provides read access to the commit ID cache within the server. For example:
+
+  /_query_commit/master
+
+  will return the commit ID stored under the commit key "master" within the
+  commit cache. Currently "master" is the only named commit we cache, and it
+  corresponds to the commit ID whose data currently populates the data cache
+  used by live instances.
+  '''
+  def __init__(self, request):
+    Servlet.__init__(self, request)
+
+  def Get(self):
+    object_store_creator = ObjectStoreCreator(start_empty=False)
+    commit_tracker = CommitTracker(object_store_creator)
+
+    def generate_response(result):
+      commit_id, history = result
+      history_log = ''.join('%s: %s<br>' % (entry.datetime, entry.commit_id)
+                            for entry in reversed(history))
+      response = 'Current commit: %s<br><br>Most recent commits:<br>%s' % (
+          commit_id, history_log)
+      return response
+
+    commit_name = self._request.path
+    id_future = commit_tracker.Get(commit_name)
+    history_future = commit_tracker.GetHistory(commit_name)
+    return Response.Ok(
+        All((id_future, history_future)).Then(generate_response).Get())
+
+
+class DumpRefreshServlet(Servlet):
+  def __init__(self, request):
+    Servlet.__init__(self, request)
+
+  def Get(self):
+    object_store_creator = ObjectStoreCreator(start_empty=False)
+    refresh_tracker = RefreshTracker(object_store_creator)
+    commit_id = self._request.path
+    work_order = refresh_tracker._GetWorkOrder(commit_id).Get()
+    task_names = ['%s@%s' % (commit_id, task) for task in work_order.tasks]
+    completions = refresh_tracker._task_completions.GetMulti(task_names).Get()
+    missing = []
+    for task in task_names:
+      if task not in completions:
+        missing.append(task)
+    response = 'Missing:<br>%s' % ''.join('%s<br>' % task for task in missing)
+    return Response.Ok(response)
+
+class ResetCommitServlet(Servlet):
+  '''Writes a new commit ID to the commit cache. For example:
+
+  /_reset_commit/master/123456
+
+  will reset the 'master' commit ID to '123456'. The provided commit MUST be
+  in the named commit's recent history or it will be ignored.
+  '''
+
+  class Delegate(object):
+    def CreateCommitTracker(self):
+      return CommitTracker(ObjectStoreCreator(start_empty=False))
+
+  def __init__(self, request, delegate=Delegate()):
+    Servlet.__init__(self, request)
+    self._delegate = delegate
+
+  def Get(self):
+    commit_tracker = self._delegate.CreateCommitTracker()
+    commit_name, commit_id = self._request.path.split('/', 1)
+    history = commit_tracker.GetHistory(commit_name).Get()
+    if not any(entry.commit_id == commit_id for entry in history):
+      return Response.BadRequest('Commit %s not cached.' % commit_id)
+    commit_tracker.Set(commit_name, commit_id).Get()
+    return Response.Ok('Commit "%s" updated to %s' % (commit_name, commit_id))
+

chrome/common/extensions/docs/server2/admin_servlets_test.py

+#!/usr/bin/env python
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import unittest
+
+from admin_servlets import ResetCommitServlet
+from commit_tracker import CommitTracker
+from object_store_creator import ObjectStoreCreator
+from servlet import Request
+
+
+_COMMIT_HISTORY_DATA = (
+  '1234556789abcdef1234556789abcdef12345567',
+  'f00f00f00f00f00f00f00f00f00f00f00f00f00f',
+  '1010101010101010101010101010101010101010',
+  'abcdefabcdefabcdefabcdefabcdefabcdefabcd',
+  '4242424242424242424242424242424242424242',
+)
+
+
+class _ResetCommitDelegate(ResetCommitServlet.Delegate):
+  def __init__(self, commit_tracker):
+    self._commit_tracker = commit_tracker
+
+  def CreateCommitTracker(self):
+    return self._commit_tracker
+
+
+class AdminServletsTest(unittest.TestCase):
+  def setUp(self):
+    object_store_creator = ObjectStoreCreator(start_empty=True)
+    self._commit_tracker = CommitTracker(object_store_creator)
+    for id in _COMMIT_HISTORY_DATA:
+      self._commit_tracker.Set('master', id).Get()
+
+  def _ResetCommit(self, commit_name, commit_id):
+    return ResetCommitServlet(
+        Request.ForTest('%s/%s' % (commit_name, commit_id)),
+        _ResetCommitDelegate(self._commit_tracker)).Get()
+
+  def _AssertBadRequest(self, commit_name, commit_id):
+    response = self._ResetCommit(commit_name, commit_id)
+    self.assertEqual(response.status, 400,
+        'Should have failed to reset to commit %s to %s.' %
+        (commit_name, commit_id))
+
+  def _AssertOk(self, commit_name, commit_id):
+    response = self._ResetCommit(commit_name, commit_id)
+    self.assertEqual(response.status, 200,
+        'Failed to reset commit %s to %s.' % (commit_name, commit_id))
+
+  def testResetCommitServlet(self):
+    # Make sure all the valid commits can be used for reset.
+    for id in _COMMIT_HISTORY_DATA:
+      self._AssertOk('master', id)
+
+    # Non-existent commit should fail to update
+    self._AssertBadRequest('master',
+                           'b000000000000000000000000000000000000000')
+
+    # Commit 'master' should still point to the last valid entry
+    self.assertEqual(self._commit_tracker.Get('master').Get(),
+                     _COMMIT_HISTORY_DATA[-1])
+
+    # Reset to a valid commit but older
+    self._AssertOk('master', _COMMIT_HISTORY_DATA[0])
+
+    # Commit 'master' should point to the first history entry
+    self.assertEqual(self._commit_tracker.Get('master').Get(),
+                     _COMMIT_HISTORY_DATA[0])
+
+    # Add a new entry to the history and validate that it can be used for reset.
+    _NEW_ENTRY = '9999999999999999999999999999999999999999'
+    self._commit_tracker.Set('master', _NEW_ENTRY).Get()
+    self._AssertOk('master', _NEW_ENTRY)
+
+    # Add a bunch (> 50) of entries to ensure that _NEW_ENTRY has been flushed
+    # out of the history.
+    for i in xrange(0, 20):
+      for id in _COMMIT_HISTORY_DATA:
+        self._commit_tracker.Set('master', id).Get()
+
+    # Verify that _NEW_ENTRY is no longer valid for reset.
+    self._AssertBadRequest('master', _NEW_ENTRY)
+
+if __name__ == '__main__':
+  unittest.main()

chrome/common/extensions/docs/server2/app.yaml

@@ -22,6 +22,10 @@ handlers:
   script: appengine_main.py
   secure: always
   login: admin
+- url: /_reset_commit/.*
+  script: appengine_main.py
+  secure: always
+  login: admin
 - url: /.*
   script: appengine_main.py
   secure: always

chrome/common/extensions/docs/server2/availability_finder.py

@@ -174,7 +174,7 @@ class AvailabilityFinder(object):
     schema_fs = self._CreateAPISchemaFileSystem(file_system)
     api_schemas = schema_fs.GetFromFile(api_filename).Get()
     matching_schemas = [api for api in api_schemas
-                        if api['namespace'] == api_name]
+                        if api and api['namespace'] == api_name]
     # There should only be a single matching schema per file, or zero in the
     # case of no API data being found in _EXTENSION_API.
     assert len(matching_schemas) <= 1

chrome/common/extensions/docs/server2/commit_tracker.py

@@ -2,20 +2,55 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import collections
+import datetime
+
 from object_store_creator import ObjectStoreCreator
 from future import Future
 
 
+# The maximum number of commit IDs to retain in a named commit's history deque.
+_MAX_COMMIT_HISTORY_LENGTH = 50
+
+
+class CachedCommit(object):
+  '''Object type which is stored for each entry in a named commit's history.
+  |datetime| is used as a timestamp for when the commit cache was completed,
+  and is only meant to provide a loose ordering of commits for administrative
+  servlets to display.'''
+  def __init__(self, commit_id, datetime):
+    self.commit_id = commit_id
+    self.datetime = datetime
+
+
 class CommitTracker(object):
   '''Utility class for managing and querying the storage of various named commit
   IDs.'''
   def __init__(self, object_store_creator):
-    # The object store should never be created empty since the sole purpose of
+    # The object stores should never be created empty since the sole purpose of
     # this tracker is to persist named commit data across requests.
     self._store = object_store_creator.Create(CommitTracker, start_empty=False)
+    self._history_store = object_store_creator.Create(CommitTracker,
+        category='history', start_empty=False)
 
   def Get(self, key):
     return self._store.Get(key)
 
   def Set(self, key, commit):
-    return self._store.Set(key, commit)
+    return (self._store.Set(key, commit)
+      .Then(lambda _: self._UpdateHistory(key, commit)))
+
+  def GetHistory(self, key):
+    '''Fetches the commit ID history for a named commit. If the commit has no
+    history, this will return an empty collection.'''
+    return (self._history_store.Get(key)
+        .Then(lambda history: () if history is None else history))
+
+  def _UpdateHistory(self, key, commit):
+    '''Appends a commit ID to a named commit's tracked history.'''
+    def create_or_amend_history(history):
+      if history is None:
+        history = collections.deque([], maxlen=50)
+      history.append(CachedCommit(commit, datetime.datetime.now()))
+      return self._history_store.Set(key, history)
+    return self._history_store.Get(key).Then(create_or_amend_history)

chrome/common/extensions/docs/server2/handler.py

@@ -4,11 +4,10 @@
 
 import time
 
-from appengine_wrappers import taskqueue
-from commit_tracker import CommitTracker
+from admin_servlets import (DumpRefreshServlet, EnqueueServlet,
+    QueryCommitServlet, ResetCommitServlet)
 from cron_servlet import CronServlet
 from instance_servlet import InstanceServlet
-from object_store_creator import ObjectStoreCreator
 from patch_servlet import PatchServlet
 from refresh_servlet import RefreshServlet
 from servlet import Servlet, Request, Response
@@ -18,55 +17,15 @@ from test_servlet import TestServlet
 _DEFAULT_SERVLET = InstanceServlet.GetConstructor()
 
 
-class _EnqueueServlet(Servlet):
-  '''This Servlet can be used to manually enqueue tasks on the default
-  taskqueue. Useful for when an admin wants to manually force a specific
-  DataSource refresh, but the refresh operation takes longer than the 60 sec
-  timeout of a non-taskqueue request. For example, you might query
-
-  /_enqueue/_refresh/content_providers/cr-native-client?commit=123ff65468dcafff0
-
-  which will enqueue a task (/_refresh/content_providers/cr-native-client) to
-  refresh the NaCl documentation cache for commit 123ff65468dcafff0.
-
-  Access to this servlet should always be restricted to administrative users.
-  '''
-  def __init__(self, request):
-    Servlet.__init__(self, request)
-
-  def Get(self):
-    queue = taskqueue.Queue()
-    queue.add(taskqueue.Task(url='/%s' % self._request.path,
-                             params=self._request.arguments))
-    return Response.Ok('Task enqueued.')
-
-
-class _QueryCommitServlet(Servlet):
-  '''Provides read access to the commit ID cache within the server. For example:
-
-  /_query_commit/master
-
-  will return the commit ID stored under the commit key "master" within the
-  commit cache. Currently "master" is the only named commit we cache, and it
-  corresponds to the commit ID whose data currently populates the data cache
-  used by live instances.
-  '''
-  def __init__(self, request):
-    Servlet.__init__(self, request)
-
-  def Get(self):
-    object_store_creator = ObjectStoreCreator(start_empty=False)
-    commit_tracker = CommitTracker(object_store_creator)
-    return Response.Ok(commit_tracker.Get(self._request.path).Get())
-
-
 _SERVLETS = {
   'cron': CronServlet,
-  'enqueue': _EnqueueServlet,
+  'enqueue': EnqueueServlet,
   'patch': PatchServlet,
-  'query_commit': _QueryCommitServlet,
+  'query_commit': QueryCommitServlet,
   'refresh': RefreshServlet,
+  'reset_commit': ResetCommitServlet,
   'test': TestServlet,
+  'dump_refresh': DumpRefreshServlet,
 }
 
 

chrome/common/extensions/docs/server2/servlet.py

@@ -95,6 +95,12 @@ class Response(object):
     status = 301 if permanent else 302
     return Response(headers={'Location': url}, status=status)
 
+  @staticmethod
+  def BadRequest(content, headers=None):
+    '''Returns a bad request (400) response.
+    '''
+    return Response(content=content, headers=headers, status=400)
+
   @staticmethod
   def NotFound(content, headers=None):
     '''Returns a not found (404) response.