Extensions Docs Server: Integration testing

A test that makes sure all the pages in the templates/public directory render
without erroring.

BUG=131095


Review URL: https://chromiumcodereview.appspot.com/10828042

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@149101 0039d316-1c4b-4281-b951-d872f2087c98

chrome/common/extensions/api/file_browser_handler.json

@@ -61,7 +61,7 @@
                   "type": "string",
                   "description": "Suggested name for the file."
               }
-            } 
+            }
           },
           {
             "name": "callback",
@@ -80,6 +80,7 @@
                   "entry": {
                     "type": "object",
                     "constructor": "Entry",
+                    "additionalProperties": { "type": "any" },
                     "optional": true,
                     "description": "Selected file entry. It will be null if a file hasn't been selected."
                   }

chrome/common/extensions/api/input_ime.json

@@ -435,7 +435,7 @@
         "description": "This event is sent when focus enters a text box. It is sent to all extensions that are listening to this event, and enabled by the user.",
         "parameters": [
           {
-            "type": "InputContext",
+            "$ref": "InputContext",
             "name": "context",
             "description": "Describes the text field that has acquired focus."
           }
@@ -459,7 +459,7 @@
         "description": "This event is sent when the properties of the current InputContext change, such as the the type. It is sent to all extensions that are listening to this event, and enabled by the user.",
         "parameters": [
           {
-            "type": "InputContext",
+            "$ref": "InputContext",
             "name": "context",
             "description": "An InputContext object describing the text field that has changed."
           }
@@ -482,7 +482,7 @@
             "description": "ID of the engine receiving the event"
           },
           {
-            "type": "KeyboardEvent",
+            "$ref": "KeyboardEvent",
             "name": "keyData",
             "description": "Data on the key event"
           }

chrome/common/extensions/docs/extensions/input.ime.html

@@ -2488,7 +2488,7 @@
               <h4>onFocus</h4>
               <div class="summary">
                 <!-- Note: intentionally longer 80 columns -->
-                <span class="subdued">chrome.input.ime.</span><span>onFocus</span><span class="subdued">.addListener</span>(function(<span>InputContext context</span>) <span class="subdued">{...}</span><span></span>);
+                <span class="subdued">chrome.input.ime.</span><span>onFocus</span><span class="subdued">.addListener</span>(function(<span>input.ime.InputContext context</span>) <span class="subdued">{...}</span><span></span>);
               </div>
               <div class="description">
                 <p>This event is sent when focus enters a text box. It is sent to all extensions that are listening to this event, and enabled by the user.</p>
@@ -2506,7 +2506,7 @@
                   (
                     <span id="typeTemplate">
                       <span>
-                        <span>InputContext</span>
+                        <a href="input.ime.html#type-input.ime.InputContext">input.ime.InputContext</a>
                       </span>
                     </span>
                   )
@@ -2533,7 +2533,7 @@
               <h4>onInputContextUpdate</h4>
               <div class="summary">
                 <!-- Note: intentionally longer 80 columns -->
-                <span class="subdued">chrome.input.ime.</span><span>onInputContextUpdate</span><span class="subdued">.addListener</span>(function(<span>InputContext context</span>) <span class="subdued">{...}</span><span></span>);
+                <span class="subdued">chrome.input.ime.</span><span>onInputContextUpdate</span><span class="subdued">.addListener</span>(function(<span>input.ime.InputContext context</span>) <span class="subdued">{...}</span><span></span>);
               </div>
               <div class="description">
                 <p>This event is sent when the properties of the current InputContext change, such as the the type. It is sent to all extensions that are listening to this event, and enabled by the user.</p>
@@ -2551,7 +2551,7 @@
                   (
                     <span id="typeTemplate">
                       <span>
-                        <span>InputContext</span>
+                        <a href="input.ime.html#type-input.ime.InputContext">input.ime.InputContext</a>
                       </span>
                     </span>
                   )
@@ -2578,7 +2578,7 @@
               <h4>onKeyEvent</h4>
               <div class="summary">
                 <!-- Note: intentionally longer 80 columns -->
-                <span class="subdued">chrome.input.ime.</span><span>onKeyEvent</span><span class="subdued">.addListener</span>(function(<span>string engineID, KeyboardEvent keyData</span>) <span class="subdued">{...}</span><span></span>);
+                <span class="subdued">chrome.input.ime.</span><span>onKeyEvent</span><span class="subdued">.addListener</span>(function(<span>string engineID, input.ime.KeyboardEvent keyData</span>) <span class="subdued">{...}</span><span></span>);
               </div>
               <div class="description">
                 <p>This event is sent if this extension owns the active IME.</p>
@@ -2619,7 +2619,7 @@
                   (
                     <span id="typeTemplate">
                       <span>
-                        <span>KeyboardEvent</span>
+                        <a href="input.ime.html#type-input.ime.KeyboardEvent">input.ime.KeyboardEvent</a>
                       </span>
                     </span>
                   )

chrome/common/extensions/docs/server2/api_data_source.py

@@ -71,4 +71,4 @@ class APIDataSource(object):
             self._base_path + '/' + idl_path), path)
       except Exception as e:
         logging.warn(e)
-        return None
+        raise

chrome/common/extensions/docs/server2/api_data_source_test.py

@@ -29,8 +29,7 @@ class APIDataSourceTest(unittest.TestCase):
     self.assertEqual(expected, data_source['test_file'])
     self.assertEqual(expected, data_source['testFile'])
     self.assertEqual(expected, data_source['testFile.html'])
-
-    self.assertEqual(None, data_source['junk'])
+    self.assertRaises(OSError, data_source.get, 'junk')
 
 if __name__ == '__main__':
   unittest.main()

chrome/common/extensions/docs/server2/app.yaml

@@ -6,7 +6,7 @@ threadsafe: false
 
 handlers:
 - url: /.*
-  script: echo_handler.py
+  script: appengine_main.py
 
 inbound_services:
 - warmup

chrome/common/extensions/docs/server2/appengine_main.py

+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import sys
+
+# Add the original server location to sys.path so we are able to import
+# modules from there.
+SERVER_PATH = 'chrome/common/extensions/docs/server2'
+if os.path.abspath(SERVER_PATH) not in sys.path:
+  sys.path.append(os.path.abspath(SERVER_PATH))
+
+from google.appengine.ext import webapp
+from google.appengine.ext.webapp.util import run_wsgi_app
+
+from handler import Handler
+
+def main():
+  run_wsgi_app(webapp.WSGIApplication([('/.*', Handler)], debug=False))
+
+if __name__ == '__main__':
+  main()

chrome/common/extensions/docs/server2/appengine_memcache.py

@@ -2,6 +2,8 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+from appengine_wrappers import memcache
+
 MEMCACHE_FILE_SYSTEM_READ = 'MemcacheFileSystem.Get'
 MEMCACHE_FILE_SYSTEM_STAT = 'MemcacheFileSystem.Stat'
 MEMCACHE_BRANCH_UTILITY = 'BranchUtility'
@@ -11,18 +13,17 @@ class AppEngineMemcache(object):
   use. Uses a branch to make sure there are no key collisions if separate
   branches cache the same file.
   """
-  def __init__(self, branch, memcache):
+  def __init__(self, branch):
     self._branch = branch
-    self._memcache = memcache
 
   def Set(self, key, value, namespace, time=60):
-    return self._memcache.set(key,
-                              value,
-                              namespace=self._branch + '.' + namespace,
-                              time=time)
+    return memcache.set(key,
+                        value,
+                        namespace=self._branch + '.' + namespace,
+                        time=time)
 
   def Get(self, key, namespace):
-    return self._memcache.get(key, namespace=self._branch + '.' + namespace)
+    return memcache.get(key, namespace=self._branch + '.' + namespace)
 
   def Delete(self, key, namespace):
-    return self._memcache.delete(key, namespace=self._branch + '.' + namespace)
+    return memcache.delete(key, namespace=self._branch + '.' + namespace)

chrome/common/extensions/docs/server2/appengine_url_fetcher.py

@@ -2,8 +2,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-from google.appengine.api import urlfetch
-
+from appengine_wrappers import urlfetch
 from future import Future
 
 class _AsyncFetchDelegate(object):

chrome/common/extensions/docs/server2/appengine_wrappers.py

+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from in_memory_memcache import InMemoryMemcache
+
+# This will attempt to import the actual Appengine modules, and if it fails,
+# they will be replaced with fake modules. This is useful during testing.
+try:
+  import google.appengine.ext.webapp as webapp
+  import google.appengine.api.memcache as memcache
+  import google.appengine.api.urlfetch as urlfetch
+except ImportError:
+  class FakeUrlFetch(object):
+    def fetch(self, url):
+      raise NotImplementedError()
+
+    def create_rpc(self):
+      raise NotImplementedError()
+
+    def make_fetch_call(self):
+      raise NotImplementedError()
+  urlfetch = FakeUrlFetch()
+
+  # Use an in-memory memcache if Appengine memcache isn't available.
+  memcache = InMemoryMemcache()
+
+  # A fake webapp.RequestHandler class for Handler to extend.
+  class webapp(object):
+    class RequestHandler(object):
+      def __init__(self, request, response):
+        self.request = request
+        self.response = response

chrome/common/extensions/docs/server2/branch_utility_test.py

@@ -3,9 +3,9 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+from appengine_memcache import AppEngineMemcache
 from branch_utility import BranchUtility
 from fake_url_fetcher import FakeUrlFetcher
-from in_memory_memcache import InMemoryMemcache
 import unittest
 
 class BranchUtilityTest(unittest.TestCase):
@@ -13,7 +13,7 @@ class BranchUtilityTest(unittest.TestCase):
     self._branch_util = BranchUtility('branch_utility/first.json',
                                       'stable',
                                       FakeUrlFetcher('test_data'),
-                                      InMemoryMemcache())
+                                      AppEngineMemcache(''))
 
   def testSplitChannelNameFromPath(self):
     self.assertEquals(('dev', 'hello/stuff.html'),

chrome/common/extensions/docs/server2/handlebar_dict_generator.py

@@ -46,7 +46,8 @@ class HandlebarDictGenerator(object):
     try:
       self._namespace = model.Namespace(clean_json, clean_json['namespace'])
     except Exception as e:
-      logging.info(e)
+      logging.error(e)
+      raise
 
   def _StripPrefix(self, name):
     if name.startswith(self._namespace.name + '.'):
@@ -84,7 +85,8 @@ class HandlebarDictGenerator(object):
         'properties': self._GenerateProperties(self._namespace.properties)
       }
     except Exception as e:
-      logging.info(e)
+      logging.error(e)
+      raise
 
   def _GenerateType(self, type_):
     type_dict = {

chrome/common/extensions/docs/server2/echo_handler.py → chrome/common/extensions/docs/server2/handler.py

-#!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
@@ -7,19 +6,14 @@ import logging
 import os
 import sys
 
-# Add the original server location to sys.path so we are able to import
-# modules from there.
-SERVER_PATH = 'chrome/common/extensions/docs/server2'
-if os.path.abspath(SERVER_PATH) not in sys.path:
-  sys.path.append(os.path.abspath(SERVER_PATH))
-
-from google.appengine.ext import webapp
-from google.appengine.api import memcache
-from google.appengine.ext.webapp.util import run_wsgi_app
+from appengine_wrappers import webapp
+from appengine_wrappers import memcache
+from appengine_wrappers import urlfetch
 
 from api_data_source import APIDataSource
 from api_list_data_source import APIListDataSource
 from appengine_memcache import AppEngineMemcache
+from appengine_url_fetcher import AppEngineUrlFetcher
 from branch_utility import BranchUtility
 from example_zipper import ExampleZipper
 from file_system_cache import FileSystemCache
@@ -32,10 +26,22 @@ from subversion_file_system import SubversionFileSystem
 from template_data_source import TemplateDataSource
 from appengine_url_fetcher import AppEngineUrlFetcher
 
+# The branch that the server will default to when no branch is specified in the
+# URL. This is necessary because it is not possible to pass flags to the script
+# handler.
+DEFAULT_BRANCH = 'local'
+
 SVN_URL = 'http://src.chromium.org/chrome'
 TRUNK_URL = SVN_URL + '/trunk'
 BRANCH_URL = SVN_URL + '/branches'
 
+OMAHA_PROXY_URL = 'http://omahaproxy.appspot.com/json'
+BRANCH_UTILITY = BranchUtility(OMAHA_PROXY_URL,
+                               DEFAULT_BRANCH,
+                               AppEngineUrlFetcher(''),
+                               AppEngineMemcache('branch_utility'))
+
+STATIC_DIR_PREFIX = 'docs/server2'
 EXTENSIONS_PATH = 'chrome/common/extensions'
 DOCS_PATH = 'docs'
 API_PATH = 'api'
@@ -46,19 +52,50 @@ PRIVATE_TEMPLATE_PATH = DOCS_PATH + '/server2/templates/private'
 EXAMPLES_PATH = DOCS_PATH + '/examples'
 FULL_EXAMPLES_PATH = DOCS_PATH + '/' + EXAMPLES_PATH
 
-# The branch that the server will default to when no branch is specified in the
-# URL. This is necessary because it is not possible to pass flags to the script
-# handler.
-DEFAULT_BRANCH = 'local'
-
 # Global cache of instances because Handler is recreated for every request.
 SERVER_INSTANCES = {}
 
-OMAHA_PROXY_URL = 'http://omahaproxy.appspot.com/json'
-BRANCH_UTILITY = BranchUtility(OMAHA_PROXY_URL,
-                               DEFAULT_BRANCH,
-                               AppEngineUrlFetcher(''),
-                               AppEngineMemcache('branch_utility', memcache))
+def _GetInstanceForBranch(branch, local_path):
+  if branch in SERVER_INSTANCES:
+    return SERVER_INSTANCES[branch]
+  if branch == 'local':
+    file_system = LocalFileSystem(local_path)
+  else:
+    fetcher = AppEngineUrlFetcher(
+        _GetURLFromBranch(branch) + '/' + EXTENSIONS_PATH)
+    file_system = MemcacheFileSystem(SubversionFileSystem(fetcher),
+                                     AppEngineMemcache(branch))
+
+  cache_builder = FileSystemCache.Builder(file_system)
+  api_data_source = APIDataSource(cache_builder, API_PATH)
+  api_list_data_source = APIListDataSource(cache_builder,
+                                           file_system,
+                                           API_PATH,
+                                           PUBLIC_TEMPLATE_PATH)
+  intro_data_source = IntroDataSource(cache_builder,
+                                      [INTRO_PATH, ARTICLE_PATH])
+  samples_data_source_factory = SamplesDataSource.Factory(branch,
+                                                          file_system,
+                                                          cache_builder,
+                                                          EXAMPLES_PATH)
+  template_data_source_factory = TemplateDataSource.Factory(
+      branch,
+      api_data_source,
+      api_list_data_source,
+      intro_data_source,
+      samples_data_source_factory,
+      cache_builder,
+      PUBLIC_TEMPLATE_PATH,
+      PRIVATE_TEMPLATE_PATH)
+  example_zipper = ExampleZipper(file_system,
+                                 cache_builder,
+                                 DOCS_PATH,
+                                 EXAMPLES_PATH)
+  SERVER_INSTANCES[branch] = ServerInstance(
+      template_data_source_factory,
+      example_zipper,
+      cache_builder)
+  return SERVER_INSTANCES[branch]
 
 def _GetURLFromBranch(branch):
     if branch == 'trunk':
@@ -66,56 +103,28 @@ def _GetURLFromBranch(branch):
     return BRANCH_URL + '/' + branch + '/src'
 
 class Handler(webapp.RequestHandler):
-  def _GetInstanceForBranch(self, branch):
-    if branch in SERVER_INSTANCES:
-      return SERVER_INSTANCES[branch]
-    if branch == 'local':
-      file_system = LocalFileSystem(EXTENSIONS_PATH)
-    else:
-      fetcher = AppEngineUrlFetcher(
-          _GetURLFromBranch(branch) + '/' + EXTENSIONS_PATH)
-      file_system = MemcacheFileSystem(SubversionFileSystem(fetcher),
-                                       AppEngineMemcache(branch, memcache))
+  def __init__(self, request, response, local_path=EXTENSIONS_PATH):
+    self._local_path = local_path
+    super(Handler, self).__init__(request, response)
 
-    cache_builder = FileSystemCache.Builder(file_system)
-    api_data_source = APIDataSource(cache_builder, API_PATH)
-    api_list_data_source = APIListDataSource(cache_builder,
-                                             file_system,
-                                             API_PATH,
-                                             PUBLIC_TEMPLATE_PATH)
-    intro_data_source = IntroDataSource(cache_builder,
-                                        [INTRO_PATH, ARTICLE_PATH])
-    samples_data_source_factory = SamplesDataSource.Factory(branch,
-                                                            file_system,
-                                                            cache_builder,
-                                                            EXAMPLES_PATH)
-    template_data_source_factory = TemplateDataSource.Factory(
-        branch,
-        api_data_source,
-        api_list_data_source,
-        intro_data_source,
-        samples_data_source_factory,
-        cache_builder,
-        PUBLIC_TEMPLATE_PATH,
-        PRIVATE_TEMPLATE_PATH)
-    example_zipper = ExampleZipper(file_system,
-                                   cache_builder,
-                                   DOCS_PATH,
-                                   EXAMPLES_PATH)
-    SERVER_INSTANCES[branch] = ServerInstance(
-        template_data_source_factory,
-        example_zipper,
-        cache_builder)
-    return SERVER_INSTANCES[branch]
+  def _NavigateToPath(self, path):
+    channel_name, real_path = BRANCH_UTILITY.SplitChannelNameFromPath(path)
+    branch = BRANCH_UTILITY.GetBranchNumberForChannelName(channel_name)
+    if real_path == '':
+      real_path = 'index.html'
+    # TODO: This leaks Server instances when branch bumps.
+    _GetInstanceForBranch(branch, self._local_path).Get(real_path,
+                                                        self.request,
+                                                        self.response)
 
   def get(self):
     path = self.request.path
     if '_ah/warmup' in path:
       logging.info('Warmup request.')
-      self.get('/chrome/extensions/trunk/samples.html')
-      self.get('/chrome/extensions/dev/samples.html')
-      self.get('/chrome/extensions/beta/samples.html')
-      self.get('/chrome/extensions/stable/samples.html')
+      self._NavigateToPath('trunk/samples.html')
+      self._NavigateToPath('dev/samples.html')
+      self._NavigateToPath('beta/samples.html')
+      self._NavigateToPath('stable/samples.html')
       return
 
     # Redirect paths like "directory" to "directory/". This is so relative file
@@ -124,18 +133,4 @@ class Handler(webapp.RequestHandler):
       self.redirect(path + '/')
     path = path.replace('/chrome/extensions/', '')
     path = path.strip('/')
-
-    channel_name, real_path = BRANCH_UTILITY.SplitChannelNameFromPath(path)
-    branch = BRANCH_UTILITY.GetBranchNumberForChannelName(channel_name)
-    if real_path == '':
-      real_path = 'index.html'
-    # TODO: This leaks Server instances when branch bumps.
-    self._GetInstanceForBranch(branch).Get(real_path,
-                                           self.request,
-                                           self.response)
-
-def main():
-  run_wsgi_app(webapp.WSGIApplication([('/.*', Handler)], debug=False))
-
-if __name__ == '__main__':
-  main()
+    self._NavigateToPath(path)

chrome/common/extensions/docs/server2/in_memory_memcache.py

@@ -9,16 +9,16 @@ class InMemoryMemcache(object):
   def __init__(self):
     self._cache = {}
 
-  def Set(self, key, value, namespace, time=60):
+  def set(self, key, value, namespace, time=60):
     if namespace not in self._cache:
       self._cache[namespace] = {}
     self._cache[namespace][key] = value
 
-  def Get(self, key, namespace):
+  def get(self, key, namespace):
     if namespace not in self._cache:
       return None
     return self._cache[namespace].get(key, None)
 
-  def Delete(self, key, namespace):
+  def delete(self, key, namespace):
     if namespace in self._cache:
       self._cache[namespace].pop(key)

chrome/common/extensions/docs/server2/integration_test.py

+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+from StringIO import StringIO
+import unittest
+
+from handler import Handler
+
+KNOWN_FAILURES = [
+  'webstore.html',
+]
+
+class _MockResponse:
+  def __init__(self):
+    self.status = 200
+    self.out = StringIO()
+
+  def set_status(self, status):
+    self.status = status
+
+class _MockRequest:
+  def __init__(self, path):
+    self.headers = {}
+    self.path = path
+
+class IntegrationTest(unittest.TestCase):
+  def testAll(self):
+    for filename in os.listdir(os.path.join('templates', 'public')):
+      if filename in KNOWN_FAILURES or filename.startswith('.'):
+        continue
+      request = _MockRequest(filename)
+      response = _MockResponse()
+      Handler(request, response, local_path='../..').get()
+      self.assertEqual(200, response.status)
+      self.assertTrue(response.out.getvalue())
+
+  def test404(self):
+    request = _MockRequest('junk.html')
+    bad_response = _MockResponse()
+    Handler(request, bad_response, local_path='../..').get()
+    self.assertEqual(404, bad_response.status)
+    self.assertTrue(bad_response.out.getvalue())
+
+if __name__ == '__main__':
+  unittest.main()

chrome/common/extensions/docs/server2/memcache_file_system_test.py

@@ -7,13 +7,13 @@ import os
 import unittest
 
 import appengine_memcache as memcache
-from in_memory_memcache import InMemoryMemcache
+from appengine_memcache import AppEngineMemcache
 from local_file_system import LocalFileSystem
 from memcache_file_system import MemcacheFileSystem
 
 class LocalFileSystemTest(unittest.TestCase):
   def setUp(self):
-    self._memcache = InMemoryMemcache()
+    self._memcache = AppEngineMemcache('')
     self._file_system = MemcacheFileSystem(
         LocalFileSystem(os.path.join('test_data', 'file_system')),
         self._memcache)

chrome/common/extensions/docs/server2/server_instance.py

@@ -7,7 +7,7 @@ import mimetypes
 import os
 
 STATIC_DIR_PREFIX = 'docs/server2'
-DOCS_PREFIX = 'docs'
+DOCS_PATH = 'docs'
 
 class ServerInstance(object):
   """This class is used to hold a data source and fetcher for an instance of a
@@ -40,7 +40,7 @@ class ServerInstance(object):
       content = self._example_zipper.Create(path[:-len('.zip')])
       response.headers['content-type'] = mimetypes.types_map['.zip']
     elif path.startswith('examples/'):
-      content = self._cache.GetFromFile(DOCS_PREFIX + '/' + path)
+      content = self._cache.GetFromFile(DOCS_PATH + '/' + path)
       response.headers['content-type'] = 'text/plain'
     elif path.startswith('static/'):
       content = self._FetchStaticResource(path, response)

chrome/common/extensions/docs/server2/start_dev_server.py

@@ -14,7 +14,7 @@ import build_server
 SERVER_PATH = sys.path[0]
 SRC_PATH = os.path.join(SERVER_PATH, os.pardir, os.pardir, os.pardir, os.pardir,
     os.pardir)
-FILENAMES = ['app.yaml', 'echo_handler.py']
+FILENAMES = ['app.yaml', 'appengine_main.py']
 
 def CleanUp(signal, frame):
   for filename in FILENAMES:

chrome/common/extensions/docs/server2/template_data_source.py

@@ -123,5 +123,5 @@ class TemplateDataSource(object):
     try:
       return self._cache.GetFromFile(base_path + '/' + real_path)
     except Exception as e:
-      logging.warn(e)
+      logging.error(e)
       return None

chrome/common/extensions/docs/server2/templates/public/experimental_inputUI.html

-{{+partials.standard_api api:apis.experimental_inputUI intro:intros.experimental_inputUI}}
\ No newline at end of file