Docserver: Factor SamplesModel out of SamplesDataSource

BUG=275039
NOTRY=True

Review URL: https://codereview.chromium.org/437323003

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@288149 0039d316-1c4b-4281-b951-d872f2087c98

chrome/common/extensions/docs/server2/api_data_source.py

@@ -9,19 +9,7 @@ from file_system import FileNotFoundError
 from future import Future, All
 from jsc_view import JSCView, GetEventByNameFromEvents
 from platform_util import GetPlatforms
-
-
-class _LazySamplesGetter(object):
-  '''This class is needed so that an extensions API page does not have to fetch
-  the apps samples page and vice versa.
-  '''
-
-  def __init__(self, api_name, samples):
-    self._api_name = api_name
-    self._samples = samples
-
-  def get(self, key):
-    return self._samples.FilterSamples(key, self._api_name)
+from samples_data_source import CreateSamplesView
 
 
 class APIDataSource(DataSource):
@@ -43,7 +31,7 @@ class APIDataSource(DataSource):
 
     # This caches the result of _LoadEventByName.
     self._event_byname_futures = {}
-    self._samples = server_instance.samples_data_source_factory.Create(request)
+    self._request = request
 
   def _LoadEventByName(self, platform):
     '''All events have some members in common. We source their description
@@ -84,9 +72,12 @@ class APIDataSource(DataSource):
       # Parsing samples on the preview server takes seconds and doesn't add
       # anything. Don't do it.
       if not IsPreviewServer():
-        jsc_view['samples'] = _LazySamplesGetter(
-            jsc_view['name'],
-            self._samples)
+        samples_model = self._platform_bundle.GetSamplesModel(platform)
+        # Creates an object that lazily gets samples.
+        jsc_view['samples'] = type('getter', (object,), {
+          'get': lambda _, platform: CreateSamplesView(
+              samples_model.FilterSamples(jsc_view['name']), self._request)
+          })()
       return jsc_view
     return Future(callback=resolve)
 

chrome/common/extensions/docs/server2/app.yaml

 application: chrome-apps-doc
-version: 3-39-1
+version: 3-39-2
 runtime: python27
 api_version: 1
 threadsafe: false

chrome/common/extensions/docs/server2/caching_file_system.py

@@ -7,7 +7,7 @@ import sys
 
 from file_system import FileSystem, StatInfo, FileNotFoundError
 from future import Future
-from path_util import IsDirectory
+from path_util import IsDirectory, ToDirectory
 from third_party.json_schema_compiler.memoize import memoize
 
 
@@ -43,8 +43,7 @@ class CachingFileSystem(FileSystem):
     # Always stat the parent directory, since it will have the stat of the child
     # anyway, and this gives us an entire directory's stat info at once.
     dir_path, file_path = posixpath.split(path)
-    if dir_path and not dir_path.endswith('/'):
-      dir_path += '/'
+    dir_path = ToDirectory(dir_path)
 
     def make_stat_info(dir_stat):
       '''Converts a dir stat into the correct resulting StatInfo; if the Stat

chrome/common/extensions/docs/server2/chained_compiled_file_system.py

@@ -6,6 +6,7 @@ from compiled_file_system import CompiledFileSystem
 from docs_server_utils import StringIdentity
 from file_system import FileNotFoundError
 from future import Future
+from path_util import ToDirectory
 
 
 class ChainedCompiledFileSystem(object):
@@ -52,8 +53,7 @@ class ChainedCompiledFileSystem(object):
         lambda compiled_fs: compiled_fs.GetFileVersion(path))
 
   def GetFromFileListing(self, path):
-    if not path.endswith('/'):
-      path += '/'
+    path = ToDirectory(path)
     return self._GetImpl(
         path,
         lambda compiled_fs: compiled_fs.GetFromFileListing(path),

chrome/common/extensions/docs/server2/compiled_file_system.py

@@ -8,7 +8,7 @@ import schema_util
 from docs_server_utils import ToUnicode
 from file_system import FileNotFoundError
 from future import Future
-from path_util import AssertIsDirectory, AssertIsFile
+from path_util import AssertIsDirectory, AssertIsFile, ToDirectory
 from third_party.handlebar import Handlebar
 from third_party.json_schema_compiler import json_parse
 from third_party.json_schema_compiler.memoize import memoize
@@ -236,8 +236,7 @@ class CompiledFileSystem(object):
     return self._file_system.Stat(path).version
 
   def GetFileListingVersion(self, path):
-    if not path.endswith('/'):
-      path += '/'
+    path = ToDirectory(path)
     cache_entry = self._list_object_store.Get(path).Get()
     if cache_entry is not None:
       return cache_entry.version

chrome/common/extensions/docs/server2/cron.yaml

@@ -2,4 +2,4 @@ cron:
 - description: Repopulates all cached data.
   url: /_cron
   schedule: every 5 minutes
-  target: 3-39-1
+  target: 3-39-2

chrome/common/extensions/docs/server2/data_source_registry.py

@@ -7,6 +7,7 @@ from api_list_data_source import APIListDataSource
 from data_source import DataSource
 from manifest_data_source import ManifestDataSource
 from permissions_data_source import PermissionsDataSource
+from samples_data_source import SamplesDataSource
 from sidenav_data_source import SidenavDataSource
 from strings_data_source import StringsDataSource
 from template_data_source import (
@@ -22,6 +23,7 @@ _all_data_sources = {
   'manifest_source': ManifestDataSource,
   'partials': PartialDataSource,
   'permissions': PermissionsDataSource,
+  'samples': SamplesDataSource,
   'sidenavs': SidenavDataSource,
   'strings': StringsDataSource,
   'whatsNew' : WhatsNewDataSource

chrome/common/extensions/docs/server2/platform_bundle.py

@@ -5,10 +5,13 @@
 from api_categorizer import APICategorizer
 from api_models import APIModels
 from availability_finder import AvailabilityFinder
+from empty_dir_file_system import EmptyDirFileSystem
+from environment import IsDevServer
 from features_bundle import FeaturesBundle
 from future import All
 from platform_util import GetPlatforms, PlatformToExtensionType
 from reference_resolver import ReferenceResolver
+from samples_model import SamplesModel
 
 
 class _PlatformData(object):
@@ -18,6 +21,7 @@ class _PlatformData(object):
     self.reference_resolver = None
     self.availability_finder = None
     self.api_categorizer = None
+    self.samples_model = None
 
 
 class PlatformBundle(object):
@@ -28,14 +32,38 @@ class PlatformBundle(object):
                compiled_fs_factory,
                host_fs_at_trunk,
                host_file_system_iterator,
-               object_store_creator):
+               object_store_creator,
+               base_path):
     self._branch_utility = branch_utility
     self._compiled_fs_factory = compiled_fs_factory
     self._host_fs_at_trunk = host_fs_at_trunk
     self._host_file_system_iterator = host_file_system_iterator
     self._object_store_creator = object_store_creator
+    self._base_path = base_path
     self._platform_data = dict((p, _PlatformData()) for p in GetPlatforms())
 
+  def GetSamplesModel(self, platform):
+    if self._platform_data[platform].samples_model is None:
+      # Note: samples are super slow in the dev server because it doesn't
+      # support async fetch, so disable them.
+      if IsDevServer():
+        extension_samples_fs = EmptyDirFileSystem()
+        app_samples_fs = EmptyDirFileSystem()
+      else:
+        extension_samples_fs = self._host_fs_at_trunk
+        # TODO(kalman): Re-enable the apps samples, see http://crbug.com/344097.
+        app_samples_fs = EmptyDirFileSystem()
+        #app_samples_fs = github_file_system_provider.Create(
+        #    'GoogleChrome', 'chrome-app-samples')
+      self._platform_data[platform].samples_model = SamplesModel(
+          extension_samples_fs,
+          app_samples_fs,
+          self._compiled_fs_factory,
+          self.GetReferenceResolver(platform),
+          self._base_path,
+          platform)
+    return self._platform_data[platform].samples_model
+
   def GetFeaturesBundle(self, platform):
     if self._platform_data[platform].features_bundle is None:
       self._platform_data[platform].features_bundle = FeaturesBundle(

chrome/common/extensions/docs/server2/samples_data_source_test.py

-#!/usr/bin/env python
-# Copyright (c) 2012 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import json
-import os
-import sys
-import unittest
-
-from samples_data_source import SamplesDataSource
-from servlet import Request
-from test_util import Server2Path
-
-
-class SamplesDataSourceTest(unittest.TestCase):
-  def setUp(self):
-    self._base_path = Server2Path('test_data', 'samples_data_source')
-
-  def _ReadLocalFile(self, filename):
-    with open(os.path.join(self._base_path, filename), 'r') as f:
-      return f.read()
-
-  def _FakeGet(self, key):
-    return json.loads(self._ReadLocalFile(key))
-
-  def testFilterSamples(self):
-    sds = SamplesDataSource({}, {}, '.', Request.ForTest('/'))
-    sds.get = self._FakeGet
-    self.assertEquals(json.loads(self._ReadLocalFile('expected.json')),
-                      sds.FilterSamples('samples.json', 'bobaloo'))
-
-if __name__ == '__main__':
-  unittest.main()

chrome/common/extensions/docs/server2/samples_model.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import logging
+import posixpath
+import re
+
+from extensions_paths import EXAMPLES
+from samples_data_source import SamplesDataSource
+import third_party.json_schema_compiler.json_comment_eater as json_comment_eater
+import url_constants
+
+
+_DEFAULT_ICON_PATH = 'images/sample-default-icon.png'
+
+
+def _GetAPIItems(js_file):
+  chrome_pattern = r'chrome[\w.]+'
+  # Add API calls that appear normally, like "chrome.runtime.connect".
+  calls = set(re.findall(chrome_pattern, js_file))
+  # Add API calls that have been assigned into variables, like
+  # "var storageArea = chrome.storage.sync; storageArea.get", which should
+  # be expanded like "chrome.storage.sync.get".
+  for match in re.finditer(r'var\s+(\w+)\s*=\s*(%s);' % chrome_pattern,
+                           js_file):
+    var_name, api_prefix = match.groups()
+    for var_match in re.finditer(r'\b%s\.([\w.]+)\b' % re.escape(var_name),
+                                 js_file):
+      api_suffix, = var_match.groups()
+      calls.add('%s.%s' % (api_prefix, api_suffix))
+  return calls
+
+
+class SamplesModel(object):
+  def __init__(self,
+               extension_samples_file_system,
+               app_samples_file_system,
+               compiled_fs_factory,
+               reference_resolver,
+               base_path,
+               platform):
+    self._samples_fs = (extension_samples_file_system if
+                        platform == 'extensions' else app_samples_file_system)
+    self._samples_cache = compiled_fs_factory.Create(
+        self._samples_fs,
+        self._MakeSamplesList,
+        SamplesDataSource,
+        category=platform)
+    self._text_cache = compiled_fs_factory.ForUnicode(self._samples_fs)
+    self._reference_resolver = reference_resolver
+    self._base_path = base_path
+    self._platform = platform
+
+  def GetCache(self):
+    return self._samples_cache
+
+  def FilterSamples(self, api_name):
+    '''Fetches and filters the list of samples for this platform, returning
+    only the samples that use the API |api_name|.
+    '''
+    samples_list = self._samples_cache.GetFromFileListing(
+        '' if self._platform == 'apps' else EXAMPLES).Get()
+    return [sample for sample in samples_list if any(
+        call['name'].startswith(api_name + '.')
+        for call in sample['api_calls'])]
+
+  def _GetDataFromManifest(self, path, file_system):
+    manifest = self._text_cache.GetFromFile(path + '/manifest.json').Get()
+    try:
+      manifest_json = json.loads(json_comment_eater.Nom(manifest))
+    except ValueError as e:
+      logging.error('Error parsing manifest.json for %s: %s' % (path, e))
+      return None
+    l10n_data = {
+      'name': manifest_json.get('name', ''),
+      'description': manifest_json.get('description', None),
+      'icon': manifest_json.get('icons', {}).get('128', None),
+      'default_locale': manifest_json.get('default_locale', None),
+      'locales': {}
+    }
+    if not l10n_data['default_locale']:
+      return l10n_data
+    locales_path = path + '/_locales/'
+    locales_dir = file_system.ReadSingle(locales_path).Get()
+    if locales_dir:
+      def load_locale_json(path):
+        return (path, json.loads(self._text_cache.GetFromFile(path).Get()))
+
+      try:
+        locales_json = [load_locale_json(locales_path + f + 'messages.json')
+                        for f in locales_dir]
+      except ValueError as e:
+        logging.error('Error parsing locales files for %s: %s' % (path, e))
+      else:
+        for path, json_ in locales_json:
+          l10n_data['locales'][path[len(locales_path):].split('/')[0]] = json_
+    return l10n_data
+
+  def _MakeSamplesList(self, base_path, files):
+    samples_list = []
+    for filename in sorted(files):
+      if filename.rsplit('/')[-1] != 'manifest.json':
+        continue
+
+      # This is a little hacky, but it makes a sample page.
+      sample_path = filename.rsplit('/', 1)[-2]
+      sample_files = [path for path in files
+                      if path.startswith(sample_path + '/')]
+      js_files = [path for path in sample_files if path.endswith('.js')]
+      js_contents = [self._text_cache.GetFromFile(
+          posixpath.join(base_path, js_file)).Get()
+          for js_file in js_files]
+      api_items = set()
+      for js in js_contents:
+        api_items.update(_GetAPIItems(js))
+
+      api_calls = []
+      for item in sorted(api_items):
+        if len(item.split('.')) < 3:
+          continue
+        if item.endswith('.removeListener') or item.endswith('.hasListener'):
+          continue
+        if item.endswith('.addListener'):
+          item = item[:-len('.addListener')]
+        if item.startswith('chrome.'):
+          item = item[len('chrome.'):]
+        ref_data = self._reference_resolver.GetLink(item)
+        # TODO(kalman): What about references like chrome.storage.sync.get?
+        # That should link to either chrome.storage.sync or
+        # chrome.storage.StorageArea.get (or probably both).
+        # TODO(kalman): Filter out API-only references? This can happen when
+        # the API namespace is assigned to a variable, but it's very hard to
+        # to disambiguate.
+        if ref_data is None:
+          continue
+        api_calls.append({
+          'name': ref_data['text'],
+          'link': ref_data['href']
+        })
+
+      if self._platform == 'apps':
+        url = url_constants.GITHUB_BASE + '/' + sample_path
+        icon_base = url_constants.RAW_GITHUB_BASE + '/' + sample_path
+        download_url = url
+      else:
+        extension_sample_path = posixpath.join('examples', sample_path)
+        url = extension_sample_path
+        icon_base = extension_sample_path
+        download_url = extension_sample_path + '.zip'
+
+      manifest_data = self._GetDataFromManifest(
+          posixpath.join(base_path, sample_path),
+          self._samples_fs)
+      if manifest_data['icon'] is None:
+        icon_path = posixpath.join(
+            self._base_path, 'static', _DEFAULT_ICON_PATH)
+      else:
+        icon_path = '%s/%s' % (icon_base, manifest_data['icon'])
+      manifest_data.update({
+        'icon': icon_path,
+        'download_url': download_url,
+        'url': url,
+        'files': [f.replace(sample_path + '/', '') for f in sample_files],
+        'api_calls': api_calls
+      })
+      samples_list.append(manifest_data)
+
+    return samples_list

chrome/common/extensions/docs/server2/samples_model_test.py

+#!/usr/bin/env python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import os
+import unittest
+
+from server_instance import ServerInstance
+from test_file_system import TestFileSystem
+from test_util import Server2Path
+
+
+def _ReadLocalFile(filename):
+  base_path = Server2Path('test_data', 'samples_data_source')
+  with open(os.path.join(base_path, filename), 'r') as f:
+    return f.read()
+
+
+class _FakeCache(object):
+  def __init__(self, obj):
+    self._cache = obj
+
+  def GetFromFileListing(self, _):
+    getter = lambda: 0
+    getter.Get = lambda: self._cache
+    return getter
+
+
+class SamplesModelSourceTest(unittest.TestCase):
+  def setUp(self):
+    server_instance = ServerInstance.ForTest(file_system=TestFileSystem({}))
+    self._samples_model = server_instance.platform_bundle.GetSamplesModel(
+        'apps')
+    self._samples_model._samples_cache = _FakeCache(json.loads(_ReadLocalFile(
+        'samples.json')))
+
+  def testFilterSamples(self):
+    self.assertEquals(json.loads(_ReadLocalFile('expected.json')),
+                      self._samples_model.FilterSamples('bobaloo'))
+
+if __name__ == '__main__':
+  unittest.main()

chrome/common/extensions/docs/server2/server_instance.py

@@ -73,25 +73,8 @@ class ServerInstance(object):
         self.compiled_fs_factory,
         host_fs_at_trunk,
         self.host_file_system_iterator,
-        self.object_store_creator)
-
-    # Note: samples are super slow in the dev server because it doesn't support
-    # async fetch, so disable them.
-    if IsDevServer():
-      extension_samples_fs = EmptyDirFileSystem()
-      app_samples_fs = EmptyDirFileSystem()
-    else:
-      extension_samples_fs = host_fs_at_trunk
-      # TODO(kalman): Re-enable the apps samples, see http://crbug.com/344097.
-      app_samples_fs = EmptyDirFileSystem()
-      #app_samples_fs = github_file_system_provider.Create(
-      #    'GoogleChrome', 'chrome-app-samples')
-    self.samples_data_source_factory = SamplesDataSource.Factory(
-        extension_samples_fs,
-        app_samples_fs,
-        CompiledFileSystem.Factory(object_store_creator),
-        self.platform_bundle,
-        base_path)
+        self.object_store_creator,
+        self.base_path)
 
     self.content_providers = ContentProviders(
         object_store_creator,

chrome/common/extensions/docs/server2/template_renderer.py

@@ -28,7 +28,7 @@ class TemplateRenderer(object):
     rendering the template.
     '''
     assert isinstance(template, Handlebar), type(template)
-    render_context = self._CreateDataSources(request)
+    render_context = CreateDataSources(self._server_instance, request)
     if data_sources is not None:
       render_context = dict((name, d) for name, d in render_context.iteritems()
                             if name in data_sources)
@@ -38,15 +38,6 @@ class TemplateRenderer(object):
       'extensions_samples_url': EXTENSIONS_SAMPLES,
       'static': self._server_instance.base_path + 'static',
     })
-    if additional_context:
-      render_context.update(additional_context)
+    render_context.update(additional_context or {})
     render_data = template.Render(render_context)
     return render_data.text, render_data.errors
-
-  def _CreateDataSources(self, request):
-    server_instance = self._server_instance
-    data_sources = CreateDataSources(server_instance, request=request)
-    data_sources.update({
-      'samples': server_instance.samples_data_source_factory.Create(request),
-    })
-    return data_sources