Docserver: Modify doc inlining so all relevant data is present in availability_finder.py

Not inlining schema nodes for availability finder causes data to go missing and
availability to not be found. Inlining schema nodes means that availability data
gets messed up when a node is no longer inlined, because inlined nodes are
deleted from the schema. Solution: Inline nodes, but do not remove them from the
schema when used in availability_finder.py. This provides all the necessary data,
and doesn't interfere with availability data generation when a node is no longer
inlined because it will be still be present in the schema graphs.

BUG=233982
NOTRY=True

Review URL: https://codereview.chromium.org/385703005

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@283579 0039d316-1c4b-4281-b951-d872f2087c98

chrome/common/extensions/docs/server2/api_data_source.py

@@ -217,9 +217,7 @@ class _APINodeCursor(object):
       return None
     node_availability = self._LookupAvailability(self._lookup_path)
     if node_availability is None:
-      if not IsReleaseServer():
-        # Bad, and happens a lot :(
-        logging.warning('No availability found for: %s' % self)
+      logging.warning('No availability found for: %s' % self)
       return None
 
     parent_node_availability = self._LookupAvailability(self._GetParentPath())

chrome/common/extensions/docs/server2/api_models.py

@@ -38,7 +38,7 @@ class APIModels(object):
               node['extension_types'] != 'all' and
               self._platform not in node['extension_types'])
 
-    schema = ProcessSchema(path, data, inline=True)[0]
+    schema = ProcessSchema(path, data)[0]
     if not schema:
       raise ValueError('No schema for %s' % path)
     return Namespace(DeleteNodes(

chrome/common/extensions/docs/server2/app.yaml

 application: chrome-apps-doc
-version: 3-30-0
+version: 3-31-0
 runtime: python27
 api_version: 1
 threadsafe: false

chrome/common/extensions/docs/server2/availability_finder.py

@@ -6,9 +6,11 @@ import posixpath
 
 from api_schema_graph import APISchemaGraph
 from branch_utility import BranchUtility, ChannelInfo
+from compiled_file_system import CompiledFileSystem, SingleFile, Unicode
 from extensions_paths import API_PATHS, JSON_TEMPLATES
 from features_bundle import FeaturesBundle
 from file_system import FileNotFoundError
+from schema_util import ProcessSchema
 from third_party.json_schema_compiler.memoize import memoize
 from third_party.json_schema_compiler.model import UnixName
 
@@ -142,6 +144,21 @@ class AvailabilityFinder(object):
     return AvailabilityInfo(
         self._branch_utility.GetChannelInfo(api_info['channel']))
 
+  @memoize
+  def _CreateAPISchemaFileSystem(self, file_system):
+    '''Creates a CompiledFileSystem for parsing raw JSON or IDL API schema
+    data and formatting it so that it can be used to create APISchemaGraphs.
+    '''
+    # When processing the API schemas, we retain inlined types in the schema
+    # so that there are not missing nodes in the APISchemaGraphs when trying
+    # to lookup availability.
+    def process_schema(path, data):
+      return ProcessSchema(path, data, retain_inlined_types=True)
+    return self._compiled_fs_factory.Create(file_system,
+                                            SingleFile(Unicode(process_schema)),
+                                            CompiledFileSystem,
+                                            category='api-schema')
+
   def _GetAPISchema(self, api_name, file_system, version):
     '''Searches |file_system| for |api_name|'s API schema data, and processes
     and returns it if found.
@@ -151,7 +168,7 @@ class AvailabilityFinder(object):
       # No file for the API could be found in the given |file_system|.
       return None
 
-    schema_fs = self._compiled_fs_factory.ForAPISchema(file_system)
+    schema_fs = self._CreateAPISchemaFileSystem(file_system)
     api_schemas = schema_fs.GetFromFile(api_filename).Get()
     matching_schemas = [api for api in api_schemas
                         if api['namespace'] == api_name]

chrome/common/extensions/docs/server2/availability_finder_test.py

@@ -191,112 +191,120 @@ class AvailabilityFinderTest(unittest.TestCase):
     for platform in GetPlatforms():
       # Allow the LookupResult constructions below to take just one line.
       lookup_result = api_schema_graph.LookupResult
-      availability_graph = self._create_availability_finder(
+      avail_finder = self._create_availability_finder(
           self._node_fs_creator,
           self._node_fs_iterator,
-          platform).GetAPINodeAvailability('tabs')
+          platform)
+      tabs_graph = avail_finder.GetAPINodeAvailability('tabs')
+      fake_tabs_graph = avail_finder.GetAPINodeAvailability('fakeTabs')
 
       self.assertEquals(
           lookup_result(True, self._branch_utility.GetChannelInfo('trunk')),
-          availability_graph.Lookup('tabs', 'properties',
+          tabs_graph.Lookup('tabs', 'properties',
                                     'fakeTabsProperty3'))
       self.assertEquals(
           lookup_result(True, self._branch_utility.GetChannelInfo('dev')),
-          availability_graph.Lookup('tabs', 'events', 'onActivated',
+          tabs_graph.Lookup('tabs', 'events', 'onActivated',
                                     'parameters', 'activeInfo', 'properties',
                                     'windowId'))
       self.assertEquals(
           lookup_result(True, self._branch_utility.GetChannelInfo('dev')),
-          availability_graph.Lookup('tabs', 'events', 'onUpdated', 'parameters',
+          tabs_graph.Lookup('tabs', 'events', 'onUpdated', 'parameters',
                                     'tab'))
       self.assertEquals(
           lookup_result(True, self._branch_utility.GetChannelInfo('beta')),
-          availability_graph.Lookup('tabs', 'events','onActivated'))
+          tabs_graph.Lookup('tabs', 'events','onActivated'))
       self.assertEquals(
           lookup_result(True, self._branch_utility.GetChannelInfo('beta')),
-          availability_graph.Lookup('tabs', 'functions', 'get', 'parameters',
+          tabs_graph.Lookup('tabs', 'functions', 'get', 'parameters',
                                     'tabId'))
       self.assertEquals(
           lookup_result(True, self._branch_utility.GetChannelInfo('stable')),
-          availability_graph.Lookup('tabs', 'types', 'InjectDetails',
+          tabs_graph.Lookup('tabs', 'types', 'InjectDetails',
                                     'properties', 'code'))
       self.assertEquals(
           lookup_result(True, self._branch_utility.GetChannelInfo('stable')),
-          availability_graph.Lookup('tabs', 'types', 'InjectDetails',
+          tabs_graph.Lookup('tabs', 'types', 'InjectDetails',
                                     'properties', 'file'))
       self.assertEquals(
           lookup_result(True, self._branch_utility.GetStableChannelInfo(25)),
-          availability_graph.Lookup('tabs', 'types', 'InjectDetails'))
+          tabs_graph.Lookup('tabs', 'types', 'InjectDetails'))
 
       # Test inlined type.
       self.assertEquals(
           lookup_result(True, self._branch_utility.GetChannelInfo('trunk')),
-          availability_graph.Lookup('tabs', 'types', 'InlinedType'))
+          tabs_graph.Lookup('tabs', 'types', 'InlinedType'))
+
+      # Test implicitly inlined type.
+      self.assertEquals(
+          lookup_result(True, self._branch_utility.GetStableChannelInfo(25)),
+          fake_tabs_graph.Lookup('fakeTabs', 'types',
+                                 'WasImplicitlyInlinedType'))
 
       # Nothing new in version 24 or 23.
 
       self.assertEquals(
           lookup_result(True, self._branch_utility.GetStableChannelInfo(22)),
-          availability_graph.Lookup('tabs', 'types', 'Tab', 'properties',
+          tabs_graph.Lookup('tabs', 'types', 'Tab', 'properties',
                                     'windowId'))
       self.assertEquals(
           lookup_result(True, self._branch_utility.GetStableChannelInfo(21)),
-          availability_graph.Lookup('tabs', 'types', 'Tab', 'properties',
+          tabs_graph.Lookup('tabs', 'types', 'Tab', 'properties',
                                     'selected'))
 
       # Nothing new in version 20.
 
       self.assertEquals(
           lookup_result(True, self._branch_utility.GetStableChannelInfo(19)),
-          availability_graph.Lookup('tabs', 'functions', 'getCurrent'))
+          tabs_graph.Lookup('tabs', 'functions', 'getCurrent'))
       self.assertEquals(
           lookup_result(True, self._branch_utility.GetStableChannelInfo(18)),
-          availability_graph.Lookup('tabs', 'types', 'Tab', 'properties',
+          tabs_graph.Lookup('tabs', 'types', 'Tab', 'properties',
                                     'index'))
       self.assertEquals(
           lookup_result(True, self._branch_utility.GetStableChannelInfo(17)),
-          availability_graph.Lookup('tabs', 'events', 'onUpdated', 'parameters',
+          tabs_graph.Lookup('tabs', 'events', 'onUpdated', 'parameters',
                                     'changeInfo'))
 
       # Nothing new in version 16.
 
       self.assertEquals(
           lookup_result(True, self._branch_utility.GetStableChannelInfo(15)),
-          availability_graph.Lookup('tabs', 'properties',
+          tabs_graph.Lookup('tabs', 'properties',
                                     'fakeTabsProperty2'))
 
       # Everything else is available at the API's release, version 14 here.
       self.assertEquals(
           lookup_result(True, self._branch_utility.GetStableChannelInfo(14)),
-          availability_graph.Lookup('tabs', 'types', 'Tab'))
+          tabs_graph.Lookup('tabs', 'types', 'Tab'))
       self.assertEquals(
           lookup_result(True, self._branch_utility.GetStableChannelInfo(14)),
-          availability_graph.Lookup('tabs', 'types', 'Tab',
+          tabs_graph.Lookup('tabs', 'types', 'Tab',
                                     'properties', 'url'))
       self.assertEquals(
           lookup_result(True, self._branch_utility.GetStableChannelInfo(14)),
-          availability_graph.Lookup('tabs', 'properties',
+          tabs_graph.Lookup('tabs', 'properties',
                                     'fakeTabsProperty1'))
       self.assertEquals(
           lookup_result(True, self._branch_utility.GetStableChannelInfo(14)),
-          availability_graph.Lookup('tabs', 'functions', 'get', 'parameters',
+          tabs_graph.Lookup('tabs', 'functions', 'get', 'parameters',
                                     'callback'))
       self.assertEquals(
           lookup_result(True, self._branch_utility.GetStableChannelInfo(14)),
-          availability_graph.Lookup('tabs', 'events', 'onUpdated'))
+          tabs_graph.Lookup('tabs', 'events', 'onUpdated'))
 
       # Test things that aren't available.
       self.assertEqual(lookup_result(False, None),
-                       availability_graph.Lookup('tabs', 'types',
+                       tabs_graph.Lookup('tabs', 'types',
                                                  'UpdateInfo'))
       self.assertEqual(lookup_result(False, None),
-                       availability_graph.Lookup('tabs', 'functions', 'get',
+                       tabs_graph.Lookup('tabs', 'functions', 'get',
                                                  'parameters', 'callback',
                                                  'parameters', 'tab', 'id'))
       self.assertEqual(lookup_result(False, None),
-                       availability_graph.Lookup('functions'))
+                       tabs_graph.Lookup('functions'))
       self.assertEqual(lookup_result(False, None),
-                       availability_graph.Lookup('events', 'onActivated',
+                       tabs_graph.Lookup('events', 'onActivated',
                                                  'parameters', 'activeInfo',
                                                  'tabId'))
 

chrome/common/extensions/docs/server2/compiled_file_system.py

@@ -105,17 +105,6 @@ class CompiledFileSystem(object):
                          CompiledFileSystem,
                          category='json')
 
-    @memoize
-    def ForAPISchema(self, file_system):
-      '''Creates a CompiledFileSystem for parsing raw JSON or IDL API schema
-      data and formatting it so that it can be used by other classes, such
-      as Model and APISchemaGraph.
-      '''
-      return self.Create(file_system,
-                         SingleFile(Unicode(schema_util.ProcessSchema)),
-                         CompiledFileSystem,
-                         category='api-schema')
-
     @memoize
     def ForTemplates(self, file_system):
       '''Creates a CompiledFileSystem for parsing templates.

chrome/common/extensions/docs/server2/cron.yaml

@@ -2,4 +2,4 @@ cron:
 - description: Repopulates all cached data.
   url: /_cron
   schedule: every 5 minutes
-  target: 3-30-0
+  target: 3-31-0

chrome/common/extensions/docs/server2/schema_util.py

@@ -54,8 +54,10 @@ def DetectInlineableTypes(schema):
         type_['inline_doc'] = True
 
 
-def InlineDocs(schema):
+def InlineDocs(schema, retain_inlined_types):
   '''Replace '$ref's that refer to inline_docs with the json for those docs.
+  If |retain_inlined_types| is False, then the inlined nodes are removed
+  from the schema.
   '''
   types = schema.get('types')
   if types is None:
@@ -68,11 +70,13 @@ def InlineDocs(schema):
   for type_ in types:
     if type_.get('inline_doc'):
       inline_docs[type_['id']] = type_
-      for k in ('description', 'id', 'inline_doc'):
-        type_.pop(k, None)
+      if not retain_inlined_types:
+        for k in ('description', 'id', 'inline_doc'):
+          type_.pop(k, None)
     else:
       types_without_inline_doc.append(type_)
-  schema['types'] = types_without_inline_doc
+  if not retain_inlined_types:
+    schema['types'] = types_without_inline_doc
 
   def apply_inline(node):
     if isinstance(node, list):
@@ -89,10 +93,11 @@ def InlineDocs(schema):
   apply_inline(schema)
 
 
-def ProcessSchema(path, file_data, inline=False):
-  '''Parses |file_data| using a method determined by checking the extension of
-  the file at the given |path|. Then, trims 'nodoc' and if |inline| is given
-  and True, handles inlineable types from the parsed schema data.
+def ProcessSchema(path, file_data, retain_inlined_types=False):
+  '''Parses |file_data| using a method determined by checking the
+  extension of the file at the given |path|. Then, trims 'nodoc' and if
+  |retain_inlined_types| is given and False, removes inlineable types from
+  the parsed schema data.
   '''
   def trim_and_inline(schema, is_idl=False):
     '''Modifies an API schema in place by removing nodes that shouldn't be
@@ -102,10 +107,9 @@ def ProcessSchema(path, file_data, inline=False):
       # A return of True signifies that the entire schema should not be
       # documented. Otherwise, only nodes that request 'nodoc' are removed.
       return None
-    if inline:
-      if is_idl:
-        DetectInlineableTypes(schema)
-      InlineDocs(schema)
+    if is_idl:
+      DetectInlineableTypes(schema)
+    InlineDocs(schema, retain_inlined_types)
     return schema
 
   if path.endswith('.idl'):

chrome/common/extensions/docs/server2/schema_util_test.py

@@ -170,7 +170,7 @@ class SchemaUtilTest(unittest.TestCase):
     }
 
     inlined_schema = deepcopy(schema)
-    InlineDocs(inlined_schema)
+    InlineDocs(inlined_schema, False)
     self.assertEqual(expected_schema, inlined_schema)
 
   def testDetectInline(self):
@@ -201,7 +201,7 @@ class SchemaUtilTest(unittest.TestCase):
     }
 
     DetectInlineableTypes(schema)
-    InlineDocs(schema)
+    InlineDocs(schema, False)
     self.assertEqual(expected_schema, schema)
 
 

chrome/common/extensions/docs/server2/test_data/object_level_availability/tabs.py

@@ -4,9 +4,32 @@
 
 import json
 
-from extensions_paths import CHROME_EXTENSIONS
+from extensions_paths import CHROME_EXTENSIONS, SERVER2
 from test_file_system import MoveAllTo
+from test_util import ReadFile
 
+FAKE_TABS_IDL = '\n'.join([
+  '// Copyleft stuff.',
+  '',
+  '// Some description here.',
+  'namespace fakeTabs {',
+  '  dictionary WasImplicitlyInlinedType {};',
+  '  interface Functions {',
+  '    static void myFunc(WasImplicitlyInlinedType arg);',
+  '    static void anotherFunc(WasImplicitlyInlinedType arg);',
+  '  };',
+  '};'])
+
+FAKE_TABS_WITH_INLINING_IDL = '\n'.join([
+  '// Copyleft stuff.',
+  '',
+  '// Some description here.',
+  'namespace fakeTabs {',
+  '  dictionary WasImplicitlyInlinedType {};',
+  '  interface Functions {',
+  '    static void myFunc(WasImplicitlyInlinedType arg);',
+  '  };',
+  '};'])
 
 TABS_SCHEMA_BRANCHES = MoveAllTo(CHROME_EXTENSIONS, {
   'trunk': {
@@ -22,6 +45,7 @@ TABS_SCHEMA_BRANCHES = MoveAllTo(CHROME_EXTENSIONS, {
       '_api_features.json': '{}',
       '_manifest_features.json': '{}',
       '_permission_features.json': '{}',
+      'fake_tabs.idl': FAKE_TABS_IDL,
       'tabs.json': json.dumps([{
         'namespace': 'tabs',
         'types': [
@@ -168,6 +192,7 @@ TABS_SCHEMA_BRANCHES = MoveAllTo(CHROME_EXTENSIONS, {
       '_api_features.json': "{}",
       '_manifest_features.json': "{}",
       '_permission_features.json': "{}",
+      'fake_tabs.idl': FAKE_TABS_IDL,
       'tabs.json': json.dumps([{
         'namespace': 'tabs',
         'types': [
@@ -265,6 +290,7 @@ TABS_SCHEMA_BRANCHES = MoveAllTo(CHROME_EXTENSIONS, {
       '_api_features.json': "{}",
       '_manifest_features.json': "{}",
       '_permission_features.json': "{}",
+      'fake_tabs.idl': FAKE_TABS_IDL,
       'tabs.json': json.dumps([{
         'namespace': 'tabs',
         'types': [
@@ -357,6 +383,7 @@ TABS_SCHEMA_BRANCHES = MoveAllTo(CHROME_EXTENSIONS, {
     'api': {
       '_manifest_features.json': "{}",
       '_permission_features.json': "{}",
+      'fake_tabs.idl': FAKE_TABS_IDL,
       'tabs.json': json.dumps([{
         'namespace': 'tabs',
         'types': [
@@ -435,6 +462,7 @@ TABS_SCHEMA_BRANCHES = MoveAllTo(CHROME_EXTENSIONS, {
     'api': {
       '_manifest_features.json': "{}",
       '_permission_features.json': "{}",
+      'fake_tabs.idl': FAKE_TABS_WITH_INLINING_IDL,
       'tabs.json': json.dumps([{
         'namespace': 'tabs',
         'types': [