Support nullable types in mojom.

This CL updates the lexer, parser and also the internal representation of mojom modules. The changes to the generators will be in a separate CL.

This CL doesn't change data_tests.py because it has been broken for quite a while. We could fix it in a separate CL if we think that it is still useful.

BUG=324170
TEST=those updated test files.

Review URL: https://codereview.chromium.org/437643002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@287997 0039d316-1c4b-4281-b951-d872f2087c98

mojo/public/tools/bindings/pylib/mojom/generate/data.py

@@ -99,20 +99,20 @@ def KindFromData(kinds, data, scope):
   kind = LookupKind(kinds, data, scope)
   if kind:
     return kind
-  if data.startswith('a:'):
-    kind = mojom.Array()
-    kind.kind = KindFromData(kinds, data[2:], scope)
+
+  if data.startswith('?'):
+    kind = KindFromData(kinds, data[1:], scope).MakeNullableKind()
+  elif data.startswith('a:'):
+    kind = mojom.Array(KindFromData(kinds, data[2:], scope))
   elif data.startswith('r:'):
-    kind = mojom.InterfaceRequest()
-    kind.kind = KindFromData(kinds, data[2:], scope)
+    kind = mojom.InterfaceRequest(KindFromData(kinds, data[2:], scope))
   elif data.startswith('a'):
     colon = data.find(':')
     length = int(data[1:colon])
-    kind = mojom.FixedArray(length)
-    kind.kind = KindFromData(kinds, data[colon+1:], scope)
+    kind = mojom.FixedArray(length, KindFromData(kinds, data[colon+1:], scope))
   else:
-    kind = mojom.Kind()
-  kind.spec = data
+    kind = mojom.Kind(data)
+
   kinds[data] = kind
   return kind
 

mojo/public/tools/bindings/pylib/mojom/generate/module.py

@@ -11,31 +11,95 @@
 # interface = module.AddInterface('Bar')
 # method = interface.AddMethod('Tat', 0)
 # method.AddParameter('baz', 0, mojom.INT32)
-#
+
 
 class Kind(object):
   def __init__(self, spec=None):
     self.spec = spec
     self.parent_kind = None
 
+
+class ReferenceKind(Kind):
+  """ReferenceKind represents pointer types and handle types.
+     A type is nullable means that NULL (for pointer types) or invalid handle
+     (for handle types) is a legal value for the type.
+  """
+
+  def __init__(self, spec=None, is_nullable=False):
+    assert spec is None or is_nullable == spec.startswith('?')
+    Kind.__init__(self, spec)
+    self.is_nullable = is_nullable
+    self.shared_definition = {}
+
+  def MakeNullableKind(self):
+    assert not self.is_nullable
+
+    if self == STRING:
+      return NULLABLE_STRING
+    if self == HANDLE:
+      return NULLABLE_HANDLE
+    if self == DCPIPE:
+      return NULLABLE_DCPIPE
+    if self == DPPIPE:
+      return NULLABLE_DPPIPE
+    if self == MSGPIPE:
+      return NULLABLE_MSGPIPE
+    if self == SHAREDBUFFER:
+      return NULLABLE_SHAREDBUFFER
+
+    nullable_kind = type(self)()
+    nullable_kind.shared_definition = self.shared_definition
+    if self.spec is not None:
+      nullable_kind.spec = '?' + self.spec
+    nullable_kind.is_nullable = True
+
+    return nullable_kind
+
+  @classmethod
+  def AddSharedProperty(cls, name):
+    """Adds a property |name| to |cls|, which accesses the corresponding item in
+       |shared_definition|.
+
+       The reason of adding such indirection is to enable sharing definition
+       between a reference kind and its nullable variation. For example:
+         a = Struct('test_struct_1')
+         b = a.MakeNullableKind()
+         a.name = 'test_struct_2'
+         print b.name  # Outputs 'test_struct_2'.
+    """
+    def Get(self):
+      return self.shared_definition[name]
+
+    def Set(self, value):
+      self.shared_definition[name] = value
+
+    setattr(cls, name, property(Get, Set))
+
+
 # Initialize the set of primitive types. These can be accessed by clients.
-BOOL         = Kind('b')
-INT8         = Kind('i8')
-INT16        = Kind('i16')
-INT32        = Kind('i32')
-INT64        = Kind('i64')
-UINT8        = Kind('u8')
-UINT16       = Kind('u16')
-UINT32       = Kind('u32')
-UINT64       = Kind('u64')
-FLOAT        = Kind('f')
-DOUBLE       = Kind('d')
-STRING       = Kind('s')
-HANDLE       = Kind('h')
-DCPIPE       = Kind('h:d:c')
-DPPIPE       = Kind('h:d:p')
-MSGPIPE      = Kind('h:m')
-SHAREDBUFFER = Kind('h:s')
+BOOL                  = Kind('b')
+INT8                  = Kind('i8')
+INT16                 = Kind('i16')
+INT32                 = Kind('i32')
+INT64                 = Kind('i64')
+UINT8                 = Kind('u8')
+UINT16                = Kind('u16')
+UINT32                = Kind('u32')
+UINT64                = Kind('u64')
+FLOAT                 = Kind('f')
+DOUBLE                = Kind('d')
+STRING                = ReferenceKind('s')
+HANDLE                = ReferenceKind('h')
+DCPIPE                = ReferenceKind('h:d:c')
+DPPIPE                = ReferenceKind('h:d:p')
+MSGPIPE               = ReferenceKind('h:m')
+SHAREDBUFFER          = ReferenceKind('h:s')
+NULLABLE_STRING       = ReferenceKind('?s', True)
+NULLABLE_HANDLE       = ReferenceKind('?h', True)
+NULLABLE_DCPIPE       = ReferenceKind('?h:d:c', True)
+NULLABLE_DPPIPE       = ReferenceKind('?h:d:p', True)
+NULLABLE_MSGPIPE      = ReferenceKind('?h:m', True)
+NULLABLE_SHAREDBUFFER = ReferenceKind('?h:s', True)
 
 
 # Collection of all Primitive types
@@ -56,7 +120,13 @@ PRIMITIVES = (
   DCPIPE,
   DPPIPE,
   MSGPIPE,
-  SHAREDBUFFER
+  SHAREDBUFFER,
+  NULLABLE_STRING,
+  NULLABLE_HANDLE,
+  NULLABLE_DCPIPE,
+  NULLABLE_DPPIPE,
+  NULLABLE_MSGPIPE,
+  NULLABLE_SHAREDBUFFER
 )
 
 
@@ -100,16 +170,21 @@ class Field(object):
     self.default = default
 
 
-class Struct(Kind):
+class Struct(ReferenceKind):
+  ReferenceKind.AddSharedProperty('name')
+  ReferenceKind.AddSharedProperty('module')
+  ReferenceKind.AddSharedProperty('imported_from')
+  ReferenceKind.AddSharedProperty('fields')
+
   def __init__(self, name=None, module=None):
-    self.name = name
-    self.module = module
-    self.imported_from = None
     if name is not None:
       spec = 'x:' + name
     else:
       spec = None
-    Kind.__init__(self, spec)
+    ReferenceKind.__init__(self, spec)
+    self.name = name
+    self.module = module
+    self.imported_from = None
     self.fields = []
 
   def AddField(self, name, kind, ordinal=None, default=None):
@@ -118,30 +193,39 @@ class Struct(Kind):
     return field
 
 
-class Array(Kind):
+class Array(ReferenceKind):
+  ReferenceKind.AddSharedProperty('kind')
+
   def __init__(self, kind=None):
-    self.kind = kind
     if kind is not None:
-      Kind.__init__(self, 'a:' + kind.spec)
+      ReferenceKind.__init__(self, 'a:' + kind.spec)
     else:
-      Kind.__init__(self)
-
-class FixedArray(Kind):
-  def __init__(self, length, kind=None):
+      ReferenceKind.__init__(self)
     self.kind = kind
-    self.length = length
+
+
+class FixedArray(ReferenceKind):
+  ReferenceKind.AddSharedProperty('kind')
+  ReferenceKind.AddSharedProperty('length')
+
+  def __init__(self, length=-1, kind=None):
     if kind is not None:
-      Kind.__init__(self, 'a' + length + ':' + kind.spec)
+      ReferenceKind.__init__(self, 'a%d:%s' % (length, kind.spec))
     else:
-      Kind.__init__(self)
+      ReferenceKind.__init__(self)
+    self.kind = kind
+    self.length = length
+
+
+class InterfaceRequest(ReferenceKind):
+  ReferenceKind.AddSharedProperty('kind')
 
-class InterfaceRequest(Kind):
   def __init__(self, kind=None):
-    self.kind = kind
     if kind is not None:
-      Kind.__init__(self, 'r:' + kind.spec)
+      ReferenceKind.__init__(self, 'r:' + kind.spec)
     else:
-      Kind.__init__(self)
+      ReferenceKind.__init__(self)
+    self.kind = kind
 
 
 class Parameter(object):
@@ -173,16 +257,22 @@ class Method(object):
     return parameter
 
 
-class Interface(Kind):
+class Interface(ReferenceKind):
+  ReferenceKind.AddSharedProperty('module')
+  ReferenceKind.AddSharedProperty('name')
+  ReferenceKind.AddSharedProperty('imported_from')
+  ReferenceKind.AddSharedProperty('client')
+  ReferenceKind.AddSharedProperty('methods')
+
   def __init__(self, name=None, client=None, module=None):
-    self.module = module
-    self.name = name
-    self.imported_from = None
     if name is not None:
       spec = 'x:' + name
     else:
       spec = None
-    Kind.__init__(self, spec)
+    ReferenceKind.__init__(self, spec)
+    self.module = module
+    self.name = name
+    self.imported_from = None
     self.client = client
     self.methods = []
 

mojo/public/tools/bindings/pylib/mojom/generate/pack.py

@@ -13,23 +13,29 @@ import module as mojom
 
 class PackedField(object):
   kind_to_size = {
-    mojom.BOOL:         1,
-    mojom.INT8:         1,
-    mojom.UINT8:        1,
-    mojom.INT16:        2,
-    mojom.UINT16:       2,
-    mojom.INT32:        4,
-    mojom.UINT32:       4,
-    mojom.FLOAT:        4,
-    mojom.HANDLE:       4,
-    mojom.MSGPIPE:      4,
-    mojom.SHAREDBUFFER: 4,
-    mojom.DCPIPE:       4,
-    mojom.DPPIPE:       4,
-    mojom.INT64:        8,
-    mojom.UINT64:       8,
-    mojom.DOUBLE:       8,
-    mojom.STRING:       8
+    mojom.BOOL:                  1,
+    mojom.INT8:                  1,
+    mojom.UINT8:                 1,
+    mojom.INT16:                 2,
+    mojom.UINT16:                2,
+    mojom.INT32:                 4,
+    mojom.UINT32:                4,
+    mojom.FLOAT:                 4,
+    mojom.HANDLE:                4,
+    mojom.MSGPIPE:               4,
+    mojom.SHAREDBUFFER:          4,
+    mojom.DCPIPE:                4,
+    mojom.DPPIPE:                4,
+    mojom.NULLABLE_HANDLE:       4,
+    mojom.NULLABLE_MSGPIPE:      4,
+    mojom.NULLABLE_SHAREDBUFFER: 4,
+    mojom.NULLABLE_DCPIPE:       4,
+    mojom.NULLABLE_DPPIPE:       4,
+    mojom.INT64:                 8,
+    mojom.UINT64:                8,
+    mojom.DOUBLE:                8,
+    mojom.STRING:                8,
+    mojom.NULLABLE_STRING:       8
   }
 
   @classmethod

mojo/public/tools/bindings/pylib/mojom/generate/pack_tests.py

@@ -88,17 +88,33 @@ def TestPaddingPackedOverflow():
   return TestSequence(kinds, fields, offsets)
 
 
+def TestNullableTypes():
+  kinds = (mojom.STRING.MakeNullableKind(),
+           mojom.HANDLE.MakeNullableKind(),
+           mojom.Struct('test_struct').MakeNullableKind(),
+           mojom.DCPIPE.MakeNullableKind(),
+           mojom.Array().MakeNullableKind(),
+           mojom.DPPIPE.MakeNullableKind(),
+           mojom.FixedArray(5).MakeNullableKind(),
+           mojom.MSGPIPE.MakeNullableKind(),
+           mojom.Interface('test_inteface').MakeNullableKind(),
+           mojom.SHAREDBUFFER.MakeNullableKind(),
+           mojom.InterfaceRequest().MakeNullableKind())
+  fields = (1, 2, 4, 3, 5, 6, 8, 7, 9, 10, 11)
+  offsets = (0, 8, 12, 16, 24, 32, 36, 40, 48, 52, 56)
+  return TestSequence(kinds, fields, offsets)
+
+
 def TestAllTypes():
-  struct = mojom.Struct('test')
-  array = mojom.Array()
   return TestSequence(
       (mojom.BOOL, mojom.INT8, mojom.STRING, mojom.UINT8,
        mojom.INT16, mojom.DOUBLE, mojom.UINT16,
        mojom.INT32, mojom.UINT32, mojom.INT64,
        mojom.FLOAT, mojom.STRING, mojom.HANDLE,
-       mojom.UINT64, mojom.Struct('test'), mojom.Array()),
-      (1, 2, 4, 5, 7, 3, 6,  8,  9,  10, 11, 13, 12, 14, 15, 16, 17),
-      (0, 1, 2, 4, 6, 8, 16, 24, 28, 32, 40, 44, 48, 56, 64, 72, 80))
+       mojom.UINT64, mojom.Struct('test'), mojom.Array(),
+       mojom.STRING.MakeNullableKind()),
+      (1, 2, 4, 5, 7, 3, 6,  8,  9,  10, 11, 13, 12, 14, 15, 16, 17, 18),
+      (0, 1, 2, 4, 6, 8, 16, 24, 28, 32, 40, 44, 48, 56, 64, 72, 80, 88))
 
 
 def TestPaddingPackedOutOfOrderByOrdinal():
@@ -165,6 +181,7 @@ def Main(args):
   errors += RunTest(TestPaddingPackedInOrder)
   errors += RunTest(TestPaddingPackedOutOfOrder)
   errors += RunTest(TestPaddingPackedOverflow)
+  errors += RunTest(TestNullableTypes)
   errors += RunTest(TestAllTypes)
   errors += RunTest(TestPaddingPackedOutOfOrderByOrdinal)
   errors += RunTest(TestBools)

mojo/public/tools/bindings/pylib/mojom/parse/lexer.py

@@ -87,6 +87,7 @@ class Lexer(object):
     'MINUS',
     'PLUS',
     'AMP',
+    'QSTN',
 
     # Assignment
     'EQUALS',
@@ -167,6 +168,7 @@ class Lexer(object):
   t_MINUS             = r'-'
   t_PLUS              = r'\+'
   t_AMP               = r'&'
+  t_QSTN              = r'\?'
 
   # =
   t_EQUALS            = r'='

mojo/public/tools/bindings/pylib/mojom/parse/parser.py

@@ -226,10 +226,18 @@ class Parser(object):
                          filename=self.filename, lineno=p.lineno(2))
 
   def p_typename(self, p):
-    """typename : basictypename
-                | array
-                | fixed_array
-                | interfacerequest"""
+    """typename : nonnullable_typename QSTN
+                | nonnullable_typename"""
+    if len(p) == 2:
+      p[0] = p[1]
+    else:
+      p[0] = p[1] + "?"
+
+  def p_nonnullable_typename(self, p):
+    """nonnullable_typename : basictypename
+                            | array
+                            | fixed_array
+                            | interfacerequest"""
     p[0] = p[1]
 
   def p_basictypename(self, p):

mojo/public/tools/bindings/pylib/mojom/parse/translate.py

@@ -36,16 +36,25 @@ def _MapKind(kind):
                  'handle<data_pipe_producer>': 'h:d:p',
                  'handle<message_pipe>': 'h:m',
                  'handle<shared_buffer>': 'h:s'}
+  if kind.endswith('?'):
+    base_kind = _MapKind(kind[0:-1])
+    # NOTE: This doesn't rule out enum types. Those will be detected later, when
+    # cross-reference is established.
+    reference_kinds = ('s', 'h', 'a', 'r', 'x')
+    if base_kind[0] not in reference_kinds:
+      raise Exception(
+          'A type (spec "%s") cannot be made nullable' % base_kind)
+    return '?' + base_kind
   if kind.endswith('[]'):
     typename = kind[0:-2]
     if _FIXED_ARRAY_REGEXP.search(typename):
-      raise Exception("Arrays of fixed sized arrays not supported")
+      raise Exception('Arrays of fixed sized arrays not supported')
     return 'a:' + _MapKind(typename)
   if kind.endswith(']'):
     lbracket = kind.rfind('[')
     typename = kind[0:lbracket]
     if typename.find('[') != -1:
-      raise Exception("Fixed sized arrays of arrays not supported")
+      raise Exception('Fixed sized arrays of arrays not supported')
     return 'a' + kind[lbracket+1:-1] + ':' + _MapKind(typename)
   if kind.endswith('&'):
     return 'r:' + _MapKind(kind[0:-1])

mojo/public/tools/bindings/pylib/mojom_tests/parse/lexer_unittest.py

@@ -132,6 +132,8 @@ class LexerTest(unittest.TestCase):
                       _MakeLexToken("MINUS", "-"))
     self.assertEquals(self._SingleTokenForInput("&"),
                       _MakeLexToken("AMP", "&"))
+    self.assertEquals(self._SingleTokenForInput("?"),
+                      _MakeLexToken("QSTN", "?"))
     self.assertEquals(self._SingleTokenForInput("="),
                       _MakeLexToken("EQUALS", "="))
     self.assertEquals(self._SingleTokenForInput("=>"),

mojo/public/tools/bindings/pylib/mojom_tests/parse/parser_unittest.py

@@ -372,8 +372,9 @@ class ParserTest(unittest.TestCase):
         }  // my_module
         """
     with self.assertRaisesRegexp(
-        lexer.LexError,
-        r"^my_file\.mojom:4: Error: Illegal character '\?'$"):
+        parser.ParseError,
+        r"^my_file\.mojom:4: Error: Unexpected '\?':\n"
+            r" *MY_ENUM_1 = 1 \? 2 : 3$"):
       parser.Parse(source, "my_file.mojom")
 
   def testSimpleOrdinals(self):
@@ -971,6 +972,87 @@ class ParserTest(unittest.TestCase):
             r" *module {}$"):
       parser.Parse(source2, "my_file.mojom")
 
+  def testValidNullableTypes(self):
+    """Tests parsing nullable types."""
+
+    source = """\
+        struct MyStruct {
+          int32? a;  // This is actually invalid, but handled at a different
+                     // level.
+          string? b;
+          int32[] ? c;
+          string ? [] ? d;
+          int32[]?[]? e;
+          int32[1]? f;
+          string?[1]? g;
+          some_struct? h;
+          handle? i;
+          handle<data_pipe_consumer>? j;
+          handle<data_pipe_producer>? k;
+          handle<message_pipe>? l;
+          handle<shared_buffer>? m;
+          some_interface&? n;
+        };
+        """
+    expected = ast.Mojom(
+        None,
+        ast.ImportList(),
+        [ast.Struct(
+            'MyStruct',
+            None,
+            ast.StructBody(
+                [ast.StructField('a', None, 'int32?', None),
+                 ast.StructField('b', None, 'string?', None),
+                 ast.StructField('c', None, 'int32[]?', None),
+                 ast.StructField('d', None, 'string?[]?', None),
+                 ast.StructField('e', None, 'int32[]?[]?', None),
+                 ast.StructField('f', None, 'int32[1]?', None),
+                 ast.StructField('g', None, 'string?[1]?', None),
+                 ast.StructField('h', None, 'some_struct?', None),
+                 ast.StructField('i', None, 'handle?', None),
+                 ast.StructField('j', None, 'handle<data_pipe_consumer>?',
+                                 None),
+                 ast.StructField('k', None, 'handle<data_pipe_producer>?',
+                                 None),
+                 ast.StructField('l', None, 'handle<message_pipe>?', None),
+                 ast.StructField('m', None, 'handle<shared_buffer>?', None),
+                 ast.StructField('n', None, 'some_interface&?', None)]))])
+    self.assertEquals(parser.Parse(source, "my_file.mojom"), expected)
+
+  def testInvalidNullableTypes(self):
+    """Tests that invalid nullable types are correctly detected."""
+    source1 = """\
+        struct MyStruct {
+          string?? a;
+        };
+        """
+    with self.assertRaisesRegexp(
+        parser.ParseError,
+        r"^my_file\.mojom:2: Error: Unexpected '\?':\n"
+            r" *string\?\? a;$"):
+      parser.Parse(source1, "my_file.mojom")
+
+    source2 = """\
+        struct MyStruct {
+          handle?<data_pipe_consumer> a;
+        };
+        """
+    with self.assertRaisesRegexp(
+        parser.ParseError,
+        r"^my_file\.mojom:2: Error: Unexpected '<':\n"
+            r" *handle\?<data_pipe_consumer> a;$"):
+      parser.Parse(source2, "my_file.mojom")
+
+    source3 = """\
+        struct MyStruct {
+          some_interface?& a;
+        };
+        """
+    with self.assertRaisesRegexp(
+        parser.ParseError,
+        r"^my_file\.mojom:2: Error: Unexpected '&':\n"
+            r" *some_interface\?& a;$"):
+      parser.Parse(source3, "my_file.mojom")
 
 if __name__ == "__main__":
   unittest.main()