mojo: Starting serialization for python bindings.

This is a reland of https://codereview.chromium.org/570563002 with the
gyp build config updated.

R=sdefresne@chromium.org

Review URL: https://codereview.chromium.org/571353002

Cr-Commit-Position: refs/heads/master@{#295038}

mojo/mojo.gyp

@@ -598,6 +598,7 @@
             'public/python/mojo/bindings/__init__.py',
             'public/python/mojo/bindings/descriptor.py',
             'public/python/mojo/bindings/reflection.py',
+            'public/python/mojo/bindings/serialization.py',
           ],
           'dependencies': [
             'mojo_python_system',

mojo/public/python/BUILD.gn

@@ -40,6 +40,7 @@ copy("bindings") {
     "mojo/bindings/__init__.py",
     "mojo/bindings/descriptor.py",
     "mojo/bindings/reflection.py",
+    "mojo/bindings/serialization.py",
   ]
   outputs = [
     "$root_out_dir/python/mojo/bindings/{{source_file_part}}",

mojo/public/python/mojo/bindings/reflection.py

@@ -4,6 +4,12 @@
 
 """The metaclasses used by the mojo python bindings."""
 
+import itertools
+
+# pylint: disable=F0401
+import mojo.bindings.serialization as serialization
+
+
 class MojoEnumType(type):
   """Meta class for enumerations.
 
@@ -85,12 +91,22 @@ class MojoStructType(type):
       dictionary[key] = MojoEnumType(key, (object,), { 'VALUES': enums[key] })
 
     # Add fields
-    for field in descriptor.get('fields', []):
+    groups = descriptor.get('fields', [])
+
+    fields = list(
+        itertools.chain.from_iterable([group.descriptors for group in groups]))
+    for field in fields:
       dictionary[field.name] = _BuildProperty(field)
 
     # Add init
     dictionary['__init__'] = _StructInit
 
+    # Add serialization method
+    serialization_object = serialization.Serialization(groups)
+    def Serialize(self, handle_offset=0):
+      return serialization_object.Serialize(self, handle_offset)
+    dictionary['Serialize'] = Serialize
+
     return type.__new__(mcs, name, bases, dictionary)
 
   # Prevent adding new attributes, or mutating constants.

mojo/public/python/mojo/bindings/serialization.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Utility classes for serialization"""
+
+import struct
+
+
+# Format of a header for a struct or an array.
+HEADER_STRUCT = struct.Struct("=II")
+
+
+class SerializationException(Exception):
+  """Error when strying to serialize a struct."""
+  pass
+
+
+class Serialization(object):
+  """
+  Helper class to serialize/deserialize a struct.
+  """
+  def __init__(self, groups):
+    self._groups = groups
+    self.version = _GetVersion(groups)
+    main_struct = _GetStruct(groups)
+    self.size = HEADER_STRUCT.size + main_struct.size
+    self._struct_per_version = {
+      self.version: main_struct,
+    }
+
+  def _GetMainStruct(self):
+    return self._GetStruct(self.version)
+
+  def _GetStruct(self, version):
+    # If asking for a greater ver
+    version = min(version, self.version)
+    if version not in self._struct_per_version:
+      self._struct_per_version[version] = _GetStruct(_FilterGroups(self._groups,
+                                                                   version))
+    return self._struct_per_version[version]
+
+  def Serialize(self, obj, handle_offset):
+    """
+    Serialize the given obj. handle_offset is the the first value to use when
+    encoding handles.
+    """
+    handles = []
+    data = bytearray(self.size)
+    HEADER_STRUCT.pack_into(data, 0, self.size, self.version)
+    position = HEADER_STRUCT.size
+    to_pack = []
+    for group in self._groups:
+      position = position + NeededPaddingForAlignment(position,
+                                                      group.GetByteSize())
+      (entry, new_handles) = group.Serialize(
+          obj,
+          len(data) - position,
+          data,
+          handle_offset + len(handles))
+      to_pack.append(entry)
+      handles.extend(new_handles)
+      position = position + group.GetByteSize()
+    self._GetMainStruct().pack_into(data, HEADER_STRUCT.size, *to_pack)
+    return (data, handles)
+
+
+def NeededPaddingForAlignment(value, alignment=8):
+  """Returns the padding necessary to align value with the given alignment."""
+  if value % alignment:
+    return alignment - (value % alignment)
+  return 0
+
+
+def _GetVersion(groups):
+  return sum([len(x.descriptors) for x in groups])
+
+
+def _FilterGroups(groups, version):
+  return [group for group in groups if group.version < version]
+
+
+def _GetStruct(groups):
+  index = 0
+  codes = [ '=' ]
+  for group in groups:
+    code = group.GetTypeCode()
+    size = group.GetByteSize()
+    needed_padding = NeededPaddingForAlignment(index, size)
+    if needed_padding:
+      codes.append('x' * needed_padding)
+      index = index + needed_padding
+    codes.append(code)
+    index = index + size
+  alignment_needed = NeededPaddingForAlignment(index)
+  if alignment_needed:
+    codes.append('x' * alignment_needed)
+  return struct.Struct(''.join(codes))

mojo/public/tools/bindings/generators/mojom_python_generator.py

@@ -12,45 +12,61 @@ import mojom.generate.module as mojom
 from mojom.generate.template_expander import UseJinja
 
 _kind_to_type = {
-  mojom.BOOL:                  "_descriptor.TYPE_BOOL",
-  mojom.INT8:                  "_descriptor.TYPE_INT8",
-  mojom.UINT8:                 "_descriptor.TYPE_UINT8",
-  mojom.INT16:                 "_descriptor.TYPE_INT16",
-  mojom.UINT16:                "_descriptor.TYPE_UINT16",
-  mojom.INT32:                 "_descriptor.TYPE_INT32",
-  mojom.UINT32:                "_descriptor.TYPE_UINT32",
-  mojom.INT64:                 "_descriptor.TYPE_INT64",
-  mojom.UINT64:                "_descriptor.TYPE_UINT64",
-  mojom.FLOAT:                 "_descriptor.TYPE_FLOAT",
-  mojom.DOUBLE:                "_descriptor.TYPE_DOUBLE",
-  mojom.STRING:                "_descriptor.TYPE_STRING",
-  mojom.NULLABLE_STRING:       "_descriptor.TYPE_NULLABLE_STRING",
-  mojom.HANDLE:                "_descriptor.TYPE_HANDLE",
-  mojom.DCPIPE:                "_descriptor.TYPE_HANDLE",
-  mojom.DPPIPE:                "_descriptor.TYPE_HANDLE",
-  mojom.MSGPIPE:               "_descriptor.TYPE_HANDLE",
-  mojom.SHAREDBUFFER:          "_descriptor.TYPE_HANDLE",
-  mojom.NULLABLE_HANDLE:       "_descriptor.TYPE_NULLABLE_HANDLE",
-  mojom.NULLABLE_DCPIPE:       "_descriptor.TYPE_NULLABLE_HANDLE",
-  mojom.NULLABLE_DPPIPE:       "_descriptor.TYPE_NULLABLE_HANDLE",
-  mojom.NULLABLE_MSGPIPE:      "_descriptor.TYPE_NULLABLE_HANDLE",
-  mojom.NULLABLE_SHAREDBUFFER: "_descriptor.TYPE_NULLABLE_HANDLE",
+  mojom.BOOL:                  '_descriptor.TYPE_BOOL',
+  mojom.INT8:                  '_descriptor.TYPE_INT8',
+  mojom.UINT8:                 '_descriptor.TYPE_UINT8',
+  mojom.INT16:                 '_descriptor.TYPE_INT16',
+  mojom.UINT16:                '_descriptor.TYPE_UINT16',
+  mojom.INT32:                 '_descriptor.TYPE_INT32',
+  mojom.UINT32:                '_descriptor.TYPE_UINT32',
+  mojom.INT64:                 '_descriptor.TYPE_INT64',
+  mojom.UINT64:                '_descriptor.TYPE_UINT64',
+  mojom.FLOAT:                 '_descriptor.TYPE_FLOAT',
+  mojom.DOUBLE:                '_descriptor.TYPE_DOUBLE',
+  mojom.STRING:                '_descriptor.TYPE_STRING',
+  mojom.NULLABLE_STRING:       '_descriptor.TYPE_NULLABLE_STRING',
+  mojom.HANDLE:                '_descriptor.TYPE_HANDLE',
+  mojom.DCPIPE:                '_descriptor.TYPE_HANDLE',
+  mojom.DPPIPE:                '_descriptor.TYPE_HANDLE',
+  mojom.MSGPIPE:               '_descriptor.TYPE_HANDLE',
+  mojom.SHAREDBUFFER:          '_descriptor.TYPE_HANDLE',
+  mojom.NULLABLE_HANDLE:       '_descriptor.TYPE_NULLABLE_HANDLE',
+  mojom.NULLABLE_DCPIPE:       '_descriptor.TYPE_NULLABLE_HANDLE',
+  mojom.NULLABLE_DPPIPE:       '_descriptor.TYPE_NULLABLE_HANDLE',
+  mojom.NULLABLE_MSGPIPE:      '_descriptor.TYPE_NULLABLE_HANDLE',
+  mojom.NULLABLE_SHAREDBUFFER: '_descriptor.TYPE_NULLABLE_HANDLE',
 }
 
 # int64 integers are not handled by array.array. int64/uint64 array are
 # supported but storage is not optimized (ie. they are plain python list, not
 # array.array)
-_kind_to_typecode = {
-  mojom.INT8:   "'b'",
-  mojom.UINT8:  "'B'",
-  mojom.INT16:  "'h'",
-  mojom.UINT16: "'H'",
-  mojom.INT32:  "'i'",
-  mojom.UINT32: "'I'",
-  mojom.FLOAT:  "'f'",
-  mojom.DOUBLE: "'d'",
+_kind_to_typecode_for_native_array = {
+  mojom.INT8:   'b',
+  mojom.UINT8:  'B',
+  mojom.INT16:  'h',
+  mojom.UINT16: 'H',
+  mojom.INT32:  'i',
+  mojom.UINT32: 'I',
+  mojom.FLOAT:  'f',
+  mojom.DOUBLE: 'd',
 }
 
+_kind_to_typecode = dict(_kind_to_typecode_for_native_array)
+_kind_to_typecode.update({
+  mojom.INT64:                 'q',
+  mojom.UINT64:                'Q',
+  mojom.HANDLE:                'i',
+  mojom.DCPIPE:                'i',
+  mojom.DPPIPE:                'i',
+  mojom.MSGPIPE:               'i',
+  mojom.SHAREDBUFFER:          'i',
+  mojom.NULLABLE_HANDLE:       'i',
+  mojom.NULLABLE_DCPIPE:       'i',
+  mojom.NULLABLE_DPPIPE:       'i',
+  mojom.NULLABLE_MSGPIPE:      'i',
+  mojom.NULLABLE_SHAREDBUFFER: 'i',
+})
+
 
 def NameToComponent(name):
   # insert '_' between anything and a Title name (e.g, HTTPEntry2FooBar ->
@@ -99,8 +115,8 @@ def ExpressionToText(token):
     if token.value == 'double.NAN' or token.value == 'float.NAN':
       return 'float(\'nan\')';
 
-  if token in ["true", "false"]:
-    return str(token == "true")
+  if token in ['true', 'false']:
+    return str(token == 'true')
 
   return token
 
@@ -113,43 +129,54 @@ def GetStructClass(kind):
 
 def GetFieldType(kind, field=None):
   if mojom.IsAnyArrayKind(kind):
-    if kind.kind in _kind_to_typecode:
-      arguments = [ _kind_to_typecode[kind.kind] ]
-    else:
-      arguments = [ GetFieldType(kind.kind) ]
+    arguments = []
+    if kind.kind in _kind_to_typecode_for_native_array:
+      arguments.append('%r' %_kind_to_typecode_for_native_array[kind.kind])
+    elif kind.kind != mojom.BOOL:
+      arguments.append(GetFieldType(kind.kind))
     if mojom.IsNullableKind(kind):
-      arguments.append("nullable=True")
+      arguments.append('nullable=True')
     if mojom.IsFixedArrayKind(kind):
-      arguments.append("length=%d" % kind.length)
-    if kind.kind in _kind_to_typecode:
-      return "_descriptor.NativeArrayType(%s)" % ", ".join(arguments)
-    else:
-      return "_descriptor.PointerArrayType(%s)" % ", ".join(arguments)
+      arguments.append('length=%d' % kind.length)
+    array_type = 'GenericArrayType'
+    if kind.kind == mojom.BOOL:
+      array_type = 'BooleanArrayType'
+    elif kind.kind in _kind_to_typecode_for_native_array:
+      array_type = 'NativeArrayType'
+    return '_descriptor.%s(%s)' % (array_type, ', '.join(arguments))
 
   if mojom.IsStructKind(kind):
     arguments = [ GetStructClass(kind) ]
     if mojom.IsNullableKind(kind):
-      arguments.append("nullable=True")
-    return "_descriptor.StructType(%s)" % ", ".join(arguments)
+      arguments.append('nullable=True')
+    return '_descriptor.StructType(%s)' % ', '.join(arguments)
 
   if mojom.IsEnumKind(kind):
     return GetFieldType(mojom.INT32)
 
-  return _kind_to_type.get(kind, "_descriptor.TYPE_NONE")
+  return _kind_to_type.get(kind, '_descriptor.TYPE_NONE')
 
 def GetFieldDescriptor(packed_field):
   field = packed_field.field
-  arguments = [ '\'%s\'' % field.name ]
-  arguments.append(GetFieldType(field.kind, field))
-  arguments.append(str(packed_field.offset))
+  class_name = 'SingleFieldGroup'
   if field.kind == mojom.BOOL:
-    arguments.append('bit_offset=%d' % packed_field.bit)
+    class_name = 'FieldDescriptor'
+  arguments = [ '%r' % field.name ]
+  arguments.append(GetFieldType(field.kind, field))
+  arguments.append(str(packed_field.field.ordinal))
   if field.default:
     if mojom.IsStructKind(field.kind):
       arguments.append('default_value=True')
     else:
       arguments.append('default_value=%s' % ExpressionToText(field.default))
-  return '_descriptor.FieldDescriptor(%s)' % ', '.join(arguments)
+  return '_descriptor.%s(%s)' % (class_name, ', '.join(arguments))
+
+def GetFieldGroup(byte):
+  if len(byte.packed_fields) > 1:
+    descriptors = map(GetFieldDescriptor, byte.packed_fields)
+    return '_descriptor.BooleanGroup([%s])' % ', '.join(descriptors)
+  assert len(byte.packed_fields) == 1
+  return GetFieldDescriptor(byte.packed_fields[0])
 
 def ComputeStaticValues(module):
   in_progress = set()
@@ -229,11 +256,12 @@ def ComputeStaticValues(module):
 
   return module
 
+
 class Generator(generator.Generator):
 
   python_filters = {
     'expression_to_text': ExpressionToText,
-    'field_descriptor': GetFieldDescriptor,
+    'field_group': GetFieldGroup,
     'name': GetNameForElement,
   }
 

mojo/public/tools/bindings/generators/python_templates/module.py.tmpl

@@ -46,9 +46,9 @@ class {{struct|name}}(object):
 {%   if struct.fields %}
     'fields': [
 {%    for byte in struct.bytes %}
-{%     for packed_field in byte.packed_fields %}
-      {{packed_field|field_descriptor}},
-{%     endfor %}
+{%      if byte.packed_fields %}
+      {{byte|field_group}},
+{%      endif %}
 {%   endfor %}
     ],
 {%   endif %}

mojo/python/tests/bindings_serialization_deserialization_unittest.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import unittest
+
+# pylint: disable=F0401
+import mojo.system
+
+# Generated files
+# pylint: disable=F0401
+import sample_service_mojom
+
+
+def _NewHandle():
+  return mojo.system.MessagePipe().handle0
+
+def _NewFoo():
+  foo_instance = sample_service_mojom.Foo()
+  foo_instance.name = "Foo.name"
+  foo_instance.x = 23
+  foo_instance.y = -23
+  foo_instance.a = False
+  foo_instance.b = True
+  foo_instance.c = True
+  foo_instance.bar = sample_service_mojom.Bar()
+  foo_instance.extra_bars = [
+      sample_service_mojom.Bar(),
+      sample_service_mojom.Bar()
+  ]
+  foo_instance.data = 'Hello world'
+  foo_instance.source = _NewHandle()
+  foo_instance.input_streams = [ _NewHandle() ]
+  foo_instance.output_streams = [ _NewHandle(), _NewHandle() ]
+  foo_instance.array_of_array_of_bools = [ [ True, False ], [] ]
+  foo_instance.multi_array_of_strings = [
+      [
+          [ "1", "2" ],
+          [],
+          [ "3", "4" ],
+      ],
+      [],
+  ]
+  foo_instance.array_of_bools = [ True, 0, 1, 2, 0 ]
+  return foo_instance
+
+class SerializationDeserializationTest(unittest.TestCase):
+
+  def testFooSerialization(self):
+    (data, _) = _NewFoo().Serialize()
+    self.assertTrue(len(data))
+    self.assertEquals(len(data) % 8, 0)