Move main mojom parser library code

This moves Python library code out of mojo/public/tools/bindings/pylib/
and into mojo/public/tools/mojom/.

Note that duplicate nested mojom/ directory name is intentional: the
repository will be named "mojom" and the top-level directory will
contain documentation and scripts, as well as other subdirectories. The
"mojom" subdirectory is then the home of the parser's Python modules.

This is all in preparation to mirror mojo/public/tools/mojom/ in a
read-only repository.

Bug: 1060467
Change-Id: I33c32f2c6b402b227408f8e5f1fc8c40be4dd3b4
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2098849
Auto-Submit: Ken Rockot <rockot@google.com>
Commit-Queue: Oksana Zhuravlova <oksamyt@chromium.org>
Reviewed-by: Oksana Zhuravlova <oksamyt@chromium.org>
Cr-Commit-Position: refs/heads/master@{#749373}

mojo/public/tools/bindings/gen_data_files_list.py

@@ -21,10 +21,12 @@ import sys
 from cStringIO import StringIO
 from optparse import OptionParser
 
-sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)),
-                                "pylib"))
+sys.path.insert(
+    0,
+    os.path.join(
+        os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "mojom"))
 
-from mojom.generate.generator import  WriteFile
+from mojom.generate.generator import WriteFile
 
 
 def main():

mojo/public/tools/bindings/generate_type_mappings.py

@@ -63,8 +63,10 @@ import os
 import re
 import sys
 
-sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)),
-                                "pylib"))
+sys.path.insert(
+    0,
+    os.path.join(
+        os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "mojom"))
 
 from mojom.generate.generator import WriteFile
 

mojo/public/tools/bindings/mojom.gni

@@ -63,27 +63,28 @@ enable_scrambled_message_ids =
     (is_mac || is_win || (is_linux && !is_chromeos && !is_chromecast) ||
      ((enable_nacl || is_nacl || is_nacl_nonsfi) && target_os != "chromeos"))
 
-mojom_generator_root = "//mojo/public/tools/bindings"
-compile_typescript_script = "$mojom_generator_root/compile_typescript.py"
+_mojom_tools_root = "//mojo/public/tools"
+_mojom_library_root = "$_mojom_tools_root/mojom/mojom"
+mojom_generator_root = "$_mojom_tools_root/bindings"
 mojom_generator_script = "$mojom_generator_root/mojom_bindings_generator.py"
 mojom_generator_sources = [
+  "$_mojom_library_root/__init__.py",
+  "$_mojom_library_root/error.py",
+  "$_mojom_library_root/generate/__init__.py",
+  "$_mojom_library_root/generate/constant_resolver.py",
+  "$_mojom_library_root/generate/generator.py",
+  "$_mojom_library_root/generate/module.py",
+  "$_mojom_library_root/generate/pack.py",
+  "$_mojom_library_root/generate/template_expander.py",
+  "$_mojom_library_root/generate/translate.py",
+  "$_mojom_library_root/parse/__init__.py",
+  "$_mojom_library_root/parse/ast.py",
+  "$_mojom_library_root/parse/lexer.py",
+  "$_mojom_library_root/parse/parser.py",
   "$mojom_generator_root/generators/mojom_cpp_generator.py",
   "$mojom_generator_root/generators/mojom_java_generator.py",
   "$mojom_generator_root/generators/mojom_js_generator.py",
   "$mojom_generator_root/generators/mojom_ts_generator.py",
-  "$mojom_generator_root/pylib/mojom/__init__.py",
-  "$mojom_generator_root/pylib/mojom/error.py",
-  "$mojom_generator_root/pylib/mojom/generate/__init__.py",
-  "$mojom_generator_root/pylib/mojom/generate/constant_resolver.py",
-  "$mojom_generator_root/pylib/mojom/generate/generator.py",
-  "$mojom_generator_root/pylib/mojom/generate/module.py",
-  "$mojom_generator_root/pylib/mojom/generate/pack.py",
-  "$mojom_generator_root/pylib/mojom/generate/template_expander.py",
-  "$mojom_generator_root/pylib/mojom/generate/translate.py",
-  "$mojom_generator_root/pylib/mojom/parse/__init__.py",
-  "$mojom_generator_root/pylib/mojom/parse/ast.py",
-  "$mojom_generator_root/pylib/mojom/parse/lexer.py",
-  "$mojom_generator_root/pylib/mojom/parse/parser.py",
   "$mojom_generator_script",
 ]
 
@@ -1604,7 +1605,7 @@ template("mojom") {
 
     generator_js_target_name = "${target_name}_js__generator"
     action(generator_js_target_name) {
-      script = compile_typescript_script
+      script = "$mojom_generator_root/compile_typescript.py"
       sources = ts_outputs
       outputs = js_outputs
       public_deps = [ ":$generator_ts_target_name" ]

mojo/public/tools/bindings/mojom_bindings_generator.py

@@ -36,13 +36,11 @@ def _GetDirAbove(dirname):
     if tail == dirname:
       return path
 
-# Manually check for the command-line flag. (This isn't quite right, since it
-# ignores, e.g., "--", but it's close enough.)
-if "--use_bundled_pylibs" in sys.argv[1:]:
-  sys.path.insert(0, os.path.join(_GetDirAbove("mojo"), "third_party"))
 
-sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)),
-                                "pylib"))
+sys.path.insert(
+    0,
+    os.path.join(
+        os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "mojom"))
 
 from mojom.error import Error
 import mojom.fileutil as fileutil

mojo/public/tools/bindings/pylib/mojom/generate/run_tests.py

-#!/usr/bin/env python
-# Copyright 2013 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-""" Test runner for Mojom """
-
-from __future__ import print_function
-
-import subprocess
-import sys
-
-def TestMojom(testname, args):
-  print('\nRunning unit tests for %s.' % testname)
-  try:
-    args = [sys.executable, testname] + args
-    subprocess.check_call(args, stdout=sys.stdout)
-    print('Succeeded')
-    return 0
-  except subprocess.CalledProcessError as err:
-    print('Failed with %s.' % str(err))
-    return 1
-
-
-def main(args):
-  errors = 0
-  errors += TestMojom('data_tests.py', ['--test'])
-  errors += TestMojom('module_tests.py', ['--test'])
-  errors += TestMojom('pack_tests.py', ['--test'])
-
-  if errors:
-    print('\nFailed tests.')
-  return min(errors, 127)  # Make sure the return value doesn't "wrap".
-
-
-if __name__ == '__main__':
-  sys.exit(main(sys.argv[1:]))

mojo/public/tools/bindings/pylib/mojom/generate/test_support.py

-# Copyright 2013 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-from __future__ import print_function
-
-import sys
-import traceback
-
-import module as mojom
-
-# Support for writing mojom test cases.
-# RunTest(fn) will execute fn, catching any exceptions. fn should return
-# the number of errors that are encountered.
-#
-# EXPECT_EQ(a, b) and EXPECT_TRUE(b) will print error information if the
-# expectations are not true and return a non zero value. This allows test cases
-# to be written like this
-#
-# def Foo():
-#   errors = 0
-#   errors += EXPECT_EQ('test', test())
-#   ...
-#   return errors
-#
-# RunTest(foo)
-
-def FieldsAreEqual(field1, field2):
-  if field1 == field2:
-    return True
-  return field1.name == field2.name and \
-      KindsAreEqual(field1.kind, field2.kind) and \
-      field1.ordinal == field2.ordinal and \
-      field1.default == field2.default
-
-
-def KindsAreEqual(kind1, kind2):
-  if kind1 == kind2:
-    return True
-  if kind1.__class__ != kind2.__class__ or kind1.spec != kind2.spec:
-    return False
-  if kind1.__class__ == mojom.Kind:
-    return kind1.spec == kind2.spec
-  if kind1.__class__ == mojom.Struct:
-    if kind1.name != kind2.name or \
-        kind1.spec != kind2.spec or \
-        len(kind1.fields) != len(kind2.fields):
-      return False
-    for i in range(len(kind1.fields)):
-      if not FieldsAreEqual(kind1.fields[i], kind2.fields[i]):
-        return False
-    return True
-  if kind1.__class__ == mojom.Array:
-    return KindsAreEqual(kind1.kind, kind2.kind)
-  print('Unknown Kind class: ', kind1.__class__.__name__)
-  return False
-
-
-def ParametersAreEqual(parameter1, parameter2):
-  if parameter1 == parameter2:
-    return True
-  return parameter1.name == parameter2.name and \
-     parameter1.ordinal == parameter2.ordinal and \
-     parameter1.default == parameter2.default and \
-     KindsAreEqual(parameter1.kind, parameter2.kind)
-
-
-def MethodsAreEqual(method1, method2):
-  if method1 == method2:
-    return True
-  if method1.name != method2.name or \
-      method1.ordinal != method2.ordinal or \
-      len(method1.parameters) != len(method2.parameters):
-    return False
-  for i in range(len(method1.parameters)):
-    if not ParametersAreEqual(method1.parameters[i], method2.parameters[i]):
-      return False
-  return True
-
-
-def InterfacesAreEqual(interface1, interface2):
-  if interface1 == interface2:
-    return True
-  if interface1.name != interface2.name or \
-      len(interface1.methods) != len(interface2.methods):
-    return False
-  for i in range(len(interface1.methods)):
-    if not MethodsAreEqual(interface1.methods[i], interface2.methods[i]):
-      return False
-  return True
-
-
-def ModulesAreEqual(module1, module2):
-  if module1 == module2:
-    return True
-  if module1.name != module2.name or \
-      module1.namespace != module2.namespace or \
-      len(module1.structs) != len(module2.structs) or \
-      len(module1.interfaces) != len(module2.interfaces):
-    return False
-  for i in range(len(module1.structs)):
-    if not KindsAreEqual(module1.structs[i], module2.structs[i]):
-      return False
-  for i in range(len(module1.interfaces)):
-    if not InterfacesAreEqual(module1.interfaces[i], module2.interfaces[i]):
-      return False
-  return True
-
-
-# Builds and returns a Module suitable for testing/
-def BuildTestModule():
-  module = mojom.Module('test', 'testspace')
-  struct = module.AddStruct('teststruct')
-  struct.AddField('testfield1', mojom.INT32)
-  struct.AddField('testfield2', mojom.Array(mojom.INT32), 42)
-
-  interface = module.AddInterface('Server')
-  method = interface.AddMethod('Foo', 42)
-  method.AddParameter('foo', mojom.INT32)
-  method.AddParameter('bar', mojom.Array(struct))
-
-  return module
-
-
-# Tests if |module| is as built by BuildTestModule(). Returns the number of
-# errors
-def TestTestModule(module):
-  errors = 0
-
-  errors += EXPECT_EQ('test', module.name)
-  errors += EXPECT_EQ('testspace', module.namespace)
-  errors += EXPECT_EQ(1, len(module.structs))
-  errors += EXPECT_EQ('teststruct', module.structs[0].name)
-  errors += EXPECT_EQ(2, len(module.structs[0].fields))
-  errors += EXPECT_EQ('testfield1', module.structs[0].fields[0].name)
-  errors += EXPECT_EQ(mojom.INT32, module.structs[0].fields[0].kind)
-  errors += EXPECT_EQ('testfield2', module.structs[0].fields[1].name)
-  errors += EXPECT_EQ(mojom.Array, module.structs[0].fields[1].kind.__class__)
-  errors += EXPECT_EQ(mojom.INT32, module.structs[0].fields[1].kind.kind)
-
-  errors += EXPECT_EQ(1, len(module.interfaces))
-  errors += EXPECT_EQ('Server', module.interfaces[0].name)
-  errors += EXPECT_EQ(1, len(module.interfaces[0].methods))
-  errors += EXPECT_EQ('Foo', module.interfaces[0].methods[0].name)
-  errors += EXPECT_EQ(2, len(module.interfaces[0].methods[0].parameters))
-  errors += EXPECT_EQ('foo', module.interfaces[0].methods[0].parameters[0].name)
-  errors += EXPECT_EQ(mojom.INT32,
-                      module.interfaces[0].methods[0].parameters[0].kind)
-  errors += EXPECT_EQ('bar', module.interfaces[0].methods[0].parameters[1].name)
-  errors += EXPECT_EQ(
-    mojom.Array,
-    module.interfaces[0].methods[0].parameters[1].kind.__class__)
-  errors += EXPECT_EQ(
-    module.structs[0],
-    module.interfaces[0].methods[0].parameters[1].kind.kind)
-  return errors
-
-
-def PrintFailure(string):
-  stack = traceback.extract_stack()
-  frame = stack[len(stack)-3]
-  sys.stderr.write("ERROR at %s:%d, %s\n" % (frame[0], frame[1], string))
-  print("Traceback:")
-  for line in traceback.format_list(stack[:len(stack)-2]):
-    sys.stderr.write(line)
-
-
-def EXPECT_EQ(a, b):
-  if a != b:
-    PrintFailure("%s != %s" % (a, b))
-    return 1
-  return 0
-
-
-def EXPECT_TRUE(a):
-  if not a:
-    PrintFailure('Expecting True')
-    return 1
-  return 0
-
-
-def RunTest(fn):
-  sys.stdout.write('Running %s...' % fn.__name__)
-  try:
-    errors = fn()
-  except:
-    traceback.print_exc(sys.stderr)
-    errors = 1
-  if errors == 0:
-    sys.stdout.write('OK\n')
-  elif errors == 1:
-    sys.stdout.write('1 ERROR\n')
-  else:
-    sys.stdout.write('%d ERRORS\n' % errors)
-  return errors

mojo/public/tools/mojom/README.md

+# The Mojom Parser
+
+The Mojom format is an interface definition language (IDL) for describing
+interprocess communication (IPC) messages and data types for use with the
+low-level cross-platform
+[Mojo IPC library](https://chromium.googlesource.com/chromium/src/+/master/mojo/public/c/system/README.md).
+
+This directory consists of a `mojom` Python module, its tests, and supporting
+command-line tools. The Python module implements the parser used by the
+command-line tools and exposes an API to help external bindings generators emit
+useful code from the parser's outputs.
+
+TODO(https://crbug.com/1060464): Fill out this documentation once the library
+and tools have stabilized.

mojo/public/tools/bindings/pylib/mojom/fileutil.py → mojo/public/tools/mojom/mojom/fileutil.py

@@ -3,7 +3,20 @@
 # found in the LICENSE file.
 
 import errno
+import imp
 import os.path
+import sys
+
+
+def _GetDirAbove(dirname):
+  """Returns the directory "above" this file containing |dirname| (which must
+  also be "above" this file)."""
+  path = os.path.abspath(__file__)
+  while True:
+    path, tail = os.path.split(path)
+    assert tail
+    if tail == dirname:
+      return path
 
 
 def EnsureDirectoryExists(path, always_try_to_create=False):
@@ -17,3 +30,15 @@ def EnsureDirectoryExists(path, always_try_to_create=False):
       # There may have been a race to create this directory.
       if e.errno != errno.EEXIST:
         raise
+
+
+def EnsureModuleAvailable(module_name):
+  """Helper function which attempts to find the Python module named
+  |module_name| using the usual module search. If that fails, this assumes it's
+  being called within the Chromium tree, or an equivalent tree where this
+  library lives somewhere under a "mojo" directory which has a "third_party"
+  sibling."""
+  try:
+    imp.find_module(module_name)
+  except ImportError:
+    sys.path.append(os.path.join(_GetDirAbove("mojo"), "third_party"))

mojo/public/tools/bindings/pylib/mojom/generate/constant_resolver.py → mojo/public/tools/mojom/mojom/generate/constant_resolver.py

@@ -4,7 +4,8 @@
 """Resolves the values used for constants and enums."""
 
 from itertools import ifilter
-import mojom.generate.module as mojom
+
+from mojom.generate import module as mojom
 
 
 def ResolveConstants(module, expression_to_text):

mojo/public/tools/bindings/pylib/mojom/generate/generator.py → mojo/public/tools/mojom/mojom/generate/generator.py

@@ -9,9 +9,9 @@ from functools import partial
 import os.path
 import re
 
-import mojom.generate.module as mojom
-import mojom.fileutil as fileutil
-import mojom.generate.pack as pack
+from mojom import fileutil
+from mojom.generate import module as mojom
+from mojom.generate import pack
 
 
 def ExpectedArraySize(kind):
@@ -114,7 +114,7 @@ def WriteFile(contents, full_path):
 
   # Dump the data to disk.
   with open(full_path, "wb") as f:
-    if type(contents) != bytes:
+    if not isinstance(contents, bytes):
       f.write(contents.encode('utf-8'))
     else:
       f.write(contents)

mojo/public/tools/bindings/pylib/mojom/generate/pack.py → mojo/public/tools/mojom/mojom/generate/pack.py

@@ -2,7 +2,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import mojom.generate.module as mojom
+from mojom.generate import module as mojom
 
 # This module provides a mechanism for determining the packed order and offsets
 # of a mojom.Struct.
@@ -195,23 +195,23 @@ class ByteInfo(object):
 def GetByteLayout(packed_struct):
   total_payload_size = GetPayloadSizeUpToField(
       packed_struct.packed_fields[-1] if packed_struct.packed_fields else None)
-  bytes = [ByteInfo() for i in range(total_payload_size)]
+  byte_info = [ByteInfo() for i in range(total_payload_size)]
 
   limit_of_previous_field = 0
   for packed_field in packed_struct.packed_fields:
     for i in range(limit_of_previous_field, packed_field.offset):
-      bytes[i].is_padding = True
-    bytes[packed_field.offset].packed_fields.append(packed_field)
+      byte_info[i].is_padding = True
+    byte_info[packed_field.offset].packed_fields.append(packed_field)
     limit_of_previous_field = packed_field.offset + packed_field.size
 
-  for i in range(limit_of_previous_field, len(bytes)):
-    bytes[i].is_padding = True
+  for i in range(limit_of_previous_field, len(byte_info)):
+    byte_info[i].is_padding = True
 
-  for byte in bytes:
+  for byte in byte_info:
     # A given byte cannot both be padding and have a fields packed into it.
     assert not (byte.is_padding and byte.packed_fields)
 
-  return bytes
+  return byte_info
 
 
 class VersionInfo(object):

mojo/public/tools/bindings/pylib/mojom/generate/template_expander.py → mojo/public/tools/mojom/mojom/generate/template_expander.py

@@ -7,11 +7,9 @@
 import os.path
 import sys
 
-_current_dir = os.path.dirname(os.path.realpath(__file__))
-# jinja2 is in chromium's third_party directory
-# Insert at front to override system libraries, and after path[0] == script dir
-sys.path.insert(
-    1, os.path.join(_current_dir, *([os.pardir] * 7 + ['third_party'])))
+from mojom import fileutil
+
+fileutil.EnsureModuleAvailable("jinja2")
 import jinja2
 
 

mojo/public/tools/bindings/pylib/mojom/generate/translate.py → mojo/public/tools/mojom/mojom/generate/translate.py

@@ -13,7 +13,7 @@ import itertools
 import os
 import re
 
-import mojom.generate.module as mojom
+from mojom.generate import module as mojom
 from mojom.parse import ast
 
 
@@ -507,11 +507,11 @@ def _ResolveNumericEnumValues(enum_fields):
       prev_value += 1
 
     # Integral value (e.g: BEGIN = -0x1).
-    elif type(field.value) is str:
+    elif isinstance(field.value, str):
       prev_value = int(field.value, 0)
 
     # Reference to a previous enum value (e.g: INIT = BEGIN).
-    elif type(field.value) is mojom.EnumValue:
+    elif isinstance(field.value, mojom.EnumValue):
       prev_value = resolved_enum_values[field.value.mojom_name]
     else:
       raise Exception("Unresolved enum value.")

mojo/public/tools/bindings/pylib/mojom/parse/ast.py → mojo/public/tools/mojom/mojom/parse/ast.py

@@ -17,7 +17,7 @@ class NodeBase(object):
     self.lineno = lineno
 
   def __eq__(self, other):
-    return isinstance(self, other)
+    return type(self) == type(other)
 
   # Make != the inverse of ==. (Subclasses shouldn't have to override this.)
   def __ne__(self, other):

mojo/public/tools/bindings/pylib/mojom/parse/conditional_features.py → mojo/public/tools/mojom/mojom/parse/conditional_features.py

@@ -3,8 +3,8 @@
 # found in the LICENSE file.
 """Helpers for processing conditionally enabled features in a mojom."""
 
-from . import ast
-from ..error import Error
+from mojom.error import Error
+from mojom.parse import ast
 
 
 class EnableIfError(Error):

mojo/public/tools/bindings/pylib/mojom/parse/lexer.py → mojo/public/tools/mojom/mojom/parse/lexer.py

@@ -6,26 +6,12 @@ import imp
 import os.path
 import sys
 
+from mojom import fileutil
+from mojom.error import Error
 
-def _GetDirAbove(dirname):
-  """Returns the directory "above" this file containing |dirname| (which must
-  also be "above" this file)."""
-  path = os.path.abspath(__file__)
-  while True:
-    path, tail = os.path.split(path)
-    assert tail
-    if tail == dirname:
-      return path
-
-
-try:
-  imp.find_module("ply")
-except ImportError:
-  sys.path.append(os.path.join(_GetDirAbove("mojo"), "third_party"))
+fileutil.EnsureModuleAvailable("ply")
 from ply.lex import TOKEN
 
-from ..error import Error
-
 
 class LexError(Error):
   """Class for errors from the lexer."""

mojo/public/tools/bindings/pylib/mojom/parse/parser.py → mojo/public/tools/mojom/mojom/parse/parser.py

@@ -6,16 +6,15 @@
 import os.path
 import sys
 
-_current_dir = os.path.dirname(os.path.realpath(__file__))
-sys.path.insert(
-    1, os.path.join(_current_dir, *([os.pardir] * 7 + ['third_party'])))
+from mojom import fileutil
+from mojom.error import Error
+from mojom.parse import ast
+from mojom.parse.lexer import Lexer
+
+fileutil.EnsureModuleAvailable("ply")
 from ply import lex
 from ply import yacc
 
-from ..error import Error
-from . import ast
-from .lexer import Lexer
-
 _MAX_ORDINAL_VALUE = 0xffffffff
 _MAX_ARRAY_SIZE = 0xffffffff