Extract mojom parser from bindings generator

This extracts the mojom parsing logic from mojom_bindings_generator.py
into a new mojom_parser.py script within the mojom tool directory.

Apart from simply moving the parsing logic to a new script, this also
changes how the parse result is persisted by an intermediate build step
for subsequent build steps to consume.

Prior to this change, the parsing step emitted a ".p" file for each
processed .mojom file. This was nothing more than a serialized version
of the parsed AST with [EnableIf] feature filtering applied. Every
subsequent step for the mojom in question would load this AST, then
recursively process its import dependencies while loading their ASTs in
turn, before finally processing all of the aggregated data (resolving
references, computing field packing data, etc) to produce a Module
object suitable for bindings generation.

This CL rolls most of that processing into the parsing step (with more
to follow in future CLs), and the resulting computed Module object is
persisted to a ".mojom-module" file.

To generate bindings for a given .mojom file, the generator can simply
call Module.Load() on the corresponding .mojom-module produced by the
parser. The loaded Module is sufficient to inform code generation
with no need to resolve dependencies, process other modules, or
perform other complex computations on the model.

Each mojom-module is thus entirely self-contained, replicating the full
contents of all transitive import dependencies. This means the build
consumes more disk space with intermediate build artifacts, but it
also means generation steps are faster and simpler.

This CL should result in no net changes to the final build outputs.

Bug: 1060471
Change-Id: Ic21b737409ac1329f0f8a8751bf021f2de4768a4
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2128966
Commit-Queue: Ken Rockot <rockot@google.com>
Reviewed-by: Oksana Zhuravlova <oksamyt@chromium.org>
Cr-Commit-Position: refs/heads/master@{#755952}

mojo/public/tools/bindings/mojom.gni

@@ -65,9 +65,8 @@ enable_scrambled_message_ids =
 
 _mojom_tools_root = "//mojo/public/tools"
 _mojom_library_root = "$_mojom_tools_root/mojom/mojom"
-mojom_generator_root = "$_mojom_tools_root/bindings"
-mojom_generator_script = "$mojom_generator_root/mojom_bindings_generator.py"
-mojom_generator_sources = [
+mojom_parser_script = "$_mojom_tools_root/mojom/mojom_parser.py"
+mojom_parser_sources = [
   "$_mojom_library_root/__init__.py",
   "$_mojom_library_root/error.py",
   "$_mojom_library_root/generate/__init__.py",
@@ -81,12 +80,18 @@ mojom_generator_sources = [
   "$_mojom_library_root/parse/ast.py",
   "$_mojom_library_root/parse/lexer.py",
   "$_mojom_library_root/parse/parser.py",
+]
+
+mojom_generator_root = "$_mojom_tools_root/bindings"
+mojom_generator_script = "$mojom_generator_root/mojom_bindings_generator.py"
+mojom_generator_sources =
+    mojom_parser_sources + [
       "$mojom_generator_root/generators/mojom_cpp_generator.py",
       "$mojom_generator_root/generators/mojom_java_generator.py",
       "$mojom_generator_root/generators/mojom_js_generator.py",
       "$mojom_generator_root/generators/mojom_ts_generator.py",
       "$mojom_generator_script",
-]
+    ]
 
 if (enable_scrambled_message_ids) {
   declare_args() {
@@ -600,8 +605,22 @@ template("mojom") {
       (!defined(invoker.enable_fuzzing) || invoker.enable_fuzzing) &&
       enable_ipc_fuzzer && (!defined(invoker.testonly) || !invoker.testonly)
 
-  if (sources_list != []) {
   parser_target_name = "${target_name}__parser"
+  parser_deps = []
+  foreach(dep, all_deps) {
+    _label = get_label_info(dep, "label_no_toolchain")
+    parser_deps += [ "${_label}__parser" ]
+  }
+  if (defined(invoker.parser_deps)) {
+    parser_deps += invoker.parser_deps
+  }
+  if (sources_list == []) {
+    # Even without sources we generate a parser target to at least forward
+    # other parser dependencies.
+    group(parser_target_name) {
+      public_deps = parser_deps
+    }
+  } else {
     enabled_features = []
     if (defined(invoker.enabled_features)) {
       enabled_features += invoker.enabled_features
@@ -626,57 +645,46 @@ template("mojom") {
     }
 
     action(parser_target_name) {
-      script = mojom_generator_script
-      inputs = mojom_generator_sources + jinja2_sources
+      script = mojom_parser_script
+      inputs = mojom_parser_sources
       sources = sources_list
-      if (defined(invoker.parser_deps)) {
-        deps = invoker.parser_deps
-      }
+      deps = parser_deps
       outputs = []
-      filelist = []
-      foreach(source, sources_list) {
-        filelist += [ rebase_path(source, root_build_dir) ]
-      }
       foreach(base_path, output_file_base_paths) {
-        filename = get_path_info(base_path, "name")
+        filename = get_path_info(base_path, "file")
         dirname = get_path_info(base_path, "dir")
-        outputs += [ "$root_gen_dir/$dirname/$filename.p" ]
+        outputs += [ "$root_gen_dir/$dirname/${filename}-module" ]
       }
 
+      filelist = []
+      foreach(source, sources_list) {
+        filelist += [ rebase_path(source) ]
+      }
       response_file_contents = filelist
 
       args = [
-        "-o",
-        rebase_path(root_gen_dir, root_build_dir),
-        "parse",
-        "--filelist={{response_file_name}}",
-        "-d",
-        rebase_path("//", root_build_dir),
+        # Resolve relative input mojom paths against both the root src dir and
+        # the root gen dir.
+        "--input-root",
+        rebase_path("//"),
+        "--input-root",
+        rebase_path(root_gen_dir),
+
+        "--output-root",
+        rebase_path(root_gen_dir),
+
+        "--mojom-file-list={{response_file_name}}",
       ]
+
       foreach(enabled_feature, enabled_features) {
         args += [
-          "--enable_feature",
+          "--enable-feature",
           enabled_feature,
         ]
       }
     }
   }
 
-  parsed_target_name = "${target_name}__parsed"
-  group(parsed_target_name) {
-    public_deps = []
-    if (sources_list != []) {
-      public_deps += [ ":$parser_target_name" ]
-    }
-    foreach(d, all_deps) {
-      # Resolve the name, so that a target //mojo/something becomes
-      # //mojo/something:something and we can append the parsed
-      # suffix to get the mojom dependency name.
-      full_name = get_label_info("$d", "label_no_toolchain")
-      public_deps += [ "${full_name}__parsed" ]
-    }
-  }
-
   if (sources_list != []) {
     verify_deps_target_names = []
     verify_deps_target_name = "${target_name}__verify_deps"
@@ -687,7 +695,7 @@ template("mojom") {
       script = mojom_generator_script
       inputs = mojom_generator_sources + jinja2_sources
       sources = sources_list
-      deps = [ ":$parsed_target_name" ]
+      deps = [ ":$parser_target_name" ]
       if (defined(invoker.parser_deps)) {
         deps += invoker.parser_deps
       }
@@ -789,7 +797,7 @@ template("mojom") {
       inputs = mojom_generator_sources + jinja2_sources
       sources = sources_list
       deps = [
-        ":$parsed_target_name",
+        ":$parser_target_name",
         "//mojo/public/tools/bindings:precompile_templates",
       ]
       if (defined(invoker.parser_deps)) {
@@ -829,7 +837,7 @@ template("mojom") {
       inputs = mojom_generator_sources + jinja2_sources
       sources = sources_list
       deps = [
-               ":$parsed_target_name",
+               ":$parser_target_name",
                "//mojo/public/tools/bindings:precompile_templates",
              ] + verify_deps_target_names
       if (defined(invoker.parser_deps)) {
@@ -1089,7 +1097,7 @@ template("mojom") {
         inputs = mojom_generator_sources + jinja2_sources
         sources = sources_list
         deps = [
-                 ":$parsed_target_name",
+                 ":$parser_target_name",
                  ":$type_mappings_target_name",
                  "//mojo/public/tools/bindings:precompile_templates",
                ] + verify_deps_target_names
@@ -1430,7 +1438,7 @@ template("mojom") {
           inputs = mojom_generator_sources + jinja2_sources
           sources = sources_list
           deps = [
-                   ":$parsed_target_name",
+                   ":$parser_target_name",
                    ":$type_mappings_target_name",
                    "//mojo/public/tools/bindings:precompile_templates",
                  ] + verify_deps_target_names
@@ -1527,7 +1535,7 @@ template("mojom") {
         inputs = mojom_generator_sources + jinja2_sources
         sources = sources_list
         deps = [
-                 ":$parsed_target_name",
+                 ":$parser_target_name",
                  "//mojo/public/tools/bindings:precompile_templates",
                ] + verify_deps_target_names
         if (defined(invoker.parser_deps)) {
@@ -1691,7 +1699,7 @@ template("mojom") {
       inputs = mojom_generator_sources + jinja2_sources
       sources = sources_list
       deps = [
-               ":$parsed_target_name",
+               ":$parser_target_name",
                "//mojo/public/tools/bindings:precompile_templates",
              ] + verify_deps_target_names
       outputs = ts_outputs

mojo/public/tools/bindings/mojom_bindings_generator.py

@@ -9,11 +9,6 @@ from __future__ import print_function
 
 import argparse
 
-try:
-  import cPickle as pickle
-except ImportError:
-  import pickle
-
 import hashlib
 import importlib
 import json
@@ -44,11 +39,10 @@ sys.path.insert(
 
 from mojom.error import Error
 import mojom.fileutil as fileutil
+from mojom.generate.module import Module
 from mojom.generate import template_expander
 from mojom.generate import translate
 from mojom.generate.generator import AddComputedData, WriteFile
-from mojom.parse.conditional_features import RemoveDisabledDefinitions
-from mojom.parse.parser import Parse
 
 sys.path.append(
     os.path.join(_GetDirAbove("mojo"), "tools", "diagnosis"))
@@ -104,16 +98,8 @@ class RelativePath(object):
         os.path.abspath(self.path), os.path.abspath(self.root))
 
 
-def FindImportFile(args, rel_dir, file_name, search_rel_dirs):
-  """Finds |file_name| in either |rel_dir| or |search_rel_dirs|. Returns a
-  RelativePath with first file found, or an arbitrary non-existent file
-  otherwise."""
-  for rel_search_dir in [rel_dir] + search_rel_dirs:
-    path = os.path.join(rel_search_dir.path, file_name)
-    if os.path.isfile(path):
-      return RelativePath(path, rel_search_dir.root, args.output_dir)
-  return RelativePath(
-      os.path.join(rel_dir.path, file_name), rel_dir.root, args.output_dir)
+def _GetModulePath(path, output_dir):
+  return os.path.join(output_dir, path.relative_path() + '-module')
 
 
 def ScrambleMethodOrdinals(interfaces, salt):
@@ -154,7 +140,7 @@ def ReadFileContents(filename):
 
 
 class MojomProcessor(object):
-  """Parses mojom files and creates ASTs for them.
+  """Takes parsed mojom modules and generates language bindings from them.
 
   Attributes:
     _processed_files: {Dict[str, mojom.generate.module.Module]} Mapping from
@@ -189,26 +175,9 @@ class MojomProcessor(object):
           MakeImportStackMessage(imported_filename_stack + [rel_filename.path]))
       sys.exit(1)
 
-    tree = _UnpickleAST(_FindPicklePath(rel_filename, args.gen_directories +
-                                        [args.output_dir]))
-    dirname = os.path.dirname(rel_filename.path)
-
-    # Process all our imports first and collect the module object for each.
-    # We use these to generate proper type info.
-    imports = {}
-    for parsed_imp in tree.import_list:
-      rel_import_file = FindImportFile(
-          args, RelativePath(dirname, rel_filename.root, args.output_dir),
-          parsed_imp.import_filename, args.import_directories)
-      imports[parsed_imp.import_filename] = self._GenerateModule(
-          args, remaining_args, generator_modules, rel_import_file,
-          imported_filename_stack + [rel_filename.path])
-
-    # Set the module path as relative to the source root.
-    # Normalize to unix-style path here to keep the generators simpler.
-    module_path = rel_filename.relative_path().replace('\\', '/')
-
-    module = translate.OrderedModule(tree, module_path, imports)
+    module_path = _GetModulePath(rel_filename, args.output_dir)
+    with open(module_path, 'rb') as f:
+      module = Module.Load(f)
 
     if args.scrambled_message_id_salt_paths:
       salt = b''.join(
@@ -278,71 +247,6 @@ def _Generate(args, remaining_args):
   return 0
 
 
-def _FindPicklePath(rel_filename, search_dirs):
-  filename, _ = os.path.splitext(rel_filename.relative_path())
-  pickle_path = filename + '.p'
-  for search_dir in search_dirs:
-    path = os.path.join(search_dir, pickle_path)
-    if os.path.isfile(path):
-      return path
-  raise Exception("%s: Error: Could not find file in %r" %
-                  (pickle_path, search_dirs))
-
-
-def _GetPicklePath(rel_filename, output_dir):
-  filename, _ = os.path.splitext(rel_filename.relative_path())
-  pickle_path = filename + '.p'
-  return os.path.join(output_dir, pickle_path)
-
-
-def _PickleAST(ast, output_file):
-  full_dir = os.path.dirname(output_file)
-  fileutil.EnsureDirectoryExists(full_dir)
-
-  try:
-    WriteFile(pickle.dumps(ast), output_file)
-  except (IOError, pickle.PicklingError) as e:
-    print("%s: Error: %s" % (output_file, str(e)))
-    sys.exit(1)
-
-def _UnpickleAST(input_file):
-  try:
-    with open(input_file, "rb") as f:
-      return pickle.load(f)
-  except (IOError, pickle.UnpicklingError) as e:
-    print("%s: Error: %s" % (input_file, str(e)))
-    sys.exit(1)
-
-
-def _ParseFile(args, rel_filename):
-  try:
-    with open(rel_filename.path) as f:
-      source = f.read()
-  except IOError as e:
-    print("%s: Error: %s" % (rel_filename.path, e.strerror))
-    sys.exit(1)
-
-  try:
-    tree = Parse(source, rel_filename.path)
-    RemoveDisabledDefinitions(tree, args.enabled_features)
-  except Error as e:
-    print("%s: Error: %s" % (rel_filename.path, str(e)))
-    sys.exit(1)
-  _PickleAST(tree, _GetPicklePath(rel_filename, args.output_dir))
-
-
-def _Parse(args, _):
-  fileutil.EnsureDirectoryExists(args.output_dir)
-
-  if args.filelist:
-    with open(args.filelist) as f:
-      args.filename.extend(f.read().split())
-
-  for filename in args.filename:
-    _ParseFile(args, RelativePath(filename, args.depth, args.output_dir))
-  return 0
-
-
 def _Precompile(args, _):
   generator_modules = LoadGenerators(",".join(_BUILTIN_GENERATORS.keys()))
 
@@ -376,11 +280,10 @@ def _VerifyImportDeps(args, __):
 
   for filename in args.filename:
     rel_path = RelativePath(filename, args.depth, args.output_dir)
-    tree = _UnpickleAST(_GetPicklePath(rel_path, args.output_dir))
-
-    mojom_imports = set(
-      parsed_imp.import_filename for parsed_imp in tree.import_list
-      )
+    module_path = _GetModulePath(rel_path, args.output_dir)
+    with open(module_path, 'rb') as f:
+      module = Module.Load(f)
+      mojom_imports = set(imp.path for imp in module.imports)
 
     sources = set()
 
@@ -421,23 +324,6 @@ def main():
 
   subparsers = parser.add_subparsers()
 
-  parse_parser = subparsers.add_parser(
-      "parse", description="Parse mojom to AST and remove disabled definitions."
-                           " Pickle pruned AST into output_dir.")
-  parse_parser.add_argument("filename", nargs="*", help="mojom input file")
-  parse_parser.add_argument("--filelist", help="mojom input file list")
-  parse_parser.add_argument(
-      "-d", "--depth", dest="depth", default=".", help="depth from source root")
-  parse_parser.add_argument(
-      "--enable_feature",
-      dest = "enabled_features",
-      default=[],
-      action="append",
-      help="Controls which definitions guarded by an EnabledIf attribute "
-      "will be enabled. If an EnabledIf attribute does not specify a value "
-      "that matches one of the enabled features, it will be disabled.")
-  parse_parser.set_defaults(func=_Parse)
-
   generate_parser = subparsers.add_parser(
       "generate", description="Generate bindings from mojom files.")
   generate_parser.add_argument("filename", nargs="*",

mojo/public/tools/mojom/mojom/generate/module.py

@@ -12,6 +12,7 @@
 # method = interface.AddMethod('Tat', 0)
 # method.AddParameter('baz', 0, mojom.INT32)
 
+import pickle
 
 # We use our own version of __repr__ when displaying the AST, as the
 # AST currently doesn't capture which nodes are reference (e.g. to
@@ -100,6 +101,19 @@ class Kind(object):
     # Gives us a decent __repr__ for all kinds.
     return self.Repr()
 
+  def __eq__(self, rhs):
+    # pylint: disable=unidiomatic-typecheck
+    return (type(self) == type(rhs)
+            and (self.spec, self.parent_kind) == (rhs.spec, rhs.parent_kind))
+
+  def __hash__(self):
+    # TODO(crbug.com/1060471): Remove this and other __hash__ methods on Kind
+    # and its subclasses. This is to support existing generator code which uses
+    # some primitive Kinds as dict keys. The default hash (object identity)
+    # breaks these dicts when a pickled Module instance is unpickled and used
+    # during a subsequent run of the parser.
+    return hash((self.spec, self.parent_kind))
+
 
 class ReferenceKind(Kind):
   """ReferenceKind represents pointer and handle types.
@@ -173,6 +187,14 @@ class ReferenceKind(Kind):
 
     setattr(cls, name, property(Get, Set))
 
+  def __eq__(self, rhs):
+    return (isinstance(rhs, ReferenceKind)
+            and super(ReferenceKind, self).__eq__(rhs)
+            and self.is_nullable == rhs.is_nullable)
+
+  def __hash__(self):
+    return hash((super(ReferenceKind, self).__hash__(), self.is_nullable))
+
 
 # Initialize the set of primitive types. These can be accessed by clients.
 BOOL = Kind('b')
@@ -246,11 +268,19 @@ class NamedValue(object):
             (self.parent_kind and
              (self.parent_kind.mojom_name + '.') or "") + self.mojom_name)
 
+  def __eq__(self, rhs):
+    return (isinstance(rhs, NamedValue)
+            and (self.parent_kind, self.mojom_name) == (rhs.parent_kind,
+                                                        rhs.mojom_name))
+
 
 class BuiltinValue(object):
   def __init__(self, value):
     self.value = value
 
+  def __eq__(self, rhs):
+    return isinstance(rhs, BuiltinValue) and self.value == rhs.value
+
 
 class ConstantValue(NamedValue):
   def __init__(self, module, parent_kind, constant):
@@ -289,6 +319,12 @@ class Constant(object):
   def Stylize(self, stylizer):
     self.name = stylizer.StylizeConstant(self.mojom_name)
 
+  def __eq__(self, rhs):
+    return (isinstance(rhs, Constant)
+            and (self.mojom_name, self.kind, self.value,
+                 self.parent_kind) == (rhs.mojom_name, rhs.kind, rhs.value,
+                                       rhs.parent_kind))
+
 
 class Field(object):
   def __init__(self,
@@ -320,6 +356,12 @@ class Field(object):
     return self.attributes.get(ATTRIBUTE_MIN_VERSION) \
         if self.attributes else None
 
+  def __eq__(self, rhs):
+    return (isinstance(rhs, Field)
+            and (self.mojom_name, self.kind, self.ordinal, self.default,
+                 self.attributes) == (rhs.mojom_name, rhs.kind, rhs.ordinal,
+                                      rhs.default, rhs.attributes))
+
 
 class StructField(Field):
   pass
@@ -401,6 +443,15 @@ class Struct(ReferenceKind):
     for constant in self.constants:
       constant.Stylize(stylizer)
 
+  def __eq__(self, rhs):
+    return (isinstance(rhs, Struct) and
+            (self.mojom_name, self.native_only, self.fields, self.constants,
+             self.attributes) == (rhs.mojom_name, rhs.native_only, rhs.fields,
+                                  rhs.constants, rhs.attributes))
+
+  def __hash__(self):
+    return id(self)
+
 
 class Union(ReferenceKind):
   """A union of several kinds.
@@ -447,6 +498,14 @@ class Union(ReferenceKind):
     for field in self.fields:
       field.Stylize(stylizer)
 
+  def __eq__(self, rhs):
+    return (isinstance(rhs, Union) and
+            (self.mojom_name, self.fields,
+             self.attributes) == (rhs.mojom_name, rhs.fields, rhs.attributes))
+
+  def __hash__(self):
+    return id(self)
+
 
 class Array(ReferenceKind):
   """An array.
@@ -484,6 +543,13 @@ class Array(ReferenceKind):
           'is_nullable': False
       })
 
+  def __eq__(self, rhs):
+    return (isinstance(rhs, Array)
+            and (self.kind, self.length) == (rhs.kind, rhs.length))
+
+  def __hash__(self):
+    return id(self)
+
 
 class Map(ReferenceKind):
   """A map.
@@ -521,6 +587,13 @@ class Map(ReferenceKind):
     else:
       return GenericRepr(self, {'key_kind': True, 'value_kind': True})
 
+  def __eq__(self, rhs):
+    return (isinstance(rhs, Map) and
+            (self.key_kind, self.value_kind) == (rhs.key_kind, rhs.value_kind))
+
+  def __hash__(self):
+    return id(self)
+
 
 class PendingRemote(ReferenceKind):
   ReferenceKind.AddSharedProperty('kind')
@@ -536,6 +609,12 @@ class PendingRemote(ReferenceKind):
       ReferenceKind.__init__(self)
     self.kind = kind
 
+  def __eq__(self, rhs):
+    return isinstance(rhs, PendingRemote) and self.kind == rhs.kind
+
+  def __hash__(self):
+    return id(self)
+
 
 class PendingReceiver(ReferenceKind):
   ReferenceKind.AddSharedProperty('kind')
@@ -551,6 +630,12 @@ class PendingReceiver(ReferenceKind):
       ReferenceKind.__init__(self)
     self.kind = kind
 
+  def __eq__(self, rhs):
+    return isinstance(rhs, PendingReceiver) and self.kind == rhs.kind
+
+  def __hash__(self):
+    return id(self)
+
 
 class PendingAssociatedRemote(ReferenceKind):
   ReferenceKind.AddSharedProperty('kind')
@@ -566,6 +651,12 @@ class PendingAssociatedRemote(ReferenceKind):
       ReferenceKind.__init__(self)
     self.kind = kind
 
+  def __eq__(self, rhs):
+    return isinstance(rhs, PendingAssociatedRemote) and self.kind == rhs.kind
+
+  def __hash__(self):
+    return id(self)
+
 
 class PendingAssociatedReceiver(ReferenceKind):
   ReferenceKind.AddSharedProperty('kind')
@@ -581,6 +672,12 @@ class PendingAssociatedReceiver(ReferenceKind):
       ReferenceKind.__init__(self)
     self.kind = kind
 
+  def __eq__(self, rhs):
+    return isinstance(rhs, PendingAssociatedReceiver) and self.kind == rhs.kind
+
+  def __hash__(self):
+    return id(self)
+
 
 class InterfaceRequest(ReferenceKind):
   ReferenceKind.AddSharedProperty('kind')
@@ -595,6 +692,12 @@ class InterfaceRequest(ReferenceKind):
       ReferenceKind.__init__(self)
     self.kind = kind
 
+  def __eq__(self, rhs):
+    return isinstance(rhs, InterfaceRequest) and self.kind == rhs.kind
+
+  def __hash__(self):
+    return id(self)
+
 
 class AssociatedInterfaceRequest(ReferenceKind):
   ReferenceKind.AddSharedProperty('kind')
@@ -611,6 +714,12 @@ class AssociatedInterfaceRequest(ReferenceKind):
       ReferenceKind.__init__(self)
     self.kind = kind.kind if kind is not None else None
 
+  def __eq__(self, rhs):
+    return isinstance(rhs, AssociatedInterfaceRequest) and self.kind == rhs.kind
+
+  def __hash__(self):
+    return id(self)
+
 
 class Parameter(object):
   def __init__(self,
@@ -639,6 +748,12 @@ class Parameter(object):
     return self.attributes.get(ATTRIBUTE_MIN_VERSION) \
         if self.attributes else None
 
+  def __eq__(self, rhs):
+    return (isinstance(rhs, Parameter)
+            and (self.mojom_name, self.ordinal, self.kind, self.default,
+                 self.attributes) == (rhs.mojom_name, rhs.ordinal, rhs.kind,
+                                      rhs.default, rhs.attributes))
+
 
 class Method(object):
   def __init__(self, interface, mojom_name, ordinal=None, attributes=None):
@@ -707,6 +822,13 @@ class Method(object):
     return self.attributes.get(ATTRIBUTE_SYNC) \
         if self.attributes else None
 
+  def __eq__(self, rhs):
+    return (isinstance(rhs, Method) and
+            (self.mojom_name, self.ordinal, self.parameters,
+             self.response_parameters,
+             self.attributes) == (rhs.mojom_name, rhs.ordinal, rhs.parameters,
+                                  rhs.response_parameters, rhs.attributes))
+
 
 class Interface(ReferenceKind):
   ReferenceKind.AddSharedProperty('mojom_name')
@@ -753,6 +875,15 @@ class Interface(ReferenceKind):
     for constant in self.constants:
       constant.Stylize(stylizer)
 
+  def __eq__(self, rhs):
+    return (isinstance(rhs, Interface)
+            and (self.mojom_name, self.methods, self.enums, self.constants,
+                 self.attributes) == (rhs.mojom_name, rhs.methods, rhs.enums,
+                                      rhs.constants, rhs.attributes))
+
+  def __hash__(self):
+    return id(self)
+
 
 class AssociatedInterface(ReferenceKind):
   ReferenceKind.AddSharedProperty('kind')
@@ -768,6 +899,12 @@ class AssociatedInterface(ReferenceKind):
       ReferenceKind.__init__(self)
     self.kind = kind
 
+  def __eq__(self, rhs):
+    return isinstance(rhs, AssociatedInterface) and self.kind == rhs.kind
+
+  def __hash__(self):
+    return id(self)
+
 
 class EnumField(object):
   def __init__(self,
@@ -789,6 +926,12 @@ class EnumField(object):
     return self.attributes.get(ATTRIBUTE_MIN_VERSION) \
         if self.attributes else None
 
+  def __eq__(self, rhs):
+    return (isinstance(rhs, EnumField)
+            and (self.mojom_name, self.value, self.attributes,
+                 self.numeric_value) == (rhs.mojom_name, rhs.value,
+                                         rhs.attributes, rhs.numeric_value))
+
 
 class Enum(Kind):
   def __init__(self, mojom_name=None, module=None, attributes=None):
@@ -821,6 +964,16 @@ class Enum(Kind):
     return self.attributes.get(ATTRIBUTE_EXTENSIBLE, False) \
         if self.attributes else False
 
+  def __eq__(self, rhs):
+    return (isinstance(rhs, Enum) and
+            (self.mojom_name, self.native_only, self.fields, self.attributes,
+             self.min_value,
+             self.max_value) == (rhs.mojom_name, rhs.native_only, rhs.fields,
+                                 rhs.attributes, rhs.min_value, rhs.max_value))
+
+  def __hash__(self):
+    return id(self)
+
 
 class Module(object):
   def __init__(self, path=None, mojom_namespace=None, attributes=None):
@@ -841,6 +994,14 @@ class Module(object):
     # Gives us a decent __repr__ for modules.
     return self.Repr()
 
+  def __eq__(self, rhs):
+    return (isinstance(rhs, Module) and
+            (self.path, self.attributes, self.mojom_namespace, self.imports,
+             self.constants, self.enums, self.structs, self.unions,
+             self.interfaces) == (rhs.path, rhs.attributes, rhs.mojom_namespace,
+                                  rhs.imports, rhs.constants, rhs.enums,
+                                  rhs.structs, rhs.unions, rhs.interfaces))
+
   def Repr(self, as_ref=True):
     if as_ref:
       return '<%s path=%r mojom_namespace=%r>' % (
@@ -887,6 +1048,15 @@ class Module(object):
     for imported_module in self.imports:
       imported_module.Stylize(stylizer)
 
+  def Dump(self, f):
+    pickle.dump(self, f, 2)
+
+  @classmethod
+  def Load(cls, f):
+    result = pickle.load(f)
+    assert isinstance(result, Module)
+    return result
+
 
 def IsBoolKind(kind):
   return kind.spec == BOOL.spec

mojo/public/tools/mojom/mojom/generate/translate.py

@@ -292,12 +292,15 @@ def _Struct(module, parsed_struct):
             lambda enum: _Enum(module, enum, struct),
             _ElemsOfType(parsed_struct.body, ast.Enum,
                          parsed_struct.mojom_name)))
-    struct.constants = map(
+    struct.constants = list(
+        map(
             lambda constant: _Constant(module, constant, struct),
-        _ElemsOfType(parsed_struct.body, ast.Const, parsed_struct.mojom_name))
+            _ElemsOfType(parsed_struct.body, ast.Const,
+                         parsed_struct.mojom_name)))
     # Stash fields parsed_struct here temporarily.
     struct.fields_data = _ElemsOfType(parsed_struct.body, ast.StructField,
                                       parsed_struct.mojom_name)
+
   struct.attributes = _AttributeListToDict(parsed_struct.attribute_list)
 
   # Enforce that a [Native] attribute is set to make native-only struct
@@ -446,9 +449,9 @@ def _Interface(module, parsed_iface):
   interface.enums = list(
       map(lambda enum: _Enum(module, enum, interface),
           _ElemsOfType(parsed_iface.body, ast.Enum, parsed_iface.mojom_name)))
-  interface.constants = map(
-      lambda constant: _Constant(module, constant, interface),
-      _ElemsOfType(parsed_iface.body, ast.Const, parsed_iface.mojom_name))
+  interface.constants = list(
+      map(lambda constant: _Constant(module, constant, interface),
+          _ElemsOfType(parsed_iface.body, ast.Const, parsed_iface.mojom_name)))
   # Stash methods parsed_iface here temporarily.
   interface.methods_data = _ElemsOfType(parsed_iface.body, ast.Method,
                                         parsed_iface.mojom_name)

mojo/public/tools/mojom/mojom_parser.py

+#!/usr/bin/env python
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Parses mojom IDL files.
+
+This script parses one or more input mojom files and produces corresponding
+module files fully describing the definitions contained within each mojom. The
+module data is pickled and can be easily consumed by other tools to, e.g.,
+generate usable language bindings.
+"""
+
+import argparse
+import errno
+import os
+import os.path
+import sys
+from collections import defaultdict
+
+from mojom.generate import module
+from mojom.generate import translate
+from mojom.parse import parser
+from mojom.parse import conditional_features
+
+
+def _ResolveRelativeImportPath(path, roots):
+  """Attempts to resolve a relative import path against a set of possible roots.
+
+  Args:
+    path: The relative import path to resolve.
+    roots: A list of absolute paths which will be checked in descending length
+        order for a match against path.
+
+  Returns:
+    A normalized absolute path combining one of the roots with the input path if
+    and only if such a file exists.
+
+  Raises:
+    ValueError: The path could not be resolved against any of the given roots.
+  """
+  for root in reversed(sorted(roots, key=len)):
+    abs_path = os.path.join(root, path)
+    if os.path.isfile(abs_path):
+      return os.path.normpath(abs_path)
+
+  raise ValueError('"%s" does not exist in any of %s' % (path, roots))
+
+
+def _RebaseAbsolutePath(path, roots):
+  """Rewrites an absolute file path as relative to an absolute directory path in
+  roots.
+
+  Args:
+    path: The absolute path of an existing file.
+    roots: A list of absolute directory paths. The given path argument must fall
+        within one of these directories.
+
+  Returns:
+    A path equivalent to the input path, but relative to one of the provided
+    roots. If the input path falls within multiple roots, the longest root is
+    chosen (and thus the shortest relative path is returned).
+
+    Paths returned by this method always use forward slashes as a separator to
+    mirror mojom import syntax.
+
+  Raises:
+    ValueError if the given path does not fall within any of the listed roots.
+  """
+  assert os.path.isabs(path)
+  assert os.path.isfile(path)
+  assert all(map(os.path.isabs, roots))
+
+  sorted_roots = list(reversed(sorted(roots, key=len)))
+
+  def try_rebase_path(path, root):
+    head, rebased_path = os.path.split(path)
+    while head != root:
+      head, tail = os.path.split(head)
+      if not tail:
+        return None
+      rebased_path = os.path.join(tail, rebased_path)
+    return rebased_path
+
+  for root in sorted_roots:
+    relative_path = try_rebase_path(path, root)
+    if relative_path:
+      # TODO(crbug.com/953884): Use pathlib for this kind of thing once we're
+      # fully migrated to Python 3.
+      return relative_path.replace('\\', '/')
+
+  raise ValueError('%s does not fall within any of %s' % (path, sorted_roots))
+
+
+def _GetModuleFilename(mojom_filename):
+  return mojom_filename + '-module'
+
+
+def _EnsureInputLoaded(mojom_abspath, module_path, abs_paths, asts,
+                       dependencies, loaded_modules):
+  """Recursively ensures that a module and its dependencies are loaded.
+
+  Args:
+    mojom_abspath: An absolute file path pointing to a mojom file to load.
+    module_path: The relative path used to identify mojom_abspath.
+    abs_paths: A mapping from module paths to absolute file paths for all
+        inputs given to this execution of the script.
+    asts: A map from each input mojom's absolute path to its parsed AST.
+    dependencies: A mapping of which input mojoms depend on each other, indexed
+        by absolute file path.
+    loaded_modules: A mapping of all modules loaded so far, including non-input
+        modules that were pulled in as transitive dependencies of the inputs.
+    import_set: The working set of mojom imports processed so far in this
+        call stack. Used to detect circular dependencies.
+    import_stack: An ordered list of imports processed so far in this call
+        stack. Used to report circular dependencies.
+
+  Returns:
+    None
+
+    On return, loaded_modules will be populated with the loaded input mojom's
+    Module as well as the Modules of all of its transitive dependencies."""
+
+  if mojom_abspath in loaded_modules:
+    # Already done.
+    return
+
+  for dep_abspath, dep_path in dependencies[mojom_abspath]:
+    if dep_abspath not in loaded_modules:
+      _EnsureInputLoaded(dep_abspath, dep_path, abs_paths, asts, dependencies,
+                         loaded_modules)
+
+  imports = {}
+  for imp in asts[mojom_abspath].import_list:
+    path = imp.import_filename
+    imports[path] = loaded_modules[abs_paths[path]]
+  loaded_modules[mojom_abspath] = translate.OrderedModule(
+      asts[mojom_abspath], module_path, imports)
+
+
+def ParseMojoms(mojom_files, input_root_paths, output_root_path,
+                enabled_features):
+  """Parses a set of mojom files and produces serialized module outputs.
+
+  Args:
+    mojom_files: A list of mojom files to process. Paths must be absolute paths
+        which fall within one of the input or output root paths.
+    input_root_paths: A list of absolute filesystem paths which may be used to
+        resolve relative mojom file paths.
+    output_root_path: An absolute filesystem path which will service as the root
+        for all emitted artifacts. Artifacts produced from a given mojom file
+        are based on the mojom's relative path, rebased onto this path.
+        Additionally, the script expects this root to contain already-generated
+        modules for any transitive dependencies not listed in mojom_files.
+    enabled_features: A list of enabled feature names, controlling which AST
+        nodes are filtered by [EnableIf] attributes.
+
+  Returns:
+    None.
+
+    Upon completion, a mojom-module file will be saved for each input mojom.
+  """
+  loaded_mojom_asts = {}
+  loaded_modules = {}
+  input_dependencies = defaultdict(set)
+  mojom_files_to_parse = dict((abs_path,
+                               _RebaseAbsolutePath(abs_path, input_root_paths))
+                              for abs_path in mojom_files)
+  abs_paths = dict(
+      (path, abs_path) for abs_path, path in mojom_files_to_parse.items())
+  for mojom_abspath, _ in mojom_files_to_parse.items():
+    with open(mojom_abspath) as f:
+      ast = parser.Parse(''.join(f.readlines()), mojom_abspath)
+      conditional_features.RemoveDisabledDefinitions(ast, enabled_features)
+      loaded_mojom_asts[mojom_abspath] = ast
+      for imp in ast.import_list:
+        import_abspath = _ResolveRelativeImportPath(imp.import_filename,
+                                                    input_root_paths)
+        abs_paths[imp.import_filename] = import_abspath
+        if import_abspath in mojom_files_to_parse:
+          # This import is in the input list, so we're going to translate it
+          # into a module below; however it's also a dependency of another input
+          # module. We retain record of dependencies to help with input
+          # processing later.
+          input_dependencies[mojom_abspath].add((import_abspath,
+                                                 imp.import_filename))
+        else:
+          # We have an import that isn't being parsed right now. It must already
+          # be parsed and have a module file sitting in a corresponding output
+          # location.
+          module_path = _GetModuleFilename(imp.import_filename)
+          module_abspath = _ResolveRelativeImportPath(module_path,
+                                                      [output_root_path])
+          with open(module_abspath, 'rb') as module_file:
+            loaded_modules[import_abspath] = module.Module.Load(module_file)
+
+  # At this point all transitive imports not listed as inputs have been loaded
+  # and we have a complete dependency tree of the unprocessed inputs. Now we can
+  # load all the inputs, resolving dependencies among them recursively as we go.
+  num_existing_modules_loaded = len(loaded_modules)
+  for mojom_abspath, mojom_path in mojom_files_to_parse.items():
+    _EnsureInputLoaded(mojom_abspath, mojom_path, abs_paths, loaded_mojom_asts,
+                       input_dependencies, loaded_modules)
+  assert (num_existing_modules_loaded +
+          len(mojom_files_to_parse) == len(loaded_modules))
+
+  # Now we have fully translated modules for every input and every transitive
+  # dependency. We can dump the modules to disk for other tools to use.
+  for mojom_abspath, mojom_path in mojom_files_to_parse.items():
+    module_path = os.path.join(output_root_path, _GetModuleFilename(mojom_path))
+    module_dir = os.path.dirname(module_path)
+    if not os.path.exists(module_dir):
+      try:
+        # Python 2 doesn't support exist_ok on makedirs(), so we just ignore
+        # that failure if it happens. It's possible during build due to races
+        # among build steps with module outputs in the same directory.
+        os.makedirs(module_dir)
+      except OSError as e:
+        if e.errno != errno.EEXIST:
+          raise
+    with open(module_path, 'wb') as f:
+      loaded_modules[mojom_abspath].Dump(f)
+
+
+def main():
+  arg_parser = argparse.ArgumentParser(
+      description="""
+Parses one or more mojom files and produces corresponding module outputs fully
+describing the definitions therein. The output is exhaustive, stable, and
+sufficient for another tool to consume and emit e.g. usable language
+bindings based on the original mojoms.""",
+      epilog="""
+Note that each transitive import dependency reachable from the input mojoms must
+either also be listed as an input or must have its corresponding compiled module
+already present in the provided output root.""")
+
+  arg_parser.add_argument(
+      '--input-root',
+      default=[],
+      action='append',
+      metavar='ROOT',
+      dest='input_root_paths',
+      help='Adds ROOT to the set of root paths against which relative input '
+      'paths should be resolved. Provided root paths are always searched '
+      'in order from longest absolute path to shortest.')
+  arg_parser.add_argument(
+      '--output-root',
+      action='store',
+      required=True,
+      dest='output_root_path',
+      metavar='ROOT',
+      help='Use ROOT as the root path in which the parser should emit compiled '
+      'modules for each processed input mojom. The path of emitted module is '
+      'based on the relative input path, rebased onto this root. Note that '
+      'ROOT is also searched for existing modules of any transitive imports '
+      'which were not included in the set of inputs.')
+  arg_parser.add_argument(
+      '--mojoms',
+      nargs='+',
+      dest='mojom_files',
+      default=[],
+      metavar='MOJOM_FILE',
+      help='Input mojom filename(s). Each filename must be either an absolute '
+      'path which falls within one of the given input or output roots, or a '
+      'relative path the parser will attempt to resolve using each of those '
+      'roots in unspecified order.')
+  arg_parser.add_argument(
+      '--mojom-file-list',
+      action='store',
+      metavar='LIST_FILENAME',
+      help='Input file whose contents are a list of mojoms to process. This '
+      'may be provided in lieu of --mojoms to avoid hitting command line '
+      'length limtations')
+  arg_parser.add_argument(
+      '--enable-feature',
+      dest='enabled_features',
+      default=[],
+      action='append',
+      metavar='FEATURE',
+      help='Enables a named feature when parsing the given mojoms. Features '
+      'are identified by arbitrary string values. Specifying this flag with a '
+      'given FEATURE name will cause the parser to process any syntax elements '
+      'tagged with an [EnableIf=FEATURE] attribute. If this flag is not '
+      'provided for a given FEATURE, such tagged elements are discarded by the '
+      'parser and will not be present in the compiled output.')
+
+  args, _ = arg_parser.parse_known_args()
+  if args.mojom_file_list:
+    with open(args.mojom_file_list) as f:
+      args.mojom_files.extend(f.read().split())
+
+  if not args.mojom_files:
+    raise ValueError(
+        'Must list at least one mojom file via --mojoms or --mojom-file-list')
+
+  ParseMojoms(
+      list(map(os.path.abspath, args.mojom_files)),
+      list(map(os.path.abspath, args.input_root_paths)),
+      os.path.abspath(args.output_root_path), args.enabled_features)
+
+
+if __name__ == '__main__':
+  main()

mojo/public/tools/mojom/mojom_parser_unittest.py

+#!/usr/bin/env python
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import os.path
+import shutil
+import tempfile
+import unittest
+
+import mojom_parser
+
+from mojom.generate import module
+
+
+class MojomParserTest(unittest.TestCase):
+  """Tests covering the behavior defined by the main mojom_parser.py script.
+  This includes behavior around input and output path manipulation, dependency
+  resolution, and module serialization and deserialization."""
+
+  def __init__(self, method_name):
+    super(MojomParserTest, self).__init__(method_name)
+    self._temp_dir = None
+
+  def setUp(self):
+    self._temp_dir = tempfile.mkdtemp()
+
+  def tearDown(self):
+    shutil.rmtree(self._temp_dir)
+    self._temp_dir = None
+
+  def GetPath(self, path):
+    assert not os.path.isabs(path)
+    return os.path.join(self._temp_dir, path)
+
+  def GetModulePath(self, path):
+    assert not os.path.isabs(path)
+    return os.path.join(self.GetPath('out'), path) + '-module'
+
+  def WriteFile(self, path, contents):
+    full_path = self.GetPath(path)
+    dirname = os.path.dirname(full_path)
+    if not os.path.exists(dirname):
+      os.makedirs(dirname)
+    with open(full_path, 'w') as f:
+      f.write(contents)
+
+  def LoadModule(self, mojom_path):
+    with open(self.GetModulePath(mojom_path), 'rb') as f:
+      return module.Module.Load(f)
+
+  def ParseMojoms(self, mojoms):
+    """Parse all input mojoms relative the temp dir."""
+    out_dir = self.GetPath('out')
+    mojom_parser.ParseMojoms(
+        map(lambda mojom: os.path.join(self._temp_dir, mojom), mojoms),
+        [self._temp_dir, out_dir], out_dir, [])
+
+  def testBasicParse(self):
+    """Basic test to verify that we can parse a mojom file and get a module."""
+    mojom = 'foo/bar.mojom'
+    self.WriteFile(
+        mojom, """\
+        module test;
+        enum TestEnum { kFoo };
+        """)
+    self.ParseMojoms([mojom])
+
+    m = self.LoadModule(mojom)
+    self.assertEqual('foo/bar.mojom', m.path)
+    self.assertEqual('test', m.mojom_namespace)
+    self.assertEqual(1, len(m.enums))
+
+  def testBasicParseWithAbsolutePaths(self):
+    """Verifies that we can parse a mojom file given an absolute path input."""
+    mojom = 'foo/bar.mojom'
+    self.WriteFile(
+        mojom, """\
+        module test;
+        enum TestEnum { kFoo };
+        """)
+    self.ParseMojoms([self.GetPath(mojom)])
+
+    m = self.LoadModule(mojom)
+    self.assertEqual('foo/bar.mojom', m.path)
+    self.assertEqual('test', m.mojom_namespace)
+    self.assertEqual(1, len(m.enums))
+
+  def testImport(self):
+    """Verify imports within the same set of mojom inputs."""
+    a = 'a.mojom'
+    b = 'b.mojom'
+    self.WriteFile(
+        a, """\
+        module a;
+        import "b.mojom";
+        struct Foo { b.Bar bar; };""")
+    self.WriteFile(b, """\
+        module b;
+        struct Bar {};""")
+    self.ParseMojoms([a, b])
+
+    ma = self.LoadModule(a)
+    mb = self.LoadModule(b)
+    self.assertEqual('a.mojom', ma.path)
+    self.assertEqual('b.mojom', mb.path)
+    self.assertEqual(1, len(ma.imports))
+    self.assertEqual(mb, ma.imports[0])
+
+  def testPreProcessedImport(self):
+    """Verify imports processed by a previous parser execution can be loaded
+    properly when parsing a dependent mojom."""
+    a = 'a.mojom'
+    self.WriteFile(a, """\
+        module a;
+        struct Bar {};""")
+    self.ParseMojoms([a])
+
+    b = 'b.mojom'
+    self.WriteFile(
+        b, """\
+        module b;
+        import "a.mojom";
+        struct Foo { a.Bar bar; };""")
+    self.ParseMojoms([b])
+
+  def testMissingImport(self):
+    """Verify that an import fails if the imported mojom does not exist."""
+    a = 'a.mojom'
+    self.WriteFile(
+        a, """\
+        module a;
+        import "non-existent.mojom";
+        struct Bar {};""")
+    with self.assertRaisesRegexp(ValueError, "does not exist"):
+      self.ParseMojoms([a])
+
+  def testUnparsedImport(self):
+    """Verify that an import fails if the imported mojom is not in the set of
+    mojoms provided to the parser on this execution AND there is no pre-existing
+    parsed output module already on disk for it."""
+    a = 'a.mojom'
+    b = 'b.mojom'
+    self.WriteFile(a, """\
+        module a;
+        struct Bar {};""")
+    self.WriteFile(
+        b, """\
+        module b;
+        import "a.mojom";
+        struct Foo { a.Bar bar; };""")
+
+    # a.mojom has not been parsed yet, so its import will fail when processing
+    # b.mojom here.
+    with self.assertRaisesRegexp(ValueError, "does not exist"):
+      self.ParseMojoms([b])