IDL compiler: Refactor the build environment

Stops scripts depending on web_idl module's internals, and
makes scripts only use public API of web_idl module.

Fixes (probably was missing) dependencies of build settings,
also refactors gn files (less distributed).

Bug: 839389
Change-Id: I54df6a2cbf92dc1efe5a1cac9427d067e114f329
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1710235Reviewed-by: Hitoshi Yoshida <peria@chromium.org>
Commit-Queue: Yuki Shiino <yukishiino@chromium.org>
Cr-Commit-Position: refs/heads/master@{#679917}

third_party/blink/renderer/bindings/BUILD.gn

@@ -4,6 +4,7 @@
 
 import("//third_party/blink/renderer/bindings/scripts/scripts.gni")
 import("//third_party/blink/renderer/build/scripts/scripts.gni")
+import("//third_party/blink/renderer/core/core_idl_files.gni")
 import("//third_party/blink/renderer/modules/modules_idl_files.gni")
 
 action("interfaces_info") {
@@ -46,31 +47,75 @@ generate_global_constructors("global_constructors_idls") {
   ]
 }
 
-blink_python_runner("web_idl_database") {
-  script = "$bindings_scripts_dir/build_web_idl_database.py"
+template("collect_idl_files") {
+  action(target_name) {
+    script = "${bindings_scripts_dir}/collect_idl_files.py"
+    sources = invoker.sources
+    outputs = [
+      invoker.output,
+    ]
+    deps = [
+      "${bindings_scripts_dir}:web_idl_pylib",
+    ]
+    if (defined(invoker.deps)) {
+      deps += invoker.deps
+    }
 
-  inputs =
-      [
-        "$bindings_core_output_dir/web_idl_collection_for_core.pickle",
-        "$bindings_modules_output_dir/web_idl_collection_for_modules.pickle",
-      ] + web_idl_scripts
-  outputs = [
-    "$bindings_output_dir/web_idl_database.pickle",
+    # List input file names in a temporary file.
+    response_file_contents = rebase_path(sources, root_build_dir)
+    args = [
+      "--idl-list-file",
+      "{{response_file_name}}",
+      "--component",
+      invoker.component,
+      "--output",
+      rebase_path(invoker.output, root_build_dir),
+    ]
+  }
+}
+
+collect_idl_files("web_idl_in_core") {
+  # all IDL files in 'core' component
+  sources = core_static_interface_idl_files +
+            core_generated_interface_idl_files + core_all_dependency_idl_files
+  component = "core"
+  output = "${bindings_output_dir}/web_idl_in_core.pickle"
+  deps = [
+    "core:core_global_constructors_idls",
+    "//third_party/blink/renderer/core:generated_testing_idls_internal_runtime_flags",
+    "//third_party/blink/renderer/core:generated_testing_idls_settings",
   ]
+}
 
-  args = [
-    "--output",
-    rebase_path("$bindings_output_dir/web_idl_database.pickle", root_build_dir),
-    "--",
-    rebase_path("$bindings_core_output_dir/web_idl_collection_for_core.pickle",
-                root_build_dir),
-    rebase_path(
-        "$bindings_modules_output_dir/web_idl_collection_for_modules.pickle",
-        root_build_dir),
+collect_idl_files("web_idl_in_modules") {
+  # all IDL files in 'modules' component
+  sources = modules_definition_idl_files + modules_static_interface_idl_files +
+            modules_static_dependency_idl_files
+  component = "modules"
+  output = "${bindings_output_dir}/web_idl_in_modules.pickle"
+}
+
+blink_python_runner("web_idl_database") {
+  script = "${bindings_scripts_dir}/build_web_idl_database.py"
+
+  input_data_files = get_target_outputs(":web_idl_in_core") +
+                     get_target_outputs(":web_idl_in_modules")
+  inputs = input_data_files
+
+  output_data_file = "${bindings_output_dir}/web_idl_database.pickle"
+  outputs = [
+    output_data_file,
   ]
 
   deps = [
-    "//third_party/blink/renderer/bindings/core:core_web_idl_collection",
-    "//third_party/blink/renderer/bindings/modules:modules_web_idl_collection",
+    ":web_idl_in_core",
+    ":web_idl_in_modules",
+    "${bindings_scripts_dir}:web_idl_pylib",
   ]
+
+  args = [
+           "--output",
+           rebase_path(output_data_file, root_build_dir),
+           "--",
+         ] + rebase_path(input_data_files, root_build_dir)
 }

third_party/blink/renderer/bindings/core/BUILD.gn

@@ -38,17 +38,3 @@ generate_global_constructors("core_global_constructors_idls") {
     ":core_global_objects",
   ]
 }
-
-generate_web_idl_collection("core_web_idl_collection") {
-  # |sources| contains all IDL files under core/.
-  sources = core_static_interface_idl_files +
-            core_generated_interface_idl_files + core_all_dependency_idl_files
-  output = "web_idl_collection_for_core.pickle"
-  component = "core"
-  output_dir = bindings_core_output_dir
-  deps = [
-    ":core_global_constructors_idls",
-    "//third_party/blink/renderer/core:generated_testing_idls_internal_runtime_flags",
-    "//third_party/blink/renderer/core:generated_testing_idls_settings",
-  ]
-}

third_party/blink/renderer/bindings/modules/BUILD.gn

@@ -170,12 +170,3 @@ source_set("generated") {
     "//v8",
   ]
 }
-
-generate_web_idl_collection("modules_web_idl_collection") {
-  # |sources| contains all IDL files under modules/.
-  sources = modules_definition_idl_files + modules_static_interface_idl_files +
-            modules_static_dependency_idl_files
-  output = "web_idl_collection_for_modules.pickle"
-  component = "modules"
-  output_dir = bindings_modules_output_dir
-}

third_party/blink/renderer/bindings/scripts/BUILD.gn

@@ -46,3 +46,51 @@ action("cached_jinja_templates") {
     rebase_path(stamp_file, root_build_dir),
   ]
 }
+
+# Python library that supports Web IDL compiler and database
+group("web_idl_pylib") {
+  data = [
+    "web_idl/argument.py",
+    "web_idl/ast_group.py",
+    "web_idl/attribute.py",
+    "web_idl/callback_function.py",
+    "web_idl/callback_interface.py",
+    "web_idl/common.py",
+    "web_idl/constant.py",
+    "web_idl/constructor.py",
+    "web_idl/database.py",
+    "web_idl/dictionary.py",
+    "web_idl/enumeration.py",
+    "web_idl/exposure.py",
+    "web_idl/extended_attribute.py",
+    "web_idl/identifier_ir_map.py",
+    "web_idl/idl_compiler.py",
+    "web_idl/idl_member.py",
+    "web_idl/idl_type.py",
+    "web_idl/includes.py",
+    "web_idl/interface.py",
+    "web_idl/ir_builder.py",
+    "web_idl/namespace.py",
+    "web_idl/operation.py",
+    "web_idl/reference.py",
+    "web_idl/typedef.py",
+    "web_idl/user_defined_type.py",
+    "web_idl/values.py",
+  ]
+  data += [  # dependencies to Python libraries
+    "//third_party/blink/renderer/build/scripts/blinkbuild/name_style_converter.py",
+
+    # PLY (Python Lex-Yacc)
+    "//third_party/ply/lex.py",
+    "//third_party/ply/yacc.py",
+
+    # Web IDL lexer/parser (base parser)
+    "//tools/idl_parser/idl_lexer.py",
+    "//tools/idl_parser/idl_node.py",
+    "//tools/idl_parser/idl_parser.py",
+
+    # Blink IDL lexer/parser/constructor
+    "blink_idl_lexer.py",
+    "blink_idl_parser.py",
+  ]
+}

third_party/blink/renderer/bindings/scripts/build_web_idl_database.py

-#!/usr/bin/python
-#
 # Copyright 2019 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-"""Generates a data collection of IDL information per component.
-This scripts parses IDL files and stores the result ASTs in a pickle file.
-The output file may contain information about component, too.
+"""
+Builds Web IDL database.
+
+Web IDL database is a Python object that supports a variety of accessors to
+IDL definitions such as IDL interface and IDL attribute.
 """
 
 import optparse
 
 import utilities
-
-from web_idl.identifier_ir_map import IdentifierIRMap
-from web_idl.idl_compiler import IdlCompiler
-from web_idl.idl_type import IdlTypeFactory
-from web_idl.ir_builder import load_and_register_idl_definitions
-from web_idl.reference import RefByIdFactory
+import web_idl
 
 
 def parse_options():
@@ -34,19 +29,10 @@ def parse_options():
 
 def main():
     options, filepaths = parse_options()
-    ir_map = IdentifierIRMap()
-    ref_to_idl_type_factory = RefByIdFactory()
-    ref_to_idl_def_factory = RefByIdFactory()
-    idl_type_factory = IdlTypeFactory()
-    load_and_register_idl_definitions(
-        filepaths,
-        ir_map.register,
-        ref_to_idl_type_factory.create,
-        ref_to_idl_def_factory.create,
-        idl_type_factory)
-    idl_compiler = IdlCompiler(ir_map)
-    idl_database = idl_compiler.build_database()
-    utilities.write_pickle_file(options.output, idl_database)
+
+    database = web_idl.build_database(filepaths)
+
+    utilities.write_pickle_file(options.output, database)
 
 
 if __name__ == '__main__':

third_party/blink/renderer/bindings/scripts/generate_web_idl_collection.py → third_party/blink/renderer/bindings/scripts/collect_idl_files.py

@@ -2,48 +2,55 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-"""Generates a data collection of IDL information per component.
-This scripts parses IDL files and stores the result ASTs in a pickle file.
-The output file may contain information about component, too.
+"""
+Collects Web IDL definitions in IDL files into a Python object per Blink
+component.
+
+Collected IDL definitions are parsed into ASTs and saved into a file with
+a label of Blink component.
 """
 
-import blink_idl_parser
 import optparse
+
+import blink_idl_parser
 import utilities
-from web_idl.collector import Collector
-from web_idl.collection import Collection
+import web_idl
 
 
 _VALID_COMPONENTS = ("core", "modules")
 
+
 def parse_options():
     parser = optparse.OptionParser()
     parser.add_option('--idl-list-file', type='string',
                       help='a file path which lists IDL file paths to process')
     parser.add_option('--component', type='choice', choices=_VALID_COMPONENTS,
-                      help='specify a component name where IDLs belong')
+                      help='specify a component name')
     parser.add_option('--output', type='string',
-                      help='pickle file to write down')
+                      help='the output file path')
     options, args = parser.parse_args()
 
     if options.idl_list_file is None:
-        parser.error('Must specify a file listing IDL files using --idl-list-file.')
+        parser.error('Specify a file listing IDL files with --idl-list-file.')
     if options.output is None:
-        parser.error('Must specify a pickle file to output using --output.')
+        parser.error('Specify the output file path with --output.')
     if options.component is None:
-        parser.error('Must specify a component using --component.')
+        parser.error('Specify a component with --component.')
 
     return options, args
 
 
 def main():
-    options, _ = parse_options()
-    idl_file_names = utilities.read_idl_files_list_from_file(options.idl_list_file)
+    options, args = parse_options()
+    if args:
+        raise RuntimeError('unknown arguments {}'.format(args))
 
+    filepaths = utilities.read_idl_files_list_from_file(options.idl_list_file)
     parser = blink_idl_parser.BlinkIDLParser()
-    collector = Collector(component=options.component, parser=parser)
-    collector.collect_from_idl_files(idl_file_names)
-    Collection.write_to_file(collector.get_collection(), options.output)
+    ast_group = web_idl.AstGroup(options.component)
+    for filepath in filepaths:
+        ast_group.add_ast_node(blink_idl_parser.parse_file(parser, filepath))
+    ast_group.write_to_file(options.output)
 
 
 if __name__ == '__main__':

third_party/blink/renderer/bindings/scripts/scripts.gni

@@ -2,12 +2,12 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import("//third_party/blink/renderer/build/scripts/scripts.gni")
 import("//third_party/blink/renderer/core/core_idl_files.gni")
 import("//third_party/blink/renderer/modules/modules_idl_files.gni")
 
 bindings_scripts_dir = get_path_info(".", "abspath")
-bindings_scripts_output_dir =
-    "$root_gen_dir/third_party/blink/renderer/bindings/scripts"
+bindings_scripts_output_dir = get_path_info(".", "gen_dir")
 
 jinja_module_files = [
   "//third_party/jinja2/__init__.py",
@@ -57,37 +57,6 @@ idl_compiler_files = get_path_info(
         ],
         "abspath")
 
-web_idl_scripts = get_path_info([
-                                  "web_idl/argument.py",
-                                  "web_idl/attribute.py",
-                                  "web_idl/callback_function.py",
-                                  "web_idl/callback_interface.py",
-                                  "web_idl/collection.py",
-                                  "web_idl/collector.py",
-                                  "web_idl/common.py",
-                                  "web_idl/constant.py",
-                                  "web_idl/constructor.py",
-                                  "web_idl/database.py",
-                                  "web_idl/dictionary.py",
-                                  "web_idl/enumeration.py",
-                                  "web_idl/exposure.py",
-                                  "web_idl/extended_attribute.py",
-                                  "web_idl/identifier_ir_map.py",
-                                  "web_idl/idl_compiler.py",
-                                  "web_idl/idl_member.py",
-                                  "web_idl/idl_type.py",
-                                  "web_idl/includes.py",
-                                  "web_idl/interface.py",
-                                  "web_idl/ir_builder.py",
-                                  "web_idl/namespace.py",
-                                  "web_idl/operation.py",
-                                  "web_idl/reference.py",
-                                  "web_idl/typedef.py",
-                                  "web_idl/user_defined_type.py",
-                                  "web_idl/values.py",
-                                ],
-                                "abspath")
-
 # Calls the compute_interfaces_info_individual script.
 #
 # Parameters:
@@ -505,34 +474,3 @@ template("generate_origin_trial_features") {
            invoker.deps
   }
 }
-
-template("generate_web_idl_collection") {
-  action(target_name) {
-    script = "${bindings_scripts_dir}/generate_web_idl_collection.py"
-    output_dir = invoker.output_dir
-    output_file_name = invoker.output
-    output_path = "${output_dir}/${output_file_name}"
-    inputs = [
-               script,
-               "${bindings_scripts_dir}/web_idl/collection.py",
-               "${bindings_scripts_dir}/web_idl/collector.py",
-             ] + idl_lexer_parser_files + invoker.sources
-    outputs = [
-      output_path,
-    ]
-    if (defined(invoker.deps)) {
-      deps = invoker.deps
-    }
-
-    # List input file names in a temporary file.
-    response_file_contents = rebase_path(invoker.sources, root_build_dir)
-    args = [
-      "--idl-list-file",
-      "{{response_file_name}}",
-      "--component",
-      invoker.component,
-      "--output",
-      rebase_path(output_path, root_build_dir),
-    ]
-  }
-}

third_party/blink/renderer/bindings/scripts/web_idl/__init__.py

 # Copyright 2017 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+
+from .ast_group import AstGroup
+from .database import Database
+from .database_builder import build_database
+
+__all__ = [
+    "AstGroup",
+    "Database",
+    "build_database",
+]

third_party/blink/renderer/bindings/scripts/web_idl/collection.py → third_party/blink/renderer/bindings/scripts/web_idl/ast_group.py

@@ -3,43 +3,37 @@
 # found in the LICENSE file.
 
 import pickle
-import idl_parser
+
 from .common import Component
 
 
-class Collection(object):
-    """
-    Collection class stores ASTs of Web IDL files and meta information
-    like component.
-    """
+class AstGroup(object):
+    """A set of Web IDL ASTs grouped by component."""
 
     def __init__(self, component=None):
         assert component is None or isinstance(component, Component)
-        self._asts = []
+        self._nodes = []
         self._component = component
 
-    def add_ast(self, ast):
-        assert isinstance(ast, idl_parser.idl_node.IDLNode)
-        assert ast.GetClass() == 'File', (
-            'Root node of an AST must be a File node., but is %s.' % ast.GetClass())
-        self._asts.append(ast)
+    def __iter__(self):
+        return self._nodes.__iter__()
 
     @staticmethod
-    def load_from_file(filepath):
+    def read_from_file(filepath):
         with open(filepath, 'r') as pickle_file:
-            collection = pickle.load(pickle_file)
-        assert isinstance(collection, Collection)
-        return collection
+            ast_group = pickle.load(pickle_file)
+        assert isinstance(ast_group, AstGroup)
+        return ast_group
 
-    @staticmethod
-    def write_to_file(collection, filepath):
-        assert isinstance(collection, Collection)
+    def write_to_file(self, filepath):
         with open(filepath, 'w') as pickle_file:
-            pickle.dump(collection, pickle_file)
+            pickle.dump(self, pickle_file)
 
-    @property
-    def asts(self):
-        return self._asts
+    def add_ast_node(self, node):
+        assert node.GetClass() == 'File', (
+            'Root node of an AST must be a File node, but is %s.' %
+            node.GetClass())
+        self._nodes.append(node)
 
     @property
     def component(self):

third_party/blink/renderer/bindings/scripts/web_idl/collector.py

-# Copyright 2017 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import blink_idl_parser
-from .collection import Collection
-
-
-class Collector(object):
-
-    def __init__(self, component, parser=blink_idl_parser.BlinkIDLParser()):
-        self._component = component
-        self._collection = Collection(component)
-        self._parser = parser
-
-    def collect_from_idl_files(self, filepaths):
-        if isinstance(filepaths, str):
-            filepaths = [filepaths]
-        for filepath in filepaths:
-            try:
-                ast = blink_idl_parser.parse_file(self._parser, filepath)
-                self.collect_from_ast(ast)
-            except ValueError as ve:
-                raise ValueError('%s\nin file %s' % (str(ve), filepath))
-
-    def collect_from_idl_text(self, text, filename='TEXT'):
-        ast = self._parser.ParseText(filename, text)  # pylint: disable=no-member
-        self.collect_from_ast(ast)
-
-    def collect_from_ast(self, node):
-        self._collection.add_ast(node)
-
-    def get_collection(self):
-        return self._collection

third_party/blink/renderer/bindings/scripts/web_idl/database_builder.py

+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from .identifier_ir_map import IdentifierIRMap
+from .idl_compiler import IdlCompiler
+from .idl_type import IdlTypeFactory
+from .ir_builder import load_and_register_idl_definitions
+from .reference import RefByIdFactory
+
+
+def build_database(filepaths):
+    """Compiles IDL definitions in |filepaths| and builds a database."""
+
+    ir_map = IdentifierIRMap()
+    ref_to_idl_type_factory = RefByIdFactory()
+    ref_to_idl_def_factory = RefByIdFactory()
+    idl_type_factory = IdlTypeFactory()
+    load_and_register_idl_definitions(
+        filepaths, ir_map.register, ref_to_idl_type_factory.create,
+        ref_to_idl_def_factory.create, idl_type_factory)
+    compiler = IdlCompiler(ir_map, ref_to_idl_type_factory,
+                           ref_to_idl_def_factory, idl_type_factory)
+    return compiler.build_database()

third_party/blink/renderer/bindings/scripts/web_idl/idl_compiler.py

@@ -3,6 +3,8 @@
 # found in the LICENSE file.
 
 from .identifier_ir_map import IdentifierIRMap
+from .idl_type import IdlTypeFactory
+from .reference import RefByIdFactory
 
 
 class IdlCompiler(object):
@@ -26,8 +28,27 @@ class IdlCompiler(object):
     the details.
     """
 
-    def __init__(self, ir_map):
+    def __init__(self, ir_map, ref_to_idl_type_factory, ref_to_idl_def_factory,
+                 idl_type_factory):
+        """
+        Args:
+            ir_map: IdentifierIRMap filled with the initial IRs of IDL
+                definitions.
+            ref_to_idl_type_factory: RefByIdFactory that created all references
+                to IdlType.
+            ref_to_idl_def_factory: RefByIdFactory that created all references
+                to UserDefinedType.
+            idl_type_factory: IdlTypeFactory that created all instances of
+                IdlType.
+        """
+        assert isinstance(ir_map, IdentifierIRMap)
+        assert isinstance(ref_to_idl_type_factory, RefByIdFactory)
+        assert isinstance(ref_to_idl_def_factory, RefByIdFactory)
+        assert isinstance(idl_type_factory, IdlTypeFactory)
         self._ir_map = ir_map
+        self._ref_to_idl_type_factory = ref_to_idl_type_factory
+        self._ref_to_idl_def_factory = ref_to_idl_def_factory
+        self._idl_type_factory = idl_type_factory
 
     def build_database(self):
         self._merge_partials()

third_party/blink/renderer/bindings/scripts/web_idl/ir_builder.py

@@ -3,10 +3,10 @@
 # found in the LICENSE file.
 
 from .argument import Argument
+from .ast_group import AstGroup
 from .attribute import Attribute
 from .callback_function import CallbackFunction
 from .callback_interface import CallbackInterface
-from .collection import Collection
 from .common import DebugInfo
 from .constant import Constant
 from .dictionary import Dictionary
@@ -46,12 +46,12 @@ def load_and_register_idl_definitions(filepaths, register_ir,
     assert callable(register_ir)
 
     for filepath in filepaths:
-        asts_per_component = Collection.load_from_file(filepath)
+        asts_per_component = AstGroup.read_from_file(filepath)
         component = asts_per_component.component
         builder = _IRBuilder(component, create_ref_to_idl_type,
                              create_ref_to_idl_def, idl_type_factory)
 
-        for file_node in asts_per_component.asts:
+        for file_node in asts_per_component:
             assert file_node.GetClass() == 'File'
             for top_level_node in file_node.GetChildren():
                 register_ir(builder.build_top_level_def(top_level_node))