DevTools: speed up closure dependency checking

Compiling frontend runs in ~13 seconds (including dependency check) instead of 22 seconds.

Examples of the RegEx match:
http://regexr.com/3etk5

The following CL needs to land before this CL lands:
https://codereview.chromium.org/2623743002/
BUG=673445

Review-Url: https://codereview.chromium.org/2588843002
Cr-Commit-Position: refs/heads/master@{#442754}

third_party/WebKit/Source/devtools/BUILD.gn

@@ -1041,7 +1041,9 @@ group("devtools_closure_compile") {
     "scripts/build/",
     "scripts/closure/",
     "scripts/compile_frontend.py",
+    "scripts/dependency_preprocessor.py",
     "scripts/jsdoc_validator/",
+    "scripts/special_case_namespaces.json",
     "scripts/utils.py",
     "//testing/scripts/common.py",
     "//testing/scripts/run_devtools_check.py",

third_party/WebKit/Source/devtools/front_end/Runtime.js

@@ -773,15 +773,14 @@ Runtime.Module = class {
       return Promise.resolve();
 
     // the namespace keyword confuses clang-format
+    // NOTE: Update scripts/special_case_namespaces.json if you add a special cased namespace
     // clang-format off
     // Module namespaces.
     const specialCases = {
       'sdk': 'SDK',
       'ui': 'UI',
       'perf_ui': 'PerfUI',
-      'css_tracker': 'CSSTracker',
-      'ui_lazy': 'UI',
-      'components_lazy': 'Components'
+      'css_tracker': 'CSSTracker'
     };
     var namespace = specialCases[this._name] || this._name.split('_').map(a => a.substring(0, 1).toUpperCase() + a.substring(1)).join('');
     self[namespace] = self[namespace] || {};

third_party/WebKit/Source/devtools/scripts/README.md

@@ -32,4 +32,4 @@ The easiest way to run the node.js scripts is to use `npm run` which displays al
 
 ## Closure
 
-DevTools manually rolls the closure compiler to ./closure. If you manually roll closure compiler, you will need to re-generate the closure_runner (in ./closure) and jsdoc_validator custom jars using the python scripts in their respective directory. Make sure you use JDK 7 to compile these jars, otherwise they won't run on buildbot. If you compile with JDK 7, anyone with Java 7 or Java 8 should be able to run the jar.
+DevTools manually rolls the closure compiler to ./closure. If you manually roll closure compiler, you will need to re-generate the closure_runner (in ./closure) and jsdoc_validator custom jars using the python scripts in their respective directory.
\ No newline at end of file

third_party/WebKit/Source/devtools/scripts/build/modular_build.py

@@ -8,6 +8,7 @@
 Utilities for the modular DevTools build.
 """
 
+import collections
 from os import path
 import os
 
@@ -69,8 +70,8 @@ class Descriptors:
         return json.dumps(result)
 
     def all_compiled_files(self):
-        files = {}
-        for name in self.modules:
+        files = collections.OrderedDict()
+        for name in self.sorted_modules():
             module = self.modules[name]
             skipped_files = set(module.get('skip_compilation', []))
             for script in module.get('scripts', []):

third_party/WebKit/Source/devtools/scripts/compile_frontend.py

@@ -38,6 +38,7 @@ import tempfile
 from build import modular_build
 from build import generate_protocol_externs
 
+import dependency_preprocessor
 import utils
 
 try:
@@ -45,13 +46,6 @@ try:
 except ImportError:
     import json
 
-
-if len(sys.argv) == 2 and sys.argv[1] == '--help':
-    print("Usage: %s [module_names]" % path.basename(sys.argv[0]))
-    print("  module_names    list of modules for which the Closure compilation should run.")
-    print("                  If absent, the entire frontend will be compiled.")
-    sys.exit(0)
-
 is_cygwin = sys.platform == 'cygwin'
 
 
@@ -79,9 +73,7 @@ v8_inspector_path = path.normpath(path.join(path.dirname(devtools_path), os.pard
 devtools_frontend_path = path.join(devtools_path, 'front_end')
 global_externs_file = to_platform_path(path.join(devtools_frontend_path, 'externs.js'))
 protocol_externs_file = path.join(devtools_frontend_path, 'protocol_externs.js')
-
-jsmodule_name_prefix = 'jsmodule_'
-runtime_module_name = '_runtime'
+runtime_file = to_platform_path(path.join(devtools_frontend_path, 'Runtime.js'))
 
 type_checked_jsdoc_tags_list = ['param', 'return', 'type', 'enum']
 type_checked_jsdoc_tags_or = '|'.join(type_checked_jsdoc_tags_list)
@@ -226,7 +218,6 @@ closure_compiler_jar = to_platform_path(path.join(scripts_path, 'closure', 'comp
 closure_runner_jar = to_platform_path(path.join(scripts_path, 'closure', 'closure_runner', 'closure_runner.jar'))
 jsdoc_validator_jar = to_platform_path(path.join(scripts_path, 'jsdoc_validator', 'jsdoc_validator.jar'))
 
-modules_dir = tempfile.mkdtemp()
 common_closure_args = [
     '--summary_detail_level', '3',
     '--jscomp_error', 'visibility',
@@ -238,7 +229,6 @@ common_closure_args = [
     '--extra_annotation_name', 'suppressReceiverCheck',
     '--extra_annotation_name', 'suppressGlobalPropertiesCheck',
     '--checks-only',
-    '--module_output_path_prefix', to_platform_path_exact(modules_dir + path.sep)
 ]
 
 worker_modules_by_name = {}
@@ -301,79 +291,53 @@ def check_duplicate_files():
 print 'Checking duplicate files across modules...'
 check_duplicate_files()
 
+print 'Compiling frontend...'
 
-def module_arg(module_name):
-    return ' --module ' + jsmodule_name_prefix + module_name
-
-
-def modules_to_check():
-    if len(sys.argv) == 1:
-        return descriptors.sorted_modules()
-    print 'Compiling only these modules: %s' % sys.argv[1:]
-    return [module for module in descriptors.sorted_modules() if module in set(sys.argv[1:])]
-
-
-def dump_module(name, recursively, processed_modules):
-    if name in processed_modules:
-        return ''
-    processed_modules[name] = True
-    module = modules_by_name[name]
-
-    command = ''
-    dependencies = module.get('dependencies', [])
-    if recursively:
-        for dependency in dependencies:
-            command += dump_module(dependency, recursively, processed_modules)
-    command += module_arg(name) + ':'
-    filtered_scripts = descriptors.module_compiled_files(name)
-    filtered_scripts = [path.join(devtools_frontend_path, name, script) for script in filtered_scripts]
-    if name == 'protocol':
-        filtered_scripts.append(protocol_externs_file)
-    command += str(len(filtered_scripts))
-    first_dependency = True
-    for dependency in dependencies + [runtime_module_name]:
-        if first_dependency:
-            command += ':'
-        else:
-            command += ','
-        first_dependency = False
-        command += jsmodule_name_prefix + dependency
-    for script in filtered_scripts:
-        command += ' --js ' + to_platform_path(script)
-    return command
+temp_devtools_path = tempfile.mkdtemp()
 
-print 'Compiling frontend...'
 
-compiler_args_file = tempfile.NamedTemporaryFile(mode='wt', delete=False)
-try:
-    runtime_js_path = to_platform_path(path.join(devtools_frontend_path, 'Runtime.js'))
-    checked_modules = modules_to_check()
-    for name in checked_modules:
-        closure_args = ' '.join(common_closure_args)
-        closure_args += ' --externs ' + to_platform_path(global_externs_file)
-        runtime_module = module_arg(runtime_module_name) + ':1 --js ' + runtime_js_path
-        closure_args += runtime_module + dump_module(name, True, {})
-        compiler_args_file.write('%s %s%s' % (name, closure_args, os.linesep))
-finally:
-    compiler_args_file.close()
-
-modular_compiler_proc = popen(java_exec + ['-jar', closure_runner_jar, '--compiler-args-file', to_platform_path_exact(compiler_args_file.name)])
-
-spawned_compiler_command = java_exec + [
+def prepare_closure_frontend_compile():
+    temp_frontend_path = path.join(temp_devtools_path, 'front_end')
+    checker = dependency_preprocessor.DependencyPreprocessor(descriptors, temp_frontend_path, devtools_frontend_path)
+    checker.enforce_dependencies()
+
+    command = common_closure_args + [
+        '--externs', to_platform_path(global_externs_file),
+        '--js', runtime_file,
+    ]
+
+    all_files = descriptors.all_compiled_files()
+    args = []
+    for file in all_files:
+        args.extend(['--js', file])
+        if "InspectorBackend.js" in file:
+            args.extend(['--js', protocol_externs_file])
+    command += args
+    command = [arg.replace(devtools_frontend_path, temp_frontend_path) for arg in command]
+    compiler_args_file = tempfile.NamedTemporaryFile(mode='wt', delete=False)
+    try:
+        compiler_args_file.write('devtools_frontend %s' % (' '.join(command)))
+    finally:
+        compiler_args_file.close()
+    return compiler_args_file.name
+
+compiler_args_file_path = prepare_closure_frontend_compile()
+frontend_compile_proc = popen(java_exec + ['-jar', closure_runner_jar, '--compiler-args-file', to_platform_path_exact(compiler_args_file_path)])
+
+print 'Compiling devtools_compatibility.js...'
+
+closure_compiler_command = java_exec + [
     '-jar',
     closure_compiler_jar
 ] + common_closure_args
 
-print 'Compiling devtools_compatibility.js...'
-
-command = spawned_compiler_command + [
+devtools_js_compile_command = closure_compiler_command + [
     '--externs', to_platform_path(global_externs_file),
     '--externs', to_platform_path(path.join(devtools_frontend_path, 'host', 'InspectorFrontendHostAPI.js')),
     '--jscomp_off=externsValidation',
-    '--module', jsmodule_name_prefix + 'devtools__compatibility_js' + ':1',
     '--js', to_platform_path(path.join(devtools_frontend_path, 'devtools_compatibility.js'))
 ]
-devtools_js_compile_proc = popen(command)
+devtools_js_compile_proc = popen(devtools_js_compile_command)
 
 print 'Verifying JSDoc comments...'
 errors_found |= verify_jsdoc()
@@ -388,71 +352,23 @@ if jsdoc_validator_out:
 
 os.remove(jsdoc_validator_file_list.name)
 
-(module_compile_out, _) = modular_compiler_proc.communicate()
-print 'Modular compilation output:'
-
-start_module_regex = re.compile(r'^@@ START_MODULE:(.+) @@$')
-end_module_regex = re.compile(r'^@@ END_MODULE @@$')
-
-in_module = False
-skipped_modules = {}
-error_count = 0
-
-def skip_dependents(module_name):
-    for skipped_module in dependents_by_module_name.get(module_name, []):
-        skipped_modules[skipped_module] = True
-
-has_module_output = False
-
-# pylint: disable=E1103
-for line in module_compile_out.splitlines():
-    if not in_module:
-        match = re.search(start_module_regex, line)
-        if not match:
-            continue
-        in_module = True
-        has_module_output = True
-        module_error_count = 0
-        module_output = []
-        module_name = match.group(1)
-        skip_module = skipped_modules.get(module_name)
-        if skip_module:
-            skip_dependents(module_name)
-    else:
-        match = re.search(end_module_regex, line)
-        if not match:
-            if not skip_module:
-                module_output.append(line)
-                if has_errors(line):
-                    error_count += 1
-                    module_error_count += 1
-                    skip_dependents(module_name)
-            continue
-
-        in_module = False
-        if skip_module:
-            print 'Skipping module %s...' % module_name
-        elif not module_error_count:
-            print 'Module %s compiled successfully: %s' % (module_name, module_output[0])
-        else:
-            print 'Module %s compile failed: %s errors%s' % (module_name, module_error_count, os.linesep)
-            print os.linesep.join(module_output)
-
-if not has_module_output:
-    print module_compile_out
-
-if error_count:
-    print 'Total Closure errors: %d%s' % (error_count, os.linesep)
-    errors_found = True
-
 (devtools_js_compile_out, _) = devtools_js_compile_proc.communicate()
 print 'devtools_compatibility.js compilation output:%s' % os.linesep, devtools_js_compile_out
 errors_found |= has_errors(devtools_js_compile_out)
 
-os.remove(compiler_args_file.name)
+(frontend_compile_out, _) = frontend_compile_proc.communicate()
+print 'devtools frontend compilation output:'
+for line in frontend_compile_out.splitlines():
+    if "@@ START_MODULE" in line or "@@ END_MODULE" in line:
+        continue
+    print line
+errors_found |= has_errors(frontend_compile_out)
+
 os.remove(protocol_externs_file)
-shutil.rmtree(modules_dir, True)
+os.remove(compiler_args_file_path)
+shutil.rmtree(temp_devtools_path, True)
 
 if errors_found:
     print 'ERRORS DETECTED'
     sys.exit(1)
+print 'DONE - compiled without errors'
\ No newline at end of file

third_party/WebKit/Source/devtools/scripts/dependency_preprocessor.py

+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import codecs
+import multiprocessing
+from os import path
+import re
+import shutil
+
+try:
+    import simplejson as json
+except ImportError:
+    import json
+
+special_case_namespaces_path = path.join(path.dirname(path.abspath(__file__)), 'special_case_namespaces.json')
+
+
+class DependencyPreprocessor(object):
+
+    def __init__(self, descriptors, temp_frontend_path, devtools_frontend_path):
+        self.descriptors = descriptors
+        self.temp_frontend_path = temp_frontend_path
+        self.module_descriptors = descriptors.modules
+        self.modules = set(self.descriptors.sorted_modules())
+        shutil.copytree(devtools_frontend_path, self.temp_frontend_path)
+        with open(special_case_namespaces_path) as json_file:
+            self._special_case_namespaces = json.load(json_file)
+
+    def enforce_dependencies(self):
+        arg_list = []
+        for module in self.modules:
+            dependencies = set(self.descriptors.sorted_dependencies_closure(module))
+            excluded_modules = self.modules - {module} - dependencies - self._transitive_implicit_dependencies(module)
+            excluded_namespaces = [self._map_module_to_namespace(m) for m in excluded_modules]
+            file_paths = [path.join(self.temp_frontend_path, module, file_name)
+                          for file_name in self.descriptors.module_compiled_files(module)]
+            arg = {
+                'excluded_namespaces': excluded_namespaces,
+                'file_paths': file_paths,
+            }
+            arg_list.append(arg)
+        parallelize(poison_module, arg_list)
+
+    def _transitive_implicit_dependencies(self, module):
+        """Finds implicit dependencies for workers (which cherry-pick files from other modules)"""
+        explicit_dependencies = self.descriptors.sorted_dependencies_closure(module)
+        implicit_dependencies = set()
+        for explicit_dependency in explicit_dependencies:
+            implicit_dependencies |= self._implicit_dependencies_for_module(explicit_dependency)
+        return implicit_dependencies
+
+    def _implicit_dependencies_for_module(self, module):
+        implicit_dependencies = set()
+        for module_file in self.descriptors.module_compiled_files(module):
+            if "../" in module_file:
+                components = module_file.split('/')
+                implicit_dependencies.add(components[1])
+        return implicit_dependencies
+
+    def _map_module_to_namespace(self, module):
+        return self._special_case_namespaces.get(module, self._to_camel_case(module))
+
+    def _to_camel_case(self, snake_string):
+        components = snake_string.split('_')
+        return ''.join(x.title() for x in components)
+
+
+def poison_module(target):
+    excluded_namespaces = target['excluded_namespaces']
+    file_paths = target['file_paths']
+    for file_path in file_paths:
+        with codecs.open(file_path, 'r', 'utf-8') as file:
+            file_contents = file.read()
+        file_contents = poison_contents_for_namespaces(file_contents, excluded_namespaces)
+        with codecs.open(file_path, 'w', 'utf-8') as file:
+            file.write(file_contents)
+
+
+def poison_contents_for_namespaces(file_contents, namespaces):
+    # Technically, should be [^.]\s*\b + NAMESPACES + \b\s*[^:]
+    # but we rely on clang-format to format like this:
+    #   SomeModule
+    #     .Console
+    regex = r'([^.]\b)(' + '|'.join(namespaces) + r')(\b[^:])'
+    replace = r'\1$$UndeclaredDependency_\2$$\3'
+    return re.sub(regex, replace, file_contents)
+
+
+def parallelize(fn, arg_list):
+    number_of_processes = min(multiprocessing.cpu_count(), 8)
+    pool = multiprocessing.Pool(number_of_processes)
+    pool.map(fn, arg_list)
+    pool.close()
+    pool.join()

third_party/WebKit/Source/devtools/scripts/special_case_namespaces.json

@@ -2,7 +2,5 @@
   "sdk": "SDK",
   "ui": "UI",
   "perf_ui": "PerfUI",
-  "css_tracker": "CSSTracker",
-  "ui_lazy": "UI",
-  "components_lazy": "Components"
+  "css_tracker": "CSSTracker"
 }
\ No newline at end of file