Commit 54238fda authored by Henrique Nakashima's avatar Henrique Nakashima Committed by Commit Bot

[Android] Parallelize jdeps on each JAR when generating dependency json

Use multiprocess.Pool to spawn CPU# / 2 processes and accelerate
generate_json_dependency_graph.py when possible.

Based on a single run:

Before:
real	5m38.769s
user	29m9.770s
sys	1m38.073s

After:
real	1m11.504s
user	23m35.792s
sys	1m5.959s

Bug: 1081840
Change-Id: Ib5d1e24c37d1fadeae5df0536c7fd1b20e7a1b60
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2285379
Commit-Queue: Henrique Nakashima <hnakashima@chromium.org>
Reviewed-by: default avatarPeter Wen <wnwen@chromium.org>
Cr-Commit-Position: refs/heads/master@{#786438}
parent 8e532f91
...@@ -5,6 +5,9 @@ ...@@ -5,6 +5,9 @@
"""Command-line tool to run jdeps and process its output into a JSON file.""" """Command-line tool to run jdeps and process its output into a JSON file."""
import argparse import argparse
import functools
import math
import multiprocessing
import pathlib import pathlib
import subprocess import subprocess
...@@ -86,6 +89,7 @@ class JavaClassJdepsParser(object): ...@@ -86,6 +89,7 @@ class JavaClassJdepsParser(object):
def _run_jdeps(jdeps_path: str, filepath: pathlib.Path): def _run_jdeps(jdeps_path: str, filepath: pathlib.Path):
"""Runs jdeps on the given filepath and returns the output.""" """Runs jdeps on the given filepath and returns the output."""
print(f'Running jdeps and parsing output for {filepath}')
jdeps_res = subprocess.run([jdeps_path, '-R', '-verbose:class', filepath], jdeps_res = subprocess.run([jdeps_path, '-R', '-verbose:class', filepath],
capture_output=True, capture_output=True,
text=True, text=True,
...@@ -176,11 +180,17 @@ def main(): ...@@ -176,11 +180,17 @@ def main():
target_jars: JarTargetList = list_original_targets_and_jars( target_jars: JarTargetList = list_original_targets_and_jars(
gn_desc_output, arguments.build_output_dir) gn_desc_output, arguments.build_output_dir)
print('Running jdeps and parsing output...') print('Running jdeps...')
# jdeps already has some parallelism
jdeps_process_number = math.ceil(multiprocessing.cpu_count() / 2)
with multiprocessing.Pool(jdeps_process_number) as pool:
jar_paths = [target_jar for _, target_jar in target_jars]
jdeps_outputs = pool.map(
functools.partial(_run_jdeps, arguments.jdeps_path), jar_paths)
print('Parsing jdeps output...')
jdeps_parser = JavaClassJdepsParser() jdeps_parser = JavaClassJdepsParser()
for build_target, target_jar in target_jars: for raw_jdeps_output, (build_target, _) in zip(jdeps_outputs, target_jars):
print(f'Running jdeps and parsing output for {target_jar}')
raw_jdeps_output = _run_jdeps(arguments.jdeps_path, target_jar)
jdeps_parser.parse_raw_jdeps_output(build_target, raw_jdeps_output) jdeps_parser.parse_raw_jdeps_output(build_target, raw_jdeps_output)
class_graph = jdeps_parser.graph class_graph = jdeps_parser.graph
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment