[Android] Process all dependency jars when generating dependency json

generate_json_dependency_graph.py now takes as parameter a root build
target and parses all jars it depends on, directly or indirectly, rather
than parsing only a single jar.

Bug: 1081840
Change-Id: Ia05313ca178d917cb8e17a23af535f47525cfe96
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2285382
Commit-Queue: Henrique Nakashima <hnakashima@chromium.org>
Reviewed-by: Peter Wen <wnwen@chromium.org>
Cr-Commit-Position: refs/heads/master@{#786386}

tools/android/dependency_analysis/README.md

@@ -4,28 +4,35 @@ As part of Chrome Modularization, this directory contains various tools for
 analyzing the dependencies contained within the Chrome Android project.
 
 ## Usage
-Start by generating a JSON dependency file with a snapshot of the dependencies 
+Start by generating a JSON dependency file with a snapshot of the dependencies
 for your JAR using the **JSON dependency generator** command-line tool.
 
-This snapshot file can then be used as input for various other 
+This snapshot file can then be used as input for various other
 analysis tools listed below.
 
 ## Command-line tools
-The usage information for any of the following tools is also accessible via 
+The usage information for any of the following tools is also accessible via
 `toolname -h` or `toolname --help`.
 #### JSON Dependency Generator
-Runs [jdeps](https://docs.oracle.com/javase/8/docs/technotes/tools/unix/jdeps.html) 
+Runs [jdeps](https://docs.oracle.com/javase/8/docs/technotes/tools/unix/jdeps.html)
 on a given JAR and writes the resulting dependency graph into a JSON file.
 ```
-usage: generate_json_dependency_graph.py [-h] -t TARGET -o OUTPUT [-j JDEPS_PATH]
+usage: generate_json_dependency_graph.py [-h] -C BUILD_OUTPUT_DIR -o OUTPUT
+                                         [-t TARGET] [-j JDEPS_PATH]
+
+Runs jdeps (dependency analysis tool) on all JARs a root build target depends
+on and writes the resulting dependency graph into a JSON file. The default
+root build target is chrome/android:monochrome_public_bundle.
 
 optional arguments:
+  -t TARGET, --target TARGET
+                        Root build target.
   -j JDEPS_PATH, --jdeps-path JDEPS_PATH
                         Path to the jdeps executable.
 
 required arguments:
-  -t TARGET, --target TARGET
-                        Path to the JAR file to run jdeps on.
+  -C BUILD_OUTPUT_DIR, --build_output_dir BUILD_OUTPUT_DIR
+                        Build output directory.
   -o OUTPUT, --output OUTPUT
                         Path to the file to write JSON output to. Will be
                         created if it does not yet exist and overwrite
@@ -62,14 +69,12 @@ required arguments:
 ```
 
 ## Example Usage
-This Linux example assumes Chromium is contained in a directory `~/cr` 
-and that Chromium has been built as per the instructions 
+This Linux example assumes Chromium is contained in a directory `~/cr`
+and that Chromium has been built as per the instructions
 [here](https://chromium.googlesource.com/chromium/src/+/master/docs/linux/build_instructions.md),
 although the only things these assumptions affect are the file paths.
 ```
-cd ~/cr/src/tools/android/dependency_analysis
-
-./generate_json_dependency_graph.py --target ~/cr/src/out/Default/obj/chrome/android/chrome_java__process_prebuilt.desugar.jar --output ./json_graph.txt
+$ tools/android/dependency_analysis/generate_json_dependency_graph.py -C out/Debug -o ~/json_graph.txt
 >>> Running jdeps and parsing output...
 >>> Parsed class-level dependency graph, got 3239 nodes and 19272 edges.
 >>> Created package-level dependency graph, got 500 nodes and 4954 edges.
@@ -89,4 +94,4 @@ cd ~/cr/src/tools/android/dependency_analysis
 >>> 	1 class edge(s) comprising the dependency:
 >>> 		AboutChromeSettings -> ChromeVersionInfo
 >>> ...
-```
\ No newline at end of file
+```

tools/android/dependency_analysis/class_dependency.py

+# Lint as: python3
 # Copyright 2020 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 """Implementation of the graph module for a [Java class] dependency graph."""
 
 import re
-from typing import Tuple
+from typing import List, Tuple
 
 import graph
 import class_json_consts
@@ -51,6 +52,7 @@ class JavaClass(graph.Node):
         self._class_name = class_name
 
         self._nested_classes = set()
+        self._build_targets = set()
 
     @property
     def package(self):
@@ -70,13 +72,21 @@ class JavaClass(graph.Node):
         """A set of nested classes contained within this class."""
         return self._nested_classes
 
+    @property
+    def build_targets(self) -> List[str]:
+        """Which build target(s) contain the class."""
+        return self._build_targets
+
     @nested_classes.setter
     def nested_classes(self, other):
         self._nested_classes = other
 
-    def add_nested_class(self, nested: str):  # pylint: disable=missing-function-docstring
+    def add_nested_class(self, nested: str):
         self._nested_classes.add(nested)
 
+    def add_build_target(self, build_target: str) -> None:
+        self._build_targets.add(build_target)
+
     def get_node_metadata(self):
         """Generates JSON metadata for the current node.
 
@@ -85,12 +95,14 @@ class JavaClass(graph.Node):
         {
             'package': str,
             'class': str,
+            'build_targets' [ str, ... ]
             'nested_classes': [ class_key, ... ],
         }
         """
         return {
             class_json_consts.PACKAGE: self.package,
             class_json_consts.CLASS: self.class_name,
+            class_json_consts.BUILD_TARGETS: sorted(self.build_targets),
             class_json_consts.NESTED_CLASSES: sorted(self.nested_classes),
         }
 
@@ -106,6 +118,3 @@ class JavaClassDependencyGraph(graph.Graph):
         package = re_match.group('package')
         class_name = re_match.group('class_name')
         return JavaClass(package, class_name)
-
-    def add_nested_class_to_key(self, key: str, nested: str):  # pylint: disable=missing-function-docstring
-        self.get_node_by_key(key).add_nested_class(nested)

tools/android/dependency_analysis/class_dependency_unittest.py

@@ -107,14 +107,6 @@ class TestJavaClassDependencyGraph(unittest.TestCase):
         self.assertEqual(created_node.class_name, 'class')
         self.assertEqual(created_node.name, 'package.class')
 
-    def test_add_nested_class_to_key(self):
-        """Tests adding a nested class to an existing node."""
-        added = self.test_graph.add_node_if_new('package.class')
-        added.add_nested_class = unittest.mock.Mock()
-
-        self.test_graph.add_nested_class_to_key('package.class', 'nested')
-        added.add_nested_class.assert_called_once_with('nested')
-
 
 if __name__ == '__main__':
     unittest.main()

tools/android/dependency_analysis/class_json_consts.py

@@ -6,4 +6,5 @@
 # Node-specific constants
 PACKAGE = 'package'  # The package component of a class's full name.
 CLASS = 'class'  # The class component of a class's full name.
+BUILD_TARGETS = 'build_targets'  # Which build targets the class is in
 NESTED_CLASSES = 'nested_classes'  # Nested classes of a class.

tools/android/dependency_analysis/generate_json_dependency_graph.py

@@ -8,12 +8,15 @@ import argparse
 import pathlib
 import subprocess
 
+from typing import List, Tuple
+
 import class_dependency
 import package_dependency
 import serialization
 
 SRC_PATH = pathlib.Path(__file__).resolve().parents[3]  # src/
 JDEPS_PATH = SRC_PATH.joinpath('third_party/jdk/current/bin/jdeps')
+DEFAULT_ROOT_TARGET = 'chrome/android:monochrome_public_bundle'
 
 
 def class_is_interesting(name: str):
@@ -23,7 +26,8 @@ def class_is_interesting(name: str):
     return False
 
 
-class JavaClassJdepsParser(object):  # pylint: disable=useless-object-inheritance
+# pylint: disable=useless-object-inheritance
+class JavaClassJdepsParser(object):
     """A parser for jdeps class-level dependency output."""
     def __init__(self):  # pylint: disable=missing-function-docstring
         self._graph = class_dependency.JavaClassDependencyGraph()
@@ -36,12 +40,12 @@ class JavaClassJdepsParser(object):  # pylint: disable=useless-object-inheritanc
         """
         return self._graph
 
-    def parse_raw_jdeps_output(self, jdeps_output: str):
+    def parse_raw_jdeps_output(self, build_target: str, jdeps_output: str):
         """Parses the entirety of the jdeps output."""
         for line in jdeps_output.split('\n'):
-            self.parse_line(line)
+            self.parse_line(build_target, line)
 
-    def parse_line(self, line: str):
+    def parse_line(self, build_target: str, line: str):
         """Parses a line of jdeps output.
 
         The assumed format of the line starts with 'name_1 -> name_2'.
@@ -66,17 +70,21 @@ class JavaClassJdepsParser(object):  # pylint: disable=useless-object-inheritanc
         key_to, nested_to = class_dependency.split_nested_class_from_key(
             dep_to)
 
-        self._graph.add_node_if_new(key_from)
+        from_node: class_dependency.JavaClass = self._graph.add_node_if_new(
+            key_from)
+
         self._graph.add_node_if_new(key_to)
         if key_from != key_to:  # Skip self-edges (class-nested dependency)
             self._graph.add_edge_if_new(key_from, key_to)
         if nested_from is not None:
-            self._graph.add_nested_class_to_key(key_from, nested_from)
+            from_node.add_nested_class(nested_from)
         if nested_to is not None:
-            self._graph.add_nested_class_to_key(key_from, nested_to)
+            from_node.add_nested_class(nested_to)
+
+        from_node.add_build_target(build_target)
 
 
-def run_jdeps(jdeps_path: str, filepath: str):
+def _run_jdeps(jdeps_path: str, filepath: pathlib.Path):
     """Runs jdeps on the given filepath and returns the output."""
     jdeps_res = subprocess.run([jdeps_path, '-R', '-verbose:class', filepath],
                                capture_output=True,
@@ -85,17 +93,66 @@ def run_jdeps(jdeps_path: str, filepath: str):
     return jdeps_res.stdout
 
 
+def _run_gn_desc_list_dependencies(build_output_dir: str, target: str):
+    """Runs gn desc to list all jars that a target depends on.
+
+    This includes direct and indirect dependencies."""
+    gn_desc_res = subprocess.run(
+        ['gn', 'desc', '--all', build_output_dir, target, 'deps'],
+        capture_output=True,
+        text=True,
+        check=True)
+    return gn_desc_res.stdout
+
+
+JarTargetList = List[Tuple[str, pathlib.Path]]
+
+
+def list_original_targets_and_jars(gn_desc_output: str,
+                                   build_output_dir: str) -> JarTargetList:
+    """Parses gn desc output to list original java targets and output jar paths.
+
+    Returns a list of tuples (build_target: str, jar_path: str), where:
+    - build_target is the original java dependency target in the form
+      "//path/to:target"
+    - jar_path is the path to the built jar in the build_output_dir,
+      including the path to the output dir
+    """
+    jar_tuples: JarTargetList = []
+    for build_target_line in gn_desc_output.split('\n'):
+        if not build_target_line.endswith('__compile_java'):
+            continue
+        build_target = build_target_line.strip()
+        original_build_target = build_target.replace('__compile_java', '')
+        jar_path = _get_jar_path_for_target(build_output_dir, build_target)
+        jar_tuples.append((original_build_target, jar_path))
+    return jar_tuples
+
+
+def _get_jar_path_for_target(build_output_dir: str, build_target: str) -> str:
+    """Calculates the output location of a jar for a java build target."""
+    target_path, target_name = build_target.split(':')
+    assert target_path.startswith('//'), \
+        f'Build target should start with "//" but is: "{build_target}"'
+    jar_dir = target_path[len('//'):]
+    jar_name = target_name.replace('__compile_java', '.javac.jar')
+    return pathlib.Path(build_output_dir) / 'obj' / jar_dir / jar_name
+
+
 def main():
-    """Runs jdeps and creates a JSON file from the output."""
+    """Runs jdeps on all JARs a build target depends on.
+
+    Creates a JSON file from the jdeps output."""
     arg_parser = argparse.ArgumentParser(
-        description='Runs jdeps (dependency analysis tool) on a given JAR and '
-        'writes the resulting dependency graph into a JSON file.')
+        description='Runs jdeps (dependency analysis tool) on all JARs a root '
+        'build target depends on and writes the resulting dependency graph '
+        'into a JSON file. The default root build target is '
+        'chrome/android:monochrome_public_bundle.')
     required_arg_group = arg_parser.add_argument_group('required arguments')
-    required_arg_group.add_argument(
-        '-t',
-        '--target',
-        required=True,
-        help='Path to the JAR file to run jdeps on.')
+    required_arg_group.add_argument('-C',
+                                    '--build_output_dir',
+                                    required=True,
+                                    help='Build output directory.')
     required_arg_group.add_argument(
         '-o',
         '--output',
@@ -103,16 +160,28 @@ def main():
         help='Path to the file to write JSON output to. Will be created '
         'if it does not yet exist and overwrite existing '
         'content if it does.')
+    arg_parser.add_argument('-t',
+                            '--target',
+                            default=DEFAULT_ROOT_TARGET,
+                            help='Root build target.')
     arg_parser.add_argument('-j',
                             '--jdeps-path',
                             default=JDEPS_PATH,
                             help='Path to the jdeps executable.')
     arguments = arg_parser.parse_args()
 
+    print('Getting list of dependency jars...')
+    gn_desc_output = _run_gn_desc_list_dependencies(arguments.build_output_dir,
+                                                    arguments.target)
+    target_jars: JarTargetList = list_original_targets_and_jars(
+        gn_desc_output, arguments.build_output_dir)
+
     print('Running jdeps and parsing output...')
-    raw_jdeps_output = run_jdeps(arguments.jdeps_path, arguments.target)
     jdeps_parser = JavaClassJdepsParser()
-    jdeps_parser.parse_raw_jdeps_output(raw_jdeps_output)
+    for build_target, target_jar in target_jars:
+        print(f'Running jdeps and parsing output for {target_jar}')
+        raw_jdeps_output = _run_jdeps(arguments.jdeps_path, target_jar)
+        jdeps_parser.parse_raw_jdeps_output(build_target, raw_jdeps_output)
 
     class_graph = jdeps_parser.graph
     print(f'Parsed class-level dependency graph, '

tools/android/dependency_analysis/generate_json_dependency_graph_unittest.py

@@ -4,9 +4,26 @@
 # found in the LICENSE file.
 """Unit tests for dependency_analysis.generate_json_dependency_graph."""
 
+import pathlib
 import unittest
+
 import generate_json_dependency_graph
 
+GN_DESC_OUTPUT = """
+//path/to/dep1:java
+//path/to/dep1:java__build_config_crbug_908819
+//path/to/dep1:java__compile_java
+//path/to/dep1:java__dex
+//path/to/dep2:java
+//path/to/dep2:java__build_config_crbug_908819
+//path/to/dep2:java__compile_java
+//path/to/dep2:java__dex
+//path/to/root:java
+//path/to/root:java__build_config_crbug_908819
+//path/to/root:java__compile_java
+//path/to/root:java__dex
+"""
+
 
 class TestHelperFunctions(unittest.TestCase):
     """Unit tests for module-level helper functions."""
@@ -34,11 +51,31 @@ class TestHelperFunctions(unittest.TestCase):
             generate_json_dependency_graph.class_is_interesting(
                 'java.lang.Object'))
 
+    def test_list_original_targets_and_jars(self):
+        result = generate_json_dependency_graph.list_original_targets_and_jars(
+            GN_DESC_OUTPUT, 'out/Test')
+        self.assertEqual(len(result), 3)
+        self.assertEqual(
+            result[0],
+            ('//path/to/dep1:java',
+             pathlib.Path('out/Test/obj/path/to/dep1/java.javac.jar')))
+        self.assertEqual(
+            result[1],
+            ('//path/to/dep2:java',
+             pathlib.Path('out/Test/obj/path/to/dep2/java.javac.jar')))
+        self.assertEqual(
+            result[2],
+            ('//path/to/root:java',
+             pathlib.Path('out/Test/obj/path/to/root/java.javac.jar')))
+
 
 class TestJavaClassJdepsParser(unittest.TestCase):
     """Unit tests for
         dependency_analysis.class_dependency.JavaClassJdepsParser.
     """
+
+    BUILD_TARGET = '//build/target:1'
+
     def setUp(self):
         """Sets up a new JavaClassJdepsParser."""
         self.parser = generate_json_dependency_graph.JavaClassJdepsParser()
@@ -46,19 +83,20 @@ class TestJavaClassJdepsParser(unittest.TestCase):
     def test_parse_line(self):
         """Tests that new nodes + edges are added after a successful parse."""
         self.parser.parse_line(
+            self.BUILD_TARGET,
             'org.chromium.a -> org.chromium.b org.chromium.c')
         self.assertEqual(self.parser.graph.num_nodes, 2)
         self.assertEqual(self.parser.graph.num_edges, 1)
 
     def test_parse_line_not_interesting(self):
         """Tests that nothing is changed if there is an uninteresting class."""
-        self.parser.parse_line('org.chromium.a -> b c')
+        self.parser.parse_line(self.BUILD_TARGET, 'org.chromium.a -> b c')
         self.assertEqual(self.parser.graph.num_nodes, 0)
         self.assertEqual(self.parser.graph.num_edges, 0)
 
     def test_parse_line_too_short(self):
         """Tests that nothing is changed if the line is too short."""
-        self.parser.parse_line('org.chromium.a -> b')
+        self.parser.parse_line(self.BUILD_TARGET, 'org.chromium.a -> b')
         self.assertEqual(self.parser.graph.num_nodes, 0)
         self.assertEqual(self.parser.graph.num_edges, 0)
 
@@ -66,19 +104,20 @@ class TestJavaClassJdepsParser(unittest.TestCase):
         """Tests that nothing is changed if the line contains `not found`
             as the second class.
         """
-        self.parser.parse_line('org.chromium.a -> not found')
+        self.parser.parse_line(self.BUILD_TARGET,
+                               'org.chromium.a -> not found')
         self.assertEqual(self.parser.graph.num_nodes, 0)
         self.assertEqual(self.parser.graph.num_edges, 0)
 
     def test_parse_line_empty_string(self):
         """Tests that nothing is changed if the line is empty."""
-        self.parser.parse_line('')
+        self.parser.parse_line(self.BUILD_TARGET, '')
         self.assertEqual(self.parser.graph.num_nodes, 0)
         self.assertEqual(self.parser.graph.num_edges, 0)
 
     def test_parse_line_bad_input(self):
         """Tests that nothing is changed if the line is nonsensical"""
-        self.parser.parse_line('bad_input')
+        self.parser.parse_line(self.BUILD_TARGET, 'bad_input')
         self.assertEqual(self.parser.graph.num_nodes, 0)
         self.assertEqual(self.parser.graph.num_edges, 0)
 

tools/android/dependency_analysis/serialization_unittest.py

@@ -20,6 +20,8 @@ class TestSerialization(unittest.TestCase):
     CLASS_1 = 'p1.c1'
     CLASS_2 = 'p1.c2'
     CLASS_3 = 'p2.c3'
+    BUILD_TARGET_1 = '//build/target:one'
+    BUILD_TARGET_2 = '//build/target:two'
     CLASS_1_NESTED_1 = 'abc'
     CLASS_1_NESTED_2 = 'def'
     CLASS_2_NESTED_1 = 'ghi'
@@ -35,6 +37,7 @@ class TestSerialization(unittest.TestCase):
                     'p1',
                     class_json_consts.CLASS:
                     'c1',
+                    class_json_consts.BUILD_TARGETS: [BUILD_TARGET_1],
                     class_json_consts.NESTED_CLASSES:
                     [CLASS_1_NESTED_1, CLASS_1_NESTED_2],
                 },
@@ -44,14 +47,19 @@ class TestSerialization(unittest.TestCase):
                 json_consts.META: {
                     class_json_consts.PACKAGE: 'p1',
                     class_json_consts.CLASS: 'c2',
+                    class_json_consts.BUILD_TARGETS: [],
                     class_json_consts.NESTED_CLASSES: [CLASS_2_NESTED_1],
                 },
             },
             {
                 json_consts.NAME: CLASS_3,
                 json_consts.META: {
-                    class_json_consts.PACKAGE: 'p2',
-                    class_json_consts.CLASS: 'c3',
+                    class_json_consts.PACKAGE:
+                    'p2',
+                    class_json_consts.CLASS:
+                    'c3',
+                    class_json_consts.BUILD_TARGETS:
+                    [BUILD_TARGET_1, BUILD_TARGET_2],
                     class_json_consts.NESTED_CLASSES: [],
                 },
             },
@@ -116,9 +124,18 @@ class TestSerialization(unittest.TestCase):
         test_graph.add_edge_if_new(self.CLASS_1, self.CLASS_2)
         test_graph.add_edge_if_new(self.CLASS_1, self.CLASS_3)
         test_graph.add_edge_if_new(self.CLASS_2, self.CLASS_3)
-        test_graph.add_nested_class_to_key(self.CLASS_1, self.CLASS_1_NESTED_1)
-        test_graph.add_nested_class_to_key(self.CLASS_1, self.CLASS_1_NESTED_2)
-        test_graph.add_nested_class_to_key(self.CLASS_2, self.CLASS_2_NESTED_1)
+        test_graph.get_node_by_key(self.CLASS_1).add_nested_class(
+            self.CLASS_1_NESTED_1)
+        test_graph.get_node_by_key(self.CLASS_1).add_nested_class(
+            self.CLASS_1_NESTED_2)
+        test_graph.get_node_by_key(self.CLASS_2).add_nested_class(
+            self.CLASS_2_NESTED_1)
+        test_graph.get_node_by_key(self.CLASS_1).add_build_target(
+            self.BUILD_TARGET_1)
+        test_graph.get_node_by_key(self.CLASS_3).add_build_target(
+            self.BUILD_TARGET_1)
+        test_graph.get_node_by_key(self.CLASS_3).add_build_target(
+            self.BUILD_TARGET_2)
 
         test_json_obj = serialization.create_json_obj_from_graph(test_graph)
 
@@ -130,12 +147,12 @@ class TestSerialization(unittest.TestCase):
         class_graph.add_edge_if_new(self.CLASS_1, self.CLASS_2)
         class_graph.add_edge_if_new(self.CLASS_1, self.CLASS_3)
         class_graph.add_edge_if_new(self.CLASS_2, self.CLASS_3)
-        class_graph.add_nested_class_to_key(self.CLASS_1,
-                                            self.CLASS_1_NESTED_1)
-        class_graph.add_nested_class_to_key(self.CLASS_1,
-                                            self.CLASS_1_NESTED_2)
-        class_graph.add_nested_class_to_key(self.CLASS_2,
-                                            self.CLASS_2_NESTED_1)
+        class_graph.get_node_by_key(self.CLASS_1).add_nested_class(
+            self.CLASS_1_NESTED_1)
+        class_graph.get_node_by_key(self.CLASS_1).add_nested_class(
+            self.CLASS_1_NESTED_2)
+        class_graph.get_node_by_key(self.CLASS_2).add_nested_class(
+            self.CLASS_2_NESTED_1)
 
         package_graph = package_dependency.JavaPackageDependencyGraph(
             class_graph)