Add support for Java xrefs

Bug: 994805
Change-Id: I367fe4c751d494fff091860eca4329659a72a05b
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2062798
Commit-Queue: Josip Sokcevic <sokcevic@google.com>
Reviewed-by: Andrew Grieve <agrieve@chromium.org>
Reviewed-by: Scott Violet <sky@chromium.org>
Cr-Commit-Position: refs/heads/master@{#744845}

DEPS

@@ -30,6 +30,7 @@ gclient_gn_args_file = 'src/build/config/gclient_args.gni'
 gclient_gn_args = [
   'build_with_chromium',
   'checkout_android',
+  'checkout_android_prebuilts_build_tools',
   'checkout_android_native_support',
   'checkout_google_benchmark',
   'checkout_ios_webkit',
@@ -57,6 +58,9 @@ vars = {
   # TODO(ehmaldonado): Remove this once the bug in gclient is fixed.
   'checkout_android': False,
 
+  # Pull in Android prebuilts build tools so we can create Java xrefs
+  'checkout_android_prebuilts_build_tools': False,
+
   # Pull in Android native toolchain dependencies for Chrome OS too, so we can
   # build ARC++ support libraries.
   'checkout_android_native_support': 'checkout_android or checkout_chromeos',
@@ -1522,6 +1526,11 @@ deps = {
       'dep_type': 'cipd',
   },
 
+  'src/third_party/android_prebuilts/build_tools': {
+      'url': Var('android_git') + '/platform/prebuilts/build-tools.git' + '@' + '5794e96eb8bae47bb48feee915d99583573b3887',
+      'condition': 'checkout_android_prebuilts_build_tools',
+  },
+
   # === ANDROID_DEPS Generated Code Start ===
   # Generated by //third_party/android_deps/fetch_all.py
   'src/third_party/android_deps/libs/android_arch_core_common': {

build/android/gyp/compile_java.py

@@ -23,6 +23,10 @@ sys.path.insert(
     os.path.join(build_utils.DIR_SOURCE_ROOT, 'third_party', 'colorama', 'src'))
 import colorama
 
+_JAVAC_EXTRACTOR = os.path.join(build_utils.DIR_SOURCE_ROOT, 'third_party',
+                                'android_prebuilts', 'build_tools', 'common',
+                                'framework', 'javac_extractor.jar')
+
 # Full list of checks: https://errorprone.info/bugpatterns
 ERRORPRONE_WARNINGS_TO_TURN_OFF = [
     # This one should really be turned on.
@@ -355,15 +359,48 @@ def _CreateJarFile(jar_path, provider_configurations, additional_jar_files,
   logging.info('Completed jar file: %s', jar_path)
 
 
-def _OnStaleMd5(options, javac_cmd, java_files, classpath):
+def _OnStaleMd5(options, javac_cmd, javac_args, java_files):
   logging.info('Starting _OnStaleMd5')
+  if options.enable_kythe_annotations:
+    # Kythe requires those env variables to be set and compile_java.py does the
+    # same
+    if not os.environ.get('KYTHE_ROOT_DIRECTORY') or \
+        not os.environ.get('KYTHE_OUTPUT_DIRECTORY'):
+      raise Exception('--enable-kythe-annotations requires '
+                      'KYTHE_ROOT_DIRECTORY and KYTHE_OUTPUT_DIRECTORY '
+                      'environment variables to be set.')
+    javac_extractor_cmd = [
+        build_utils.JAVA_PATH,
+        '-jar',
+        _JAVAC_EXTRACTOR,
+    ]
+    try:
+      _RunCompiler(options, javac_extractor_cmd + javac_args, java_files,
+                   options.classpath, options.jar_path + '.javac_extractor',
+                   save_outputs=False),
+    except build_utils.CalledProcessError as e:
+      # Having no index for particular target is better than failing entire
+      # codesearch. Log and error and move on.
+      logging.error('Could not generate kzip: %s', e)
+
+  # Compiles with Error Prone take twice as long to run as pure javac. Thus GN
+  # rules run both in parallel, with Error Prone only used for checks.
+  _RunCompiler(options, javac_cmd + javac_args, java_files,
+               options.classpath, options.jar_path,
+               save_outputs=not options.enable_errorprone)
+  logging.info('Completed all steps in _OnStaleMd5')
+
+
+def _RunCompiler(options, javac_cmd, java_files, classpath, jar_path,
+                 save_outputs=True):
+  logging.info('Starting _RunCompiler')
 
   # Compiles with Error Prone take twice as long to run as pure javac. Thus GN
   # rules run both in parallel, with Error Prone only used for checks.
   save_outputs = not options.enable_errorprone
 
   # Use jar_path's directory to ensure paths are relative (needed for goma).
-  temp_dir = options.jar_path + '.staging'
+  temp_dir = jar_path + '.staging'
   shutil.rmtree(temp_dir, True)
   os.makedirs(temp_dir)
   try:
@@ -431,14 +468,14 @@ def _OnStaleMd5(options, javac_cmd, java_files, classpath):
       if annotation_processor_java_files:
         info_file_context.SubmitFiles(annotation_processor_java_files)
 
-      _CreateJarFile(options.jar_path, options.provider_configurations,
+      _CreateJarFile(jar_path, options.provider_configurations,
                      options.additional_jar_files, classes_dir)
 
-      info_file_context.Commit(options.jar_path + '.info')
+      info_file_context.Commit(jar_path + '.info')
     else:
-      build_utils.Touch(options.jar_path)
+      build_utils.Touch(jar_path)
 
-    logging.info('Completed all steps in _OnStaleMd5')
+    logging.info('Completed all steps in _RunCompiler')
   finally:
     shutil.rmtree(temp_dir)
 
@@ -519,6 +556,11 @@ def _ParseOptions(argv):
       action='append',
       default=[],
       help='Additional arguments to pass to javac.')
+  parser.add_option(
+      '--enable-kythe-annotations',
+      action='store_true',
+      help='Enable generation of Kythe kzip, used for codesearch. Ensure '
+      'proper environment variables are set before using this flag.')
 
   options, args = parser.parse_args(argv)
   build_utils.CheckOptions(options, parser, required=('jar_path', ))
@@ -554,14 +596,13 @@ def main(argv):
 
   argv = build_utils.ExpandFileArgs(argv)
   options, java_files = _ParseOptions(argv)
-  javac_path = build_utils.JAVAC_PATH
 
   javac_cmd = []
   if options.gomacc_path:
     javac_cmd.append(options.gomacc_path)
+  javac_cmd.append(build_utils.JAVAC_PATH)
 
-  javac_cmd += [
-      javac_path,
+  javac_args = [
       '-g',
       # Chromium only allows UTF8 source files.  Being explicit avoids
       # javac pulling a default encoding from the user's environment.
@@ -582,10 +623,10 @@ def main(argv):
       errorprone_flags.append('-Xep:{}:ERROR'.format(warning))
     if not options.warnings_as_errors:
       errorprone_flags.append('-XepAllErrorsAsWarnings')
-    javac_cmd += ['-XDcompilePolicy=simple', ' '.join(errorprone_flags)]
+    javac_args += ['-XDcompilePolicy=simple', ' '.join(errorprone_flags)]
 
   if options.java_version:
-    javac_cmd.extend([
+    javac_args.extend([
         '-source',
         options.java_version,
         '-target',
@@ -596,26 +637,26 @@ def main(argv):
     options.bootclasspath.append(build_utils.RT_JAR_PATH)
 
   if options.warnings_as_errors:
-    javac_cmd.extend(['-Werror'])
+    javac_args.extend(['-Werror'])
   else:
     # XDignore.symbol.file makes javac compile against rt.jar instead of
     # ct.sym. This means that using a java internal package/class will not
     # trigger a compile warning or error.
-    javac_cmd.extend(['-XDignore.symbol.file'])
+    javac_args.extend(['-XDignore.symbol.file'])
 
   if options.processors:
-    javac_cmd.extend(['-processor', ','.join(options.processors)])
+    javac_args.extend(['-processor', ','.join(options.processors)])
 
   if options.bootclasspath:
-    javac_cmd.extend(['-bootclasspath', ':'.join(options.bootclasspath)])
+    javac_args.extend(['-bootclasspath', ':'.join(options.bootclasspath)])
 
   if options.processorpath:
-    javac_cmd.extend(['-processorpath', ':'.join(options.processorpath)])
+    javac_args.extend(['-processorpath', ':'.join(options.processorpath)])
   if options.processor_args:
     for arg in options.processor_args:
-      javac_cmd.extend(['-A%s' % arg])
+      javac_args.extend(['-A%s' % arg])
 
-  javac_cmd.extend(options.javac_arg)
+  javac_args.extend(options.javac_arg)
 
   classpath_inputs = (
       options.bootclasspath + options.classpath + options.processorpath)
@@ -631,11 +672,12 @@ def main(argv):
       options.jar_path + '.info',
   ]
 
-  input_strings = javac_cmd + options.classpath + java_files
+  input_strings = javac_cmd + javac_args + options.classpath + java_files
   if options.jar_info_exclude_globs:
     input_strings.append(options.jar_info_exclude_globs)
+
   md5_check.CallAndWriteDepfileIfStale(
-      lambda: _OnStaleMd5(options, javac_cmd, java_files, options.classpath),
+      lambda: _OnStaleMd5(options, javac_cmd, javac_args, java_files),
       options,
       depfile_deps=depfile_deps,
       input_paths=input_paths,

build/config/android/internal_rules.gni

@@ -11,6 +11,7 @@ import("//build/config/dcheck_always_on.gni")
 import("//build/config/python.gni")
 import("//build/config/sanitizers/sanitizers.gni")
 import("//build/toolchain/goma.gni")
+import("//build/toolchain/kythe.gni")
 import("//build/util/generate_wrapper.gni")
 import("//build_overrides/build.gni")
 assert(is_android)
@@ -2796,6 +2797,11 @@ if (enable_java_templates) {
         # Override the default action_pool when goma is enabled.
         pool = "//build/config/android:goma_javac_pool"
       }
+      # Flag enable_kythe_annotations requires
+      # checkout_android_prebuilts_build_tools=True in .gclient.
+      if (enable_kythe_annotations && !invoker.enable_errorprone) {
+        args += [ "--enable-kythe-annotations" ]
+      }
       if (defined(invoker.srcjar_filearg)) {
         args += [ "--java-srcjars=${invoker.srcjar_filearg}" ]
       }

build/toolchain/kythe.gni

+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This file defines configuration for Kythe, an indexer and cross-referencer
+# that powers codesearch.
+
+declare_args() {
+  # Enables Kythe annotations necessary to build cross references.
+  enable_kythe_annotations = false
+}

mojo/public/tools/bindings/mojom.gni

@@ -15,6 +15,7 @@ import("//ui/webui/webui_features.gni")
 import("//build/config/chrome_build.gni")
 import("//build/config/chromecast_build.gni")
 import("//build/config/nacl/config.gni")
+import("//build/toolchain/kythe.gni")
 import("//components/nacl/features.gni")
 import("//third_party/jinja2/jinja2.gni")
 import("//tools/ipc_fuzzer/ipc_fuzzer.gni")
@@ -40,10 +41,6 @@ declare_args() {
   # corresponding "foo_js_library_for_compile" target generated.
   enable_mojom_closure_compile = closure_compile && optimize_webui
 
-  # Enable adding annotations to generated C++ headers that are used for
-  # cross-references in CodeSearch.
-  enable_kythe_annotations = false
-
   # Enables generating Typescript bindings and compiling them to JS bindings.
   enable_typescript_bindings = false
 }
@@ -709,6 +706,8 @@ template("mojom") {
         ]
       }
 
+      # Enable adding annotations to generated C++ headers that are used for
+      # cross-references in CodeSearch.
       if (enable_kythe_annotations) {
         args += [ "--enable_kythe_annotations" ]
       }

third_party/.gitignore

@@ -14,10 +14,11 @@
 /android_build_tools/art/profman
 /android_build_tools/bundletool/*.jar
 /android_ndk/
+/android_prebuilts/build_tools
+/android_protobuf/src
 /android_sdk/androidx_browser/src
 /android_sdk/public/
 /android_sdk/sources/
-/android_protobuf/src
 /android_support_test_runner/lib/
 /android_system_sdk/*.jar
 /android_tools_internal/