generate_gradle.py: Allow multiple --target and add --all

While --all does what it says, Android Studio was not happy trying to
import the swath of generated projects. The flag will be available only
for the adventurous :).

BUG=620034
NOTRY=true

Review-Url: https://codereview.chromium.org/2362613002
Cr-Commit-Position: refs/heads/master@{#420426}

build/android/gradle/build.gradle.jinja

@@ -19,7 +19,11 @@ apply plugin: "java"
 
 sourceSets {
     main {
-        java.srcDirs = {{ java_dirs }}
+        java.srcDirs = [
+{% for path in java_dirs %}
+            "{{ path }}",
+{% endfor %}
+        ]
     }
 }
 

build/android/gradle/generate_gradle.py

@@ -9,6 +9,7 @@ import argparse
 import codecs
 import logging
 import os
+import re
 import shutil
 import subprocess
 import sys
@@ -34,6 +35,15 @@ _JINJA_TEMPLATE_PATH = os.path.join(
 _JAVA_SUBDIR = 'symlinked-java'
 _SRCJARS_SUBDIR = 'extracted-srcjars'
 
+_DEFAULT_TARGETS = [
+    '//android_webview:system_webview_apk',
+    '//android_webview/test:android_webview_apk',
+    '//android_webview/test:android_webview_test_apk',
+    '//chrome/android:chrome_public_apk',
+    '//chrome/android:chrome_public_test_apk',
+    '//chrome/android:chrome_sync_shell_apk',
+    '//chrome/android:chrome_sync_shell_test_apk',
+]
 
 def _RebasePath(path_or_list, new_cwd=None, old_cwd=None):
   """Makes the given path(s) relative to new_cwd, or absolute if not specified.
@@ -67,13 +77,28 @@ def _WriteFile(path, data):
     output_file.write(data)
 
 
-def _RunNinja(output_dir, ninja_targets):
+def _RunNinja(output_dir, args):
   cmd = ['ninja', '-C', output_dir, '-j50']
-  cmd.extend(ninja_targets)
+  cmd.extend(args)
   logging.info('Running: %r', cmd)
   subprocess.check_call(cmd)
 
 
+def _QueryForAllGnTargets(output_dir):
+  # Query ninja rather than GN since it's faster.
+  cmd = ['ninja', '-C', output_dir, '-t', 'targets']
+  logging.info('Running: %r', cmd)
+  ninja_output = build_utils.CheckOutput(cmd)
+  ret = []
+  SUFFIX_LEN = len('__build_config')
+  for line in ninja_output.splitlines():
+    ninja_target = line.rsplit(':', 1)[0]
+    # Ignore root aliases by ensure a : exists.
+    if ':' in ninja_target and ninja_target.endswith('__build_config'):
+      ret.append('//' + ninja_target[:-SUFFIX_LEN])
+  return ret
+
+
 class _ProjectEntry(object):
   """Helper class for various path transformations."""
   def __init__(self, gn_target):
@@ -164,7 +189,7 @@ def _CreateSymlinkTree(entry_output_dir, symlink_dir, desired_files,
     os.symlink(relpath, symlinked_path)
 
 
-def _CreateJavaSourceDir(entry_output_dir, java_sources_file):
+def _CreateJavaSourceDir(output_dir, entry_output_dir, java_sources_file):
   """Computes and constructs when necessary the list of java source directories.
 
   1. Computes the root java source directories from the list of files.
@@ -182,11 +207,15 @@ def _CreateJavaSourceDir(entry_output_dir, java_sources_file):
     found_java_files = build_utils.FindInDirectories(java_dirs, '*.java')
     unwanted_java_files = set(found_java_files) - set(java_files)
     missing_java_files = set(java_files) - set(found_java_files)
+    # Warn only about non-generated files that are missing.
+    missing_java_files = [p for p in missing_java_files
+                          if not p.startswith(output_dir)]
     if unwanted_java_files:
       logging.debug('Target requires .java symlinks: %s', entry_output_dir)
       symlink_dir = os.path.join(entry_output_dir, _JAVA_SUBDIR)
       _CreateSymlinkTree(entry_output_dir, symlink_dir, java_files, java_dirs)
       java_dirs = [symlink_dir]
+
     if missing_java_files:
       logging.warning('Some java files were not found: %s', missing_java_files)
 
@@ -202,7 +231,7 @@ def _GenerateLocalProperties(sdk_dir):
 
 
 def _GenerateGradleFile(build_config, config_json, java_dirs, relativize,
-                        use_gradle_process_resources):
+                        use_gradle_process_resources, jinja_processor):
   """Returns the data for a project's build.gradle."""
   deps_info = build_config['deps_info']
   gradle = build_config['gradle']
@@ -235,15 +264,13 @@ def _GenerateGradleFile(build_config, config_json, java_dirs, relativize,
           for p in gradle['dependent_java_projects']]
   variables['java_project_deps'] = [d.ProjectName() for d in deps]
 
-  processor = jinja_template.JinjaProcessor(host_paths.DIR_SOURCE_ROOT)
-  return processor.Render(_JINJA_TEMPLATE_PATH, variables)
+  return jinja_processor.Render(_JINJA_TEMPLATE_PATH, variables)
 
 
-def _GenerateRootGradle():
+def _GenerateRootGradle(jinja_processor):
   """Returns the data for the root project's build.gradle."""
   variables = {'template_type': 'root'}
-  processor = jinja_template.JinjaProcessor(host_paths.DIR_SOURCE_ROOT)
-  return processor.Render(_JINJA_TEMPLATE_PATH, variables)
+  return jinja_processor.Render(_JINJA_TEMPLATE_PATH, variables)
 
 
 def _GenerateSettingsGradle(project_entries):
@@ -277,10 +304,10 @@ def _ExtractSrcjars(entry_output_dir, srcjar_tuples):
       z.extractall(extracted_path)
 
 
-def _FindAllProjectEntries(main_entry):
+def _FindAllProjectEntries(main_entries):
   """Returns the list of all _ProjectEntry instances given the root project."""
   found = set()
-  to_scan = [main_entry]
+  to_scan = list(main_entries)
   while to_scan:
     cur_entry = to_scan.pop()
     if cur_entry in found:
@@ -304,11 +331,16 @@ def main():
                       action='count',
                       help='Verbose level')
   parser.add_argument('--target',
-                      help='GN target to generate project for.',
-                      default='//chrome/android:chrome_public_test_apk')
+                      dest='targets',
+                      action='append',
+                      help='GN target to generate project for. '
+                           'May be repeated.')
   parser.add_argument('--project-dir',
                       help='Root of the output project.',
                       default=os.path.join('$CHROMIUM_OUTPUT_DIR', 'gradle'))
+  parser.add_argument('--all',
+                      action='store_true',
+                      help='Generate all java targets (slows down IDE)')
   parser.add_argument('--use-gradle-process-resources',
                       action='store_true',
                       help='Have gradle generate R.java rather than ninja')
@@ -324,18 +356,26 @@ def main():
       args.project_dir.replace('$CHROMIUM_OUTPUT_DIR', output_dir))
   logging.warning('Creating project at: %s', gradle_output_dir)
 
-  target = args.target
-  # TODO(agrieve): See if it makes sense to utilize Gradle's test constructs for
-  # our instrumentation tests.
-  if target.endswith('_test_apk'):
-    target += '__apk'
-  main_entry = _ProjectEntry(target)
+  if args.all:
+    # Run GN gen if necessary (faster than running "gn gen" in the no-op case).
+    _RunNinja(output_dir, ['build.ninja'])
+    # Query ninja for all __build_config targets.
+    targets = _QueryForAllGnTargets(output_dir)
+  else:
+    targets = args.targets or _DEFAULT_TARGETS
+    # TODO(agrieve): See if it makes sense to utilize Gradle's test constructs
+    # for our instrumentation tests.
+    targets = [re.sub(r'_test_apk$', '_test_apk__apk', t) for t in targets]
+
+  main_entries = [_ProjectEntry(t) for t in targets]
   logging.warning('Building .build_config files...')
-  _RunNinja(output_dir, [main_entry.NinjaBuildConfigTarget()])
+  _RunNinja(output_dir, [e.NinjaBuildConfigTarget() for e in main_entries])
 
-  all_entries = _FindAllProjectEntries(main_entry)
+  all_entries = _FindAllProjectEntries(main_entries)
   logging.info('Found %d dependent build_config targets.', len(all_entries))
 
+  logging.warning('Writing .gradle files...')
+  jinja_processor = jinja_template.JinjaProcessor(host_paths.DIR_SOURCE_ROOT)
   config_json = build_utils.ReadJson(
       os.path.join(output_dir, 'gradle', 'config.json'))
   project_entries = []
@@ -356,12 +396,14 @@ def main():
     if java_sources_file:
       java_sources_file = _RebasePath(java_sources_file)
 
-    java_dirs = _CreateJavaSourceDir(entry_output_dir, java_sources_file)
+    java_dirs = _CreateJavaSourceDir(output_dir, entry_output_dir,
+                                     java_sources_file)
     if srcjars:
       java_dirs.append(os.path.join(entry_output_dir, _SRCJARS_SUBDIR))
 
     data = _GenerateGradleFile(build_config, config_json, java_dirs, relativize,
-                               args.use_gradle_process_resources)
+                               args.use_gradle_process_resources,
+                               jinja_processor)
     if data:
       project_entries.append(entry)
       srcjar_tuples.extend(
@@ -369,7 +411,7 @@ def main():
       _WriteFile(os.path.join(entry_output_dir, 'build.gradle'), data)
 
   _WriteFile(os.path.join(gradle_output_dir, 'build.gradle'),
-             _GenerateRootGradle())
+             _GenerateRootGradle(jinja_processor))
 
   _WriteFile(os.path.join(gradle_output_dir, 'settings.gradle'),
              _GenerateSettingsGradle(project_entries))

build/android/gyp/jinja_template.py

@@ -44,10 +44,14 @@ class JinjaProcessor(object):
     self.env.line_comment_prefix = '##'
     self.env.trim_blocks = True
     self.env.lstrip_blocks = True
+    self._template_cache = {}  # Map of path -> Template
 
   def Render(self, input_filename, variables=None):
     input_rel_path = os.path.relpath(input_filename, self.loader_base_dir)
-    template = self.env.get_template(input_rel_path)
+    template = self._template_cache.get(input_rel_path)
+    if not template:
+      template = self.env.get_template(input_rel_path)
+      self._template_cache[input_rel_path] = template
     return template.render(variables or self.variables)
 
   def GetLoadedTemplates(self):

docs/android_studio.md

@@ -5,10 +5,21 @@
 ## Usage
 
 ```shell
-build/android/gradle/generate_gradle.py --output-directory out-gn/Debug --target //chrome/android:chrome_public_test_apk
+build/android/gradle/generate_gradle.py --output-directory out-gn/Debug
 ```
 
-This creates a project at `out-gn/Debug/gradle`. To create elsewhere: `--project-dir foo`
+This creates a project at `out-gn/Debug/gradle`. To create elsewhere:
+
+```shell
+build/android/gradle/generate_gradle.py --output-directory out-gn/Debug --project-dir my-project
+```
+
+By default, only common targets are generated. To customize the list of targets
+to generate projects for:
+
+```shell
+build/android/gradle/generate_gradle.py --output-directory out-gn/Debug --target //some:target_apk --target //some/other:target_apk
+```
 
 For first-time Android Studio users: