[Build] Add Android build integration tests

This CL introduces android_nocompile_test_suite() and
android_nocompile_library() build rules.

The android_nocompile_test_suite() template tries to compile its
dependent android_nocompile_library() targets and checks that:
- The compile fails
- The compile output contains the error message specified in the
android_nocompile_library() template. The android_nocompile_library()
uses a temporary output directory so that compiling "all" still works.

The CL converts |errorprone_plugin_tests| to use
android_nocompile_test_suite().

BUG=1132014

Change-Id: I80a372a06a1fa7fa54546ac3ca0a1c48e8ee96ba
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2538313Reviewed-by: Dirk Pranke <dpranke@google.com>
Reviewed-by: Andrew Grieve <agrieve@chromium.org>
Commit-Queue: Peter Kotwicz <pkotwicz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#832489}

BUILD.gn

@@ -357,7 +357,7 @@ group("gn_all") {
       "//tools/android:push_apps_to_background",
       "//tools/android/audio_focus_grabber:audio_focus_grabber_apk",
       "//tools/android/customtabs_benchmark:customtabs_benchmark_apk",
-      "//tools/android/errorprone_plugin:errorprone_plugin_tests",
+      "//tools/android/errorprone_plugin/test:errorprone_plugin_tests",
       "//tools/android/kerberos/SpnegoAuthenticator:spnego_authenticator_apk",
       "//ui/android:ui_junit_tests",
       "//weblayer/public/java:client_aar",

build/android/gyp/compile_java.py

@@ -245,16 +245,6 @@ def ProcessJavacOutput(output):
   return '\n'.join(lines)
 
 
-def CheckErrorproneStderrWarning(jar_path, expected_warning_regex,
-                                 javac_output):
-  if not re.search(expected_warning_regex, javac_output):
-    raise Exception('Expected `{}` warning when compiling `{}`'.format(
-        expected_warning_regex, os.path.basename(jar_path)))
-
-  # Do not print warning
-  return ''
-
-
 def _ParsePackageAndClassNames(java_file):
   package_name = ''
   class_names = []
@@ -502,22 +492,12 @@ def _RunCompiler(options, javac_cmd, java_files, classpath, jar_path,
         f.write(' '.join(java_files))
       cmd += ['@' + java_files_rsp_path]
 
-
-      # |errorprone_expected_warning_regex| is used in tests for errorprone
-      # warnings. Fail compile if expected warning is not present.
-      stderr_filter = ProcessJavacOutput
-      if (options.enable_errorprone
-          and options.errorprone_expected_warning_regex):
-        stderr_filter = functools.partial(
-            CheckErrorproneStderrWarning, options.jar_path,
-            options.errorprone_expected_warning_regex)
-
       logging.debug('Build command %s', cmd)
       start = time.time()
       build_utils.CheckOutput(cmd,
                               print_stdout=options.chromium_code,
                               stdout_filter=ProcessJavacOutput,
-                              stderr_filter=stderr_filter,
+                              stderr_filter=ProcessJavacOutput,
                               fail_on_output=options.warnings_as_errors)
       end = time.time() - start
       logging.info('Java compilation took %ss', end)
@@ -601,10 +581,6 @@ def _ParseOptions(argv):
       '--enable-errorprone',
       action='store_true',
       help='Enable errorprone checks')
-  parser.add_option(
-      '--errorprone-expected-warning-regex',
-      help='When set, throws an exception if the errorprone compile does not '
-      'log a warning which matches the regex.')
   parser.add_option(
       '--warnings-as-errors',
       action='store_true',

build/android/gyp/nocompile_test.py

+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Checks that compiling targets in BUILD.gn file fails."""
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+from util import build_utils
+
+_CHROMIUM_SRC = os.path.normpath(os.path.join(__file__, '..', '..', '..', '..'))
+_NINJA_PATH = os.path.join(_CHROMIUM_SRC, 'third_party', 'depot_tools', 'ninja')
+
+# Relative to _CHROMIUM_SRC
+_GN_SRC_REL_PATH = os.path.join('third_party', 'depot_tools', 'gn')
+
+
+def _raise_command_exception(args, returncode, output):
+  """Raises an exception whose message describes a command failure.
+
+    Args:
+      args: shell command-line (as passed to subprocess.Popen())
+      returncode: status code.
+      output: command output.
+    Raises:
+      a new Exception.
+    """
+  message = 'Command failed with status {}: {}\n' \
+      'Output:-----------------------------------------\n{}\n' \
+      '------------------------------------------------\n'.format(
+          returncode, args, output)
+  raise Exception(message)
+
+
+def _run_command(args, cwd=None):
+  """Runs shell command. Raises exception if command fails."""
+  p = subprocess.Popen(args,
+                       stdout=subprocess.PIPE,
+                       stderr=subprocess.STDOUT,
+                       cwd=cwd)
+  pout, _ = p.communicate()
+  if p.returncode != 0:
+    _raise_command_exception(args, p.returncode, pout)
+
+
+def _run_command_get_output(args, success_output):
+  """Runs shell command and returns command output."""
+  p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+  pout, _ = p.communicate()
+  if p.returncode == 0:
+    return success_output
+
+  # For Python3 only:
+  if isinstance(pout, bytes) and sys.version_info >= (3, ):
+    pout = pout.decode('utf-8')
+  return pout
+
+
+def _copy_and_append_gn_args(src_args_path, dest_args_path, extra_args):
+  """Copies args.gn.
+
+    Args:
+      src_args_path: args.gn file to copy.
+      dest_args_path: Copy file destination.
+      extra_args: Text to append to args.gn after copy.
+    """
+  with open(src_args_path) as f_in, open(dest_args_path, 'w') as f_out:
+    f_out.write(f_in.read())
+    f_out.write('\n')
+    f_out.write('\n'.join(extra_args))
+
+
+def _find_lines_after_prefix(text, prefix, num_lines):
+  """Searches |text| for a line which starts with |prefix|.
+
+  Args:
+    text: String to search in.
+    prefix: Prefix to search for.
+    num_lines: Number of lines, starting with line with prefix, to return.
+  Returns:
+    Matched lines. Returns None otherwise.
+  """
+  lines = text.split('\n')
+  for i, line in enumerate(lines):
+    if line.startswith(prefix):
+      return '\n'.join(lines[i:i + num_lines])
+  return None
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--gn-args-path',
+                      required=True,
+                      help='Path to args.gn file.')
+  parser.add_argument('--test-configs-path',
+                      required=True,
+                      help='Path to file with test configurations')
+  parser.add_argument('--out-dir',
+                      required=True,
+                      help='Path to output directory to use for compilation.')
+  parser.add_argument('--stamp', help='Path to touch.')
+  options = parser.parse_args()
+
+  with open(options.test_configs_path) as f:
+    test_configs = json.loads(f.read())
+
+  if not os.path.exists(options.out_dir):
+    os.makedirs(options.out_dir)
+
+  out_gn_args_path = os.path.join(options.out_dir, 'args.gn')
+  extra_gn_args = [
+      'enable_android_nocompile_tests = true', 'treat_warnings_as_errors = true'
+  ]
+  _copy_and_append_gn_args(options.gn_args_path, out_gn_args_path,
+                           extra_gn_args)
+
+  # As all of the test targets are declared in the same BUILD.gn file, it does
+  # not matter which test target is used as the root target.
+  gn_args = [
+      _GN_SRC_REL_PATH, '--root-target=' + test_configs[0]['target'], 'gen',
+      os.path.relpath(options.out_dir, _CHROMIUM_SRC)
+  ]
+  _run_command(gn_args, cwd=_CHROMIUM_SRC)
+
+  error_messages = []
+  for config in test_configs:
+    # Strip leading '//'
+    gn_path = config['target'][2:]
+    expectation = config['expect']
+    ninja_args = [_NINJA_PATH, '-C', options.out_dir, gn_path]
+
+    # Purpose of quotes at beginning of message is to make it clear that
+    # "Compile successful." is not a compiler log message.
+    test_output = _run_command_get_output(ninja_args, '""\nCompile successful.')
+
+    failure_message = _find_lines_after_prefix(test_output, 'FAILED:', 5)
+    if not failure_message or expectation not in failure_message:
+      error_message = '//{} failed.\nExpected compile output:\n'\
+          '{}\nActual compile output:\n{}'.format(
+              gn_path, expectation, test_output)
+      error_messages.append(error_message)
+
+  if error_messages:
+    raise Exception('\n'.join(error_messages))
+
+  if options.stamp:
+    build_utils.Touch(options.stamp)
+
+
+if __name__ == '__main__':
+  main()

build/config/android/android_nocompile.gni

+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build/config/android/rules.gni")
+import("//build/config/python.gni")
+
+declare_args() {
+  # Used by tests to enable generating build files for GN targets which should
+  # not compile.
+  enable_android_nocompile_tests = false
+}
+
+# Defines a test suite which checks that the 'test targets' fail to compile. The
+# test suite runs 'gn gen' with a custom output directory and attempts to compile
+# each test target.
+#
+# All of the tests should be defined in the same dedicated BUILD.gn file in order
+# to minimize the number of targets that are processed by 'gn gen'.
+#
+# Variables
+#   tests: List of test configurations. A test configuration has the following
+#     keys:
+#     'target': The GN target which should not compile when
+#       enable_android_nocompile_tests=true The target should compile when
+#       enable_android_nocompile_tests=false.
+#     'expected_compile_output': Error message to test for when compile fails.
+#     'nocompile_sources': Source files which do not compile. This ensures that
+#       the test suite is re-run when one of these files change (as the test
+#       targets might not depend of the files when
+#       enable_android_nocompile_tests=false).
+template("android_nocompile_test_suite") {
+  assert(!enable_android_nocompile_tests)
+
+  python2_action(target_name) {
+    testonly = true
+    script = "//build/android/gyp/nocompile_test.py"
+
+    _tests = invoker.tests
+    _test0 = _tests[0]
+    _test0_dir = get_label_info(_test0["target"], "dir")
+    foreach(_test_config, _tests) {
+      assert(
+          _test0_dir == get_label_info(_test_config["target"], "dir"),
+          "To avoid running 'gn gen' for each test, all tests in an " +
+          "android_nocompile_test_suite() should be declared in same BUILD.gn file")
+    }
+
+    deps = []
+    if (defined(invoker.deps)) {
+      deps = invoker.deps
+    }
+
+    sources = []
+    _json_test_configs = []
+    foreach(_test_config, _tests) {
+      _test = _test_config["target"]
+      deps += [ _test ]
+      sources += _test_config["nocompile_sources"]
+      _dep_dir = get_label_info(_test, "dir")
+      _dep_name = get_label_info(_test, "name")
+      _json_test_configs += [
+        {
+          target = "${_dep_dir}:${_dep_name}"
+          expect = _test_config["expected_compile_output"]
+        },
+      ]
+    }
+
+    _config_path = "$target_gen_dir/${target_name}.nocompile_config"
+    write_file(_config_path, _json_test_configs, "json")
+
+    _stamp_path = "${target_gen_dir}/${target_name}.stamp"
+    args = [
+      "--gn-args-path",
+      "args.gn",
+      "--out-dir",
+      rebase_path("${target_out_dir}/${target_name}/nocompile_out",
+                  root_build_dir),
+      "--test-configs-path",
+      rebase_path(_config_path, root_build_dir),
+      "--stamp",
+      rebase_path(_stamp_path, root_build_dir),
+    ]
+    outputs = [ _stamp_path ]
+  }
+}

build/config/android/internal_rules.gni

@@ -3245,8 +3245,7 @@ if (enable_java_templates) {
       }
       if (_chromium_code) {
         args += [ "--chromium-code=1" ]
-        if (treat_warnings_as_errors &&
-            !defined(invoker.errorprone_expected_warning_regex)) {
+        if (treat_warnings_as_errors) {
           args += [ "--warnings-as-errors" ]
         }
       }
@@ -3266,9 +3265,6 @@ if (enable_java_templates) {
           "--processorpath=@FileArg($_rebased_errorprone_buildconfig:deps_info:host_classpath)",
           "--enable-errorprone",
         ]
-        if (defined(invoker.errorprone_expected_warning_regex)) {
-          args += [ "--errorprone-expected-warning-regex=${invoker.errorprone_expected_warning_regex}" ]
-        }
       }
       foreach(e, _processor_args) {
         args += [ "--processor-arg=" + e ]
@@ -3375,8 +3371,6 @@ if (enable_java_templates) {
   #  java_files: Optional list of Java source file paths for this target.
   #  javac_args: Optional list of extra arguments to pass to javac.
   #  errorprone_args: Optional list of extra arguments to pass to.
-  #  errorprone_expected_warning_regex: When set, throws an exception if the
-  #    errorprone compile does not log a warning which matches the regex.
   #  srcjar_deps: Optional list of .srcjar targets (not file paths). The Java
   #    source files they contain will also be compiled for this target.
   #  java_sources_file: Optional path to a file which will be written with
@@ -3923,10 +3917,6 @@ if (enable_java_templates) {
             }
             javac_args += invoker.errorprone_args
           }
-          if (defined(invoker.errorprone_expected_warning_regex)) {
-            errorprone_expected_warning_regex =
-                invoker.errorprone_expected_warning_regex
-          }
           deps = [ ":$_header_target_name" ]
           header_jar_path = _final_ijar_path
           generated_jar_path = _generated_jar_path

build/config/android/rules.gni

@@ -1474,8 +1474,6 @@ if (enable_java_templates) {
   #
   #   javac_args: Additional arguments to pass to javac.
   #   errorprone_args: Additional arguments to pass to errorprone.
-  #   errorprone_expected_warning_regex: When set, throws an exception if the
-  #     errorprone compile does not log a warning which matches the regex.
   #
   #   data_deps, testonly
   #

testing/buildbot/gn_isolate_map.pyl

@@ -822,7 +822,7 @@
     "type": "additional_compile_target",
   },
   "errorprone_plugin_tests": {
-    "label": "//tools/android/errorprone_plugin:errorprone_plugin_tests",
+    "label": "//tools/android/errorprone_plugin/test:errorprone_plugin_tests",
     "type": "additional_compile_target",
   },
   "events_unittests": {

tools/android/errorprone_plugin/BUILD.gn

@@ -35,31 +35,3 @@ java_binary("errorprone_plugin") {
     "//third_party/android_deps:com_google_errorprone_javac_java",
   ]
 }
-
-android_library("no_redundant_field_init_check_int_test_java") {
-  testonly = true
-  enable_errorprone = true
-  errorprone_expected_warning_regex = "NoRedundantFieldInitCheck"
-  sources = [ "test/src/org/chromium/tools/errorprone/plugin/NoRedundantFieldInitCheckIntTest.java" ]
-  deps = []
-}
-
-android_library("test_class_name_check_test_java") {
-  testonly = true
-  enable_errorprone = true
-  errorprone_expected_warning_regex = "TestClassNameCheck"
-  sources = [ "test/src/org/chromium/tools/errorprone/plugin/TestClassNameCheckTesting.java" ]
-  deps = [
-    "//base:base_java_test_support",
-    "//third_party/android_support_test_runner:runner_java",
-    "//third_party/junit:junit",
-  ]
-}
-
-group("errorprone_plugin_tests") {
-  testonly = true
-  deps = [
-    ":no_redundant_field_init_check_int_test_java",
-    ":test_class_name_check_test_java",
-  ]
-}

tools/android/errorprone_plugin/test/BUILD.gn

+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build/config/android/android_nocompile.gni")
+import("nocompile_gn/nocompile_sources.gni")
+
+android_nocompile_test_suite("errorprone_plugin_tests") {
+  # Depend on errorprone_plugin so that the action is re-run whenever any custom
+  # errorprone check is modified.
+  deps = [ "//tools/android/errorprone_plugin" ]
+  tests = [
+    {
+      target = "nocompile_gn:no_redundant_field_init_check_int_test_java"
+      nocompile_sources =
+          rebase_path(no_redundant_field_init_check_int_test_nocompile_sources,
+                      "",
+                      "nocompile_gn")
+      expected_compile_output = "warning: [NoRedundantFieldInit]"
+    },
+    {
+      target = "nocompile_gn:test_class_name_check_test_java"
+      nocompile_sources =
+          rebase_path(test_class_name_check_test_nocompile_sources,
+                      "",
+                      "nocompile_gn")
+      expected_compile_output = "warning: [TestClassNameCheck]"
+    },
+  ]
+}

tools/android/errorprone_plugin/test/nocompile_gn/BUILD.gn

+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build/config/android/android_nocompile.gni")
+import("//build/config/android/rules.gni")
+import("nocompile_sources.gni")
+
+empty_java = "../src/org/chromium/tools/errorprone/plugin/Empty.java"
+
+android_library("no_redundant_field_init_check_int_test_java") {
+  testonly = true
+  enable_errorprone = true
+  sources = [ empty_java ]
+  if (enable_android_nocompile_tests) {
+    sources += no_redundant_field_init_check_int_test_nocompile_sources
+  }
+}
+
+android_library("test_class_name_check_test_java") {
+  testonly = true
+  enable_errorprone = true
+  sources = [ empty_java ]
+  if (enable_android_nocompile_tests) {
+    sources += test_class_name_check_test_nocompile_sources
+  }
+  deps = [
+    "//base:base_java_test_support",
+    "//third_party/android_support_test_runner:runner_java",
+    "//third_party/junit:junit",
+  ]
+}

tools/android/errorprone_plugin/test/nocompile_gn/nocompile_sources.gni

+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+no_redundant_field_init_check_int_test_nocompile_sources = [ "../src/org/chromium/tools/errorprone/plugin/NoRedundantFieldInitCheckIntTest.java" ]
+
+test_class_name_check_test_nocompile_sources = [
+  "../src/org/chromium/tools/errorprone/plugin/TestClassNameCheckTesting.java",
+]

tools/android/errorprone_plugin/test/src/org/chromium/tools/errorprone/plugin/Empty.java

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package org.chromium.tools.errorprone.plugin;
+
+/**
+ * Empty class.
+ */
+public class Empty {}