Fix script and doc to generate Java coverage report by Jacoco

Bug: 843307, 961804
Change-Id: I14eb6329d84b86d655c8689249356ffa542c14c1
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1610650
Commit-Queue: Yun Liu <yliuyliu@google.com>
Reviewed-by: Eric Stevenson <estevenson@chromium.org>
Cr-Commit-Position: refs/heads/master@{#659996}

build/android/docs/coverage.md

@@ -5,52 +5,59 @@ instrumentation and junit tests.
 
 [TOC]
 
-## How EMMA coverage works
+## How Jacoco coverage works
 
-In order to use EMMA code coverage, we need to create build time **.em** files
-and runtime **.ec** files. Then we need to process them using the
-build/android/generate_emma_html.py script.
+In order to use Jacoco code coverage, we need to create build time pre-instrumented
+files and runtime **.exec** files. Then we need to process them using the
+**build/android/generate_jacoco_report.py** script.
 
-## How to collect EMMA coverage data
+## How to collect Jacoco coverage data
 
 1. Use the following GN build arguments:
 
-```gn
-target_os = "android"
-emma_coverage = true
-emma_filter = "org.chromium.chrome.browser.ntp.*,-*Test*,-*Fake*,-*Mock*"
-```
-
-The filter syntax is as documented for the [EMMA coverage
-filters](http://emma.sourceforge.net/reference/ch02s06s02.html).
+   ```gn
+   target_os = "android"
+   jacoco_coverage = true
+   ```
 
-Now when building, **.em** files will be created in the build directory.
+Now when building, pre-instrumented files will be created in the build directory.
 
 2. Run tests, with option `--coverage-dir <directory>`, to specify where to save
-   the .ec file. For example, you can run chrome junit tests:
+   the .exec file. For example, you can run chrome junit tests:
    `out/Debug/bin/run_chrome_junit_tests --coverage-dir /tmp/coverage`.
 
-3. Turn off strict mode when running instrumentation tests by adding
-   `--strict-mode=off` because the EMMA code causes strict mode violations by
-   accessing disk.
-
-4. Use a pre-L Android OS (running Dalvik) because code coverage is not
-   supported in ART.
-
-5. The coverage results of junit and instrumentation tests will be merged
+3. The coverage results of junit and instrumentation tests will be merged
    automatically if they are in the same directory.
 
-6. Now we have both .em and .ec files. We can create a html report using
-   `generate_emma_html.py`, for example:
+4. Now we have generated .exec files already. We can create a html/xml/csv report
+   using `generate_jacoco_report.py`, for example:
 
    ```shell
-   build/android/generate_emma_html.py \
+   build/android/generate_jacoco_report.py \
+       --format html \
+       --output-dir tmp/coverage_report/ \
        --coverage-dir /tmp/coverage/ \
        --metadata-dir out/Debug/ \
-       --output example.html
    ```
-   Then an example.html containing coverage info will be created:
+   Then an index.html containing coverage info will be created in output directory:
+
+   ```
+   [INFO] Loading execution data file /tmp/coverage/testTitle.exec.
+   [INFO] Loading execution data file /tmp/coverage/testSelected.exec.
+   [INFO] Loading execution data file /tmp/coverage/testClickToSelect.exec.
+   [INFO] Loading execution data file /tmp/coverage/testClickToClose.exec.
+   [INFO] Loading execution data file /tmp/coverage/testThumbnail.exec.
+   [INFO] Analyzing 58 classes.
+   ```
+
+   For xml and csv reports, we need to specify `--output-file` instead of `--output-dir` since
+   only one file will be generated as xml or csv report.
+   ```
+   --format xml \
+   --output-file tmp/coverage_report/report.xml \
+   ```
 
    ```
-   EMMA: writing [html] report to [<your_current_directory>/example.html] ...
+   --format csv \
+   --output-file tmp/coverage_report/report.csv \
    ```

build/android/generate_emma_html.py

-#!/usr/bin/env python
-
-# Copyright 2013 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Aggregates EMMA coverage files to produce html output."""
-
-from __future__ import print_function
-
-import fnmatch
-import json
-import optparse
-import os
-import sys
-
-import devil_chromium
-from devil.utils import cmd_helper
-from pylib import constants
-from pylib.constants import host_paths
-
-
-def _GetFilesWithExt(root_dir, ext):
-  """Gets all files with a given extension.
-
-  Args:
-    root_dir: Directory in which to search for files.
-    ext: Extension to look for (including dot)
-
-  Returns:
-    A list of absolute paths to files that match.
-  """
-  files = []
-  for root, _, filenames in os.walk(root_dir):
-    basenames = fnmatch.filter(filenames, '*.' + ext)
-    files.extend([os.path.join(root, basename)
-                  for basename in basenames])
-
-  return files
-
-
-def main():
-  option_parser = optparse.OptionParser()
-  option_parser.add_option('--output', help='HTML output filename.')
-  option_parser.add_option('--coverage-dir', default=None,
-                           help=('Root of the directory in which to search for '
-                                 'coverage data (.ec) files.'))
-  option_parser.add_option('--metadata-dir', default=None,
-                           help=('Root of the directory in which to search for '
-                                 'coverage metadata (.em) files.'))
-  option_parser.add_option('--cleanup', action='store_true',
-                           help=('If set, removes coverage files generated at '
-                                 'runtime.'))
-  options, _ = option_parser.parse_args()
-
-  devil_chromium.Initialize()
-
-  if not (options.coverage_dir and options.metadata_dir and options.output):
-    option_parser.error('One or more mandatory options are missing.')
-
-  coverage_files = _GetFilesWithExt(options.coverage_dir, 'ec')
-  metadata_files = _GetFilesWithExt(options.metadata_dir, 'em')
-  # Filter out zero-length files. These are created by emma_instr.py when a
-  # target has no classes matching the coverage filter.
-  metadata_files = [f for f in metadata_files if os.path.getsize(f)]
-  print('Found coverage files: %s' % str(coverage_files))
-  print('Found metadata files: %s' % str(metadata_files))
-
-  sources = []
-  for f in metadata_files:
-    sources_file = os.path.splitext(f)[0] + '_sources.txt'
-    with open(sources_file, 'r') as sf:
-      sources.extend(json.load(sf))
-
-  # Source paths should be passed to EMMA in a way that the relative file paths
-  # reflect the class package name.
-  PARTIAL_PACKAGE_NAMES = ['com/google', 'org/chromium', 'com/chrome']
-  fixed_source_paths = set()
-
-  for path in sources:
-    for partial in PARTIAL_PACKAGE_NAMES:
-      if partial in path:
-        fixed_path = os.path.join(
-            host_paths.DIR_SOURCE_ROOT, path[:path.index(partial)])
-        fixed_source_paths.add(fixed_path)
-        break
-
-  sources = list(fixed_source_paths)
-
-  input_args = []
-  for f in coverage_files + metadata_files:
-    input_args.append('-in')
-    input_args.append(f)
-
-  output_args = ['-Dreport.html.out.file', options.output,
-                 '-Dreport.html.out.encoding', 'UTF-8']
-  source_args = ['-sp', ','.join(sources)]
-
-  exit_code = cmd_helper.RunCmd(
-      ['java', '-cp',
-       os.path.join(constants.ANDROID_SDK_ROOT, 'tools', 'lib', 'emma.jar'),
-       'emma', 'report', '-r', 'html']
-      + input_args + output_args + source_args)
-
-  if options.cleanup:
-    for f in coverage_files:
-      os.remove(f)
-
-  # Command tends to exit with status 0 when it actually failed.
-  if not exit_code and not os.path.exists(options.output):
-    exit_code = 1
-
-  return exit_code
-
-
-if __name__ == '__main__':
-  sys.exit(main())

build/android/generate_jacoco_report.py

+#!/usr/bin/env python
+
+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Aggregates Jacoco coverage files to produce output."""
+
+from __future__ import print_function
+
+import argparse
+import fnmatch
+import json
+import os
+import sys
+
+import devil_chromium
+from devil.utils import cmd_helper
+from pylib.constants import host_paths
+
+# Source paths should be passed to Jacoco in a way that the relative file paths
+# reflect the class package name.
+_PARTIAL_PACKAGE_NAMES = ['com/google', 'org/chromium']
+
+# The sources_json_file is generated by jacoco_instr.py with source directories
+# and input path to non-instrumented jars.
+# e.g.
+# 'source_dirs': [
+# "chrome/android/java/src/org/chromium/chrome/browser/toolbar/bottom",
+# "chrome/android/java/src/org/chromium/chrome/browser/ui/system",
+# ...]
+# 'input_path':
+#   '$CHROMIUM_OUTPUT_DIR/\
+#    obj/chrome/android/features/tab_ui/java__process_prebuilt-filtered.jar'
+_SOURCES_JSON_FILES_SUFFIX = '__jacoco_sources.json'
+
+
+def _GetFilesWithSuffix(root_dir, suffix):
+  """Gets all files with a given suffix.
+
+  Args:
+    root_dir: Directory in which to search for files.
+    suffix: Suffix to look for.
+
+  Returns:
+    A list of absolute paths to files that match.
+  """
+  files = []
+  for root, _, filenames in os.walk(root_dir):
+    basenames = fnmatch.filter(filenames, '*' + suffix)
+    files.extend([os.path.join(root, basename) for basename in basenames])
+
+  return files
+
+
+def _AddArguments(parser):
+  """Adds arguments related to parser.
+
+  Args:
+    parser: ArgumentParser object.
+  """
+  parser.add_argument(
+      '--format',
+      required=True,
+      choices=['html', 'xml', 'csv'],
+      help='Output report format. Choose one from html, xml and csv.')
+  parser.add_argument('--output-dir', help='html report output directory.')
+  parser.add_argument('--output-file', help='xml or csv report output file.')
+  parser.add_argument(
+      '--coverage-dir',
+      required=True,
+      help=('Root of the directory in which to search for '
+            'coverage data (.exec) files.'))
+  parser.add_argument(
+      '--metadata-dir',
+      help=('Root of the directory in which to search for '
+            '*-filtered.jar and *__jacoco_sources.txt files.'))
+  parser.add_argument(
+      '--class-files',
+      nargs='+',
+      help='Location of Java non-instrumented class files. '
+      'Use non-instrumented jars instead of instrumented jars. '
+      'e.g. use chrome_java__process_prebuilt-filtered.jar instead of'
+      'chrome_java__process_prebuilt-instrumented.jar')
+  parser.add_argument(
+      '--sources',
+      nargs='+',
+      help='Location of the source files. '
+      'Specified source folders must be the direct parent of the folders '
+      'that define the Java packages.'
+      'e.g. <src_dir>/chrome/android/java/src/')
+  parser.add_argument(
+      '--cleanup',
+      action='store_true',
+      help=('If set, removes coverage files generated at '
+            'runtime.'))
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  _AddArguments(parser)
+  args = parser.parse_args()
+
+  if args.format == 'html':
+    if not args.output_dir:
+      parser.error('--output-dir needed for html report.')
+  elif not args.output_file:
+    parser.error('--output-file needed for xml or csv report.')
+
+  if not (args.metadata_dir or args.class_files):
+    parser.error('At least either --metadata-dir or --class-files needed.')
+
+  devil_chromium.Initialize()
+
+  coverage_files = _GetFilesWithSuffix(args.coverage_dir, '.exec')
+  if not coverage_files:
+    parser.error('Found no coverage file under %s' % args.coverage_dir)
+  print('Found coverage files: %s' % str(coverage_files))
+
+  class_files = []
+  sources = []
+  if args.metadata_dir:
+    sources_json_files = _GetFilesWithSuffix(args.metadata_dir,
+                                             _SOURCES_JSON_FILES_SUFFIX)
+    for f in sources_json_files:
+      with open(f, 'r') as json_file:
+        data = json.load(json_file)
+        class_files.append(data['input_path'])
+        sources.extend(data['source_dirs'])
+
+  fixed_source_paths = set()
+
+  for path in sources:
+    for partial in _PARTIAL_PACKAGE_NAMES:
+      if partial in path:
+        fixed_path = os.path.join(host_paths.DIR_SOURCE_ROOT,
+                                  path[:path.index(partial)])
+        fixed_source_paths.add(fixed_path)
+        break
+
+  sources = list(fixed_source_paths)
+
+  if args.class_files:
+    class_files += args.class_files
+  if args.sources:
+    sources += args.sources
+
+  cmd = [
+      'java', '-jar',
+      os.path.join(host_paths.DIR_SOURCE_ROOT, 'third_party', 'jacoco', 'lib',
+                   'jacococli.jar'), 'report'
+  ] + coverage_files
+  for f in class_files:
+    cmd += ['--classfiles', f]
+  for source in sources:
+    cmd += ['--sourcefiles', source]
+  if args.format == 'html':
+    cmd += ['--html', args.output_dir]
+  elif args.format == 'xml':
+    cmd += ['--xml', args.output_file]
+  else:
+    cmd += ['--csv', args.output_file]
+
+  exit_code = cmd_helper.RunCmd(cmd)
+
+  if args.cleanup:
+    for f in coverage_files:
+      os.remove(f)
+
+  # Command tends to exit with status 0 when it actually failed.
+  if not exit_code:
+    if args.format == 'html':
+      if not os.path.exists(args.output_dir) or not os.listdir(args.output_dir):
+        print('No report generated at %s' % args.output_dir)
+        exit_code = 1
+    elif not os.path.exists(args.output_file):
+      print('No report generated at %s' % args.output_file)
+      exit_code = 1
+
+  return exit_code
+
+
+if __name__ == '__main__':
+  sys.exit(main())

build/android/gyp/jacoco_instr.py

@@ -34,18 +34,19 @@ def _AddArguments(parser):
   parser.add_argument(
       '--input-path',
       required=True,
-      help=('Path to input file(s). Either the classes '
-            'directory, or the path to a jar.'))
+      help='Path to input file(s). Either the classes '
+      'directory, or the path to a jar.')
   parser.add_argument(
       '--output-path',
       required=True,
-      help=('Path to output final file(s) to. Either the '
-            'final classes directory, or the directory in '
-            'which to place the instrumented/copied jar.'))
+      help='Path to output final file(s) to. Either the '
+      'final classes directory, or the directory in '
+      'which to place the instrumented/copied jar.')
   parser.add_argument(
-      '--sources-list-file',
+      '--sources-json-file',
       required=True,
-      help='File to create with the list of sources.')
+      help='File to create with the list of source directories '
+      'and input path.')
   parser.add_argument(
       '--java-sources-file',
       required=True,
@@ -66,12 +67,16 @@ def _GetSourceDirsFromSourceFiles(source_files):
   return list(set(os.path.dirname(source_file) for source_file in source_files))
 
 
-def _CreateSourcesListFile(source_dirs, sources_list_file, src_root):
-  """Adds all normalized source directories to |sources_list_file|.
+def _CreateSourcesJsonFile(source_dirs, input_path, sources_json_file,
+                           src_root):
+  """Adds all normalized source directories and input path to
+  |sources_json_file|.
 
   Args:
     source_dirs: List of source directories.
-    sources_list_file: File into which to write the JSON list of sources.
+    input_path: The input path to non-instrumented class files.
+    sources_json_file: File into which to write the list of source directories
+    and input path.
     src_root: Root which sources added to the file should be relative to.
 
   Returns:
@@ -89,8 +94,11 @@ def _CreateSourcesListFile(source_dirs, sources_list_file, src_root):
 
     relative_sources.append(rel_source)
 
-  with open(sources_list_file, 'w') as f:
-    json.dump(relative_sources, f)
+  data = {}
+  data['source_dirs'] = relative_sources
+  data['input_path'] = os.path.abspath(input_path)
+  with open(sources_json_file, 'w') as f:
+    json.dump(data, f)
 
 
 def _RunInstrumentCommand(parser):
@@ -134,7 +142,7 @@ def _RunInstrumentCommand(parser):
   # TODO(GYP): In GN, we are passed the list of sources, detecting source
   # directories, then walking them to re-establish the list of sources.
   # This can obviously be simplified!
-  _CreateSourcesListFile(source_dirs, args.sources_list_file,
+  _CreateSourcesJsonFile(source_dirs, args.input_path, args.sources_json_file,
                          build_utils.DIR_SOURCE_ROOT)
 
   return 0

build/config/android/internal_rules.gni

@@ -1502,7 +1502,7 @@ if (enable_java_templates) {
                                "testonly",
                              ])
 
-      _source_dirs_listing_file = "$target_out_dir/${target_name}_sources.txt"
+      _sources_json_file = "$target_out_dir/${target_name}_sources.json"
       _jacococli_jar = "//third_party/jacoco/lib/jacococli.jar"
 
       script = "//build/android/gyp/jacoco_instr.py"
@@ -1511,7 +1511,7 @@ if (enable_java_templates) {
                  invoker.input_jar_path,
                ]
       outputs = [
-        _source_dirs_listing_file,
+        _sources_json_file,
         invoker.output_jar_path,
       ]
       args = [
@@ -1519,8 +1519,8 @@ if (enable_java_templates) {
         rebase_path(invoker.input_jar_path, root_build_dir),
         "--output-path",
         rebase_path(invoker.output_jar_path, root_build_dir),
-        "--sources-list-file",
-        rebase_path(_source_dirs_listing_file, root_build_dir),
+        "--sources-json-file",
+        rebase_path(_sources_json_file, root_build_dir),
         "--java-sources-file",
         rebase_path(invoker.java_sources_file, root_build_dir),
         "--jacococli-jar",