[Code Coverage] Update output dir structure for multi-platform support.

Also write output of llvm-cov export program on the disk (needed for the bot).

Plus some refactoring for logs from target programs:
- redirect stderr from targets into stdout to be written into a .log
- write logs into its own subdir for better output structure

Not output folder structure look as follows:

$ ls -l out/zreport15/
total 8
drwxr-x--- 5 mmoroz primarygroup 4096 May  4 10:53 linux
-rw-r--r-- 1 mmoroz primarygroup 2519 May  4 10:53 style.css

$ ls -l out/zreport15/linux/
total 48
drwxr-xr-x 2 mmoroz primarygroup 4096 May  4 10:53 components
-rw-r--r-- 1 mmoroz primarygroup 1627 May  4 10:53 component_view_index.html
-rw-r--r-- 1 mmoroz primarygroup 4672 May  4 10:53 coverage.profdata
-rw-r--r-- 1 mmoroz primarygroup  241 May  4 10:53 directory_view_index.html
-rw-r--r-- 1 mmoroz primarygroup 7416 May  4 10:53 file_view_index.html
-rw-r--r-- 1 mmoroz primarygroup  200 May  4 10:53 index.html
drwxr-xr-x 2 mmoroz primarygroup 4096 May  4 10:53 logs
-rw-r--r-- 1 mmoroz primarygroup 7483 May  4 10:53 summary.json
drwxr-x--- 3 mmoroz primarygroup 4096 May  4 10:53 usr


Bug: 784464, 836663
Change-Id: I3283f16b62cfeb267bb2b3092fa469d5ff39ef14
Reviewed-on: https://chromium-review.googlesource.com/1044652
Commit-Queue: Abhishek Arya <inferno@chromium.org>
Reviewed-by: Abhishek Arya <inferno@chromium.org>
Cr-Commit-Position: refs/heads/master@{#556348}

tools/code_coverage/coverage.py

@@ -110,7 +110,10 @@ PROFRAW_FILE_EXTENSION = 'profraw'
 # Name of the final profdata file, and this file needs to be passed to
 # "llvm-cov" command in order to call "llvm-cov show" to inspect the
 # line-by-line coverage of specific files.
-PROFDATA_FILE_NAME = 'coverage.profdata'
+PROFDATA_FILE_NAME = os.extsep.join(['coverage', 'profdata'])
+
+# Name of the file with summary information generated by llvm-cov export.
+SUMMARY_FILE_NAME = os.extsep.join(['summary', 'json'])
 
 # Build arg required for generating code coverage data.
 CLANG_COVERAGE_BUILD_ARG = 'use_clang_coverage'
@@ -119,9 +122,12 @@ CLANG_COVERAGE_BUILD_ARG = 'use_clang_coverage'
 DIRECTORY_COVERAGE_HTML_REPORT_NAME = os.extsep.join(['report', 'html'])
 
 # Name of the html index files for different views.
-DIRECTORY_VIEW_INDEX_FILE = os.extsep.join(['directory_view_index', 'html'])
 COMPONENT_VIEW_INDEX_FILE = os.extsep.join(['component_view_index', 'html'])
+DIRECTORY_VIEW_INDEX_FILE = os.extsep.join(['directory_view_index', 'html'])
 FILE_VIEW_INDEX_FILE = os.extsep.join(['file_view_index', 'html'])
+INDEX_HTML_FILE = os.extsep.join(['index', 'html'])
+
+LOGS_DIR_NAME = 'logs'
 
 # Used to extract a mapping between directories and components.
 COMPONENT_MAPPING_URL = 'https://storage.googleapis.com/chromium-owners/component_map.json'
@@ -187,7 +193,7 @@ class _CoverageReportHtmlGenerator(object):
                         header. For example: 'Path', 'Component'.
     """
     css_file_name = os.extsep.join(['style', 'css'])
-    css_absolute_path = os.path.abspath(os.path.join(OUTPUT_DIR, css_file_name))
+    css_absolute_path = os.path.join(OUTPUT_DIR, css_file_name)
     assert os.path.exists(css_absolute_path), (
         'css file doesn\'t exit. Please make sure "llvm-cov show -format=html" '
         'is called first, and the css file is generated at: "%s"' %
@@ -294,9 +300,10 @@ class _CoverageReportHtmlGenerator(object):
     self._table_entries = sorted(self._table_entries, cmp=EntryCmp)
 
     css_path = os.path.join(OUTPUT_DIR, os.extsep.join(['style', 'css']))
-    directory_view_path = os.path.join(OUTPUT_DIR, DIRECTORY_VIEW_INDEX_FILE)
-    component_view_path = os.path.join(OUTPUT_DIR, COMPONENT_VIEW_INDEX_FILE)
-    file_view_path = os.path.join(OUTPUT_DIR, FILE_VIEW_INDEX_FILE)
+
+    directory_view_path = _GetDirectoryViewPath()
+    component_view_path = _GetComponentViewPath()
+    file_view_path = _GetFileViewPath()
 
     html_header = self._header_template.render(
         css_path=_GetRelativePathToDirectoryOfFile(css_path, self._output_path),
@@ -504,17 +511,15 @@ def _GeneratePerFileLineByLineCoverageInHtml(binary_paths, profdata_file_path,
   # the platform name instead, as it simplifies the report dir structure when
   # the same report is generated for different platforms.
   default_report_subdir_path = os.path.join(OUTPUT_DIR, 'coverage')
-  platform_report_subdir_path = os.path.join(OUTPUT_DIR, _GetHostPlatform())
-  if os.path.exists(platform_report_subdir_path):
-    shutil.rmtree(platform_report_subdir_path)
-  os.rename(default_report_subdir_path, platform_report_subdir_path)
+  platform_report_subdir_path = _GetCoverageReportRootDirPath()
+  _MergeTwoDirectories(default_report_subdir_path, platform_report_subdir_path)
 
   logging.debug('Finished running "llvm-cov show" command')
 
 
 def _GenerateFileViewHtmlIndexFile(per_file_coverage_summary):
   """Generates html index file for file view."""
-  file_view_index_file_path = os.path.join(OUTPUT_DIR, FILE_VIEW_INDEX_FILE)
+  file_view_index_file_path = _GetFileViewPath()
   logging.debug('Generating file view html index file as: "%s".',
                 file_view_index_file_path)
   html_generator = _CoverageReportHtmlGenerator(file_view_index_file_path,
@@ -600,8 +605,7 @@ def _GenerateDirectoryViewHtmlIndexFile():
   file simply redirects to it, and the reason of this extra layer is for
   structural consistency with other views.
   """
-  directory_view_index_file_path = os.path.join(OUTPUT_DIR,
-                                                DIRECTORY_VIEW_INDEX_FILE)
+  directory_view_index_file_path = _GetDirectoryViewPath()
   logging.debug('Generating directory view html index file as: "%s".',
                 directory_view_index_file_path)
   src_root_html_report_path = _GetCoverageHtmlReportPathForDirectory(
@@ -686,8 +690,7 @@ def _GenerateCoverageInHtmlForComponent(
 
 def _GenerateComponentViewHtmlIndexFile(per_component_coverage_summary):
   """Generates the html index file for component view."""
-  component_view_index_file_path = os.path.join(OUTPUT_DIR,
-                                                COMPONENT_VIEW_INDEX_FILE)
+  component_view_index_file_path = _GetComponentViewPath()
   logging.debug('Generating component view html index file as: "%s".',
                 component_view_index_file_path)
   html_generator = _CoverageReportHtmlGenerator(component_view_index_file_path,
@@ -707,12 +710,20 @@ def _GenerateComponentViewHtmlIndexFile(per_component_coverage_summary):
   logging.debug('Finished generating component view html index file.')
 
 
+def _MergeTwoDirectories(src_path, dst_path):
+  """Merge src_path directory into dst_path directory."""
+  for filename in os.listdir(src_path):
+    dst_path = os.path.join(dst_path, filename)
+    if os.path.exists(dst_path):
+      shutil.rmtree(dst_path)
+    os.rename(os.path.join(src_path, filename), dst_path)
+  shutil.rmtree(src_path)
+
+
 def _OverwriteHtmlReportsIndexFile():
   """Overwrites the root index file to redirect to the default view."""
-  html_index_file_path = os.path.join(OUTPUT_DIR,
-                                      os.extsep.join(['index', 'html']))
-  directory_view_index_file_path = os.path.join(OUTPUT_DIR,
-                                                DIRECTORY_VIEW_INDEX_FILE)
+  html_index_file_path = _GetHtmlIndexPath()
+  directory_view_index_file_path = _GetDirectoryViewPath()
   _WriteRedirectHtmlFile(html_index_file_path, directory_view_index_file_path)
 
 
@@ -732,6 +743,14 @@ def _WriteRedirectHtmlFile(from_html_path, to_html_path):
     f.write(content)
 
 
+def _CleanUpOutputDir():
+  """Perform a cleanup of the output dir."""
+  # Remove the default index.html file produced by llvm-cov.
+  index_path = os.path.join(OUTPUT_DIR, INDEX_HTML_FILE)
+  if os.path.exists(index_path):
+    os.remove(index_path)
+
+
 def _GetCoverageHtmlReportPathForFile(file_path):
   """Given a file path, returns the corresponding html report path."""
   assert os.path.isfile(file_path), '"%s" is not a file' % file_path
@@ -765,7 +784,44 @@ def _GetCoverageHtmlReportPathForComponent(component_name):
 
 def _GetCoverageReportRootDirPath():
   """The root directory that contains all generated coverage html reports."""
-  return os.path.join(os.path.abspath(OUTPUT_DIR), _GetHostPlatform())
+  return os.path.join(OUTPUT_DIR, _GetHostPlatform())
+
+
+def _GetComponentViewPath():
+  """Path to the HTML file for the component view."""
+  return os.path.join(_GetCoverageReportRootDirPath(),
+                      COMPONENT_VIEW_INDEX_FILE)
+
+
+def _GetDirectoryViewPath():
+  """Path to the HTML file for the directory view."""
+  return os.path.join(_GetCoverageReportRootDirPath(),
+                      DIRECTORY_VIEW_INDEX_FILE)
+
+
+def _GetFileViewPath():
+  """Path to the HTML file for the file view."""
+  return os.path.join(_GetCoverageReportRootDirPath(), FILE_VIEW_INDEX_FILE)
+
+
+def _GetLogsDirectoryPath():
+  """Path to the logs directory."""
+  return os.path.join(_GetCoverageReportRootDirPath(), LOGS_DIR_NAME)
+
+
+def _GetHtmlIndexPath():
+  """Path to the main HTML index file."""
+  return os.path.join(_GetCoverageReportRootDirPath(), INDEX_HTML_FILE)
+
+
+def _GetProfdataFilePath():
+  """Path to the resulting .profdata file."""
+  return os.path.join(_GetCoverageReportRootDirPath(), PROFDATA_FILE_NAME)
+
+
+def _GetSummaryFilePath():
+  """The JSON file that contains coverage summary written by llvm-cov export."""
+  return os.path.join(_GetCoverageReportRootDirPath(), SUMMARY_FILE_NAME)
 
 
 def _CreateCoverageProfileDataForTargets(targets, commands, jobs_count=None):
@@ -839,16 +895,20 @@ def _GetTargetProfDataPathsByExecutingCommands(targets, commands):
   logging.debug('Executing the test commands')
 
   # Remove existing profraw data files.
-  for file_or_dir in os.listdir(OUTPUT_DIR):
+  for file_or_dir in os.listdir(_GetCoverageReportRootDirPath()):
     if file_or_dir.endswith(PROFRAW_FILE_EXTENSION):
-      os.remove(os.path.join(OUTPUT_DIR, file_or_dir))
+      os.remove(os.path.join(_GetCoverageReportRootDirPath(), file_or_dir))
+
+  # Ensure that logs directory exists.
+  if not os.path.exists(_GetLogsDirectoryPath()):
+    os.makedirs(_GetLogsDirectoryPath())
 
   profdata_file_paths = []
 
   # Run all test targets to generate profraw data files.
   for target, command in zip(targets, commands):
-    output_file_name = os.extsep.join([target + '_output', 'txt'])
-    output_file_path = os.path.join(OUTPUT_DIR, output_file_name)
+    output_file_name = os.extsep.join([target + '_output', 'log'])
+    output_file_path = os.path.join(_GetLogsDirectoryPath(), output_file_name)
 
     profdata_file_path = None
     for _ in xrange(MERGE_RETRIES):
@@ -871,9 +931,10 @@ def _GetTargetProfDataPathsByExecutingCommands(targets, commands):
       if _IsIOS():
         profraw_file_paths = _GetProfrawDataFileByParsingOutput(output)
       else:
-        for file_or_dir in os.listdir(OUTPUT_DIR):
+        for file_or_dir in os.listdir(_GetCoverageReportRootDirPath()):
           if file_or_dir.endswith(PROFRAW_FILE_EXTENSION):
-            profraw_file_paths.append(os.path.join(OUTPUT_DIR, file_or_dir))
+            profraw_file_paths.append(
+                os.path.join(_GetCoverageReportRootDirPath(), file_or_dir))
 
       assert profraw_file_paths, (
           'Running target %s failed to generate any profraw data file, '
@@ -929,12 +990,14 @@ def _ExecuteCommand(target, command):
   profile_pattern_string = '%1m' if _IsFuzzerTarget(target) else '%4m'
   expected_profraw_file_name = os.extsep.join(
       [target, profile_pattern_string, PROFRAW_FILE_EXTENSION])
-  expected_profraw_file_path = os.path.join(OUTPUT_DIR,
+  expected_profraw_file_path = os.path.join(_GetCoverageReportRootDirPath(),
                                             expected_profraw_file_name)
 
   try:
+    # Some fuzz targets or tests may write into stderr, redirect it as well.
     output = subprocess.check_output(
         shlex.split(command),
+        stderr=subprocess.STDOUT,
         env={'LLVM_PROFILE_FILE': expected_profraw_file_path})
   except subprocess.CalledProcessError as e:
     output = e.output
@@ -1017,9 +1080,8 @@ def _CreateCoverageProfileDataFromTargetProfDataFiles(profdata_file_paths):
     CalledProcessError: An error occurred merging profdata files.
   """
   logging.info('Creating the coverage profile data file')
-  logging.debug(
-      'Merging target profdata files to create coverage profdata file')
-  profdata_file_path = os.path.join(OUTPUT_DIR, PROFDATA_FILE_NAME)
+  logging.debug('Merging target profraw files to create target profdata file')
+  profdata_file_path = _GetProfdataFilePath()
   try:
     subprocess_cmd = [
         LLVM_PROFDATA_PATH, 'merge', '-o', profdata_file_path, '-sparse=true'
@@ -1091,7 +1153,13 @@ def _GeneratePerFileCoverageSummary(binary_paths, profdata_file_path, filters,
   if ignore_filename_regex:
     subprocess_cmd.append('-ignore-filename-regex=%s' % ignore_filename_regex)
 
-  json_output = json.loads(subprocess.check_output(subprocess_cmd))
+  export_output = subprocess.check_output(subprocess_cmd)
+
+  # Write output on the disk to be used by code coverage bot.
+  with open(_GetSummaryFilePath(), 'w') as f:
+    f.write(export_output)
+
+  json_output = json.loads(export_output)
   assert len(json_output['data']) == 1
   files_coverage_data = json_output['data'][0]['files']
 
@@ -1178,7 +1246,7 @@ def _VerifyTargetExecutablesAreInBuildDirectory(commands):
   for command in commands:
     binary_path = _GetBinaryPath(command)
     binary_absolute_path = os.path.abspath(os.path.normpath(binary_path))
-    assert binary_absolute_path.startswith(os.path.abspath(BUILD_DIR)), (
+    assert binary_absolute_path.startswith(BUILD_DIR), (
         'Target executable "%s" in command: "%s" is outside of '
         'the given build directory: "%s".' % (binary_path, command, BUILD_DIR))
 
@@ -1391,9 +1459,9 @@ def Main():
   _ConfigureLogging(args)
 
   global BUILD_DIR
-  BUILD_DIR = args.build_dir
+  BUILD_DIR = os.path.abspath(args.build_dir)
   global OUTPUT_DIR
-  OUTPUT_DIR = args.output_dir
+  OUTPUT_DIR = os.path.abspath(args.output_dir)
 
   assert args.command or args.profdata_file, (
       'Need to either provide commands to run using -c/--command option OR '
@@ -1413,8 +1481,8 @@ def Main():
   if args.filters:
     absolute_filter_paths = _VerifyPathsAndReturnAbsolutes(args.filters)
 
-  if not os.path.exists(OUTPUT_DIR):
-    os.makedirs(OUTPUT_DIR)
+  if not os.path.exists(_GetCoverageReportRootDirPath()):
+    os.makedirs(_GetCoverageReportRootDirPath())
 
   # Get profdate file and list of binary paths.
   if args.command:
@@ -1457,9 +1525,9 @@ def Main():
   # The default index file is generated only for the list of source files, needs
   # to overwrite it to display per directory coverage view by default.
   _OverwriteHtmlReportsIndexFile()
+  _CleanUpOutputDir()
 
-  html_index_file_path = 'file://' + os.path.abspath(
-      os.path.join(OUTPUT_DIR, 'index.html'))
+  html_index_file_path = 'file://' + os.path.abspath(_GetHtmlIndexPath())
   logging.info('Index file for html report is generated as: %s',
                html_index_file_path)