Reland "Reland "gtest perf tests: Output well formed test results json""

This reverts commit 89c5e790.

Reason for revert: Testing possible fixes

Original change's description:
> Revert "Reland "gtest perf tests: Output well formed test results json""
> 
> This reverts commit f02c92e1.
> 
> Reason for revert: still breaking perf waterfall https://ci.chromium.org/buildbot/chromium.perf/linux-perf/315
> 
> Original change's description:
> > Reland "gtest perf tests: Output well formed test results json"
> > 
> > This reverts commit 9eeaba4e.
> > 
> > Reason for revert: Will re-land with fix
> > 
> > Original change's description:
> > > Revert "gtest perf tests: Output well formed test results json"
> > > 
> > > This reverts commit 71b865f4.
> > > 
> > > Reason for revert: break many perf tests (see https://ci.chromium.org/buildbot/chromium.perf/linux-perf/307)
> > > 
> > > Original change's description:
> > > > gtest perf tests: Output well formed test results json
> > > > 
> > > > This CL changes the run_gtest_perf_test.py script to output well formed
> > > > test results json.
> > > > 
> > > > Bug: 855234
> > > > Change-Id: Ia5ce6e258865e34a2aa37635c014cd54efa62796
> > > > Reviewed-on: https://chromium-review.googlesource.com/1112635
> > > > Commit-Queue: Stephen Martinis <martiniss@chromium.org>
> > > > Reviewed-by: Dirk Pranke <dpranke@chromium.org>
> > > > Cr-Commit-Position: refs/heads/master@{#570943}
> > > 
> > > TBR=dpranke@chromium.org,nednguyen@google.com,martiniss@chromium.org,eyaich@chromium.org
> > > 
> > > Change-Id: Ia0aaff923014438005ebc0a8677c3bbbebb808ac
> > > No-Presubmit: true
> > > No-Tree-Checks: true
> > > No-Try: true
> > > Bug: 855234
> > > Reviewed-on: https://chromium-review.googlesource.com/1118238
> > > Reviewed-by: Ned Nguyen <nednguyen@google.com>
> > > Commit-Queue: Ned Nguyen <nednguyen@google.com>
> > > Cr-Commit-Position: refs/heads/master@{#571083}
> > 
> > TBR=dpranke@chromium.org,nednguyen@google.com,martiniss@chromium.org,eyaich@chromium.org
> > 
> > Change-Id: Idb255b1fafb7f254fd44d6554f93ff27608279f2
> > No-Presubmit: true
> > No-Tree-Checks: true
> > No-Try: true
> > Bug: 855234
> > Reviewed-on: https://chromium-review.googlesource.com/1118789
> > Commit-Queue: Stephen Martinis <martiniss@chromium.org>
> > Reviewed-by: Stephen Martinis <martiniss@chromium.org>
> > Cr-Commit-Position: refs/heads/master@{#571245}
> 
> TBR=dpranke@chromium.org,nednguyen@google.com,martiniss@chromium.org,eyaich@chromium.org
> 
> Change-Id: I6bc0869caa6a763dc8bb542fd5a1b3d25853ac86
> No-Presubmit: true
> No-Tree-Checks: true
> No-Try: true
> Bug: 855234
> Reviewed-on: https://chromium-review.googlesource.com/1119445
> Reviewed-by: Ned Nguyen <nednguyen@google.com>
> Commit-Queue: Ned Nguyen <nednguyen@google.com>
> Cr-Commit-Position: refs/heads/master@{#571321}

TBR=dpranke@chromium.org,nednguyen@google.com,martiniss@chromium.org,eyaich@chromium.org

Change-Id: I25bc4bb1f77031d95dd70347bfed886112b76218
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: 855234
Reviewed-on: https://chromium-review.googlesource.com/1120798Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Reviewed-by: Stephen Martinis <martiniss@chromium.org>
Commit-Queue: Stephen Martinis <martiniss@chromium.org>
Cr-Commit-Position: refs/heads/master@{#571968}

testing/scripts/common.py

@@ -167,6 +167,24 @@ def parse_common_test_results(json_results, test_separator='/'):
 
 def get_gtest_summary_passes(output):
   """Returns a mapping of test to boolean indicating if the test passed.
+  """
+  if not output:
+    return {}
+
+  mapping = {}
+
+  for test_suite in output.get('testsuites', []):
+    suite_name = test_suite['name']
+    for test in test_suite['testsuite']:
+      full_name = '%s.%s' % (suite_name, test['name'])
+
+      mapping[full_name] = 'failures' in test
+
+  return mapping
+
+
+def get_chromium_gtest_summary_passes(output):
+  """Returns a mapping of test to boolean indicating if the test passed.
 
   Only partially parses the format. This code is based on code in tools/build,
   specifically

testing/scripts/run_gtest_perf_test.py

@@ -32,6 +32,7 @@ import json
 import os
 import shutil
 import sys
+import time
 import tempfile
 import traceback
 
@@ -82,7 +83,7 @@ def main():
 
   args, rest_args = parser.parse_known_args()
 
-  rc, charts, output_json = execute_perf_test(args, rest_args)
+  rc, charts, output_json = execute_perf_test(args, rest_args, True)
 
   # TODO(eakuefner): Make isolated_script_test_perf_output mandatory after
   # flipping flag in swarming.
@@ -100,7 +101,7 @@ def main():
   return rc
 
 
-def execute_perf_test(args, rest_args):
+def execute_perf_test(args, rest_args, chromium_gtest):
   env = os.environ.copy()
   # Assume we want to set up the sandbox environment variables all the
   # time; doing so is harmless on non-Linux platforms and is needed
@@ -108,53 +109,95 @@ def execute_perf_test(args, rest_args):
   env[CHROME_SANDBOX_ENV] = CHROME_SANDBOX_PATH
 
   rc = 0
-  try:
-    executable = rest_args[0]
-    extra_flags = []
-    if len(rest_args) > 1:
-      extra_flags = rest_args[1:]
-
-    # These flags are to make sure that test output perf metrics in the log.
-    if not '--verbose' in extra_flags:
-      extra_flags.append('--verbose')
-    if not '--test-launcher-print-test-stdio=always' in extra_flags:
-      extra_flags.append('--test-launcher-print-test-stdio=always')
-    if args.isolated_script_test_filter:
-      filter_list = common.extract_filter_list(
-        args.isolated_script_test_filter)
-      extra_flags.append('--gtest_filter=' + ':'.join(filter_list))
-
-    if IsWindows():
-      executable = '.\%s.exe' % executable
-    else:
-      executable = './%s' % executable
-    with common.temporary_file() as tempfile_path:
-      env['CHROME_HEADLESS'] = '1'
-      cmd = [executable] + extra_flags
-
-      if args.xvfb:
-        rc = xvfb.run_executable(cmd, env, stdoutfile=tempfile_path)
+  start_time = time.time()
+  test_results = {}
+  with common.temporary_file() as results_path:
+    try:
+      executable = rest_args[0]
+      extra_flags = []
+      if len(rest_args) > 1:
+        extra_flags = rest_args[1:]
+
+      if chromium_gtest:
+        output_flag = '--test-launcher-summary-output'
+        output_file = results_path
       else:
-        rc = test_env.run_command_with_output(cmd, env=env,
-                                              stdoutfile=tempfile_path)
-
-      # Now get the correct json format from the stdout to write to the perf
-      # results file
-      results_processor = (
-          generate_legacy_perf_dashboard_json.LegacyResultsProcessor())
-      charts = results_processor.GenerateJsonResults(tempfile_path)
-  except Exception:
-    traceback.print_exc()
-    rc = 1
-
-  valid = (rc == 0)
-  failures = [] if valid else ['(entire test suite)']
+        output_flag = '--gtest_output'
+        output_file = 'json:%s' % results_path
+
+      assert not any(output_flag in flag for flag in extra_flags), (
+          'Duplicate %s flag detected.' % output_flag)
+      extra_flags.append('%s=%s' % (output_flag, output_file))
+
+      # These flags are to make sure that test output perf metrics in the log.
+      if not '--verbose' in extra_flags:
+        extra_flags.append('--verbose')
+      if not '--test-launcher-print-test-stdio=always' in extra_flags:
+        extra_flags.append('--test-launcher-print-test-stdio=always')
+      if args.isolated_script_test_filter:
+        filter_list = common.extract_filter_list(
+          args.isolated_script_test_filter)
+        extra_flags.append('--gtest_filter=' + ':'.join(filter_list))
+
+      if IsWindows():
+        executable = '.\%s.exe' % executable
+      else:
+        executable = './%s' % executable
+      with common.temporary_file() as tempfile_path:
+        env['CHROME_HEADLESS'] = '1'
+        cmd = [executable] + extra_flags
+
+        print ' '.join(cmd)
+        if args.xvfb:
+          rc = xvfb.run_executable(cmd, env, stdoutfile=tempfile_path)
+        else:
+          rc = test_env.run_command_with_output(cmd, env=env,
+                                                stdoutfile=tempfile_path)
+
+        # Now get the correct json format from the stdout to write to the perf
+        # results file
+        results_processor = (
+            generate_legacy_perf_dashboard_json.LegacyResultsProcessor())
+        charts = results_processor.GenerateJsonResults(tempfile_path)
+    except Exception:
+      traceback.print_exc()
+      rc = 1
+
+    if os.path.exists(results_path):
+      with open(results_path) as f:
+        if chromium_gtest:
+          func = common.get_chromium_gtest_summary_passes
+        else:
+          func = common.get_gtest_summary_passes
+        test_results = func(json.load(f))
+
   output_json = {
-      'valid': valid,
-      'failures': failures,
+    'version': 3,
+    'interrupted': False,
+    'path_delimiter': '/',
+    'seconds_since_epoch': start_time,
+    'num_failures_by_type': {
+      'PASS': sum(1 for success in test_results.values() if success),
+      'FAIL': sum(1 for success in test_results.values() if not success),
+    },
+    'tests': {
+        test: test_result_entry(success) for (
+            test, success) in test_results.items()
     }
+  }
+
   return rc, charts, output_json
 
+
+def test_result_entry(success):
+  test = {
+    'expected': 'PASS',
+    'actual': 'PASS' if success else 'FAIL',
+  }
+  if not success:
+    test['unexpected'] = True
+  return test
+
 # This is not really a "script test" so does not need to manually add
 # any additional compile targets.
 def main_compile_targets(args):

testing/scripts/run_performance_tests.py

@@ -197,7 +197,7 @@ def main():
     # For non telemetry tests the benchmark name is the name of the executable.
     benchmark_name = rest_args[0]
     return_code, charts, output_json = run_gtest_perf_test.execute_perf_test(
-        args, rest_args)
+        args, rest_args, False)
 
     write_results(benchmark_name, charts, output_json,
                   benchmark_log='Not available for C++ perf test',