Skip to content

GitLab

T
tangled
Commit 3cf9eadf authored by Emily Hanley's avatar Emily Hanley Committed by Commit Bot
Browse files
Options

Revert "Reland "Reland "gtest perf tests: Output well formed test results json""" 

This reverts commit 4da4a0c8.

Reason for revert: Failing on mac and windows.  I will add more comments to crbug.com/855234

Original change's description:
> Reland "Reland "gtest perf tests: Output well formed test results json""
> 
> This reverts commit 89c5e790.
> 
> Reason for revert: Testing possible fixes
> 
> Original change's description:
> > Revert "Reland "gtest perf tests: Output well formed test results json""
> > 
> > This reverts commit f02c92e1.
> > 
> > Reason for revert: still breaking perf waterfall https://ci.chromium.org/buildbot/chromium.perf/linux-perf/315
> > 
> > Original change's description:
> > > Reland "gtest perf tests: Output well formed test results json"
> > > 
> > > This reverts commit 9eeaba4e.
> > > 
> > > Reason for revert: Will re-land with fix
> > > 
> > > Original change's description:
> > > > Revert "gtest perf tests: Output well formed test results json"
> > > > 
> > > > This reverts commit 71b865f4.
> > > > 
> > > > Reason for revert: break many perf tests (see https://ci.chromium.org/buildbot/chromium.perf/linux-perf/307)
> > > > 
> > > > Original change's description:
> > > > > gtest perf tests: Output well formed test results json
> > > > > 
> > > > > This CL changes the run_gtest_perf_test.py script to output well formed
> > > > > test results json.
> > > > > 
> > > > > Bug: 855234
> > > > > Change-Id: Ia5ce6e258865e34a2aa37635c014cd54efa62796
> > > > > Reviewed-on: https://chromium-review.googlesource.com/1112635
> > > > > Commit-Queue: Stephen Martinis <martiniss@chromium.org>
> > > > > Reviewed-by: Dirk Pranke <dpranke@chromium.org>
> > > > > Cr-Commit-Position: refs/heads/master@{#570943}
> > > > 
> > > > TBR=dpranke@chromium.org,nednguyen@google.com,martiniss@chromium.org,eyaich@chromium.org
> > > > 
> > > > Change-Id: Ia0aaff923014438005ebc0a8677c3bbbebb808ac
> > > > No-Presubmit: true
> > > > No-Tree-Checks: true
> > > > No-Try: true
> > > > Bug: 855234
> > > > Reviewed-on: https://chromium-review.googlesource.com/1118238
> > > > Reviewed-by: Ned Nguyen <nednguyen@google.com>
> > > > Commit-Queue: Ned Nguyen <nednguyen@google.com>
> > > > Cr-Commit-Position: refs/heads/master@{#571083}
> > > 
> > > TBR=dpranke@chromium.org,nednguyen@google.com,martiniss@chromium.org,eyaich@chromium.org
> > > 
> > > Change-Id: Idb255b1fafb7f254fd44d6554f93ff27608279f2
> > > No-Presubmit: true
> > > No-Tree-Checks: true
> > > No-Try: true
> > > Bug: 855234
> > > Reviewed-on: https://chromium-review.googlesource.com/1118789
> > > Commit-Queue: Stephen Martinis <martiniss@chromium.org>
> > > Reviewed-by: Stephen Martinis <martiniss@chromium.org>
> > > Cr-Commit-Position: refs/heads/master@{#571245}
> > 
> > TBR=dpranke@chromium.org,nednguyen@google.com,martiniss@chromium.org,eyaich@chromium.org
> > 
> > Change-Id: I6bc0869caa6a763dc8bb542fd5a1b3d25853ac86
> > No-Presubmit: true
> > No-Tree-Checks: true
> > No-Try: true
> > Bug: 855234
> > Reviewed-on: https://chromium-review.googlesource.com/1119445
> > Reviewed-by: Ned Nguyen <nednguyen@google.com>
> > Commit-Queue: Ned Nguyen <nednguyen@google.com>
> > Cr-Commit-Position: refs/heads/master@{#571321}
> 
> TBR=dpranke@chromium.org,nednguyen@google.com,martiniss@chromium.org,eyaich@chromium.org
> 
> Change-Id: I25bc4bb1f77031d95dd70347bfed886112b76218
> No-Presubmit: true
> No-Tree-Checks: true
> No-Try: true
> Bug: 855234
> Reviewed-on: https://chromium-review.googlesource.com/1120798
> Reviewed-by: Dirk Pranke <dpranke@chromium.org>
> Reviewed-by: Stephen Martinis <martiniss@chromium.org>
> Commit-Queue: Stephen Martinis <martiniss@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#571968}

TBR=dpranke@chromium.org,nednguyen@google.com,martiniss@chromium.org,eyaich@chromium.org

Change-Id: I031c5d6204e0b53fa2719168a38292455d676167
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: 855234
Reviewed-on: https://chromium-review.googlesource.com/1124539Reviewed-by: default avatarEmily Hanley <eyaich@chromium.org>
Commit-Queue: Emily Hanley <eyaich@chromium.org>
Cr-Commit-Position: refs/heads/master@{#572183}
parent deb4b5a5
Hide whitespace changes
Inline Side-by-side
Showing with 45 additions and 106 deletions
testing/scripts/common.py
View file @ 3cf9eadf
......@@ -167,24 +167,6 @@ def parse_common_test_results(json_results, test_separator='/'):
def get_gtest_summary_passes(output):
"""Returns a mapping of test to boolean indicating if the test passed.
"""
if not output:
return {}
mapping = {}
for test_suite in output.get('testsuites', []):
suite_name = test_suite['name']
for test in test_suite['testsuite']:
full_name = '%s.%s' % (suite_name, test['name'])
mapping[full_name] = 'failures' in test
return mapping
def get_chromium_gtest_summary_passes(output):
"""Returns a mapping of test to boolean indicating if the test passed.
Only partially parses the format. This code is based on code in tools/build,
specifically
......
testing/scripts/run_gtest_perf_test.py
View file @ 3cf9eadf
......@@ -32,7 +32,6 @@ import json
import os
import shutil
import sys
import time
import tempfile
import traceback
......@@ -83,7 +82,7 @@ def main():
args, rest_args = parser.parse_known_args()
rc, charts, output_json = execute_perf_test(args, rest_args, True)
rc, charts, output_json = execute_perf_test(args, rest_args)
# TODO(eakuefner): Make isolated_script_test_perf_output mandatory after
# flipping flag in swarming.
......@@ -101,7 +100,7 @@ def main():
return rc
def execute_perf_test(args, rest_args, chromium_gtest):
def execute_perf_test(args, rest_args):
env = os.environ.copy()
# Assume we want to set up the sandbox environment variables all the
# time; doing so is harmless on non-Linux platforms and is needed
......@@ -109,95 +108,53 @@ def execute_perf_test(args, rest_args, chromium_gtest):
env[CHROME_SANDBOX_ENV] = CHROME_SANDBOX_PATH
rc = 0
start_time = time.time()
test_results = {}
with common.temporary_file() as results_path:
try:
executable = rest_args[0]
extra_flags = []
if len(rest_args) > 1:
extra_flags = rest_args[1:]
if chromium_gtest:
output_flag = '--test-launcher-summary-output'
output_file = results_path
try:
executable = rest_args[0]
extra_flags = []
if len(rest_args) > 1:
extra_flags = rest_args[1:]
# These flags are to make sure that test output perf metrics in the log.
if not '--verbose' in extra_flags:
extra_flags.append('--verbose')
if not '--test-launcher-print-test-stdio=always' in extra_flags:
extra_flags.append('--test-launcher-print-test-stdio=always')
if args.isolated_script_test_filter:
filter_list = common.extract_filter_list(
args.isolated_script_test_filter)
extra_flags.append('--gtest_filter=' + ':'.join(filter_list))
if IsWindows():
executable = '.\%s.exe' % executable
else:
executable = './%s' % executable
with common.temporary_file() as tempfile_path:
env['CHROME_HEADLESS'] = '1'
cmd = [executable] + extra_flags
if args.xvfb:
rc = xvfb.run_executable(cmd, env, stdoutfile=tempfile_path)
else:
output_flag = '--gtest_output'
output_file = 'json:%s' % results_path
assert not any(output_flag in flag for flag in extra_flags), (
'Duplicate %s flag detected.' % output_flag)
extra_flags.append('%s=%s' % (output_flag, output_file))
# These flags are to make sure that test output perf metrics in the log.
if not '--verbose' in extra_flags:
extra_flags.append('--verbose')
if not '--test-launcher-print-test-stdio=always' in extra_flags:
extra_flags.append('--test-launcher-print-test-stdio=always')
if args.isolated_script_test_filter:
filter_list = common.extract_filter_list(
args.isolated_script_test_filter)
extra_flags.append('--gtest_filter=' + ':'.join(filter_list))
if IsWindows():
executable = '.\%s.exe' % executable
else:
executable = './%s' % executable
with common.temporary_file() as tempfile_path:
env['CHROME_HEADLESS'] = '1'
cmd = [executable] + extra_flags
print ' '.join(cmd)
if args.xvfb:
rc = xvfb.run_executable(cmd, env, stdoutfile=tempfile_path)
else:
rc = test_env.run_command_with_output(cmd, env=env,
stdoutfile=tempfile_path)
# Now get the correct json format from the stdout to write to the perf
# results file
results_processor = (
generate_legacy_perf_dashboard_json.LegacyResultsProcessor())
charts = results_processor.GenerateJsonResults(tempfile_path)
except Exception:
traceback.print_exc()
rc = 1
if os.path.exists(results_path):
with open(results_path) as f:
if chromium_gtest:
func = common.get_chromium_gtest_summary_passes
else:
func = common.get_gtest_summary_passes
test_results = func(json.load(f))
rc = test_env.run_command_with_output(cmd, env=env,
stdoutfile=tempfile_path)
# Now get the correct json format from the stdout to write to the perf
# results file
results_processor = (
generate_legacy_perf_dashboard_json.LegacyResultsProcessor())
charts = results_processor.GenerateJsonResults(tempfile_path)
except Exception:
traceback.print_exc()
rc = 1
valid = (rc == 0)
failures = [] if valid else ['(entire test suite)']
output_json = {
'version': 3,
'interrupted': False,
'path_delimiter': '/',
'seconds_since_epoch': start_time,
'num_failures_by_type': {
'PASS': sum(1 for success in test_results.values() if success),
'FAIL': sum(1 for success in test_results.values() if not success),
},
'tests': {
test: test_result_entry(success) for (
test, success) in test_results.items()
'valid': valid,
'failures': failures,
}
}
return rc, charts, output_json
def test_result_entry(success):
test = {
'expected': 'PASS',
'actual': 'PASS' if success else 'FAIL',
}
if not success:
test['unexpected'] = True
return test
# This is not really a "script test" so does not need to manually add
# any additional compile targets.
def main_compile_targets(args):
......
testing/scripts/run_performance_tests.py
View file @ 3cf9eadf
......@@ -197,7 +197,7 @@ def main():
# For non telemetry tests the benchmark name is the name of the executable.
benchmark_name = rest_args[0]
return_code, charts, output_json = run_gtest_perf_test.execute_perf_test(
args, rest_args, False)
args, rest_args)
write_results(benchmark_name, charts, output_json,
benchmark_log='Not available for C++ perf test',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment