Skip to content

GitLab

T
tangled
Commit b2636748 authored by Zhaoyang Li's avatar Zhaoyang Li Committed by Commit Bot
Browse files
Options

[iOS][test runner] Copy artifacts for flaky / failed tests. 

Test runner will copy attachments from xcresult to output folder
(isolated output in infra) for all failed tests in each attempt.
Screenshots will be collected only from failing steps to avoid noise.
Other attachments will be collected from every step.
- Put the copy logic in private methods and called these in public
collect_test_results.
- Improved and unified variable naming where touched.
- Removed unused plist check where touched.
- Fixed and added tests for xcode_log_parser.

Bug: 1047704
Change-Id: I448c785a256702d93698e5c73ef89b7477efbc87
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2473499Reviewed-by: default avatarJustin Cohen <justincohen@chromium.org>
Commit-Queue: Zhaoyang Li <zhaoyangli@chromium.org>
Cr-Commit-Position: refs/heads/master@{#819543}
parent 169874f9
Show whitespace changes
Inline Side-by-side
Showing with 550 additions and 127 deletions
ios/build/bots/scripts/PRESUBMIT.py
View file @ b2636748
...@@ -22,7 +22,7 @@ def _RunTestRunnerUnitTests(input_api, output_api): ...@@ -22,7 +22,7 @@ def _RunTestRunnerUnitTests(input_api, output_api):
'test_apps_test.py', 'test_apps_test.py',
# 'test_runner_test.py', # 'test_runner_test.py',
'wpr_runner_test.py', 'wpr_runner_test.py',
# 'xcode_log_parser_test.py', 'xcode_log_parser_test.py',
# 'xcodebuild_runner_test.py', # 'xcodebuild_runner_test.py',
] ]
......
ios/build/bots/scripts/xcode_log_parser.py
View file @ b2636748
...@@ -166,6 +166,7 @@ class Xcode11LogParser(object): ...@@ -166,6 +166,7 @@ class Xcode11LogParser(object):
xcresult: (str) A path to xcresult. xcresult: (str) A path to xcresult.
results: (dict) A dictionary with passed and failed tests. results: (dict) A dictionary with passed and failed tests.
""" """
# See TESTS_REF in xcode_log_parser_test.py for an example of |root|.
root = json.loads(Xcode11LogParser._xcresulttool_get(xcresult, 'testsRef')) root = json.loads(Xcode11LogParser._xcresulttool_get(xcresult, 'testsRef'))
for summary in root['summaries']['_values'][0][ for summary in root['summaries']['_values'][0][
'testableSummaries']['_values']: 'testableSummaries']['_values']:
...@@ -187,7 +188,8 @@ class Xcode11LogParser(object): ...@@ -187,7 +188,8 @@ class Xcode11LogParser(object):
if test['testStatus']['_value'] == 'Success': if test['testStatus']['_value'] == 'Success':
results['passed'].append(test_name) results['passed'].append(test_name)
else: else:
# Parse data for failed test by its id. # Parse data for failed test by its id. See SINGLE_TEST_SUMMARY_REF
# in xcode_log_parser_test.py for an example of |rootFailure|.
rootFailure = json.loads( rootFailure = json.loads(
Xcode11LogParser._xcresulttool_get( Xcode11LogParser._xcresulttool_get(
xcresult, test['summaryRef']['id']['_value'])) xcresult, test['summaryRef']['id']['_value']))
...@@ -204,11 +206,12 @@ class Xcode11LogParser(object): ...@@ -204,11 +206,12 @@ class Xcode11LogParser(object):
results['failed'][test_name] = failure_message results['failed'][test_name] = failure_message
@staticmethod @staticmethod
def collect_test_results(xcresult, output): def collect_test_results(output_path, output):
"""Gets test result and diagnostic data from xcresult. """Gets test result, diagnostic data & artifacts from xcresult.
Args: Args:
xcresult: (str) A path to xcresult. output_path: (str) An output path passed in --resultBundlePath when
running xcodebuild.
output: [str] An output of test run. output: [str] An output of test run.
Returns: Returns:
...@@ -220,26 +223,26 @@ class Xcode11LogParser(object): ...@@ -220,26 +223,26 @@ class Xcode11LogParser(object):
} }
} }
""" """
LOGGER.info('Reading %s' % xcresult) LOGGER.info('Reading %s' % output_path)
test_results = { test_results = {
'passed': [], 'passed': [],
'failed': {} 'failed': {}
} }
if not os.path.exists(xcresult):
test_results['failed']['TESTS_DID_NOT_START'] = [
'%s with test results does not exist.' % xcresult]
return test_results
# During a run `xcodebuild .. -resultBundlePath %output_path%` # During a run `xcodebuild .. -resultBundlePath %output_path%`
# that generates output_path folder, # that generates output_path folder,
# but Xcode 11+ generates `output_path.xcresult` and `output_path` # but Xcode 11+ generates `output_path.xcresult` and `output_path`
# where output_path.xcresult is a folder with results and `output_path` # where output_path.xcresult is a folder with results and `output_path`
# is symlink to the `output_path.xcresult` folder. # is symlink to the `output_path.xcresult` folder.
# `xcresulttool` with folder/symlink behaves # `xcresulttool` with folder/symlink behaves in different way on laptop and
# in different way on laptop and on bots. # on bots. This piece of code uses .xcresult folder.
# To support debugging added this check. xcresult = output_path + '.xcresult'
if not xcresult.endswith('.xcresult'):
xcresult += '.xcresult' if not os.path.exists(xcresult):
test_results['failed']['TESTS_DID_NOT_START'] = [
'%s with test results does not exist.' % xcresult
]
return test_results
plist_path = os.path.join(xcresult, 'Info.plist') plist_path = os.path.join(xcresult, 'Info.plist')
if not os.path.exists(plist_path): if not os.path.exists(plist_path):
...@@ -248,6 +251,7 @@ class Xcode11LogParser(object): ...@@ -248,6 +251,7 @@ class Xcode11LogParser(object):
test_results['passed'] = parse_passed_tests_for_interrupted_run(output) test_results['passed'] = parse_passed_tests_for_interrupted_run(output)
return test_results return test_results
# See XCRESULT_ROOT in xcode_log_parser_test.py for an example of |root|.
root = json.loads(Xcode11LogParser._xcresulttool_get(xcresult)) root = json.loads(Xcode11LogParser._xcresulttool_get(xcresult))
metrics = root['metrics'] metrics = root['metrics']
# In case of test crash both numbers of run and failed tests are equal to 0. # In case of test crash both numbers of run and failed tests are equal to 0.
...@@ -259,35 +263,123 @@ class Xcode11LogParser(object): ...@@ -259,35 +263,123 @@ class Xcode11LogParser(object):
test_results['failed'] = Xcode11LogParser._list_of_failed_tests(root) test_results['failed'] = Xcode11LogParser._list_of_failed_tests(root)
Xcode11LogParser._get_test_statuses(xcresult, test_results) Xcode11LogParser._get_test_statuses(xcresult, test_results)
Xcode11LogParser._export_diagnostic_data(xcresult) Xcode11LogParser._export_diagnostic_data(xcresult)
Xcode11LogParser._copy_artifacts(xcresult)
# Remove the symbol link file.
if os.path.islink(output_path):
os.unlink(output_path)
Xcode11LogParser._zip_and_remove_folder(xcresult)
return test_results return test_results
@staticmethod @staticmethod
def copy_screenshots(output_folder): def _copy_artifacts(xcresult):
"""Copy screenshots of failed tests to output folder. """Copy screenshots, crash logs of failed tests to output folder.
Args: Args:
output_folder: (str) A full path to folder where xcresult: (str) A path to xcresult directory.
""" """
plist_path = os.path.join(output_folder + '.xcresult', 'Info.plist') if not os.path.exists(xcresult):
if not os.path.exists(plist_path): LOGGER.warn('%s does not exist.' % xcresult)
LOGGER.info('%s does not exist.' % plist_path)
return return
root = json.loads(Xcode11LogParser._xcresulttool_get(output_folder)) root = json.loads(Xcode11LogParser._xcresulttool_get(xcresult))
if 'testFailureSummaries' not in root['issues']: if 'testFailureSummaries' not in root.get('issues', {}):
LOGGER.info('No failures in %s' % output_folder) LOGGER.info('No failures in %s' % xcresult)
return return
for failure_summary in root['issues']['testFailureSummaries']['_values']: # See TESTS_REF['summaries']['_values'] in xcode_log_parser_test.py.
test_case = failure_summary['testCaseName']['_value'] test_summaries = json.loads(
test_case_folder = os.path.join(output_folder, 'failures', Xcode11LogParser._xcresulttool_get(xcresult, 'testsRef')).get(
format_test_case(test_case)) 'summaries', {}).get('_values', [])
copy_screenshots_for_failed_test(failure_summary['message']['_value'],
test_case_folder) test_summary_refs = {}
for summaries in test_summaries:
for summary in summaries.get('testableSummaries', {}).get('_values', []):
for all_tests in summary.get('tests', {}).get('_values', []):
for test_suite in all_tests.get('subtests', {}).get('_values', []):
for test_case in test_suite.get('subtests', {}).get('_values', []):
for test in test_case.get('subtests', {}).get('_values', []):
if test['testStatus']['_value'] != 'Success':
test_summary_refs[
test['identifier']
['_value']] = test['summaryRef']['id']['_value']
def extract_attachments(test,
test_activities,
xcresult,
include_jpg=True,
attachment_index=0):
"""Exrtact attachments from xcretult folder.
Copies all attachments under test_activities and nested subactivities(if
any) to the same directory as xcresult directory. Uses incremental
attachment_index starting from attachment_index + 1.
Args:
test: (str) Test name.
test_activities: (list) List of test activities (dict) that
store data about each test step.
xcresult: (str) A path to test results.
include_jpg: (bool) Whether include jpg or jpeg attachments.
attachment_index: (int) An attachment index, used as an incremental id
for file names in format
`attempt_%d_TestCase_testMethod_attachment_index`:
attempt_0_TestCase_testMethod_1.jpg
....
attempt_0_TestCase_testMethod_3.crash
Returns:
Last used attachment_index.
"""
for activity_summary in test_activities:
if 'subactivities' in activity_summary:
attachment_index = extract_attachments(
test,
activity_summary.get('subactivities', {}).get('_values', []),
xcresult, attachment_index)
for attachment in activity_summary.get('attachments',
{}).get('_values', []):
payload_ref = attachment['payloadRef']['id']['_value']
_, file_name_extension = os.path.splitext(
attachment['filename']['_value'])
if not include_jpg and file_name_extension in ['.jpg', '.jpeg']:
continue
attachment_index += 1
attachment_filename = (
'%s_%s_%d%s' %
(os.path.splitext(os.path.basename(xcresult))[0],
test.replace('/', '_'), attachment_index, file_name_extension))
# Extracts attachment to the same folder containing xcresult.
attachment_output_path = os.path.abspath(
os.path.join(xcresult, os.pardir, attachment_filename))
Xcode11LogParser._export_data(xcresult, payload_ref, 'file',
attachment_output_path)
return attachment_index
for test, summaryRef in test_summary_refs.iteritems():
# See SINGLE_TEST_SUMMARY_REF in xcode_log_parser_test.py for an example
# of |test_summary|.
test_summary = json.loads(
Xcode11LogParser._xcresulttool_get(xcresult, summaryRef))
# Extract all attachments except for screenshots from each step of the
# failed test.
index = extract_attachments(
test,
test_summary.get('activitySummaries', {}).get('_values', []),
xcresult,
include_jpg=False)
# Extract all attachments for at the failure step.
extract_attachments(
test,
test_summary.get('failureSummaries', {}).get('_values', []),
xcresult,
include_jpg=True,
attachment_index=index)
@staticmethod @staticmethod
def _export_diagnostic_data(xcresult): def _export_diagnostic_data(xcresult):
"""Exports diagnostic data from xcresult to xcresult_diagnostic folder. """Exports diagnostic data from xcresult to xcresult_diagnostic.zip.
Since Xcode 11 format of result bundles changed, to get diagnostic data Since Xcode 11 format of result bundles changed, to get diagnostic data
need to run command below: need to run command below:
...@@ -297,22 +389,53 @@ class Xcode11LogParser(object): ...@@ -297,22 +389,53 @@ class Xcode11LogParser(object):
Args: Args:
xcresult: (str) A path to xcresult directory. xcresult: (str) A path to xcresult directory.
""" """
plist_path = os.path.join(xcresult, 'Info.plist') if not os.path.exists(xcresult):
if not (os.path.exists(xcresult) and os.path.exists(plist_path)): LOGGER.warn('%s does not exist.' % xcresult)
return return
root = json.loads(Xcode11LogParser._xcresulttool_get(xcresult)) root = json.loads(Xcode11LogParser._xcresulttool_get(xcresult))
try: try:
diagnostics_ref = root['actions']['_values'][0]['actionResult'][ diagnostics_ref = root['actions']['_values'][0]['actionResult'][
'diagnosticsRef']['id']['_value'] 'diagnosticsRef']['id']['_value']
export_command = ['xcresulttool', 'export', diagnostic_folder = '%s_diagnostic' % xcresult
'--type', 'directory', Xcode11LogParser._export_data(xcresult, diagnostics_ref, 'directory',
'--id', diagnostics_ref, diagnostic_folder)
'--path', xcresult, Xcode11LogParser._zip_and_remove_folder(diagnostic_folder)
'--output-path', '%s_diagnostic' % xcresult]
subprocess.check_output(export_command).strip()
except KeyError: except KeyError:
LOGGER.warn('Did not parse diagnosticsRef from %s!' % xcresult) LOGGER.warn('Did not parse diagnosticsRef from %s!' % xcresult)
@staticmethod
def _export_data(xcresult, ref_id, output_type, output_path):
"""Exports data from xcresult using xcresulttool.
Since Xcode 11 format of result bundles changed, to get diagnostic data
need to run command below:
xcresulttool export --type directory --id DIAGNOSTICS_REF --output-path
./export_folder --path ./RB.xcresult
Args:
xcresult: (str) A path to xcresult directory.
ref_id: (str) A reference id of exporting entity.
output_type: (str) An export type (can be directory or file).
output_path: (str) An output location.
"""
export_command = [
'xcresulttool', 'export', '--type', output_type, '--id', ref_id,
'--path', xcresult, '--output-path', output_path
]
subprocess.check_output(export_command).strip()
@staticmethod
def _zip_and_remove_folder(dir_path):
"""Zips folder to the parent folder and then removes original folder.
Args:
dir_path: (str) A path to directory.
"""
shutil.make_archive(
os.path.join(os.path.dirname(dir_path), os.path.basename(dir_path)),
'zip', dir_path)
shutil.rmtree(dir_path)
class XcodeLogParser(object): class XcodeLogParser(object):
"""Xcode log parser. Parses logs for Xcode until version 11.""" """Xcode log parser. Parses logs for Xcode until version 11."""
...@@ -335,10 +458,7 @@ class XcodeLogParser(object): ...@@ -335,10 +458,7 @@ class XcodeLogParser(object):
} }
""" """
root_summary = plistlib.readPlist(summary_plist) root_summary = plistlib.readPlist(summary_plist)
status_summary = { status_summary = {'passed': [], 'failed': {}}
'passed': [],
'failed': {}
}
for summary in root_summary['TestableSummaries']: for summary in root_summary['TestableSummaries']:
failed_egtests = {} # Contains test identifier and message failed_egtests = {} # Contains test identifier and message
passed_egtests = [] passed_egtests = []
...@@ -366,7 +486,7 @@ class XcodeLogParser(object): ...@@ -366,7 +486,7 @@ class XcodeLogParser(object):
@staticmethod @staticmethod
def collect_test_results(output_folder, output): def collect_test_results(output_folder, output):
"""Gets test result data from Info.plist. """Gets test result data from Info.plist and copies artifacts.
Args: Args:
output_folder: (str) A path to output folder. output_folder: (str) A path to output folder.
...@@ -380,14 +500,12 @@ class XcodeLogParser(object): ...@@ -380,14 +500,12 @@ class XcodeLogParser(object):
} }
} }
""" """
test_results = { test_results = {'passed': [], 'failed': {}}
'passed': [],
'failed': {}
}
plist_path = os.path.join(output_folder, 'Info.plist') plist_path = os.path.join(output_folder, 'Info.plist')
if not os.path.exists(plist_path): if not os.path.exists(plist_path):
test_results['failed']['BUILD_INTERRUPTED'] = [ test_results['failed']['BUILD_INTERRUPTED'] = [
'%s with test results does not exist.' % plist_path] + output '%s with test results does not exist.' % plist_path
] + output
test_results['passed'] = parse_passed_tests_for_interrupted_run(output) test_results['passed'] = parse_passed_tests_for_interrupted_run(output)
return test_results return test_results
...@@ -395,25 +513,26 @@ class XcodeLogParser(object): ...@@ -395,25 +513,26 @@ class XcodeLogParser(object):
for action in root['Actions']: for action in root['Actions']:
action_result = action['ActionResult'] action_result = action['ActionResult']
if ((root['TestsCount'] == 0 and if ((root['TestsCount'] == 0 and root['TestsFailedCount'] == 0) or
root['TestsFailedCount'] == 0) 'TestSummaryPath' not in action_result):
or 'TestSummaryPath' not in action_result):
test_results['failed']['TESTS_DID_NOT_START'] = [] test_results['failed']['TESTS_DID_NOT_START'] = []
if ('ErrorSummaries' in action_result if ('ErrorSummaries' in action_result and
and action_result['ErrorSummaries']): action_result['ErrorSummaries']):
test_results['failed']['TESTS_DID_NOT_START'].append('\n'.join( test_results['failed']['TESTS_DID_NOT_START'].append('\n'.join(
error_summary['Message'] error_summary['Message']
for error_summary in action_result['ErrorSummaries'])) for error_summary in action_result['ErrorSummaries']))
else: else:
summary_plist = os.path.join(os.path.dirname(plist_path), summary_plist = os.path.join(
action_result['TestSummaryPath']) os.path.dirname(plist_path), action_result['TestSummaryPath'])
summary = XcodeLogParser._test_status_summary(summary_plist) summary = XcodeLogParser._test_status_summary(summary_plist)
test_results['failed'] = summary['failed'] test_results['failed'] = summary['failed']
test_results['passed'] = summary['passed'] test_results['passed'] = summary['passed']
XcodeLogParser._copy_screenshots(output_folder)
return test_results return test_results
@staticmethod @staticmethod
def copy_screenshots(output_folder): def _copy_screenshots(output_folder):
"""Copy screenshots of failed tests to output folder. """Copy screenshots of failed tests to output folder.
Args: Args:
......
ios/build/bots/scripts/xcode_log_parser_test.py
View file @ b2636748
...@@ -14,49 +14,93 @@ import test_runner_test ...@@ -14,49 +14,93 @@ import test_runner_test
import xcode_log_parser import xcode_log_parser
_XTEST_RESULT = '/tmp/temp_file.xcresult' OUTPUT_PATH = '/tmp/attempt_0'
XCRESULT_PATH = '/tmp/attempt_0.xcresult'
XCODE11_DICT = { XCODE11_DICT = {
'path': '/Users/user1/Xcode.app', 'path': '/Users/user1/Xcode.app',
'version': '11.0', 'version': '11.0',
'build': '11M336w', 'build': '11M336w',
} }
REF_ID = """ # A sample of json result when executing xcresulttool on .xcresult dir without
# --id. Some unused keys and values were removed.
XCRESULT_ROOT = """
{
"_type" : {
"_name" : "ActionsInvocationRecord"
},
"actions" : {
"_values" : [
{ {
"actions": { "actionResult" : {
"_values": [{ "_type" : {
"actionResult": { "_name" : "ActionResult"
"testsRef": { },
"id": { "diagnosticsRef" : {
"_value": "REF_ID" "id" : {
"_value" : "DIAGNOSTICS_REF_ID"
} }
},
"logRef" : {
"id" : {
"_value" : "0~6jr1GkZxoWVzWfcUNA5feff3l7g8fPHJ1rqKetCBa3QXhCGY74PnEuRwzktleMTFounMfCdDpSr1hRfhUGIUEQ=="
} }
},
"testsRef" : {
"id" : {
"_value" : "0~iRbOkDnmtKVIvHSV2jkeuNcg4RDTUaCLZV7KijyxdCqvhqtp08MKxl0MwjBAPpjmruoI7qNHzBR1RJQAlANNHA=="
} }
}]
} }
}""" }
}
ACTIONS_RECORD_FAILED_TEST = """ ]
},
"issues" : {
"testFailureSummaries" : {
"_values" : [
{ {
"issues": { "documentLocationInCreatingWorkspace" : {
"testFailureSummaries": { "url" : {
"_values": [{ "_value" : "file:\/\/\/..\/..\/ios\/web\/shell\/test\/page_state_egtest.mm#CharacterRangeLen=0&EndingLineNumber=130&StartingLineNumber=130"
"documentLocationInCreatingWorkspace": {
"url": {
"_value": "file://<unknown>#CharacterRangeLen=0"
} }
}, },
"message": { "message" : {
"_value": "Fail. Screenshots: {\\n\\"Failure\\": \\"path.png\\"\\n}" "_value": "Fail. Screenshots: {\\n\\"Failure\\": \\"path.png\\"\\n}"
}, },
"testCaseName": { "testCaseName" : {
"_value": "-[WebUITestCase testBackForwardFromWebURL]" "_value": "-[PageStateTestCase testZeroContentOffsetAfterLoad]"
} }
}]
} }
]
}
},
"metrics" : {
"testsCount" : {
"_value" : "2"
},
"testsFailedCount" : {
"_value" : "1"
}
}
}"""
REF_ID = """
{
"actions": {
"_values": [{
"actionResult": {
"testsRef": {
"id": {
"_value": "REF_ID"
}
}
}
}]
} }
}""" }"""
PASSED_TESTS = """ # A sample of json result when executing xcresulttool on .xcresult dir with
# "testsRef" as --id input. Some unused keys and values were removed.
TESTS_REF = """
{ {
"summaries": { "summaries": {
"_values": [{ "_values": [{
...@@ -70,31 +114,54 @@ PASSED_TESTS = """ ...@@ -70,31 +114,54 @@ PASSED_TESTS = """
"_name": "Array" "_name": "Array"
}, },
"_values": [{ "_values": [{
"identifier" : {
"_value" : "All tests"
},
"name" : {
"_value" : "All tests"
},
"subtests": { "subtests": {
"_values": [{ "_values": [{
"identifier" : {
"_value" : "ios_web_shell_eg2tests_module.xctest"
},
"name" : {
"_value" : "ios_web_shell_eg2tests_module.xctest"
},
"subtests": { "subtests": {
"_values": [{ "_values": [{
"identifier" : {
"_value" : "PageStateTestCase"
},
"name" : {
"_value" : "PageStateTestCase"
},
"subtests": { "subtests": {
"_values": [{ "_values": [{
"testStatus": { "testStatus": {
"_value": "Success" "_value": "Success"
}, },
"identifier": { "identifier": {
"_value": "TestCase1/testMethod1" "_value": "PageStateTestCase/testMethod1"
}, },
"name": { "name": {
"_value": "testMethod1" "_value": "testMethod1"
} }
}, },
{ {
"summaryRef": {
"id": {
"_value": "0~7Q_uAuUSJtx9gtHM08psXFm3g_xiTTg5bpdoDO88nMXo_iMwQTXpqlrlMe5AtkYmnZ7Ux5uEgAe83kJBfoIckw=="
}
},
"testStatus": { "testStatus": {
"_value": "Failure" "_value": "Failure"
}, },
"identifier": { "identifier": {
"_value": "TestCase1/testFailed1" "_value": "PageStateTestCase\/testZeroContentOffsetAfterLoad"
}, },
"name": { "name": {
"_value": "testFailed1" "_value": "testZeroContentOffsetAfterLoad"
} }
}, },
{ {
...@@ -102,10 +169,10 @@ PASSED_TESTS = """ ...@@ -102,10 +169,10 @@ PASSED_TESTS = """
"_value": "Success" "_value": "Success"
}, },
"identifier": { "identifier": {
"_value": "TestCase2/testMethod1" "_value": "PageStateTestCase/testMethod2"
}, },
"name": { "name": {
"_value": "testMethod1" "_value": "testMethod2"
} }
}] }]
} }
...@@ -122,6 +189,194 @@ PASSED_TESTS = """ ...@@ -122,6 +189,194 @@ PASSED_TESTS = """
} }
""" """
# A sample of json result when executing xcresulttool on .xcresult dir with
# a single test summaryRef id value as --id input. Some unused keys and values
# were removed.
SINGLE_TEST_SUMMARY_REF = """
{
"_type" : {
"_name" : "ActionTestSummary",
"_supertype" : {
"_name" : "ActionTestSummaryIdentifiableObject",
"_supertype" : {
"_name" : "ActionAbstractTestSummary"
}
}
},
"activitySummaries" : {
"_values" : [
{
"attachments" : {
"_values" : [
{
"filename" : {
"_value" : "Screenshot_25659115-F3E4-47AE-AA34-551C94333D7E.jpg"
},
"payloadRef" : {
"id" : {
"_value" : "SCREENSHOT_REF_ID_1"
}
}
}
]
},
"title" : {
"_value" : "Start Test at 2020-10-19 14:12:58.111"
}
},
{
"subactivities" : {
"_values" : [
{
"attachments" : {
"_values" : [
{
"filename" : {
"_value" : "Screenshot_23D95D0E-8B97-4F99-BE3C-A46EDE5999D7.jpg"
},
"payloadRef" : {
"id" : {
"_value" : "SCREENSHOT_REF_ID_2"
}
}
}
]
},
"subactivities" : {
"_values" : [
{
"subactivities" : {
"_values" : [
{
"attachments" : {
"_values" : [
{
"filename" : {
"_value" : "Crash_3F0A2B1C-7ADA-436E-A54C-D4C39B8411F8.crash"
},
"payloadRef" : {
"id" : {
"_value" : "CRASH_REF_ID_IN_ACTIVITY_SUMMARIES"
}
}
}
]
},
"title" : {
"_value" : "Wait for org.chromium.ios-web-shell-eg2tests to idle"
}
}
]
},
"title" : {
"_value" : "Activate org.chromium.ios-web-shell-eg2tests"
}
}
]
},
"title" : {
"_value" : "Open org.chromium.ios-web-shell-eg2tests"
}
}
]
},
"title" : {
"_value" : "Set Up"
}
},
{
"title" : {
"_value" : "Find the Target Application 'org.chromium.ios-web-shell-eg2tests'"
}
},
{
"attachments" : {
"_values" : [
{
"filename" : {
"_value" : "Screenshot_278BA84B-2196-4CCD-9D31-2C07DDDC9DFC.jpg"
},
"payloadRef" : {
"id" : {
"_value" : "SCREENSHOT_REF_ID_3"
}
}
}
]
},
"title" : {
"_value" : "Uncaught Exception at page_state_egtest.mm:131: \\nCannot scroll, the..."
}
},
{
"title" : {
"_value" : "Uncaught Exception: Immediately halt execution of testcase (EarlGreyInternalTestInterruptException)"
}
},
{
"title" : {
"_value" : "Tear Down"
}
}
]
},
"failureSummaries" : {
"_values" : [
{
"attachments" : {
"_values" : [
{
"filename" : {
"_value" : "kXCTAttachmentLegacyScreenImageData_1_6CED1FE5-96CA-47EA-9852-6FADED687262.jpeg"
},
"payloadRef" : {
"id" : {
"_value" : "SCREENSHOT_REF_ID_IN_FAILURE_SUMMARIES"
}
}
}
]
},
"fileName" : {
"_value" : "\/..\/..\/ios\/web\/shell\/test\/page_state_egtest.mm"
},
"lineNumber" : {
"_value" : "131"
},
"message" : {
"_value" : "Some logs."
}
},
{
"message" : {
"_value" : "Immediately halt execution of testcase (EarlGreyInternalTestInterruptException)"
}
}
]
},
"identifier" : {
"_value" : "PageStateTestCase\/testZeroContentOffsetAfterLoad"
},
"name" : {
"_value" : "testZeroContentOffsetAfterLoad"
},
"testStatus" : {
"_value" : "Failure"
}
}"""
def _xcresulttool_get_side_effect(xcresult_path, ref_id=None):
"""Side effect for _xcresulttool_get in Xcode11LogParser tested."""
if ref_id is None:
return XCRESULT_ROOT
if ref_id == 'testsRef':
return TESTS_REF
# Other situation in use cases of xcode_log_parser is asking for single test
# summary ref.
return SINGLE_TEST_SUMMARY_REF
class XCode11LogParserTest(test_runner_test.TestCase): class XCode11LogParserTest(test_runner_test.TestCase):
"""Test case to test Xcode11LogParser.""" """Test case to test Xcode11LogParser."""
...@@ -154,29 +409,35 @@ class XCode11LogParserTest(test_runner_test.TestCase): ...@@ -154,29 +409,35 @@ class XCode11LogParserTest(test_runner_test.TestCase):
def testXcresulttoolListFailedTests(self): def testXcresulttoolListFailedTests(self):
failure_message = [ failure_message = [
'file://<unknown>#CharacterRangeLen=0' 'file:///../../ios/web/shell/test/page_state_egtest.mm#'
'CharacterRangeLen=0&EndingLineNumber=130&StartingLineNumber=130'
] + 'Fail. Screenshots: {\n\"Failure\": \"path.png\"\n}'.splitlines() ] + 'Fail. Screenshots: {\n\"Failure\": \"path.png\"\n}'.splitlines()
expected = { expected = {
'WebUITestCase/testBackForwardFromWebURL': failure_message 'PageStateTestCase/testZeroContentOffsetAfterLoad': failure_message
} }
self.assertEqual(expected, self.assertEqual(
expected,
xcode_log_parser.Xcode11LogParser()._list_of_failed_tests( xcode_log_parser.Xcode11LogParser()._list_of_failed_tests(
json.loads(ACTIONS_RECORD_FAILED_TEST))) json.loads(XCRESULT_ROOT)))
@mock.patch('xcode_log_parser.Xcode11LogParser._xcresulttool_get') @mock.patch('xcode_log_parser.Xcode11LogParser._xcresulttool_get')
def testXcresulttoolListPassedTests(self, mock_xcresult): def testXcresulttoolListPassedTests(self, mock_xcresult):
mock_xcresult.return_value = PASSED_TESTS mock_xcresult.side_effect = _xcresulttool_get_side_effect
expected = ['TestCase1/testMethod1', 'TestCase2/testMethod1'] expected = [
self.assertEqual( 'PageStateTestCase/testMethod1', 'PageStateTestCase/testMethod2'
expected, ]
xcode_log_parser.Xcode11LogParser()._get_test_statuses(_XTEST_RESULT)) results = {'passed': [], 'failed': {}}
xcode_log_parser.Xcode11LogParser()._get_test_statuses(OUTPUT_PATH, results)
self.assertEqual(expected, results['passed'])
@mock.patch('xcode_log_parser.Xcode11LogParser._zip_and_remove_folder')
@mock.patch('xcode_log_parser.Xcode11LogParser._copy_artifacts')
@mock.patch('xcode_log_parser.Xcode11LogParser._export_diagnostic_data')
@mock.patch('os.path.exists', autospec=True) @mock.patch('os.path.exists', autospec=True)
@mock.patch('xcode_log_parser.Xcode11LogParser._xcresulttool_get') @mock.patch('xcode_log_parser.Xcode11LogParser._xcresulttool_get')
@mock.patch('xcode_log_parser.Xcode11LogParser._list_of_failed_tests') @mock.patch('xcode_log_parser.Xcode11LogParser._list_of_failed_tests')
@mock.patch('xcode_log_parser.Xcode11LogParser._list_of_passed_tests') def testCollectTestTesults(self, mock_get_failed_tests, mock_root,
def testCollectTestTesults(self, mock_get_passed_tests, mock_get_failed_tests, mock_exist_file, *args):
mock_root, mock_exist_file):
metrics_json = """ metrics_json = """
{ {
"metrics": { "metrics": {
...@@ -190,7 +451,8 @@ class XCode11LogParserTest(test_runner_test.TestCase): ...@@ -190,7 +451,8 @@ class XCode11LogParserTest(test_runner_test.TestCase):
}""" }"""
expected_test_results = { expected_test_results = {
'passed': [ 'passed': [
'TestCase1/testMethod1', 'TestCase2/testMethod1'], 'PageStateTestCase/testMethod1', 'PageStateTestCase/testMethod2'
],
'failed': { 'failed': {
'WebUITestCase/testBackForwardFromWebURL': [ 'WebUITestCase/testBackForwardFromWebURL': [
'file://<unknown>#CharacterRangeLen=0', 'file://<unknown>#CharacterRangeLen=0',
...@@ -198,59 +460,102 @@ class XCode11LogParserTest(test_runner_test.TestCase): ...@@ -198,59 +460,102 @@ class XCode11LogParserTest(test_runner_test.TestCase):
] ]
} }
} }
mock_get_passed_tests.return_value = expected_test_results['passed']
mock_get_failed_tests.return_value = expected_test_results['failed'] mock_get_failed_tests.return_value = expected_test_results['failed']
mock_root.return_value = metrics_json mock_root.side_effect = _xcresulttool_get_side_effect
mock_exist_file.return_value = True mock_exist_file.return_value = True
self.assertEqual(expected_test_results, self.assertEqual(
expected_test_results,
xcode_log_parser.Xcode11LogParser().collect_test_results( xcode_log_parser.Xcode11LogParser().collect_test_results(
_XTEST_RESULT, [])) OUTPUT_PATH, []))
@mock.patch('xcode_log_parser.Xcode11LogParser._zip_and_remove_folder')
@mock.patch('xcode_log_parser.Xcode11LogParser._copy_artifacts')
@mock.patch('xcode_log_parser.Xcode11LogParser._export_diagnostic_data')
@mock.patch('os.path.exists', autospec=True) @mock.patch('os.path.exists', autospec=True)
@mock.patch('xcode_log_parser.Xcode11LogParser._xcresulttool_get') @mock.patch('xcode_log_parser.Xcode11LogParser._xcresulttool_get')
def testCollectTestsRanZeroTests(self, mock_root, mock_exist_file): def testCollectTestsRanZeroTests(self, mock_root, mock_exist_file, *args):
metrics_json = '{"metrics": {}}' metrics_json = '{"metrics": {}}'
expected_test_results = { expected_test_results = {
'passed': [], 'passed': [],
'failed': {'TESTS_DID_NOT_START': ['0 tests executed!']}} 'failed': {'TESTS_DID_NOT_START': ['0 tests executed!']}}
mock_root.return_value = metrics_json mock_root.return_value = metrics_json
mock_exist_file.return_value = True mock_exist_file.return_value = True
self.assertEqual(expected_test_results, self.assertEqual(
expected_test_results,
xcode_log_parser.Xcode11LogParser().collect_test_results( xcode_log_parser.Xcode11LogParser().collect_test_results(
_XTEST_RESULT, [])) OUTPUT_PATH, []))
@mock.patch('os.path.exists', autospec=True) @mock.patch('os.path.exists', autospec=True)
def testCollectTestsDidNotRun(self, mock_exist_file): def testCollectTestsDidNotRun(self, mock_exist_file):
mock_exist_file.return_value = False mock_exist_file.return_value = False
expected_test_results = { expected_test_results = {
'passed': [], 'passed': [],
'failed': {'TESTS_DID_NOT_START': [ 'failed': {
'%s with test results does not exist.' % _XTEST_RESULT]}} 'TESTS_DID_NOT_START': [
self.assertEqual(expected_test_results, '%s.xcresult with test results does not exist.' % OUTPUT_PATH
]
}
}
self.assertEqual(
expected_test_results,
xcode_log_parser.Xcode11LogParser().collect_test_results( xcode_log_parser.Xcode11LogParser().collect_test_results(
_XTEST_RESULT, [])) OUTPUT_PATH, []))
@mock.patch('os.path.exists', autospec=True) @mock.patch('os.path.exists', autospec=True)
def testCollectTestsInterruptedRun(self, mock_exist_file): def testCollectTestsInterruptedRun(self, mock_exist_file):
mock_exist_file.side_effect = [True, False] mock_exist_file.side_effect = [True, False]
expected_test_results = { expected_test_results = {
'passed': [], 'passed': [],
'failed': {'BUILD_INTERRUPTED': [ 'failed': {
'%s with test results does not exist.' % os.path.join( 'BUILD_INTERRUPTED': [
_XTEST_RESULT + '.xcresult', 'Info.plist')]}} '%s with test results does not exist.' %
self.assertEqual(expected_test_results, os.path.join(OUTPUT_PATH + '.xcresult', 'Info.plist')
]
}
}
self.assertEqual(
expected_test_results,
xcode_log_parser.Xcode11LogParser().collect_test_results( xcode_log_parser.Xcode11LogParser().collect_test_results(
_XTEST_RESULT, [])) OUTPUT_PATH, []))
@mock.patch('subprocess.check_output', autospec=True)
@mock.patch('os.path.exists', autospec=True)
@mock.patch('xcode_log_parser.Xcode11LogParser._xcresulttool_get')
def testCopyScreenshots(self, mock_xcresulttool_get, mock_path_exists,
mock_process):
mock_path_exists.return_value = True
mock_xcresulttool_get.side_effect = _xcresulttool_get_side_effect
xcode_log_parser.Xcode11LogParser()._copy_artifacts(XCRESULT_PATH)
mock_process.assert_any_call([
'xcresulttool', 'export', '--type', 'file', '--id',
'SCREENSHOT_REF_ID_IN_FAILURE_SUMMARIES', '--path', XCRESULT_PATH,
'--output-path',
'/tmp/attempt_0_PageStateTestCase_testZeroContentOffsetAfterLoad_2.jpeg'
])
mock_process.assert_any_call([
'xcresulttool', 'export', '--type', 'file', '--id',
'CRASH_REF_ID_IN_ACTIVITY_SUMMARIES', '--path', XCRESULT_PATH,
'--output-path',
'/tmp/attempt_0_PageStateTestCase_testZeroContentOffsetAfterLoad_1'
'.crash'
])
# Ensures screenshots in activitySummaries are not copied.
self.assertEqual(2, mock_process.call_count)
@mock.patch('xcode_log_parser.Xcode11LogParser._zip_and_remove_folder')
@mock.patch('subprocess.check_output', autospec=True)
@mock.patch('os.path.exists', autospec=True) @mock.patch('os.path.exists', autospec=True)
@mock.patch('xcode_log_parser.Xcode11LogParser._xcresulttool_get') @mock.patch('xcode_log_parser.Xcode11LogParser._xcresulttool_get')
@mock.patch('shutil.copyfile', autospec=True) def testExportDiagnosticData(self, mock_xcresulttool_get, mock_path_exists,
def testCopyScreenshots(self, mock_copy, mock_xcresulttool_get, mock_process, _):
mock_exist_file): mock_path_exists.return_value = True
mock_exist_file.return_value = True mock_xcresulttool_get.side_effect = _xcresulttool_get_side_effect
mock_xcresulttool_get.return_value = ACTIONS_RECORD_FAILED_TEST xcode_log_parser.Xcode11LogParser._export_diagnostic_data(XCRESULT_PATH)
xcode_log_parser.Xcode11LogParser().copy_screenshots(_XTEST_RESULT) mock_process.assert_called_with([
self.assertEqual(1, mock_copy.call_count) 'xcresulttool', 'export', '--type', 'directory', '--id',
'DIAGNOSTICS_REF_ID', '--path', XCRESULT_PATH, '--output-path',
'/tmp/attempt_0.xcresult_diagnostic'
])
@mock.patch('os.path.exists', autospec=True) @mock.patch('os.path.exists', autospec=True)
def testCollectTestResults_interruptedTests(self, mock_path_exists): def testCollectTestResults_interruptedTests(self, mock_path_exists):
......
ios/build/bots/scripts/xcodebuild_runner.py
View file @ b2636748
...@@ -232,7 +232,6 @@ class LaunchCommand(object): ...@@ -232,7 +232,6 @@ class LaunchCommand(object):
if failure: if failure:
LOGGER.info('Failure for passed tests %s: %s' % (status, failure)) LOGGER.info('Failure for passed tests %s: %s' % (status, failure))
break break
self._log_parser.copy_screenshots(outdir_attempt)
# If tests are not completed(interrupted or did not start) # If tests are not completed(interrupted or did not start)
# re-run them with the same number of shards, # re-run them with the same number of shards,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment