Skip to content

GitLab

T
tangled
Commit dc9aa222 authored by zhaoyangli's avatar zhaoyangli Committed by Commit Bot
Browse files
Options

[iOS][infra] Use only xctest arg to determine running as xctest. 

In test runner: Removed logic using xctest path existence to decide
whether to run as xctest.

In test configs:
Added xctest test arg to all iOS tests in chromium. This will be
required for all unit tests. Also add this to EG2 tests for consistency
of argument meaning for iOS test runner scripts.
Comparing test configs in json files, this change:
- Added the flag to all tests ToTiOSDevice, ToTiOS, ios-simulator-cronet
who didn't have this flag in any tests but were building with .xctest.
- Added to all EG2 test targets on rest of iOS builders running tests.

We have the option to build xctest arg into test runner wrapper in gn,
but it's preferred to set at test configs for fine-grained control.

Bug: 1085603, 1006881, 1085603，1001667
Change-Id: I46d7c1995ed9f9f5d96fe0280124767360f0aa21
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2413298Reviewed-by: default avatarGarrett Beaty <gbeaty@chromium.org>
Reviewed-by: default avatarJustin Cohen <justincohen@chromium.org>
Commit-Queue: Zhaoyang Li <zhaoyangli@chromium.org>
Cr-Commit-Position: refs/heads/master@{#814876}
parent 5fbfa17e
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 303 additions and 143 deletions
ios/build/bots/scripts/test_runner.py
View file @ dc9aa222
......@@ -297,30 +297,6 @@ def get_current_xcode_info():
}
def get_xctest_from_app(app):
"""Gets xctest path for an app.
Args:
app: (str) A path to an app.
Returns:
The xctest path.
"""
plugins_dir = os.path.join(app, 'PlugIns')
if not os.path.exists(plugins_dir):
# TODO(crbug.com/1001667): Throw error when all device unit test should run
# with xctest.
LOGGER.warning('PlugIns dir doesn\'t exist in app.\n')
return None
for plugin in os.listdir(plugins_dir):
if plugin.endswith('.xctest'):
return os.path.join(plugins_dir, plugin)
# TODO(crbug.com/1001667): Throw error when all device unit test should run
# with xctest.
LOGGER.warning('.xctest doesn\'t exist in app PlugIns dir.\n')
return None
class TestRunner(object):
"""Base class containing common functionality."""
......@@ -378,8 +354,6 @@ class TestRunner(object):
self.test_args = test_args or []
self.test_cases = test_cases or []
self.xctest_path = ''
# TODO(crbug.com/1006881): Separate "running style" from "parser style"
# for XCtests and Gtests.
self.xctest = xctest
self.test_results = {}
......@@ -586,26 +560,6 @@ class TestRunner(object):
else:
raise XCTestConfigError('Wrong config. TestRunner.launch() called from'
' an unexpected class.')
# TODO(crbug.com/1085603): Remove when device unit tests have xctest in
# configs.
elif self.xctest_path:
if self.__class__.__name__ == 'DeviceTestRunner':
# When self.xctest is False and (bool)self.xctest_path is True and it's
# using a device runner, this is a XCTest hosted unit test, which is
# currently running on real devices.
# TODO(crbug.com/1006881): Separate "running style" from "parser style"
# for XCtests and Gtests.
test_app = test_apps.DeviceXCTestUnitTestsApp(
self.app_path,
included_tests=self.test_cases,
env_vars=self.env_vars,
test_args=self.test_args)
else:
raise XCTestConfigError('Trying to run a DeviceXCTestUnitTestsApp on a'
'non device runner!')
else:
test_app = test_apps.GTestsApp(
self.app_path,
......@@ -951,7 +905,7 @@ class SimulatorTestRunner(TestRunner):
A dict of environment variables.
"""
env = super(SimulatorTestRunner, self).get_launch_env()
if self.xctest_path:
if self.xctest:
env['NSUnbufferedIO'] = 'YES'
return env
......@@ -1005,10 +959,6 @@ class DeviceTestRunner(TestRunner):
if len(self.udid.splitlines()) != 1:
raise DeviceDetectionError(self.udid)
# GTest-based unittests are invoked via XCTest for all devices
# but produce GTest-style log output that is parsed with a GTestLogParser.
self.xctest_path = get_xctest_from_app(self.app_path)
self.restart = restart
def uninstall_apps(self):
......@@ -1101,9 +1051,7 @@ class DeviceTestRunner(TestRunner):
Returns:
A list of strings forming the command to launch the test.
"""
# TODO(crbug.com/1085603): Remove self.xctest_path check when device unit
# tests have xctest in configs.
if self.xctest_path or self.xctest:
if self.xctest:
return test_app.command(out_dir, destination, shards)
cmd = [
......@@ -1148,7 +1096,7 @@ class DeviceTestRunner(TestRunner):
A dict of environment variables.
"""
env = super(DeviceTestRunner, self).get_launch_env()
if self.xctest_path:
if self.xctest:
env['NSUnbufferedIO'] = 'YES'
# e.g. ios_web_shell_egtests
env['APP_TARGET_NAME'] = os.path.splitext(
......
testing/buildbot/chromium.clang.json
View file @ dc9aa222
......@@ -41158,7 +41158,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "absl_hardening_tests",
"merge": {
......@@ -41199,7 +41200,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "base_unittests",
"merge": {
......@@ -41240,7 +41242,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "boringssl_crypto_tests",
"merge": {
......@@ -41281,7 +41284,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "boringssl_ssl_tests",
"merge": {
......@@ -41322,7 +41326,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "components_unittests",
"merge": {
......@@ -41363,7 +41368,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "crypto_unittests",
"merge": {
......@@ -41404,7 +41410,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "gfx_unittests",
"merge": {
......@@ -41445,7 +41452,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "google_apis_unittests",
"merge": {
......@@ -41486,7 +41494,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "ios_chrome_unittests",
"merge": {
......@@ -41527,7 +41536,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "ios_net_unittests",
"merge": {
......@@ -41568,7 +41578,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "ios_web_inttests",
"merge": {
......@@ -41609,7 +41620,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "ios_web_unittests",
"merge": {
......@@ -41650,7 +41662,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "ios_web_view_inttests",
"merge": {
......@@ -41691,7 +41704,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "net_unittests",
"merge": {
......@@ -41732,7 +41746,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "skia_unittests",
"merge": {
......@@ -41773,7 +41788,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "sql_unittests",
"merge": {
......@@ -41814,7 +41830,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "ui_base_unittests",
"merge": {
......@@ -41855,7 +41872,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "url_unittests",
"merge": {
......@@ -41915,7 +41933,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "absl_hardening_tests",
"merge": {
......@@ -41953,7 +41972,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "base_unittests",
"merge": {
......@@ -41991,7 +42011,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "boringssl_crypto_tests",
"merge": {
......@@ -42029,7 +42050,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "boringssl_ssl_tests",
"merge": {
......@@ -42067,7 +42089,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "components_unittests",
"merge": {
......@@ -42105,7 +42128,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "crypto_unittests",
"merge": {
......@@ -42143,7 +42167,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "gfx_unittests",
"merge": {
......@@ -42181,7 +42206,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "google_apis_unittests",
"merge": {
......@@ -42219,7 +42245,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "ios_chrome_unittests",
"merge": {
......@@ -42257,7 +42284,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "ios_net_unittests",
"merge": {
......@@ -42295,7 +42323,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "ios_web_inttests",
"merge": {
......@@ -42333,7 +42362,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "ios_web_unittests",
"merge": {
......@@ -42371,7 +42401,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "ios_web_view_inttests",
"merge": {
......@@ -42409,7 +42440,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "net_unittests",
"merge": {
......@@ -42447,7 +42479,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "skia_unittests",
"merge": {
......@@ -42485,7 +42518,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "sql_unittests",
"merge": {
......@@ -42523,7 +42557,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "ui_base_unittests",
"merge": {
......@@ -42561,7 +42596,8 @@
"--out-dir",
"${ISOLATED_OUTDIR}",
"--xcode-build-version",
"12a7209"
"12a7209",
"--xctest"
],
"isolate_name": "url_unittests",
"merge": {
testing/buildbot/chromium.fyi.json
View file @ dc9aa222
This diff is collapsed.
testing/buildbot/chromium.mac.json
View file @ dc9aa222
This diff is collapsed.
testing/buildbot/test_suites.pyl
View file @ dc9aa222
......@@ -5609,9 +5609,6 @@
# ios13-beta-sim already runs 13.x and 12.4
'ios14_beta_simulator_tests': {
'ios_common_tests': {
'mixins': [
'xctest',
],
'variants': [
'SIM_IPHONE_X_14_2',
],
......@@ -5635,9 +5632,6 @@
]
},
'ios_screen_size_dependent_tests': {
'mixins': [
'xctest',
],
'variants': [
'SIM_IPHONE_6S_PLUS_14_2',
'SIM_IPHONE_X_14_2',
......@@ -5648,9 +5642,6 @@
'ios14_sdk_simulator_tests': {
'ios_common_tests': {
'mixins': [
'xctest',
],
'variants': [
# 14.2 Sims
'SIM_IPHONE_6S_14_2',
......@@ -5675,9 +5666,6 @@
]
},
'ios_screen_size_dependent_tests': {
'mixins': [
'xctest',
],
'variants': [
# 14.2 Sims
'SIM_IPHONE_6S_PLUS_14_2',
......@@ -5688,18 +5676,12 @@
'ios_asan_tests': {
'ios_common_tests': {
'mixins': [
'xctest',
],
'variants': [
'SIM_IPHONE_X_13_6',
'SIM_IPAD_AIR_2_13_6',
]
},
'ios_screen_size_dependent_tests': {
'mixins': [
'xctest',
],
'variants': [
'SIM_IPHONE_X_13_6',
'SIM_IPAD_AIR_2_13_6',
......@@ -5725,9 +5707,6 @@
'ios_code_coverage_tests': {
'ios_common_tests': {
'mixins': [
'xctest',
],
'variants': [
'SIM_IPHONE_6S_12_4',
'SIM_IPHONE_6S_13_6',
......@@ -5754,9 +5733,6 @@
],
},
'ios_screen_size_dependent_tests': {
'mixins': [
'xctest',
],
'variants': [
'SIM_IPHONE_6S_PLUS_13_6',
'SIM_IPHONE_6S_13_6',
......@@ -5800,9 +5776,6 @@
'ios_simulator_multi_window_tests': {
'ios_common_tests': {
'mixins': [
'xctest',
],
'variants': [
'SIM_IPAD_AIR_2_13_6',
]
......@@ -5820,9 +5793,6 @@
]
},
'ios_screen_size_dependent_tests': {
'mixins': [
'xctest',
],
'variants': [
'SIM_IPAD_AIR_2_13_6',
]
......@@ -5837,9 +5807,6 @@
# ios-simulator and ios-simulator-full-configs.
'ios_simulator_noncq_tests': {
'ios_common_tests': {
'mixins': [
'xctest',
],
'variants': [
'SIM_IPHONE_6S_12_4',
]
......@@ -5862,9 +5829,6 @@
'ios_simulator_tests': {
'ios_common_tests': {
'mixins': [
'xctest',
],
'variants': [
'SIM_IPHONE_6S_14_0',
'SIM_IPHONE_6S_13_6',
......@@ -5878,9 +5842,6 @@
]
},
'ios_screen_size_dependent_tests': {
'mixins': [
'xctest',
],
'variants': [
'SIM_IPHONE_6S_PLUS_13_6',
'SIM_IPHONE_6S_13_6',
......@@ -5896,9 +5857,6 @@
'ios_webkit_tot_tests': {
'ios_common_tests': {
'mixins': [
'xctest',
],
'variants': [
'SIM_IPHONE_X_13_5',
'SIM_IPAD_AIR_2_13_5',
......@@ -5919,9 +5877,6 @@
]
},
'ios_screen_size_dependent_tests': {
'mixins': [
'xctest',
],
'variants': [
'SIM_IPHONE_X_13_5',
'SIM_IPAD_AIR_2_13_5',
......
testing/buildbot/waterfalls.pyl
View file @ dc9aa222
......@@ -1598,6 +1598,7 @@
'mac_toolchain',
'out_dir_arg',
'xcode_12a7209',
'xctest',
],
'test_suites': {
'isolated_scripts': 'ios_clang_tot_sim_tests'
......@@ -1628,6 +1629,7 @@
'mac_toolchain',
'out_dir_arg',
'xcode_12a7209',
'xctest',
],
'test_suites': {
'isolated_scripts': 'ios_clang_tot_device_tests'
......@@ -2253,7 +2255,8 @@
'mac_10.15',
'mac_toolchain',
'out_dir_arg',
'xcode_12a7209'
'xcode_12a7209',
'xctest',
],
'test_suites': {
'isolated_scripts': 'ios_asan_tests',
......@@ -2266,7 +2269,8 @@
'mac_10.15',
'mac_toolchain',
'out_dir_arg',
'xcode_12a7209'
'xcode_12a7209',
'xctest',
],
'test_suites': {
'isolated_scripts': 'ios_code_coverage_tests',
......@@ -2277,7 +2281,8 @@
'mac_10.15',
'mac_toolchain',
'out_dir_arg',
'xcode_11e146'
'xcode_11e146',
'xctest',
],
'test_suites': {
'isolated_scripts': 'ios_simulator_cronet_tests',
......@@ -2288,7 +2293,8 @@
'mac_10.15',
'mac_toolchain',
'out_dir_arg',
'xcode_12a7209'
'xcode_12a7209',
'xctest',
],
'test_suites': {
'isolated_scripts': 'ios_simulator_multi_window_tests',
......@@ -2300,7 +2306,8 @@
'mac_10.15',
'mac_toolchain',
'out_dir_arg',
'xcode_11e608c'
'xcode_11e608c',
'xctest',
],
'test_suites': {
'isolated_scripts': 'ios_webkit_tot_tests',
......@@ -2332,6 +2339,7 @@
'mac_toolchain',
'out_dir_arg',
'xcode_12b5018i',
'xctest',
],
'test_suites': {
'isolated_scripts': 'ios14_beta_simulator_tests'
......@@ -2348,6 +2356,7 @@
'mac_toolchain',
'out_dir_arg',
'xcode_12b5018i',
'xctest',
],
'test_suites': {
'isolated_scripts': 'ios14_sdk_simulator_tests'
......@@ -4692,6 +4701,7 @@
'mac_toolchain',
'out_dir_arg',
'xcode_12a7209',
'xctest',
],
'test_suites': {
'isolated_scripts': 'ios_simulator_tests',
......@@ -4706,6 +4716,7 @@
'mac_toolchain',
'out_dir_arg',
'xcode_12a7209',
'xctest',
],
'test_suites': {
'isolated_scripts': 'ios_simulator_full_configs_test'
......@@ -4720,6 +4731,7 @@
'mac_toolchain',
'out_dir_arg',
'xcode_12a7209',
'xctest',
],
'test_suites': {
'isolated_scripts': 'ios_simulator_noncq_tests'
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment