[iOS][test runner] Output disabled EG tests from test runner.

Parse compile time disabled EG tests and report skipped in test result
json file. The test arg is enabled in code coverage CI builder.

Bug: 1065022
Change-Id: Ib4cec0e870dac8cf9e0e165768fde352205dc8a8
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2387116Reviewed-by: Justin Cohen <justincohen@chromium.org>
Commit-Queue: Zhaoyang Li <zhaoyangli@chromium.org>
Cr-Commit-Position: refs/heads/master@{#806842}

ios/build/bots/scripts/shard_util.py

@@ -14,14 +14,18 @@ LOGGER = logging.getLogger(__name__)
 # WARNING: THESE DUPLICATE CONSTANTS IN:
 # //build/scripts/slave/recipe_modules/ios/api.py
 
+# Regex to parse all compiled EG tests, including disabled (prepended with
+# DISABLED_ or FLAKY_).
 TEST_NAMES_DEBUG_APP_PATTERN = re.compile(
     'imp +(?:0[xX][0-9a-fA-F]+ )?-\[(?P<testSuite>[A-Za-z_][A-Za-z0-9_]'
-    '*Test[Case]*) (?P<testMethod>test[A-Za-z0-9_]*)\]')
+    '*Test[Case]*) (?P<testMethod>(?:DISABLED_|FLAKY_)?test[A-Za-z0-9_]*)\]')
 TEST_CLASS_RELEASE_APP_PATTERN = re.compile(
     r'name +0[xX]\w+ '
     '(?P<testSuite>[A-Za-z_][A-Za-z0-9_]*Test(?:Case|))\n')
+# Regex to parse all compiled EG tests, including disabled (prepended with
+# DISABLED_ or FLAKY_).
 TEST_NAME_RELEASE_APP_PATTERN = re.compile(
-    r'name +0[xX]\w+ (?P<testCase>test[A-Za-z0-9_]+)\n')
+    r'name +0[xX]\w+ (?P<testCase>(?:DISABLED_|FLAKY_)?test[A-Za-z0-9_]+)\n')
 # 'ChromeTestCase' and 'BaseEarlGreyTestCase' are parent classes
 # of all EarlGrey/EarlGrey2 test classes. They have no real tests.
 IGNORED_CLASSES = ['BaseEarlGreyTestCase', 'ChromeTestCase']
@@ -81,7 +85,7 @@ def _execute(cmd):
 
 def fetch_test_names_for_release(stdout):
   """Parse otool output to get all testMethods in all TestCases in the
-     format of (TestCase, testMethod), in release app.
+     format of (TestCase, testMethod) including disabled tests, in release app.
 
      WARNING: This logic is similar to what's found in
       //build/scripts/slave/recipe_modules/ios/api.py
@@ -90,7 +94,7 @@ def fetch_test_names_for_release(stdout):
         stdout: (string) response of 'otool -ov'
 
     Returns:
-        (list) a list of (TestCase, testMethod)
+        (list) a list of (TestCase, testMethod), containing disabled tests.
     """
   # For Release builds `otool -ov` command generates output that is
   # different from Debug builds.
@@ -114,29 +118,30 @@ def fetch_test_names_for_release(stdout):
 
 def fetch_test_names_for_debug(stdout):
   """Parse otool output to get all testMethods in all TestCases in the
-     format of (TestCase, testMethod), in debug app.
+     format of (TestCase, testMethod) including disabled tests, in debug app.
 
     Args:
         stdout: (string) response of 'otool -ov'
 
     Returns:
-        (list) a list of (TestCase, testMethod)
+        (list) a list of (TestCase, testMethod), containing disabled tests.
     """
   test_names = TEST_NAMES_DEBUG_APP_PATTERN.findall(stdout)
   return filter(lambda (test_case, _): test_case not in IGNORED_CLASSES,
                 test_names)
 
 
-def fetch_test_names(app, host_app, release=False):
+def fetch_test_names(app, host_app, release, enabled_tests_only=True):
   """Determine the list of (TestCase, testMethod) for the app.
 
     Args:
         app: (string) path to app
         host_app: (string) path to host app. None or "NO_PATH" for EG1.
         release: (bool) whether this is a release build.
+        enabled_tests_only: (bool) output only enabled tests.
 
     Returns:
-        (list) a list of (TestCase, testMethod)
+        (list) a list of (TestCase, testMethod).
     """
   # Determine what path to use
   app_path = determine_app_path(app, host_app, release)
@@ -147,9 +152,13 @@ def fetch_test_names(app, host_app, release=False):
   LOGGER.info("Ignored test classes: {}".format(IGNORED_CLASSES))
   if release:
     LOGGER.info("Release build detected. Fetching test names for release.")
-    return fetch_test_names_for_release(stdout)
-
-  return fetch_test_names_for_debug(stdout)
+  all_test_names = (
+      fetch_test_names_for_release(stdout)
+      if release else fetch_test_names_for_debug(stdout))
+  enabled_test_names = (
+      filter(lambda (_, test_method): test_method.startswith('test'),
+             all_test_names))
+  return enabled_test_names if enabled_tests_only else all_test_names
 
 
 def balance_into_sublists(test_counts, total_shards):

ios/build/bots/scripts/shard_util_test.py

@@ -23,7 +23,9 @@ DEBUG_APP_OTOOL_OUTPUT = '\n'.join([
     'name 0x1064b8438 PasswordsTestCase',
     'imp 0x1075e6887 -[PasswordsTestCase testG]',
     'name 0x1064b8438 ToolBarTestCase',
-    'imp 0x1075e6887 -[ToolBarTestCase testH]', 'version 0'
+    'imp 0x1075e6887 -[ToolBarTestCase testH]',
+    'imp 0x1075e6887 -[ToolBarTestCase DISABLED_testI]',
+    'imp 0x1075e6887 -[ToolBarTestCase FLAKY_testJ]', 'version 0'
 ])
 
 # Debug app otool output format in Xcode 11.4 toolchain.
@@ -43,7 +45,9 @@ DEBUG_APP_OTOOL_OUTPUT_114 = '\n'.join([
     '    name    0x1064b8438 PasswordsTestCase',
     '    imp     0x1075e6887 -[PasswordsTestCase testG]',
     '    name    0x1064b8438 ToolBarTestCase',
-    '    imp     0x1075e6887 -[ToolBarTestCase testH]', 'version 0'
+    '    imp     0x1075e6887 -[ToolBarTestCase testH]',
+    '    imp     0x1075e6887 -[ToolBarTestCase DISABLED_testI]',
+    '    imp     0x1075e6887 -[ToolBarTestCase FLAKY_testJ]', 'version 0'
 ])
 
 RELEASE_APP_OTOOL_OUTPUT = '\n'.join([
@@ -58,8 +62,9 @@ RELEASE_APP_OTOOL_OUTPUT = '\n'.join([
     'name 0x1075e6887 testF', 'baseProtocols 0x0',
     'name 0x1064b8438 ChromeTestCase', 'name 0x1064b8438 setUp',
     'baseProtocols 0x0', 'name 0x1064b8438 ToolBarTestCase',
-    'name 0x1075e6887 testG', 'name 0x1075e6887 testH', 'baseProtocols 0x0',
-    'version 0'
+    'name 0x1075e6887 testG', 'name 0x1075e6887 testH',
+    'name 0x1075e6887 DISABLED_testI', 'name 0x1075e6887 FLAKY_testJ',
+    'baseProtocols 0x0', 'version 0'
 ])
 
 # Release app otool output format in Xcode 11.4 toolchain.
@@ -76,7 +81,8 @@ RELEASE_APP_OTOOL_OUTPUT_114 = '\n'.join([
     '    name    0x1064b8438 ChromeTestCase', '    name    0x1064b8438 setUp',
     'baseProtocols 0x0', '    name    0x1064b8438 ToolBarTestCase',
     '    name    0x1075e6887 testG', '    name    0x1075e6887 testH',
-    'baseProtocols 0x0', 'version 0'
+    '    name    0x1075e6893 DISABLED_testI',
+    '    name    0x1075e723f FLAKY_testJ', 'baseProtocols 0x0', 'version 0'
 ])
 
 
@@ -117,92 +123,110 @@ class TestShardUtil(unittest.TestCase):
   def test_fetch_test_names_debug(self):
     """Ensures that the debug output is formatted correctly"""
     resp = shard_util.fetch_test_names_for_debug(DEBUG_APP_OTOOL_OUTPUT)
-    self.assertEqual(len(resp), 8)
-    expected_test_names = [('CacheTestCase', 'testA'), ('CacheTestCase',
-                                                        'testB'),
-                           ('CacheTestCase', 'testc'), ('TabUITestCase',
-                                                        'testD'),
-                           ('TabUITestCase', 'testE'),
-                           ('KeyboardTestCase', 'testF'),
-                           ('PasswordsTestCase', 'testG'),
-                           ('ToolBarTestCase', 'testH')]
+    self.assertEqual(len(resp), 10)
+    expected_test_names = [
+        ('CacheTestCase', 'testA'),
+        ('CacheTestCase', 'testB'),
+        ('CacheTestCase', 'testc'),
+        ('TabUITestCase', 'testD'),
+        ('TabUITestCase', 'testE'),
+        ('KeyboardTestCase', 'testF'),
+        ('PasswordsTestCase', 'testG'),
+        ('ToolBarTestCase', 'testH'),
+        ('ToolBarTestCase', 'DISABLED_testI'),
+        ('ToolBarTestCase', 'FLAKY_testJ'),
+    ]
     for test_name in expected_test_names:
       self.assertTrue(test_name in resp)
 
     test_cases = map(lambda (test_case, test_method): test_case, resp)
 
     # ({'CacheTestCase': 3, 'TabUITestCase': 2, 'PasswordsTestCase': 1,
-    # 'KeyboardTestCase': 1, 'ToolBarTestCase': 1})
+    # 'KeyboardTestCase': 1, 'ToolBarTestCase': 3})
     counts = collections.Counter(test_cases).most_common()
     name, _ = counts[0]
-    self.assertEqual(name, 'CacheTestCase')
+    self.assertEqual(name, 'ToolBarTestCase')
 
   def test_fetch_test_counts_release(self):
     """Ensures that the release output is formatted correctly"""
     resp = shard_util.fetch_test_names_for_release(RELEASE_APP_OTOOL_OUTPUT)
-    self.assertEqual(len(resp), 8)
-
-    expected_test_names = [('CacheTestCase', 'testA'), ('CacheTestCase',
-                                                        'testB'),
-                           ('CacheTestCase', 'testc'), ('KeyboardTest',
-                                                        'testD'),
-                           ('KeyboardTest', 'testE'), ('KeyboardTest', 'testF'),
-                           ('ToolBarTestCase', 'testG'),
-                           ('ToolBarTestCase', 'testH')]
+    self.assertEqual(len(resp), 10)
+
+    expected_test_names = [
+        ('CacheTestCase', 'testA'),
+        ('CacheTestCase', 'testB'),
+        ('CacheTestCase', 'testc'),
+        ('KeyboardTest', 'testD'),
+        ('KeyboardTest', 'testE'),
+        ('KeyboardTest', 'testF'),
+        ('ToolBarTestCase', 'testG'),
+        ('ToolBarTestCase', 'testH'),
+        ('ToolBarTestCase', 'DISABLED_testI'),
+        ('ToolBarTestCase', 'FLAKY_testJ'),
+    ]
     for test_name in expected_test_names:
       self.assertTrue(test_name in resp)
 
     test_cases = map(lambda (test_case, test_method): test_case, resp)
     # ({'KeyboardTest': 3, 'CacheTestCase': 3,
-    # 'ToolBarTestCase': 2})
+    # 'ToolBarTestCase': 4})
     counts = collections.Counter(test_cases).most_common()
     name, _ = counts[0]
-    self.assertEqual(name, 'KeyboardTest')
+    self.assertEqual(name, 'ToolBarTestCase')
 
   def test_fetch_test_names_debug_114(self):
     """Test the debug output from otool in Xcode 11.4"""
     resp = shard_util.fetch_test_names_for_debug(DEBUG_APP_OTOOL_OUTPUT_114)
-    self.assertEqual(len(resp), 8)
-    expected_test_names = [('CacheTestCase', 'testA'), ('CacheTestCase',
-                                                        'testB'),
-                           ('CacheTestCase', 'testc'), ('TabUITestCase',
-                                                        'testD'),
-                           ('TabUITestCase', 'testE'),
-                           ('KeyboardTestCase', 'testF'),
-                           ('PasswordsTestCase', 'testG'),
-                           ('ToolBarTestCase', 'testH')]
+    self.assertEqual(len(resp), 10)
+    expected_test_names = [
+        ('CacheTestCase', 'testA'),
+        ('CacheTestCase', 'testB'),
+        ('CacheTestCase', 'testc'),
+        ('TabUITestCase', 'testD'),
+        ('TabUITestCase', 'testE'),
+        ('KeyboardTestCase', 'testF'),
+        ('PasswordsTestCase', 'testG'),
+        ('ToolBarTestCase', 'testH'),
+        ('ToolBarTestCase', 'DISABLED_testI'),
+        ('ToolBarTestCase', 'FLAKY_testJ'),
+    ]
     for test_name in expected_test_names:
       self.assertTrue(test_name in resp)
 
     test_cases = map(lambda (test_case, test_method): test_case, resp)
 
     # ({'CacheTestCase': 3, 'TabUITestCase': 2, 'PasswordsTestCase': 1,
-    # 'KeyboardTestCase': 1, 'ToolBarTestCase': 1})
+    # 'KeyboardTestCase': 1, 'ToolBarTestCase': 3})
     counts = collections.Counter(test_cases).most_common()
     name, _ = counts[0]
-    self.assertEqual(name, 'CacheTestCase')
+    self.assertEqual(name, 'ToolBarTestCase')
 
   def test_fetch_test_counts_release_114(self):
     """Test the release output from otool in Xcode 11.4"""
     resp = shard_util.fetch_test_names_for_release(RELEASE_APP_OTOOL_OUTPUT_114)
-    self.assertEqual(len(resp), 8)
-
-    expected_test_names = [('CacheTestCase', 'testA'), ('CacheTestCase',
-                                                        'testB'),
-                           ('CacheTestCase', 'testc'), ('KeyboardTest',
-                                                        'testD'),
-                           ('KeyboardTest', 'testE'), ('KeyboardTest', 'testF'),
-                           ('ToolBarTestCase', 'testG'),
-                           ('ToolBarTestCase', 'testH')]
+    self.assertEqual(len(resp), 10)
+
+    expected_test_names = [
+        ('CacheTestCase', 'testA'),
+        ('CacheTestCase', 'testB'),
+        ('CacheTestCase', 'testc'),
+        ('KeyboardTest', 'testD'),
+        ('KeyboardTest', 'testE'),
+        ('KeyboardTest', 'testF'),
+        ('ToolBarTestCase', 'testG'),
+        ('ToolBarTestCase', 'testH'),
+        ('ToolBarTestCase', 'DISABLED_testI'),
+        ('ToolBarTestCase', 'FLAKY_testJ'),
+    ]
     for test_name in expected_test_names:
       self.assertTrue(test_name in resp)
 
     test_cases = map(lambda (test_case, test_method): test_case, resp)
     # ({'KeyboardTest': 3, 'CacheTestCase': 3,
-    # 'ToolBarTestCase': 2})
+    # 'ToolBarTestCase': 4})
     counts = collections.Counter(test_cases).most_common()
     name, _ = counts[0]
-    self.assertEqual(name, 'KeyboardTest')
+    self.assertEqual(name, 'ToolBarTestCase')
 
   def test_balance_into_sublists_debug(self):
     """Ensure the balancing algorithm works"""
@@ -217,11 +241,15 @@ class TestShardUtil(unittest.TestCase):
     sublists_3 = shard_util.balance_into_sublists(test_counts, 3)
     self.assertEqual(len(sublists_3), 3)
     # CacheTestCase has 3,
-    # TabUITestCase has 2, ToolBarTestCase has 1
+    # TabUITestCase has 2, ToolBarTestCase has 4
     # PasswordsTestCase has 1, KeyboardTestCase has 1
-    self.assertEqual(len(sublists_3[0]), 1)
-    self.assertEqual(len(sublists_3[1]), 2)
-    self.assertEqual(len(sublists_3[2]), 2)
+    # They will be balanced into:
+    # [[ToolBarTestCase], [CacheTestCase, PasswordsTestCase],
+    # [TabUITestCase, KeyboardTestCase]]
+    self.assertEqual(
+        sorted([len(sublists_3[0]),
+                len(sublists_3[1]),
+                len(sublists_3[2])]), [1, 2, 2])
 
   def test_balance_into_sublists_release(self):
     """Ensure the balancing algorithm works"""
@@ -233,7 +261,8 @@ class TestShardUtil(unittest.TestCase):
     self.assertEqual(len(sublists_3), 3)
     # KeyboardTest has 3
     # CacheTestCase has 3
-    # ToolbarTest Case has 2
+    # ToolbarTest Case has 4
+    # They will be balanced as one in each shard.
     self.assertEqual(len(sublists_3[0]), 1)
     self.assertEqual(len(sublists_3[1]), 1)
     self.assertEqual(len(sublists_3[2]), 1)

ios/build/bots/scripts/test_apps.py

@@ -12,6 +12,9 @@ import shard_util
 import test_runner
 
 
+OUTPUT_DISABLED_TESTS_TEST_ARG = '--write-compiled-tests-json-to-writable-path'
+
+
 #TODO(crbug.com/1046911): Remove usage of KIF filters.
 def get_kif_test_filter(tests, invert=False):
   """Returns the KIF test filter to filter the given test cases.
@@ -111,6 +114,7 @@ class GTestsApp(object):
       self.env_vars[env_var[0]] = None if len(env_var) == 1 else env_var[1]
     self.included_tests = included_tests or []
     self.excluded_tests = excluded_tests or []
+    self.disabled_tests = []
     self.module_name = os.path.splitext(os.path.basename(test_app))[0]
     self.release = release
     self.host_app_path = host_app_path
@@ -238,16 +242,27 @@ class GTestsApp(object):
     # but they are not test-methods.
     # TODO(crbug.com/982435): Rename not test methods with test-suffix.
     none_tests = ['ChromeTestCase/testServer', 'FindInPageTestCase/testURL']
+    # TODO(crbug.com/1123681): Move all_tests to class var. Set all_tests,
+    # disabled_tests values in initialization to avoid multiple calls to otool.
     all_tests = []
+    # Only store the tests when there is the test arg.
+    store_disabled_tests = OUTPUT_DISABLED_TESTS_TEST_ARG in self.test_args
+    self.disabled_tests = []
     for test_class, test_method in shard_util.fetch_test_names(
-        self.test_app_path, self.host_app_path, self.release):
+        self.test_app_path,
+        self.host_app_path,
+        self.release,
+        enabled_tests_only=False):
       test_name = '%s/%s' % (test_class, test_method)
       if (test_name not in none_tests and
           # inlcuded_tests contains the tests to execute, which may be a subset
           # of all tests b/c of the iOS test sharding logic in run.py. Filter by
           # self.included_tests if specified
           (test_class in self.included_tests if self.included_tests else True)):
-        all_tests.append(test_name)
+        if test_method.startswith('test'):
+          all_tests.append(test_name)
+        elif store_disabled_tests:
+          self.disabled_tests.append(test_name)
     return all_tests
 
 

ios/build/bots/scripts/xcodebuild_runner.py

@@ -403,6 +403,12 @@ class SimulatorParallelTestRunner(test_runner.SimulatorTestRunner):
       if shard_attempts[-1]['failed']:
         self.logs['failed tests'].extend(shard_attempts[-1]['failed'].keys())
 
+    # Gets disabled tests from test app object if any.
+    self.logs['disabled tests'] = []
+    for launch_command in launch_commands:
+      self.logs['disabled tests'].extend(
+          launch_command.egtests_app.disabled_tests)
+
     # Gets all failures/flakes and lists them in bot summary
     all_failures = set()
     for shard_attempts in attempts_results:
@@ -459,6 +465,8 @@ class SimulatorParallelTestRunner(test_runner.SimulatorTestRunner):
         for test in attempt_results['passed']:
           output.mark_passed(test)
 
+    output.mark_all_skipped(self.logs['disabled tests'])
+
     self.test_results['tests'] = output.tests
 
     # Test is failed if there are failures for the last run.