Modify iOS test_runner to accept shards of EarlGrey tests

Modify iOS test_runner to understand the isolated EarlGrey test shards.

Bug: 775168
Cq-Include-Trybots: master.tryserver.chromium.mac:ios-simulator-cronet;master.tryserver.chromium.mac:ios-simulator-full-configs
Change-Id: If82fe7d4f0e0358b1d10d3916ab9b5ed7f36ebdd
Reviewed-on: https://chromium-review.googlesource.com/807356
Commit-Queue: Menglu Huang <huangml@chromium.org>
Reviewed-by: Sergey Berezin <sergeyberezin@chromium.org>
Reviewed-by: smut <smut@google.com>
Cr-Commit-Position: refs/heads/master@{#522722}

ios/build/bots/scripts/run.py

@@ -49,6 +49,7 @@ def main():
         mac_toolchain=args.mac_toolchain_cmd,
         retries=args.retries,
         test_args=test_args,
+        test_cases=args.test_cases,
         xcode_path=args.xcode_path,
         xctest=args.xctest,
       )
@@ -63,6 +64,7 @@ def main():
         restart=args.restart,
         retries=args.retries,
         test_args=test_args,
+        test_cases=args.test_cases,
         xcode_path=args.xcode_path,
         xctest=args.xctest,
       )
@@ -143,6 +145,14 @@ def parse_args():
     metavar='n',
     type=int,
   )
+  parser.add_argument(
+    '-t',
+    '--test-cases',
+    action='append',
+    help=('Tests that should be included in the test run. All other tests '
+          'will be excluded from this run. If unspecified, run all tests.'),
+    metavar='testcase',
+  )
   parser.add_argument(
     '-v',
     '--version',
@@ -198,6 +208,8 @@ def parse_args():
   args.env_var = args.env_var or []
   args.env_var.extend(args_json.get('env_var', []))
   args.restart = args_json.get('restart', args.restart)
+  args.test_cases = args.test_cases or []
+  args.test_cases.extend(args_json.get('test_cases', []))
   args.xctest = args_json.get('xctest', args.xctest)
   test_args.extend(args_json.get('test_args', []))
 

ios/build/bots/scripts/test_runner.py

@@ -51,11 +51,13 @@ class DeviceDetectionError(TestRunnerError):
     super(DeviceDetectionError, self).__init__(
       'Expected one device, found %s:\n%s' % (len(udids), '\n'.join(udids)))
 
+
 class DeviceRestartError(TestRunnerError):
   """Error restarting a device."""
   def __init__(self):
     super(DeviceRestartError, self).__init__('Error restarting a device')
 
+
 class PlugInsNotFoundError(TestRunnerError):
   """The PlugIns directory was not found."""
   def __init__(self, plugins_dir):
@@ -209,6 +211,7 @@ class TestRunner(object):
     mac_toolchain='',
     retries=None,
     test_args=None,
+    test_cases=None,
     xcode_path='',
     xctest=False,
   ):
@@ -225,6 +228,8 @@ class TestRunner(object):
       retries: Number of times to retry failed test cases.
       test_args: List of strings to pass as arguments to the test when
         launching.
+      test_cases: List of tests to be included in the test run. None or [] to
+        include all tests.
       xcode_path: Path to Xcode.app folder where its contents will be installed.
       xctest: Whether or not this is an XCTest.
 
@@ -263,6 +268,7 @@ class TestRunner(object):
     self.out_dir = out_dir
     self.retries = retries or 0
     self.test_args = test_args or []
+    self.test_cases = test_cases or []
     self.xcode_version = xcode_version
     self.xctest_path = ''
 
@@ -495,6 +501,7 @@ class SimulatorTestRunner(TestRunner):
       mac_toolchain='',
       retries=None,
       test_args=None,
+      test_cases=None,
       xcode_path='',
       xctest=False,
   ):
@@ -516,6 +523,8 @@ class SimulatorTestRunner(TestRunner):
       retries: Number of times to retry failed test cases.
       test_args: List of strings to pass as arguments to the test when
         launching.
+      test_cases: List of tests to be included in the test run. None or [] to
+        include all tests.
       xcode_path: Path to Xcode.app folder where its contents will be installed.
       xctest: Whether or not this is an XCTest.
 
@@ -534,6 +543,7 @@ class SimulatorTestRunner(TestRunner):
         mac_toolchain=mac_toolchain,
         retries=retries,
         test_args=test_args,
+        test_cases=test_cases,
         xcode_path=xcode_path,
         xctest=xctest,
     )
@@ -687,6 +697,9 @@ class SimulatorTestRunner(TestRunner):
         gtest_filter = get_gtest_filter(test_filter, invert=invert)
         cmd.extend(['-e', 'GKIF_SCENARIO_FILTER=%s' % kif_filter])
         cmd.extend(['-c', '--gtest_filter=%s' % gtest_filter])
+    elif self.xctest_path and not invert:
+      for test_case in self.test_cases:
+        cmd.extend(['-t', test_case])
 
     for env_var in self.env_vars:
       cmd.extend(['-e', env_var])
@@ -725,6 +738,7 @@ class DeviceTestRunner(TestRunner):
     restart=False,
     retries=None,
     test_args=None,
+    test_cases=None,
     xctest=False,
     xcode_path='',
   ):
@@ -742,6 +756,8 @@ class DeviceTestRunner(TestRunner):
       retries: Number of times to retry failed test cases.
       test_args: List of strings to pass as arguments to the test when
         launching.
+      test_cases: List of tests to be included in the test run. None or [] to
+        include all tests.
       xctest: Whether or not this is an XCTest.
       xcode_path: Path to Xcode.app folder where its contents will be installed.
 
@@ -759,6 +775,7 @@ class DeviceTestRunner(TestRunner):
       env_vars=env_vars,
       retries=retries,
       test_args=test_args,
+      test_cases=test_cases,
       xctest=xctest,
       mac_toolchain=mac_toolchain,
       xcode_path=xcode_path,
@@ -856,6 +873,28 @@ class DeviceTestRunner(TestRunner):
     self.retrieve_crash_reports()
     self.uninstall_apps()
 
+  def set_xctest_filters(self, test_filter=None, invert=False):
+    """Sets the tests be included in the test run."""
+    if self.test_cases:
+      filter = self.test_cases
+      if test_filter:
+        # If inverted, the filter should match tests in test_cases except the
+        # ones in test_filter. Otherwise, the filter should be tests both in
+        # test_cases and test_filter. test_filter is used for test retries, it
+        # should be a subset of test_cases. If the intersection of test_cases
+        # and test_filter fails, use test_filter.
+        filter = (sorted(set(filter) - set(test_filter)) if invert
+                  else sorted(set(filter) & set(test_filter)) or test_filter)
+      self.xctestrun_data['TestTargetName'].update(
+        {'OnlyTestIdentifiers': filter})
+    elif test_filter:
+      if invert:
+        self.xctestrun_data['TestTargetName'].update(
+          {'SkipTestIdentifiers': test_filter})
+      else:
+        self.xctestrun_data['TestTargetName'].update(
+          {'OnlyTestIdentifiers': test_filter})
+
   def get_launch_command(self, test_filter=None, invert=False):
     """Returns the command that can be used to launch the test app.
 
@@ -868,13 +907,7 @@ class DeviceTestRunner(TestRunner):
       A list of strings forming the command to launch the test.
     """
     if self.xctest_path:
-      if test_filter:
-        if invert:
-          self.xctestrun_data['TestTargetName'].update(
-            {'SkipTestIdentifiers': test_filter})
-        else:
-          self.xctestrun_data['TestTargetName'].update(
-            {'OnlyTestIdentifiers': test_filter})
+      self.set_xctest_filters(test_filter, invert)
       if self.env_vars:
         self.xctestrun_data['TestTargetName'].update(
           {'EnvironmentVariables': self.env_vars})

ios/build/bots/scripts/test_runner_test.py

@@ -204,6 +204,41 @@ class SimulatorTestRunnerTest(TestCase):
     with self.assertRaises(test_runner.AppLaunchError):
       tr.launch()
 
+  def test_get_launch_command(self):
+    """Ensures test filters are set correctly for launch command"""
+    tr = test_runner.SimulatorTestRunner(
+      'fake-app',
+      'fake-iossim',
+      'platform',
+      'os',
+      'xcode-version',
+      'xcode-build',
+      'out-dir',
+    )
+    tr.xctest_path = 'fake.xctest'
+    # Cases test_filter is not empty, with empty/non-empty self.test_cases.
+    tr.test_cases = []
+    cmd = tr.get_launch_command(['a'], invert=False)
+    self.assertIn('-t', cmd)
+    self.assertIn('a', cmd)
+
+    tr.test_cases = ['a', 'b']
+    cmd = tr.get_launch_command(['a'], invert=False)
+    self.assertIn('-t', cmd)
+    self.assertIn('a', cmd)
+    self.assertNotIn('b', cmd)
+
+    # Cases test_filter is empty, with empty/non-empty self.test_cases.
+    tr.test_cases = []
+    cmd = tr.get_launch_command(test_filter=None, invert=False)
+    self.assertNotIn('-t', cmd)
+
+    tr.test_cases = ['a', 'b']
+    cmd = tr.get_launch_command(test_filter=None, invert=False)
+    self.assertIn('-t', cmd)
+    self.assertIn('a', cmd)
+    self.assertIn('b', cmd)
+
   def test_relaunch(self):
     """Ensures test is relaunched on test crash until tests complete."""
     def set_up(self):
@@ -258,5 +293,95 @@ class SimulatorTestRunnerTest(TestCase):
     self.assertTrue(tr.logs)
 
 
+class DeviceTestRunnerTest(TestCase):
+  def setUp(self):
+    super(DeviceTestRunnerTest, self).setUp()
+
+    def install_xcode(build, mac_toolchain_cmd, xcode_app_path):
+      return True
+
+    self.mock(test_runner.find_xcode, 'find_xcode',
+              lambda _: {'found': True})
+    self.mock(test_runner.find_xcode, 'get_current_xcode_info', lambda: {
+        'version': 'test version', 'build': 'test build', 'path': 'test/path'})
+    self.mock(test_runner, 'install_xcode', install_xcode)
+    self.mock(test_runner.subprocess, 'check_output',
+              lambda _: 'fake-bundle-id')
+    self.mock(os.path, 'abspath', lambda path: '/abs/path/to/%s' % path)
+    self.mock(os.path, 'exists', lambda _: True)
+
+    self.tr = test_runner.DeviceTestRunner(
+        'fake-app',
+        'xcode-version',
+        'xcode-build',
+        'out-dir',
+    )
+    self.tr.xctestrun_data = {'TestTargetName':{}}
+
+  def test_with_test_filter_without_test_cases(self):
+    """Ensures tests in the run with test_filter and no test_cases."""
+    self.tr.set_xctest_filters(['a', 'b'], invert=False)
+    self.assertEqual(
+        self.tr.xctestrun_data['TestTargetName']['OnlyTestIdentifiers'],
+        ['a', 'b']
+    )
+
+  def test_invert_with_test_filter_without_test_cases(self):
+    """Ensures tests in the run invert with test_filter and no test_cases."""
+    self.tr.set_xctest_filters(['a', 'b'], invert=True)
+    self.assertEqual(
+        self.tr.xctestrun_data['TestTargetName']['SkipTestIdentifiers'],
+        ['a', 'b']
+    )
+
+  def test_with_test_filter_with_test_cases(self):
+    """Ensures tests in the run with test_filter and test_cases."""
+    self.tr.test_cases = ['a', 'b', 'c', 'd']
+    self.tr.set_xctest_filters(['a', 'b', 'irrelevant test'], invert=False)
+    self.assertEqual(
+        self.tr.xctestrun_data['TestTargetName']['OnlyTestIdentifiers'],
+        ['a', 'b']
+    )
+
+  def test_invert_with_test_filter_with_test_cases(self):
+    """Ensures tests in the run invert with test_filter and test_cases."""
+    self.tr.test_cases = ['a', 'b', 'c', 'd']
+    self.tr.set_xctest_filters(['a', 'b', 'irrelevant test'], invert=True)
+    self.assertEqual(
+        self.tr.xctestrun_data['TestTargetName']['OnlyTestIdentifiers'],
+        ['c', 'd']
+    )
+
+  def test_without_test_filter_without_test_cases(self):
+    """Ensures tests in the run with no test_filter and no test_cases."""
+    self.tr.set_xctest_filters(test_filter=None, invert=False)
+    self.assertIsNone(
+        self.tr.xctestrun_data['TestTargetName'].get('OnlyTestIdentifiers'))
+
+  def test_invert_without_test_filter_without_test_cases(self):
+    """Ensures tests in the run invert with no test_filter and no test_cases."""
+    self.tr.set_xctest_filters(test_filter=None, invert=True)
+    self.assertIsNone(
+        self.tr.xctestrun_data['TestTargetName'].get('OnlyTestIdentifiers'))
+
+  def test_without_test_filter_with_test_cases(self):
+    """Ensures tests in the run with no test_filter but test_cases."""
+    self.tr.test_cases = ['a', 'b', 'c', 'd']
+    self.tr.set_xctest_filters(test_filter=None, invert=False)
+    self.assertEqual(
+        self.tr.xctestrun_data['TestTargetName']['OnlyTestIdentifiers'],
+        ['a', 'b', 'c', 'd']
+    )
+
+  def test_invert_without_test_filter_with_test_cases(self):
+    """Ensures tests in the run invert with no test_filter but test_cases."""
+    self.tr.test_cases = ['a', 'b', 'c', 'd']
+    self.tr.set_xctest_filters(test_filter=None, invert=True)
+    self.assertEqual(
+        self.tr.xctestrun_data['TestTargetName']['OnlyTestIdentifiers'],
+        ['a', 'b', 'c', 'd']
+    )
+
+
 if __name__ == '__main__':
   unittest.main()