Added parsing test output for interrupted builds.

If tests run is interrupted(e.g. lost connection to testservice,
test was stuck and no output for 3 mins and was killed then),
xcodebuild_runner needs to parse test_output, collect passed tests
and re-run test_bundle by filtering already passed tests
(e.g. https://chromium-swarm.appspot.com/task?id=45c892feae862110).
Removed _make_cmd_list_for_failed_tests because now tests
will filter by passed tests for both cases
when tests were interrupted and not.

At the end of run also add a check whether all tests from test-bundle
were executed and if not add 'not executed tests' record.

Created the radar
'After primaryInstrumentsServerWithError xcodebuild did not finish its execution'
https://feedbackassistant.apple.com/feedback/6476415

Bug: 979267
Change-Id: I596945e10bd8382c41487633456a3c80a3569419
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1680973Reviewed-by: Rohit Rao <rohitrao@chromium.org>
Reviewed-by: John Budorick <jbudorick@chromium.org>
Commit-Queue: Maksym Onufriienko <monufriienko@chromium.org>
Cr-Commit-Position: refs/heads/master@{#675844}

ios/build/bots/scripts/test_runner.py

@@ -334,6 +334,23 @@ def get_current_xcode_info():
   }
 
 
+def get_test_names(app_path):
+  """Gets list of tests from test app.
+
+  Args:
+     app_path: A path to test target bundle.
+
+  Returns:
+     List of tests.
+  """
+  cmd = ['otool', '-ov', app_path]
+  test_pattern = re.compile(
+      'imp (?:0[xX][0-9a-fA-F]+ )?-\['
+      '(?P<testSuite>[A-Za-z_][A-Za-z0-9_]*Test(?:Case)?)\s'
+      '(?P<testMethod>test[A-Za-z0-9_]*)\]')
+  return test_pattern.findall(subprocess.check_output(cmd))
+
+
 def shard_xctest(object_path, shards, test_cases=None):
   """Gets EarlGrey test methods inside a test target and splits them into shards
 
@@ -345,12 +362,7 @@ def shard_xctest(object_path, shards, test_cases=None):
   Returns:
     A list of test shards.
   """
-  cmd = ['otool', '-ov', object_path]
-  test_pattern = re.compile(
-    'imp -\[(?P<testSuite>[A-Za-z_][A-Za-z0-9_]*Test[Case]*) '
-    '(?P<testMethod>test[A-Za-z0-9_]*)\]')
-  test_names = test_pattern.findall(subprocess.check_output(cmd))
-
+  test_names = get_test_names(object_path)
   # If test_cases are passed in, only shard the intersection of them and the
   # listed tests.  Format of passed-in test_cases can be either 'testSuite' or
   # 'testSuite/testMethod'.  The listed tests are tuples of ('testSuite',

ios/build/bots/scripts/test_runner_test.py

@@ -8,6 +8,7 @@
 import collections
 import glob
 import logging
+import mock
 import os
 import subprocess
 import unittest
@@ -604,6 +605,37 @@ class DeviceTestRunnerTest(TestCase):
         ['a', 'b', 'c', 'd']
     )
 
+  @mock.patch('subprocess.check_output', autospec=True)
+  def test_get_test_names(self, mock_subprocess):
+    otool_output = (
+        'imp 0x102492020 -[BrowserViewControllerTestCase testJavaScript]'
+        'name 0x105ee8b84 testFixForCrbug801165'
+        'types 0x105f0c842 v16 @ 0:8'
+        'name 0x105ee8b9a testOpenURLFromNTP'
+        'types 0x105f0c842 v16 @ 0:8'
+        'imp 0x102493b30 -[BrowserViewControllerTestCase testOpenURLFromNTP]'
+        'name 0x105ee8bad testOpenURLFromTab'
+        'types 0x105f0c842 v16 @ 0:8'
+        'imp 0x102494180 -[BrowserViewControllerTestCase testOpenURLFromTab]'
+        'name 0x105ee8bc0 testOpenURLFromTabSwitcher'
+        'types 0x105f0c842 v16 @ 0:8'
+        'imp 0x102494f70 -[BrowserViewControllerTestCase testTabSwitch]'
+        'types 0x105f0c842 v16 @ 0:8'
+        'imp 0x102494f70 -[BrowserViewControllerTestCase helper]'
+        'imp 0x102494f70 -[BrowserViewControllerTestCCCCCCCCC testMethod]'
+    )
+    mock_subprocess.return_value = otool_output
+    tests = test_runner.get_test_names('')
+    self.assertEqual(
+        [
+            ('BrowserViewControllerTestCase', 'testJavaScript'),
+            ('BrowserViewControllerTestCase', 'testOpenURLFromNTP'),
+            ('BrowserViewControllerTestCase', 'testOpenURLFromTab'),
+            ('BrowserViewControllerTestCase', 'testTabSwitch')
+        ],
+        tests
+    )
+
 
 if __name__ == '__main__':
   logging.basicConfig(format='[%(asctime)s:%(levelname)s] %(message)s',

ios/build/bots/scripts/xcode_log_parser.py

@@ -18,6 +18,28 @@ import test_runner
 LOGGER = logging.getLogger(__name__)
 
 
+def parse_passed_tests_for_interrupted_run(output):
+  """Parses xcode runner output to get passed tests only.
+
+  Args:
+    output: [str] An output of test run.
+
+  Returns:
+    The list of passed tests only that will be a filter for next attempt.
+  """
+  passed_tests = []
+  # Test has format:
+  # [09:04:42:INFO] Test case '-[Test_class test_method]' passed.
+  passed_test_regex = re.compile(r'Test case \'\-\[(.+?)\s(.+?)\]\' passed')
+
+  for test_line in output:
+    m_test = passed_test_regex.search(test_line)
+    if m_test:
+      passed_tests.append('%s/%s' % (m_test.group(1), m_test.group(2)))
+  LOGGER.info('%d passed tests for interrupted build.' % len(passed_tests))
+  return passed_tests
+
+
 def format_test_case(test_case):
   """Format test case from `-[TestClass TestMethod]` to `TestClass_TestMethod`.
 
@@ -162,11 +184,12 @@ class Xcode11LogParser(object):
     return passed_tests
 
   @staticmethod
-  def collect_test_results(xcresult):
+  def collect_test_results(xcresult, output):
     """Gets test result data from xcresult.
 
     Args:
       xcresult: (str) A path to xcresult.
+      output: [str] An output of test run.
 
     Returns:
       Test result as a map:
@@ -190,7 +213,8 @@ class Xcode11LogParser(object):
     plist_path = os.path.join(xcresult + '.xcresult', 'Info.plist')
     if not os.path.exists(plist_path):
       test_results['failed']['BUILD_INTERRUPTED'] = [
-          '%s with test results does not exist.' % plist_path]
+          '%s with test results does not exist.' % plist_path] + output
+      test_results['passed'] = parse_passed_tests_for_interrupted_run(output)
       return test_results
 
     root = json.loads(Xcode11LogParser._xcresulttool_get(xcresult))
@@ -280,11 +304,12 @@ class XcodeLogParser(object):
     return status_summary
 
   @staticmethod
-  def collect_test_results(output_folder):
+  def collect_test_results(output_folder, output):
     """Gets test result data from Info.plist.
 
     Args:
       output_folder: (str) A path to output folder.
+      output: [str] An output of test run.
     Returns:
       Test result as a map:
         {
@@ -301,7 +326,8 @@ class XcodeLogParser(object):
     plist_path = os.path.join(output_folder, 'Info.plist')
     if not os.path.exists(plist_path):
       test_results['failed']['BUILD_INTERRUPTED'] = [
-          '%s with test results does not exist.' % plist_path]
+          '%s with test results does not exist.' % plist_path] + output
+      test_results['passed'] = parse_passed_tests_for_interrupted_run(output)
       return test_results
 
     root = plistlib.readPlist(plist_path)

ios/build/bots/scripts/xcode_log_parser_test.py

@@ -203,7 +203,7 @@ class XCode11LogParserTest(test_runner_test.TestCase):
     mock_exist_file.return_value = True
     self.assertEqual(expected_test_results,
                      xcode_log_parser.Xcode11LogParser().collect_test_results(
-                         _XTEST_RESULT))
+                         _XTEST_RESULT, []))
 
   @mock.patch('os.path.exists', autospec=True)
   @mock.patch('xcode_log_parser.Xcode11LogParser._xcresulttool_get')
@@ -216,7 +216,7 @@ class XCode11LogParserTest(test_runner_test.TestCase):
     mock_exist_file.return_value = True
     self.assertEqual(expected_test_results,
                      xcode_log_parser.Xcode11LogParser().collect_test_results(
-                         _XTEST_RESULT))
+                         _XTEST_RESULT, []))
 
   @mock.patch('os.path.exists', autospec=True)
   def testCollectTestsDidNotRun(self, mock_exist_file):
@@ -227,7 +227,7 @@ class XCode11LogParserTest(test_runner_test.TestCase):
             '%s with test results does not exist.' % _XTEST_RESULT]}}
     self.assertEqual(expected_test_results,
                      xcode_log_parser.Xcode11LogParser().collect_test_results(
-                         _XTEST_RESULT))
+                         _XTEST_RESULT, []))
 
   @mock.patch('os.path.exists', autospec=True)
   def testCollectTestsInterruptedRun(self, mock_exist_file):
@@ -239,7 +239,7 @@ class XCode11LogParserTest(test_runner_test.TestCase):
                 _XTEST_RESULT + '.xcresult', 'Info.plist')]}}
     self.assertEqual(expected_test_results,
                      xcode_log_parser.Xcode11LogParser().collect_test_results(
-                         _XTEST_RESULT))
+                         _XTEST_RESULT, []))
 
   @mock.patch('os.path.exists', autospec=True)
   @mock.patch('xcode_log_parser.Xcode11LogParser._xcresulttool_get')
@@ -250,3 +250,22 @@ class XCode11LogParserTest(test_runner_test.TestCase):
     mock_xcresulttool_get.return_value = ACTIONS_RECORD_FAILED_TEST
     xcode_log_parser.Xcode11LogParser().copy_screenshots(_XTEST_RESULT)
     self.assertEqual(1, mock_copy.call_count)
+
+  @mock.patch('os.path.exists', autospec=True)
+  def testCollectTestResults_interruptedTests(self, mock_path_exists):
+    mock_path_exists.side_effect = [True, False]
+    output = [
+        '[09:03:42:INFO] Test case \'-[TestCase1 method1]\' passed on device.',
+        '[09:06:40:INFO] Test case \'-[TestCase2 method1]\' passed on device.',
+        '[09:09:00:INFO] Test case \'-[TestCase2 method1]\' failed on device.',
+        '** BUILD INTERRUPTED **',
+    ]
+    not_found_message = [
+        'Info.plist.xcresult/Info.plist with test results does not exist.']
+    res = xcode_log_parser.Xcode11LogParser().collect_test_results(
+        'Info.plist', output)
+    self.assertIn('BUILD_INTERRUPTED', res['failed'])
+    self.assertEqual(not_found_message + output,
+                     res['failed']['BUILD_INTERRUPTED'])
+    self.assertEqual(['TestCase1/method1', 'TestCase2/method1'],
+                     res['passed'])

ios/build/bots/scripts/xcodebuild_runner.py

@@ -57,21 +57,22 @@ class EgtestsApp(object):
     egtests_app: full path to egtests app.
     project_path: root project folder.
     module_name: egtests module name.
-    filtered_tests: List of tests to include or exclude, depending on `invert`.
-    invert: type of filter(True - inclusive, False - exclusive).
+    included_tests: List of tests to run.
+    excluded_tests: List of tests not to run.
   """
 
-  def __init__(self, egtests_app, filtered_tests=None, invert=False,
+  def __init__(self, egtests_app, included_tests=None, excluded_tests=None,
                test_args=None, env_vars=None, host_app_path=None):
     """Initialize Egtests.
 
     Args:
       egtests_app: (str) full path to egtests app.
-      filtered_tests: (list) Specific tests to run
-        (it can inclusive/exclusive based on invert parameter).
+      included_tests: (list) Specific tests to run
          E.g.
           [ 'TestCaseClass1/testMethod1', 'TestCaseClass2/testMethod2']
-      invert: type of filter(True - inclusive, False - exclusive).
+      excluded_tests: (list) Specific tests not to run
+         E.g.
+          [ 'TestCaseClass1', 'TestCaseClass2/testMethod2']
       test_args: List of strings to pass as arguments to the test when
         launching.
       env_vars: List of environment variables to pass to the test itself.
@@ -85,8 +86,8 @@ class EgtestsApp(object):
     self.egtests_path = egtests_app
     self.project_path = os.path.dirname(self.egtests_path)
     self.module_name = os.path.splitext(os.path.basename(egtests_app))[0]
-    self.filter = filtered_tests
-    self.invert = invert
+    self.included_tests = included_tests or []
+    self.excluded_tests = excluded_tests or []
     self.test_args = test_args
     self.env_vars = env_vars
     self.host_app_path = host_app_path
@@ -121,15 +122,15 @@ class EgtestsApp(object):
     """
     module = self.module_name + '_module'
     module_data = {
-            'TestBundlePath': '__TESTHOST__%s' % self._xctest_path(),
-            'TestHostPath': '%s' % self.egtests_path,
-            'TestingEnvironmentVariables': {
-                'DYLD_INSERT_LIBRARIES': (
-                    '__PLATFORMS__/iPhoneSimulator.platform/Developer/'
-                    'usr/lib/libXCTestBundleInject.dylib'),
-                'DYLD_LIBRARY_PATH': self.project_path,
-                'DYLD_FRAMEWORK_PATH': self.project_path + ':',
-                'XCInjectBundleInto': '__TESTHOST__/%s' % self.module_name
+        'TestBundlePath': '__TESTHOST__%s' % self._xctest_path(),
+        'TestHostPath': '%s' % self.egtests_path,
+        'TestingEnvironmentVariables': {
+            'DYLD_INSERT_LIBRARIES': (
+                '__PLATFORMS__/iPhoneSimulator.platform/Developer/'
+                'usr/lib/libXCTestBundleInject.dylib'),
+            'DYLD_LIBRARY_PATH': self.project_path,
+            'DYLD_FRAMEWORK_PATH': self.project_path + ':',
+            'XCInjectBundleInto': '__TESTHOST__/%s' % self.module_name
             }
         }
     # Add module data specific to EG2 or EG1 tests
@@ -140,9 +141,9 @@ class EgtestsApp(object):
       module_data['UITargetAppPath'] = '%s' % self.host_app_path
       # Special handling for Xcode10.2
       dependent_products = [
-        module_data['UITargetAppPath'],
-        module_data['TestBundlePath'],
-        module_data['TestHostPath']
+          module_data['UITargetAppPath'],
+          module_data['TestBundlePath'],
+          module_data['TestHostPath']
       ]
       module_data['DependentProductPaths'] = dependent_products
     # EG1 tests
@@ -152,13 +153,12 @@ class EgtestsApp(object):
     xctestrun_data = {
         module: module_data
     }
-    if self.filter:
-      if self.invert:
-        xctestrun_data[module].update(
-            {'SkipTestIdentifiers': self.filter})
-      else:
-        xctestrun_data[module].update(
-            {'OnlyTestIdentifiers': self.filter})
+    if self.excluded_tests:
+      xctestrun_data[module].update(
+          {'SkipTestIdentifiers': self.excluded_tests})
+    if self.included_tests:
+      xctestrun_data[module].update(
+          {'OnlyTestIdentifiers': self.included_tests})
     if self.env_vars:
       xctestrun_data[module].update(
           {'EnvironmentVariables': self.env_vars})
@@ -207,34 +207,6 @@ class LaunchCommand(object):
     else:
       self._log_parser = xcode_log_parser.XcodeLogParser()
 
-  def _make_cmd_list_for_failed_tests(self, failed_results, out_dir,
-                                      test_args=None, env_vars=None):
-    """Makes cmd list based on failure results.
-
-    Args:
-      failed_results: Map of failed tests, where key is name of egtests_app and
-        value is a list of failed_test_case/test_methods:
-          {
-              'failed_test_case/test_methods': ['StackTrace']
-          }
-      out_dir: (str) An output path.
-      test_args: List of strings to pass as arguments to the test when
-        launching.
-      env_vars: List of environment variables to pass to the test itself.
-
-    Returns:
-      List of Launch commands to re-run failed tests.
-      Every destination will run on separate clone of a stimulator.
-    """
-    eg_app = EgtestsApp(
-        egtests_app=self.egtests_app.egtests_path,
-        filtered_tests=[test.replace(' ', '/') for test in failed_results],
-        test_args=test_args,
-        env_vars=env_vars,
-        host_app_path=self.egtests_app.host_app_path)
-    # Regenerates xctest run and gets a command.
-    return self.command(eg_app, out_dir, self.destination, shards=1)
-
   def summary_log(self):
     """Calculates test summary - how many passed, failed and error tests.
 
@@ -266,8 +238,10 @@ class LaunchCommand(object):
 
     Returns:
       returncode - return code of command run.
+      output - command output as list of strings.
     """
     LOGGER.info('Launching %s with env %s' % (cmd, self.env))
+    output = []
     proc = subprocess.Popen(
         cmd,
         env=self.env,
@@ -291,49 +265,47 @@ class LaunchCommand(object):
         break
       line = line.rstrip()
       LOGGER.info(line)
+      output.append(line)
       sys.stdout.flush()
 
     proc.wait()
     LOGGER.info('Command %s finished with %d' % (cmd, proc.returncode))
-    return proc.returncode
+    return proc.returncode, output
 
   def launch(self):
     """Launches tests using xcodebuild."""
     cmd_list = []
     self.test_results['attempts'] = []
+    cancelled_statuses = {'TESTS_DID_NOT_START', 'BUILD_INTERRUPTED'}
+    shards = self.shards
 
     # total number of attempts is self.retries+1
     for attempt in range(self.retries + 1):
       outdir_attempt = os.path.join(self.out_dir, 'attempt_%d' % attempt)
-      # Create a command for the 1st run or if tests did not start,
-      # re-run the same command but with different output folder.
-      # (http://crbug.com/916620) If tests did not start, repeat the command.
-      if (not self.test_results['attempts'] or
-          {'TESTS_DID_NOT_START', 'BUILD_INTERRUPTED'}.intersection(
-              self.test_results['attempts'][-1]['failed'].keys())):
-        cmd_list = self.command(self.egtests_app,
-                                outdir_attempt,
-                                self.destination,
-                                self.shards)
-      # Re-init the command based on list of failed tests.
-      else:
-        cmd_list = self._make_cmd_list_for_failed_tests(
-            self.test_results['attempts'][-1]['failed'],
-            outdir_attempt,
-            test_args=self.egtests_app.test_args,
-            env_vars=self.egtests_app.env_vars)
-
+      cmd_list = self.command(self.egtests_app,
+                              outdir_attempt,
+                              self.destination,
+                              shards)
       # TODO(crbug.com/914878): add heartbeat logging to xcodebuild_runner.
       LOGGER.info('Start test attempt #%d for command [%s]' % (
           attempt, ' '.join(cmd_list)))
-      self.launch_attempt(cmd_list, outdir_attempt)
+      _, output = self.launch_attempt(cmd_list, outdir_attempt)
       self.test_results['attempts'].append(
-          self._log_parser.collect_test_results(outdir_attempt))
+          self._log_parser.collect_test_results(outdir_attempt, output))
       if self.retries == attempt or not self.test_results[
           'attempts'][-1]['failed']:
         break
       self._log_parser.copy_screenshots(outdir_attempt)
-
+      # Exclude passed tests in next test attempt.
+      self.egtests_app.excluded_tests += self.test_results['attempts'][-1][
+          'passed']
+      # If tests are not completed(interrupted or did not start)
+      # re-run them with the same number of shards,
+      # otherwise re-run with shards=1 and exclude passed tests.
+      cancelled_attempt = cancelled_statuses.intersection(
+          self.test_results['attempts'][-1]['failed'].keys())
+      if not cancelled_attempt:
+        shards = 1
     self.test_results['end_run'] = int(time.time())
     self.summary_log()
 
@@ -514,7 +486,7 @@ class SimulatorParallelTestRunner(test_runner.SimulatorTestRunner):
     launch_commands = []
     for params in self.sharding_data:
       launch_commands.append(LaunchCommand(
-          EgtestsApp(params['app'], filtered_tests=params['test_cases'],
+          EgtestsApp(params['app'], included_tests=params['test_cases'],
                      env_vars=self.env_vars, test_args=self.test_args,
                      host_app_path=params['host']),
           params['destination'],
@@ -557,6 +529,12 @@ class SimulatorParallelTestRunner(test_runner.SimulatorTestRunner):
     self.logs['flaked tests'] = list(
         all_failures - set(self.logs['failed tests']))
 
+    # Gets not-started/interrupted tests
+    aborted_tests = list(set(self.get_all_tests()) - set(
+        self.logs['failed tests']) - set(self.logs['passed tests']))
+    aborted_tests.sort()
+    self.logs['aborted tests'] = aborted_tests
+
     # Test is failed if there are failures for the last run.
     return not self.logs['failed tests']
 
@@ -567,3 +545,20 @@ class SimulatorParallelTestRunner(test_runner.SimulatorTestRunner):
     """
     LOGGER.info('Erasing all simulators.')
     subprocess.call(['xcrun', 'simctl', 'erase', 'all'])
+
+  def get_all_tests(self):
+    """Gets all tests from test bundle."""
+    test_app_bundle = os.path.join(self.app_path, os.path.splitext(
+        os.path.basename(self.app_path))[0])
+    # Method names that starts with test* and also are in *TestCase classes
+    # but they are not test-methods.
+    # TODO(crbug.com/982435): Rename not test methods with test-suffix.
+    not_tests = ['ChromeTestCase/testServer', 'FindInPageTestCase/testURL']
+    all_tests = []
+    for test_class, test_method in test_runner.get_test_names(test_app_bundle):
+      test_name = '%s/%s' % (test_class, test_method)
+      if (test_name not in not_tests and
+          # Filter by self.test_cases if specified
+          (test_class in self.test_cases if self.test_cases else True)):
+        all_tests.append(test_name)
+    return all_tests

ios/build/bots/scripts/xcodebuild_runner_test.py

@@ -141,7 +141,7 @@ class XCodebuildRunnerTest(test_runner_test.TestCase):
     filtered_tests = ['TestCase1/testMethod1', 'TestCase1/testMethod2',
                       'TestCase2/testMethod1', 'TestCase1/testMethod2']
     egtest_node = xcodebuild_runner.EgtestsApp(
-        _EGTESTS_APP_PATH, filtered_tests=filtered_tests).xctestrun_node()[
+        _EGTESTS_APP_PATH, included_tests=filtered_tests).xctestrun_node()[
             'any_egtests_module']
     self.assertEqual(filtered_tests, egtest_node['OnlyTestIdentifiers'])
     self.assertNotIn('SkipTestIdentifiers', egtest_node)
@@ -153,8 +153,8 @@ class XCodebuildRunnerTest(test_runner_test.TestCase):
     skipped_tests = ['TestCase1/testMethod1', 'TestCase1/testMethod2',
                      'TestCase2/testMethod1', 'TestCase1/testMethod2']
     egtest_node = xcodebuild_runner.EgtestsApp(
-        _EGTESTS_APP_PATH, filtered_tests=skipped_tests,
-        invert=True).xctestrun_node()['any_egtests_module']
+        _EGTESTS_APP_PATH, excluded_tests=skipped_tests
+        ).xctestrun_node()['any_egtests_module']
     self.assertEqual(skipped_tests, egtest_node['SkipTestIdentifiers'])
     self.assertNotIn('OnlyTestIdentifiers', egtest_node)
 
@@ -195,42 +195,6 @@ class XCodebuildRunnerTest(test_runner_test.TestCase):
       xcodebuild_runner.LaunchCommand([], 'destination', shards=1, retries=1,
                                       out_dir=_OUT_DIR).fill_xctest_run([])
 
-  @mock.patch('xcodebuild_runner.LaunchCommand.fill_xctest_run', autospec=True)
-  def testLaunchCommand_make_cmd_list_for_failed_tests(self,
-                                                       fill_xctest_run_mock):
-    fill_xctest_run_mock.side_effect = [
-        '/var/folders/tmpfile1'
-    ]
-    egtest_app = 'module_1_egtests.app'
-    egtest_app_path = '%s/%s' % (_ROOT_FOLDER_PATH, egtest_app)
-    host_app_path = '%s/%s' % (_ROOT_FOLDER_PATH, egtest_app)
-    failed_tests = {
-        egtest_app: [
-            'TestCase1_1/TestMethod1',
-            'TestCase1_1/TestMethod2',
-            'TestCase1_2/TestMethod1',
-        ]
-    }
-    expected_egtests = xcodebuild_runner.EgtestsApp(
-        egtest_app_path, filtered_tests=failed_tests[egtest_app])
-    mock_egtest = mock.MagicMock(spec=xcodebuild_runner.EgtestsApp)
-    type(mock_egtest).egtests_path = mock.PropertyMock(
-        return_value=egtest_app_path)
-    type(mock_egtest).host_app_path = mock.PropertyMock(
-        return_value=host_app_path)
-    cmd = xcodebuild_runner.LaunchCommand(
-        egtests_app=mock_egtest,
-        destination=_DESTINATION,
-        out_dir='out/dir/attempt_2/iPhone X 12.0',
-        shards=1,
-        retries=1
-    )
-    cmd._make_cmd_list_for_failed_tests(
-        failed_tests, os.path.join(_OUT_DIR, 'attempt_2'))
-    self.assertEqual(1, len(fill_xctest_run_mock.mock_calls))
-    self.assertItemsEqual(expected_egtests.__dict__,
-                          fill_xctest_run_mock.mock_calls[0][1][1].__dict__)
-
   @mock.patch('os.listdir', autospec=True)
   @mock.patch('test_runner.get_current_xcode_info', autospec=True)
   @mock.patch('xcode_log_parser.XcodeLogParser.collect_test_results')
@@ -240,7 +204,7 @@ class XCodebuildRunnerTest(test_runner_test.TestCase):
     egtests = xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH)
     xcode_version.return_value = {'version': '10.2.1'}
     mock_collect_results.side_effect = [
-        {'failed': {'TESTS_DID_NOT_START': ['not started']}},
+        {'failed': {'TESTS_DID_NOT_START': ['not started']}, 'passed': []},
         {'failed': {}, 'passed': ['passedTest1']}
     ]
     launch_command = xcodebuild_runner.LaunchCommand(egtests,