Stopped test-run if all tests passed for interrupted build.

Base on https://chromium-swarm.appspot.com/task?id=46420ba12b52e810:
 1. test started running on 3 simulators, but one of them, Clone 2 of iPad Air,
    did not start running tests and was stuck with error
    'Test runner exited before starting test execution.'
    In Xcode10 for such a case entire run stopped,
    but Xcode11 runs tests on other 'good' Simulators.
 2. tests finished but xcodebuild did not finish its execution since
    Clone 2 of iPad Air was irresponsible
 3. in 3 mins test_runner killed xcodebuild
 4. test_runner parses passed tests and start a new attempt using passed-tests filter
 5. xcode ran 0 tests then and marked tests as succeeded but because 0 tests ran,
    test_runner assumed that TESTS_DID_NOT_START and marked results as failed.

Fixed step 4 to stop execution in case all tests passed even for interrupted build.

Bug: 987664
Change-Id: Idcd451d7feef518dc5613cb5f1bc42e8bbe84e7b
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1717919
Commit-Queue: Maksym Onufriienko <monufriienko@chromium.org>
Reviewed-by: Justin Cohen <justincohen@chromium.org>
Cr-Commit-Position: refs/heads/master@{#682502}

ios/build/bots/scripts/xcodebuild_runner.py

@@ -306,8 +306,8 @@ class LaunchCommand(object):
     self.test_results['attempts'] = []
     cancelled_statuses = {'TESTS_DID_NOT_START', 'BUILD_INTERRUPTED'}
     shards = self.shards
-    running_tests = get_all_tests(self.egtests_app.egtests_path,
-                                  self.egtests_app.included_tests)
+    running_tests = set(get_all_tests(self.egtests_app.egtests_path,
+                                      self.egtests_app.included_tests))
 
     # total number of attempts is self.retries+1
     for attempt in range(self.retries + 1):
@@ -330,10 +330,20 @@ class LaunchCommand(object):
       if self.retries == attempt or not self.test_results[
           'attempts'][-1]['failed']:
         break
-      self._log_parser.copy_screenshots(outdir_attempt)
       # Exclude passed tests in next test attempt.
       self.egtests_app.excluded_tests += self.test_results['attempts'][-1][
           'passed']
+      # crbug.com/987664 - for the case when
+      # all tests passed but build was interrupted,
+      # excluded(passed) tests are equal to tests to run.
+      if set(self.egtests_app.excluded_tests) == running_tests:
+        for status in cancelled_statuses:
+          failure = self.test_results['attempts'][-1]['failed'].pop(
+              status, None)
+          if failure:
+            LOGGER.info('Failure for passed tests %s: %s' % (status, failure))
+        break
+      self._log_parser.copy_screenshots(outdir_attempt)
       # If tests are not completed(interrupted or did not start)
       # re-run them with the same number of shards,
       # otherwise re-run with shards=1 and exclude passed tests.
@@ -400,7 +410,7 @@ class LaunchCommand(object):
            '-xctestrun', self.fill_xctest_run(egtests_app),
            '-destination', destination,
            '-resultBundlePath', out_dir]
-    if self.shards > 1:
+    if shards > 1:
       cmd += ['-parallel-testing-enabled', 'YES',
               '-parallel-testing-worker-count', str(shards)]
     return cmd
@@ -567,9 +577,9 @@ class SimulatorParallelTestRunner(test_runner.SimulatorTestRunner):
         all_failures - set(self.logs['failed tests']))
 
     # Gets not-started/interrupted tests
-    aborted_tests = list(
-        set(get_all_tests(self.app_path, self.test_cases)) - set(
-            self.logs['failed tests']) - set(self.logs['passed tests']))
+    all_tests_to_run = set(get_all_tests(self.app_path, self.test_cases))
+    aborted_tests = list(all_tests_to_run - set(self.logs['failed tests']) -
+                         set(self.logs['passed tests']))
     aborted_tests.sort()
     self.logs['aborted tests'] = aborted_tests
 

ios/build/bots/scripts/xcodebuild_runner_test.py

@@ -32,6 +32,9 @@ class XCodebuildRunnerTest(test_runner_test.TestCase):
     self.mock(os.path, 'exists', lambda _: True)
     self.mock(xcode_log_parser.XcodeLogParser, 'copy_screenshots',
               lambda _1, _2: None)
+    self.mock(os, 'listdir', lambda _: ['any_egtests.xctest'])
+    self.mock(xcodebuild_runner, 'get_all_tests',
+              lambda _1, _2: ['Class1/passedTest1', 'Class1/passedTest2'])
     self.tmpdir = tempfile.mkdtemp()
 
   def tearDown(self):
@@ -112,9 +115,7 @@ class XCodebuildRunnerTest(test_runner_test.TestCase):
     with self.assertRaises(test_runner.PlugInsNotFoundError):
       egtests._xctest_path()
 
-  @mock.patch('os.listdir', autospec=True)
-  def testEgtests_found_xctest(self, mock_listdir):
-    mock_listdir.return_value = ['any_egtests.xctest']
+  def testEgtests_found_xctest(self):
     self.assertEqual('/PlugIns/any_egtests.xctest',
                      xcodebuild_runner.EgtestsApp(
                          _EGTESTS_APP_PATH)._xctest_path())
@@ -126,18 +127,13 @@ class XCodebuildRunnerTest(test_runner_test.TestCase):
     with self.assertRaises(test_runner.XCTestPlugInNotFoundError):
       egtest._xctest_path()
 
-  @mock.patch('os.listdir', autospec=True)
-  def testEgtests_xctestRunNode_without_filter(self, mock_listdir):
-    mock_listdir.return_value = ['any_egtests.xctest']
+  def testEgtests_xctestRunNode_without_filter(self):
     egtest_node = xcodebuild_runner.EgtestsApp(
         _EGTESTS_APP_PATH).xctestrun_node()['any_egtests_module']
     self.assertNotIn('OnlyTestIdentifiers', egtest_node)
     self.assertNotIn('SkipTestIdentifiers', egtest_node)
 
-  @mock.patch('os.listdir', autospec=True)
-  def testEgtests_xctestRunNode_with_filter_only_identifiers(self,
-                                                             mock_listdir):
-    mock_listdir.return_value = ['any_egtests.xctest']
+  def testEgtests_xctestRunNode_with_filter_only_identifiers(self):
     filtered_tests = ['TestCase1/testMethod1', 'TestCase1/testMethod2',
                       'TestCase2/testMethod1', 'TestCase1/testMethod2']
     egtest_node = xcodebuild_runner.EgtestsApp(
@@ -146,10 +142,7 @@ class XCodebuildRunnerTest(test_runner_test.TestCase):
     self.assertEqual(filtered_tests, egtest_node['OnlyTestIdentifiers'])
     self.assertNotIn('SkipTestIdentifiers', egtest_node)
 
-  @mock.patch('os.listdir', autospec=True)
-  def testEgtests_xctestRunNode_with_filter_skip_identifiers(self,
-                                                             mock_listdir):
-    mock_listdir.return_value = ['any_egtests.xctest']
+  def testEgtests_xctestRunNode_with_filter_skip_identifiers(self):
     skipped_tests = ['TestCase1/testMethod1', 'TestCase1/testMethod2',
                      'TestCase2/testMethod1', 'TestCase1/testMethod2']
     egtest_node = xcodebuild_runner.EgtestsApp(
@@ -195,17 +188,15 @@ class XCodebuildRunnerTest(test_runner_test.TestCase):
       xcodebuild_runner.LaunchCommand([], 'destination', shards=1, retries=1,
                                       out_dir=_OUT_DIR).fill_xctest_run([])
 
-  @mock.patch('os.listdir', autospec=True)
   @mock.patch('test_runner.get_current_xcode_info', autospec=True)
   @mock.patch('xcode_log_parser.XcodeLogParser.collect_test_results')
   def testLaunchCommand_restartFailed1stAttempt(self, mock_collect_results,
-                                                xcode_version, mock_listdir):
-    mock_listdir.return_value = ['any_egtests.xctest']
+                                                xcode_version):
     egtests = xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH)
     xcode_version.return_value = {'version': '10.2.1'}
     mock_collect_results.side_effect = [
         {'failed': {'TESTS_DID_NOT_START': ['not started']}, 'passed': []},
-        {'failed': {}, 'passed': ['passedTest1']}
+        {'failed': {}, 'passed': ['Class1/passedTest1', 'Class1/passedTest2']}
     ]
     launch_command = xcodebuild_runner.LaunchCommand(egtests,
                                                      _DESTINATION,
@@ -215,3 +206,22 @@ class XCodebuildRunnerTest(test_runner_test.TestCase):
     self.fake_launch_attempt(launch_command, ['not_started', 'pass'])
     launch_command.launch()
     self.assertEqual(2, len(launch_command.test_results))
+
+  @mock.patch('test_runner.get_current_xcode_info', autospec=True)
+  @mock.patch('xcode_log_parser.XcodeLogParser.collect_test_results')
+  def testLaunchCommand_notRestartPassedTest(self, mock_collect_results,
+                                             xcode_version):
+    egtests = xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH)
+    xcode_version.return_value = {'version': '10.2.1'}
+    mock_collect_results.side_effect = [
+        {'failed': {'BUILD_INTERRUPTED': 'BUILD_INTERRUPTED: attempt # 0'},
+         'passed': ['Class1/passedTest1', 'Class1/passedTest2']}
+    ]
+    launch_command = xcodebuild_runner.LaunchCommand(egtests,
+                                                     _DESTINATION,
+                                                     shards=1,
+                                                     retries=3,
+                                                     out_dir=self.tmpdir)
+    self.fake_launch_attempt(launch_command, ['pass'])
+    launch_command.launch()
+    self.assertEqual(2, len(launch_command.test_results))