[iOS test runners]Modularizing and updating test_runners.

Changes in this cl:
- added test_apps(Gtest and EGtests) to reduce code duplication (
 filling xctest_run files for EG1 and EG2 tests, filtering tests,
 creating commands to run tests).
-Removes shard_xctest method and other EG sharding code from test_runner
because shards only use in xcodebuild_runner.
- updated *runner.py to support test_apps
- replaced `iossim` with `xcodebuild` command.

In following cls:
1. iossim parameter will be removed in all test-runners
   and unittests and then
2. ios/api.py remove iossim.

Bug: 1011498
Change-Id: I9966ba1152eabeee56457fa8db3b6e9f585df3c9
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1961166
Commit-Queue: Maksym Onufriienko <monufriienko@chromium.org>
Reviewed-by: Rohit Rao <rohitrao@chromium.org>
Cr-Commit-Position: refs/heads/master@{#742746}

ios/build/bots/scripts/iossim_util_test.py

@@ -8,82 +8,12 @@ import mock
 import test_runner
 import test_runner_test
 
-SIMULATORS_LIST = {
-    'devices': {
-        'com.apple.CoreSimulator.SimRuntime.iOS-11-4': [{
-            'isAvailable': True,
-            'name': 'iPhone 5s',
-            'state': 'Shutdown',
-            'udid': 'E4E66320-177A-450A-9BA1-488D85B7278E'
-        }],
-        'com.apple.CoreSimulator.SimRuntime.iOS-13-2': [
-            {
-                'isAvailable': True,
-                'name': 'iPhone X',
-                'state': 'Shutdown',
-                'udid': 'E4E66321-177A-450A-9BA1-488D85B7278E'
-            },
-            {
-                'isAvailable': True,
-                'name': 'iPhone 11',
-                'state': 'Shutdown',
-                'udid': 'A4E66321-177A-450A-9BA1-488D85B7278E'
-            }
-        ]
-    },
-    'devicetypes': [
-        {
-            'name': 'iPhone 5s',
-            'bundlePath': '/path/iPhone 4s/Content',
-            'identifier': 'com.apple.CoreSimulator.SimDeviceType.iPhone-5s'
-        },
-        {
-            'name': 'iPhone X',
-            'bundlePath': '/path/iPhone X/Content',
-            'identifier': 'com.apple.CoreSimulator.SimDeviceType.iPhone-X'
-        },
-        {
-            'name': 'iPhone 11',
-            'bundlePath': '/path/iPhone 11/Content',
-            'identifier': 'com.apple.CoreSimulator.SimDeviceType.iPhone-11'
-        },
-    ],
-    'pairs': [],
-    'runtimes': [
-        {
-            "buildversion": "15F79",
-            "bundlePath": "/path/Runtimes/iOS 11.4.simruntime",
-            "identifier": "com.apple.CoreSimulator.SimRuntime.iOS-11-4",
-            "isAvailable": True,
-            "name": "iOS 11.4",
-            "version": "11.4"
-        },
-        {
-            "buildversion": "17A844",
-            "bundlePath": "/path/Runtimes/iOS 13.1.simruntime",
-            "identifier": "com.apple.CoreSimulator.SimRuntime.iOS-13-1",
-            "isAvailable": True,
-            "name": "iOS 13.1",
-            "version": "13.1"
-        },
-        {
-            "buildversion": "17B102",
-            "bundlePath": "/path/Runtimes/iOS.simruntime",
-            "identifier": "com.apple.CoreSimulator.SimRuntime.iOS-13-2",
-            "isAvailable": True,
-            "name": "iOS 13.2",
-            "version": "13.2.2"
-        },
-    ]
-}
-
 
 class GetiOSSimUtil(test_runner_test.TestCase):
   """Tests for iossim_util.py."""
 
   def setUp(self):
     super(GetiOSSimUtil, self).setUp()
-    self.mock(iossim_util, 'get_simulator_list', lambda: SIMULATORS_LIST)
 
   def test_get_simulator_runtime_by_version(self):
     """Ensures correctness of filter."""

ios/build/bots/scripts/test_apps_test.py

+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Unittests for test_apps.py."""
+
+import test_apps
+import test_runner_test
+
+
+class GetKIFTestFilterTest(test_runner_test.TestCase):
+  """Tests for test_runner.get_kif_test_filter."""
+
+  def test_correct(self):
+    """Ensures correctness of filter."""
+    tests = [
+      'KIF.test1',
+      'KIF.test2',
+    ]
+    expected = 'NAME:test1|test2'
+
+    self.assertEqual(expected, test_apps.get_kif_test_filter(tests))
+
+  def test_correct_inverted(self):
+    """Ensures correctness of inverted filter."""
+    tests = [
+      'KIF.test1',
+      'KIF.test2',
+    ]
+    expected = '-NAME:test1|test2'
+
+    self.assertEqual(expected,
+                    test_apps.get_kif_test_filter(tests, invert=True))
+
+
+class GetGTestFilterTest(test_runner_test.TestCase):
+  """Tests for test_runner.get_gtest_filter."""
+
+  def test_correct(self):
+    """Ensures correctness of filter."""
+    tests = [
+      'test.1',
+      'test.2',
+    ]
+    expected = 'test.1:test.2'
+
+    self.assertEqual(test_apps.get_gtest_filter(tests), expected)
+
+  def test_correct_inverted(self):
+    """Ensures correctness of inverted filter."""
+    tests = [
+      'test.1',
+      'test.2',
+    ]
+    expected = '-test.1:test.2'
+
+    self.assertEqual(test_apps.get_gtest_filter(tests, invert=True), expected)

ios/build/bots/scripts/wpr_runner.py

@@ -12,8 +12,8 @@ import subprocess
 import sys
 
 import gtest_utils
+import test_apps
 import test_runner
-import xcodebuild_runner
 import xctest_utils
 
 LOGGER = logging.getLogger(__name__)
@@ -158,7 +158,7 @@ class WprProxySimulatorTestRunner(test_runner.SimulatorTestRunner):
         '--enable-features=AutofillShowTypePredictions',
         '-autofillautomation=%s' % recipe_path,
     ]
-    wpr_egtests_app = xcodebuild_runner.EgtestsApp(
+    wpr_egtests_app = test_apps.EgtestsApp(
         self.app_path,
         included_tests=["AutofillAutomationTestCase"],
         env_vars=self.env_vars,
@@ -174,11 +174,7 @@ class WprProxySimulatorTestRunner(test_runner.SimulatorTestRunner):
         self.version, self.platform, test_name,
         self.test_attempt_count[test_name])
     out_dir = os.path.join(self.out_dir, destination_folder)
-
-    launch_command = xcodebuild_runner.LaunchCommand(
-        wpr_egtests_app, destination, self.shards, self.retries)
-    return launch_command.command(wpr_egtests_app, out_dir, destination,
-                                  self.shards)
+    return wpr_egtests_app.command(out_dir, destination, self.shards)
 
   def get_launch_env(self):
     """Returns a dict of environment variables to use to launch the test app.
@@ -381,30 +377,37 @@ class WprProxySimulatorTestRunner(test_runner.SimulatorTestRunner):
 
     self.deleteSimulator(udid)
 
-    # iossim can return 5 if it exits noncleanly even if all tests passed.
+    # xcodebuild can return 5 if it exits noncleanly even if all tests passed.
     # Therefore we cannot rely on process exit code to determine success.
 
     # NOTE: total_returncode is 0 OR the last non-zero return code from a test.
     result.finalize(total_returncode, completed_without_failure)
     return result
 
-  def get_launch_command(self, test_filter=[], invert=False):
+  def get_launch_command(self, test_app=None, out_dir=None,
+                         destination=None, shards=1):
     """Returns a config dict for the test, instead of the real launch command.
     Normally this is passed into _run as the command it should use, but since
     the WPR runner builds its own cmd, we use this to configure the function.
 
     Args:
-      test_filter: List of test cases to filter.
-      invert: Whether to invert the filter or not. Inverted, the filter will
-      match everything except the given test cases.
+      test_app: A test app needed to run.
+      out_dir: (str) A path for results.
+      destination: (str) A destination of device/simulator.
+      shards: (int) How many shards the tests should be divided into.
 
     Returns:
       A dict forming the configuration for the test.
     """
 
     test_config = {}
-    test_config['invert'] = invert
-    test_config['test_filter'] = test_filter
+    if test_app:
+      if test_app.included_tests:
+        test_config['invert'] = False
+        test_config['test_filter'] = test_app.included_tests
+      elif test_app.excluded_tests:
+        test_config['invert'] = True
+        test_config['test_filter'] = test_app.excluded_tests
     return test_config
 
   def proxy_start(self):

ios/build/bots/scripts/wpr_runner_test.py

@@ -9,6 +9,7 @@ import os
 import subprocess
 import unittest
 
+import iossim_util
 import test_runner
 import test_runner_test
 import wpr_runner
@@ -38,6 +39,8 @@ class WprProxySimulatorTestRunnerTest(test_runner_test.TestCase):
               lambda a, b: True)
     self.mock(wpr_runner.WprProxySimulatorTestRunner,
               'copy_trusted_certificate', lambda a: True)
+    self.mock(iossim_util, 'get_simulator',
+              lambda a, b: 'E4E66320-177A-450A-9BA1-488D85B7278E')
 
   def test_app_not_found(self):
     """Ensures AppNotFoundError is raised."""
@@ -178,7 +181,7 @@ class WprProxySimulatorTestRunnerTest(test_runner_test.TestCase):
     self.mock(subprocess, 'Popen', popen)
 
     tr.xctest_path = 'fake.xctest'
-    cmd = tr.get_launch_command(test_filter=test_filter, invert=invert)
+    cmd = {'invert': invert, 'test_filter': test_filter}
     return tr._run(cmd=cmd, shards=1)
 
   def test_run_no_filter(self):

ios/build/bots/scripts/xcodebuild_runner_test.py

@@ -7,9 +7,11 @@
 import mock
 import os
 
+import iossim_util
 import plistlib
 import shutil
 import tempfile
+import test_apps
 import test_runner
 import test_runner_test
 import xcode_log_parser
@@ -18,7 +20,7 @@ import xcodebuild_runner
 
 _ROOT_FOLDER_PATH = 'root/folder'
 _XCODE_BUILD_VERSION = '10B61'
-_DESTINATION = 'platform=iOS Simulator,OS=12.0,name=iPhone X'
+_DESTINATION = 'A4E66321-177A-450A-9BA1-488D85B7278E'
 _OUT_DIR = 'out/dir'
 _XTEST_RUN = '/tmp/temp_file.xctestrun'
 _EGTESTS_APP_PATH = '%s/any_egtests.app' % _ROOT_FOLDER_PATH
@@ -36,6 +38,8 @@ class XCodebuildRunnerTest(test_runner_test.TestCase):
     self.mock(xcodebuild_runner, 'get_all_tests',
               lambda _1, _2: ['Class1/passedTest1', 'Class1/passedTest2'])
     self.tmpdir = tempfile.mkdtemp()
+    self.mock(iossim_util, 'get_simulator_list',
+              lambda: test_runner_test.SIMULATORS_LIST)
 
   def tearDown(self):
     shutil.rmtree(self.tmpdir, ignore_errors=True)
@@ -107,92 +111,54 @@ class XCodebuildRunnerTest(test_runner_test.TestCase):
   def testEgtests_not_found_egtests_app(self):
     self.mock(os.path, 'exists', lambda _: False)
     with self.assertRaises(test_runner.AppNotFoundError):
-      xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH)
+      test_apps.EgtestsApp(_EGTESTS_APP_PATH)
 
   def testEgtests_not_found_plugins(self):
-    egtests = xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH)
+    egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH)
     self.mock(os.path, 'exists', lambda _: False)
     with self.assertRaises(test_runner.PlugInsNotFoundError):
       egtests._xctest_path()
 
   def testEgtests_found_xctest(self):
     self.assertEqual('/PlugIns/any_egtests.xctest',
-                     xcodebuild_runner.EgtestsApp(
-                         _EGTESTS_APP_PATH)._xctest_path())
+                     test_apps.EgtestsApp(_EGTESTS_APP_PATH)._xctest_path())
 
   @mock.patch('os.listdir', autospec=True)
   def testEgtests_not_found_xctest(self, mock_listdir):
     mock_listdir.return_value = ['random_file']
-    egtest = xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH)
+    egtest = test_apps.EgtestsApp(_EGTESTS_APP_PATH)
     with self.assertRaises(test_runner.XCTestPlugInNotFoundError):
       egtest._xctest_path()
 
   def testEgtests_xctestRunNode_without_filter(self):
-    egtest_node = xcodebuild_runner.EgtestsApp(
-        _EGTESTS_APP_PATH).xctestrun_node()['any_egtests_module']
+    egtest_node = test_apps.EgtestsApp(
+        _EGTESTS_APP_PATH).fill_xctestrun_node()['any_egtests_module']
     self.assertNotIn('OnlyTestIdentifiers', egtest_node)
     self.assertNotIn('SkipTestIdentifiers', egtest_node)
 
   def testEgtests_xctestRunNode_with_filter_only_identifiers(self):
     filtered_tests = ['TestCase1/testMethod1', 'TestCase1/testMethod2',
                       'TestCase2/testMethod1', 'TestCase1/testMethod2']
-    egtest_node = xcodebuild_runner.EgtestsApp(
-        _EGTESTS_APP_PATH, included_tests=filtered_tests).xctestrun_node()[
-            'any_egtests_module']
+    egtest_node = test_apps.EgtestsApp(
+        _EGTESTS_APP_PATH,
+        included_tests=filtered_tests).fill_xctestrun_node()['any_egtests_module']
     self.assertEqual(filtered_tests, egtest_node['OnlyTestIdentifiers'])
     self.assertNotIn('SkipTestIdentifiers', egtest_node)
 
   def testEgtests_xctestRunNode_with_filter_skip_identifiers(self):
     skipped_tests = ['TestCase1/testMethod1', 'TestCase1/testMethod2',
                      'TestCase2/testMethod1', 'TestCase1/testMethod2']
-    egtest_node = xcodebuild_runner.EgtestsApp(
-        _EGTESTS_APP_PATH, excluded_tests=skipped_tests
-        ).xctestrun_node()['any_egtests_module']
+    egtest_node = test_apps.EgtestsApp(
+        _EGTESTS_APP_PATH,
+        excluded_tests=skipped_tests).fill_xctestrun_node()['any_egtests_module']
     self.assertEqual(skipped_tests, egtest_node['SkipTestIdentifiers'])
     self.assertNotIn('OnlyTestIdentifiers', egtest_node)
 
-  @mock.patch('xcodebuild_runner.LaunchCommand.fill_xctest_run', autospec=True)
-  def testLaunchCommand_command(self, mock_fill_xctestrun):
-    mock_fill_xctestrun.return_value = _XTEST_RUN
-    mock_egtest = mock.MagicMock(spec=xcodebuild_runner.EgtestsApp)
-    type(mock_egtest).egtests_path = mock.PropertyMock(
-        return_value=_EGTESTS_APP_PATH)
-    cmd = xcodebuild_runner.LaunchCommand(
-        mock_egtest, _DESTINATION, shards=3, retries=1, out_dir=_OUT_DIR)
-    self.assertEqual(['xcodebuild', 'test-without-building',
-                      '-xctestrun', '/tmp/temp_file.xctestrun',
-                      '-destination',
-                      'platform=iOS Simulator,OS=12.0,name=iPhone X',
-                      '-resultBundlePath', 'out/dir',
-                      '-parallel-testing-enabled', 'YES',
-                      '-parallel-testing-worker-count', '3'],
-                     cmd.command(egtests_app=mock_egtest,
-                                 out_dir=_OUT_DIR,
-                                 destination=_DESTINATION,
-                                 shards=3))
-
-  @mock.patch('plistlib.writePlist', autospec=True)
-  @mock.patch('os.path.join', autospec=True)
-  @mock.patch('test_runner.get_current_xcode_info', autospec=True)
-  def testFill_xctest_run(self, xcode_version, mock_path_join, _):
-    mock_path_join.return_value = _XTEST_RUN
-    mock_egtest = mock.MagicMock(spec=xcodebuild_runner.EgtestsApp)
-    xcode_version.return_value = {'version': '10.2.1'}
-    launch_command = xcodebuild_runner.LaunchCommand(
-        mock_egtest, _DESTINATION, shards=1, retries=1, out_dir=_OUT_DIR)
-    self.assertEqual(_XTEST_RUN, launch_command.fill_xctest_run(mock_egtest))
-    self.assertEqual([mock.call.xctestrun_node()], mock_egtest.method_calls)
-
-  def testFill_xctest_run_exception(self):
-    with self.assertRaises(test_runner.AppNotFoundError):
-      xcodebuild_runner.LaunchCommand([], 'destination', shards=1, retries=1,
-                                      out_dir=_OUT_DIR).fill_xctest_run([])
-
   @mock.patch('test_runner.get_current_xcode_info', autospec=True)
   @mock.patch('xcode_log_parser.XcodeLogParser.collect_test_results')
   def testLaunchCommand_restartFailed1stAttempt(self, mock_collect_results,
                                                 xcode_version):
-    egtests = xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH)
+    egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH)
     xcode_version.return_value = {'version': '10.2.1'}
     mock_collect_results.side_effect = [
         {'failed': {'TESTS_DID_NOT_START': ['not started']}, 'passed': []},
@@ -211,7 +177,7 @@ class XCodebuildRunnerTest(test_runner_test.TestCase):
   @mock.patch('xcode_log_parser.XcodeLogParser.collect_test_results')
   def testLaunchCommand_notRestartPassedTest(self, mock_collect_results,
                                              xcode_version):
-    egtests = xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH)
+    egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH)
     xcode_version.return_value = {'version': '10.2.1'}
     mock_collect_results.side_effect = [
         {'failed': {'BUILD_INTERRUPTED': 'BUILD_INTERRUPTED: attempt # 0'},