Implement iOS test sharding with multiple simulators on test_runner.

Details in doc:
https://docs.google.com/document/d/1QaAZgmYuEImL1wAYJQxPcbex-fPLVd3rx8oMMKIgZX8/edit#

Bug: 808267
Cq-Include-Trybots: master.tryserver.chromium.mac:ios-simulator-cronet;master.tryserver.chromium.mac:ios-simulator-full-configs
Change-Id: I55fc27fcf31120a7c9d39e8953f72a39dd6394a7
Reviewed-on: https://chromium-review.googlesource.com/917309
Commit-Queue: Menglu Huang <huangml@chromium.org>
Reviewed-by: Sergey Berezin <sergeyberezin@chromium.org>
Cr-Commit-Position: refs/heads/master@{#537731}

ios/build/bots/scripts/run.py

@@ -149,7 +149,7 @@ def parse_args():
   parser.add_argument(
     '-s',
     '--shards',
-    help='Number of shards to split test cases. (Not implemented yet)',
+    help='Number of shards to split test cases.',
     metavar='n',
     type=int,
   )

ios/build/bots/scripts/test_runner.py

@@ -4,17 +4,23 @@
 
 """Test runners for iOS."""
 
+from multiprocessing import pool
+
 import argparse
 import collections
 import errno
+import json
 import os
 import plistlib
+import re
 import shutil
 import subprocess
 import sys
 import tempfile
 import time
 
+from multiprocessing import pool
+
 import find_xcode
 import gtest_utils
 import xctest_utils
@@ -198,6 +204,46 @@ def install_xcode(xcode_build_version, mac_toolchain_cmd, xcode_app_path):
   return True
 
 
+def shard_xctest(object_path, shards, test_cases=None):
+  """Gets EarlGrey test methods inside a test target and splits them into shards
+
+  Args:
+    object_path: Path of the test target bundle.
+    shards: Number of shards to split tests.
+    test_cases: Passed in test cases to run.
+
+  Returns:
+    A list of test shards.
+  """
+  cmd = ['otool', '-ov', object_path]
+  test_pattern = re.compile(
+    'imp -\[(?P<testSuite>[A-Za-z_][A-Za-z0-9_]*Test[Case]*) '
+    '(?P<testMethod>test[A-Za-z0-9_]*)\]')
+  test_names = test_pattern.findall(subprocess.check_output(cmd))
+
+  # If test_cases are passed in, only shard the intersection of them and the
+  # listed tests.  Format of passed-in test_cases can be either 'testSuite' or
+  # 'testSuite/testMethod'.  The listed tests are tuples of ('testSuite',
+  # 'testMethod').  The intersection includes both test suites and test methods.
+  tests_set = set()
+  if test_cases:
+    for test in test_names:
+      test_method = '%s/%s' % (test[0], test[1])
+      if test[0] in test_cases or test_method in test_cases:
+        tests_set.add(test_method)
+  else:
+    for test in test_names:
+      # 'ChromeTestCase' is the parent class of all EarlGrey test classes. It
+      # has no real tests.
+      if 'ChromeTestCase' != test[0]:
+        tests_set.add('%s/%s' % (test[0], test[1]))
+
+  tests = sorted(tests_set)
+  shard_len = len(tests)/shards  + (len(tests) % shards > 0)
+  test_shards=[tests[i:i + shard_len] for i in range(0, len(tests), shard_len)]
+  return test_shards
+
+
 class TestRunner(object):
   """Base class containing common functionality."""
 
@@ -349,7 +395,20 @@ class TestRunner(object):
       shutil.rmtree(DERIVED_DATA)
       os.mkdir(DERIVED_DATA)
 
-  def _run(self, cmd):
+  def run_tests(self, test_shard=None):
+    """Runs passed-in tests.
+
+    Args:
+      test_shard: Test cases to be included in the run.
+
+    Return:
+      out: (list) List of strings of subprocess's output.
+      udid: (string) Name of the simulator device in the run.
+      returncode: (int) Return code of subprocess.
+    """
+    raise NotImplementedError
+
+  def _run(self, cmd, shards=1):
     """Runs the specified command, parsing GTest output.
 
     Args:
@@ -358,33 +417,51 @@ class TestRunner(object):
     Returns:
       GTestResult instance.
     """
-    print ' '.join(cmd)
-    print
-
     result = gtest_utils.GTestResult(cmd)
     if self.xctest_path:
       parser = xctest_utils.XCTestLogParser()
     else:
       parser = gtest_utils.GTestLogParser()
 
-    proc = subprocess.Popen(
-        cmd,
-        env=self.get_launch_env(),
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-    )
-
-    while True:
-      line = proc.stdout.readline()
-      if not line:
-        break
-      line = line.rstrip()
-      parser.ProcessLine(line)
-      print line
+    if shards > 1:
+      test_shards = shard_xctest(
+        os.path.join(self.app_path, self.app_name),
+        shards,
+        self.test_cases
+      )
+
+      thread_pool = pool.ThreadPool(processes=shards)
+      for out, name, ret in thread_pool.imap_unordered(
+        self.run_tests, test_shards):
+        print "Simulator %s" % name
+        for line in out:
+          print line
+          parser.ProcessLine(line)
+        returncode = ret if ret else 0
+      thread_pool.close()
+      thread_pool.join()
+    else:
+      # TODO(crbug.com/812705): Implement test sharding for unit tests.
+      # TODO(crbug.com/812712): Use thread pool for DeviceTestRunner as well.
+      proc = subprocess.Popen(
+          cmd,
+          env=self.get_launch_env(),
+          stdout=subprocess.PIPE,
+          stderr=subprocess.STDOUT,
+      )
+      while True:
+        line = proc.stdout.readline()
+        if not line:
+          break
+        line = line.rstrip()
+        parser.ProcessLine(line)
+        print line
+        sys.stdout.flush()
+
+      proc.wait()
       sys.stdout.flush()
 
-    proc.wait()
-    sys.stdout.flush()
+      returncode = proc.returncode
 
     for test in parser.FailedTests(include_flaky=True):
       # Test cases are named as <test group>.<test case>. If the test case
@@ -396,12 +473,12 @@ class TestRunner(object):
 
     result.passed_tests.extend(parser.PassedTests(include_flaky=True))
 
-    print '%s returned %s' % (cmd[0], proc.returncode)
+    print '%s returned %s' % (cmd[0], returncode)
     print
 
     # iossim can return 5 if it exits noncleanly even if all tests passed.
     # Therefore we cannot rely on process exit code to determine success.
-    result.finalize(proc.returncode, parser.CompletedWithoutFailure())
+    result.finalize(returncode, parser.CompletedWithoutFailure())
     return result
 
   def launch(self):
@@ -409,7 +486,7 @@ class TestRunner(object):
     self.set_up()
     cmd = self.get_launch_command()
     try:
-      result = self._run(cmd)
+      result = self._run(cmd=cmd, shards=self.shards or 1)
       if result.crashed and not result.crashed_test:
         # If the app crashed but not during any particular test case, assume
         # it crashed on startup. Try one more time.
@@ -558,7 +635,6 @@ class SimulatorTestRunner(TestRunner):
     self.platform = platform
     self.start_time = None
     self.version = version
-    # TODO(crbug.com/808267): Implement iOS test sharding.
     self.shards = shards
 
   @staticmethod
@@ -672,7 +748,81 @@ class SimulatorTestRunner(TestRunner):
       shutil.rmtree(self.homedir, ignore_errors=True)
       self.homedir = ''
 
-  def get_launch_command(self, test_filter=None, invert=False):
+  def run_tests(self, test_shard=None):
+    """Runs passed-in tests. Builds a command and create a simulator to
+      run tests.
+    Args:
+      test_shard: Test cases to be included in the run.
+
+    Return:
+      out: (list) List of strings of subprocess's output.
+      udid: (string) Name of the simulator device in the run.
+      returncode: (int) Return code of subprocess.
+    """
+    udid = self.getSimulator()
+    cmd = self.sharding_cmd[:]
+    cmd.extend(['-u', udid])
+    if test_shard:
+      for test in test_shard:
+        cmd.extend(['-t', test])
+
+    cmd.append(self.app_path)
+    if self.xctest_path:
+      cmd.append(self.xctest_path)
+
+    proc = subprocess.Popen(
+      cmd,
+      env=self.get_launch_env(),
+      stdout=subprocess.PIPE,
+      stderr=subprocess.STDOUT,
+    )
+
+    out = []
+    while True:
+      line = proc.stdout.readline()
+      if not line:
+        break
+      out.append(line.rstrip())
+
+    self.deleteSimulator(udid)
+    return (out, udid, proc.returncode)
+
+  def getSimulator(self):
+    """Creates a simulator device by device types and runtimes. Returns the
+      udid for the created simulator instance.
+
+    Returns:
+      An udid of a simulator device.
+    """
+    simctl_list = json.loads(subprocess.check_output(
+                             ['xcrun', 'simctl', 'list', '-j']))
+    runtimes = simctl_list['runtimes']
+    devices = simctl_list['devicetypes']
+
+    device_type_id = ''
+    for device in devices:
+      if device['name'] == self.platform:
+        device_type_id = device['identifier']
+
+    runtime_id = ''
+    for runtime in runtimes:
+      if runtime['name'] == 'iOS %s' % self.version:
+        runtime_id = runtime['identifier']
+
+    name = '%s test' % self.platform
+    print 'creating simulator %s' % name
+    udid = subprocess.check_output([
+      'xcrun', 'simctl', 'create', name, device_type_id, runtime_id]).rstrip()
+    print udid
+    return udid
+
+  def deleteSimulator(self, udid=None):
+    """Removes dynamically created simulator devices."""
+    if udid:
+      print 'deleting simulator %s' % udid
+      subprocess.check_output(['xcrun', 'simctl', 'delete', udid])
+
+  def get_launch_command(self, test_filter=None, invert=False, test_shard=None):
     """Returns the command that can be used to launch the test app.
 
     Args:
@@ -689,26 +839,30 @@ class SimulatorTestRunner(TestRunner):
         '-s', self.version,
     ]
 
-    if test_filter:
-      if self.xctest_path:
+    for env_var in self.env_vars:
+      cmd.extend(['-e', env_var])
+
+    for test_arg in self.test_args:
+      cmd.extend(['-c', test_arg])
+
+    if self.xctest_path:
+      self.sharding_cmd = cmd[:]
+      if test_filter:
         # iossim doesn't support inverted filters for XCTests.
         if not invert:
           for test in test_filter:
             cmd.extend(['-t', test])
-      else:
+      elif test_shard:
+        for test in test_shard:
+          cmd.extend(['-t', test])
+      elif not invert:
+        for test_case in self.test_cases:
+          cmd.extend(['-t', test_case])
+    elif test_filter:
         kif_filter = get_kif_test_filter(test_filter, invert=invert)
         gtest_filter = get_gtest_filter(test_filter, invert=invert)
         cmd.extend(['-e', 'GKIF_SCENARIO_FILTER=%s' % kif_filter])
         cmd.extend(['-c', '--gtest_filter=%s' % gtest_filter])
-    elif self.xctest_path and not invert:
-      for test_case in self.test_cases:
-        cmd.extend(['-t', test_case])
-
-    for env_var in self.env_vars:
-      cmd.extend(['-e', env_var])
-
-    for test_arg in self.test_args:
-      cmd.extend(['-c', test_arg])
 
     cmd.append(self.app_path)
     if self.xctest_path:

ios/build/bots/scripts/test_runner_test.py

@@ -181,7 +181,7 @@ class SimulatorTestRunnerTest(TestCase):
       return
 
     @staticmethod
-    def _run(command):
+    def _run(cmd, shards=None):
       return collections.namedtuple('result', ['crashed', 'crashed_test'])(
           crashed=True, crashed_test=None)
 
@@ -204,8 +204,45 @@ class SimulatorTestRunnerTest(TestCase):
     with self.assertRaises(test_runner.AppLaunchError):
       tr.launch()
 
+  def test_run(self):
+    """Ensures the _run method is correct with test sharding."""
+    def shard_xctest(object_path, shards, test_cases=None):
+      return [['a/1', 'b/2'], ['c/3', 'd/4'], ['e/5']]
+
+    def run_tests(self, test_shard=None):
+      out = []
+      for test in test_shard:
+        testname = test.split('/')
+        out.append('Test Case \'-[%s %s]\' started.' %
+                   (testname[0], testname[1]))
+        out.append('Test Case \'-[%s %s]\' passed (0.1 seconds)' %
+                   (testname[0], testname[1]))
+      return (out, 0, 0)
+
+    tr = test_runner.SimulatorTestRunner(
+      'fake-app',
+      'fake-iossim',
+      'platform',
+      'os',
+      'xcode-version',
+      'xcode-build',
+      'out-dir',
+    )
+    self.mock(test_runner, 'shard_xctest', shard_xctest)
+    self.mock(test_runner.SimulatorTestRunner, 'run_tests', run_tests)
+
+    tr.xctest_path = 'fake.xctest'
+    cmd = tr.get_launch_command()
+    result = tr._run(cmd=cmd, shards=3)
+    self.assertIn('a/1', result.passed_tests)
+    self.assertIn('b/2', result.passed_tests)
+    self.assertIn('c/3', result.passed_tests)
+    self.assertIn('d/4', result.passed_tests)
+    self.assertIn('e/5', result.passed_tests)
+
   def test_get_launch_command(self):
-    """Ensures test filters are set correctly for launch command"""
+    """Ensures launch command is correct with test_filters, test sharding and
+      test_cases."""
     tr = test_runner.SimulatorTestRunner(
       'fake-app',
       'fake-iossim',
@@ -245,7 +282,7 @@ class SimulatorTestRunnerTest(TestCase):
       return
 
     @staticmethod
-    def _run(command):
+    def _run(cmd, shards=None):
       result = collections.namedtuple(
           'result', [
               'crashed',
@@ -255,7 +292,7 @@ class SimulatorTestRunnerTest(TestCase):
               'passed_tests',
           ],
       )
-      if '-e' not in command:
+      if '-e' not in cmd:
         # First run, has no test filter supplied. Mock a crash.
         return result(
             crashed=True,