Swarming-based sharding for iOS

iOS utilizes otool to analyze and distribute test case execution
against x number of shards. This logic was run in the recipe,
such that the sublist of test cases would be passed into the isolate
for execution. Chromium does not support recipe-level sharding.

To mitigate, we clone the otool sharding logic into the test runner.
Each swarming task (shard) is provided with two keys via env vars:
shard index, and total number of shards.

A given test on a given run should be broken down identically across
each swarming task, because the test bundle does not change and the
number of total shards is the same. Because each task also is provided
the index, it can select the corresponding index from the sublist
and execute that sublist of test cases.

There is minimal performance hit, and this logic is only invoked when
those env vars are present, which are only set by swaring for shards
greater than 1.

Change-Id: Id3b4c96cc68ef97b62ed79afc145bbbe55156063
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2040371Reviewed-by: Rohit Rao <rohitrao@chromium.org>
Commit-Queue: Jeff Yoon <jeffyoon@chromium.org>
Cr-Commit-Position: refs/heads/master@{#739211}

ios/build/bots/scripts/run.py

@@ -24,6 +24,7 @@ import os
 import sys
 import traceback
 
+import shard_util
 import test_runner
 import wpr_runner
 import xcodebuild_runner
@@ -52,6 +53,19 @@ class Runner():
     """
     self.parse_args(args)
 
+    # GTEST_SHARD_INDEX and GTEST_TOTAL_SHARDS are additional test environment
+    # variables, set by Swarming, that are only set for a swarming task
+    # shard count is > 1.
+    #
+    # For a given test on a given run, otool should return the same total
+    # counts and thus, should generate the same sublists. With the shard index,
+    # each shard would then know the exact test case to run.
+    gtest_shard_index = os.getenv('GTEST_SHARD_INDEX', 0)
+    gtest_total_shards = os.getenv('GTEST_TOTAL_SHARDS', 0)
+    if gtest_shard_index and gtest_total_shards:
+      self.args.test_cases = shard_util.shard_test_cases(
+          self.args, gtest_shard_index, gtest_total_shards)
+
     summary = {}
     tr = None
 

ios/build/bots/scripts/shard_util.py

+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import collections
+import logging
+import os
+import re
+import subprocess
+import test_runner as tr
+
+LOGGER = logging.getLogger(__name__)
+
+# WARNING: THESE DUPLICATE CONSTANTS IN:
+# //build/scripts/slave/recipe_modules/ios/api.py
+
+TEST_NAMES_DEBUG_APP_PATTERN = re.compile(
+    'imp (?:0[xX][0-9a-fA-F]+ )?-\[(?P<testSuite>[A-Za-z_][A-Za-z0-9_]'
+    '*Test[Case]*) (?P<testMethod>test[A-Za-z0-9_]*)\]')
+TEST_CLASS_RELEASE_APP_PATTERN = re.compile(
+    r'name 0[xX]\w+ '
+    '(?P<testSuite>[A-Za-z_][A-Za-z0-9_]*Test(?:Case|))\n')
+TEST_NAME_RELEASE_APP_PATTERN = re.compile(
+    r'name 0[xX]\w+ (?P<testCase>test[A-Za-z0-9_]+)\n')
+# 'ChromeTestCase' and 'BaseEarlGreyTestCase' are parent classes
+# of all EarlGrey/EarlGrey2 test classes. They have no real tests.
+IGNORED_CLASSES = ['BaseEarlGreyTestCase', 'ChromeTestCase']
+
+
+def determine_app_path(app, host_app=None):
+  """String manipulate args.app and args.host to determine what path to use
+    for otools
+
+    Args:
+        app: (string) args.app
+        host_app: (string) args.host_app
+
+    Returns:
+        (string) path to app for otools to analyze
+    """
+  # run.py invoked via ../../ios/build/bots/scripts/, so we reverse this
+  dirname = os.path.dirname(os.path.abspath(__file__))
+  # location of app: /b/s/w/ir/out/Debug/test.app
+  full_app_path = os.path.join(dirname, '../../../..', 'out/Debug', app)
+
+  # ie/ if app_path = "../../some.app", app_name = some
+  start_idx = 0
+  if '/' in app:
+    start_idx = app.rindex('/')
+  app_name = app[start_idx:app.rindex('.app')]
+
+  # Default app_path looks like /b/s/w/ir/out/Debug/test.app/test
+  app_path = os.path.join(full_app_path, app_name)
+
+  if host_app:
+    LOGGER.debug("Detected EG2 test while building application path.")
+    # EG2 tests always end in -Runner, so we split that off
+    app_name = app[:app.rindex('-Runner')]
+    app_path = os.path.join(full_app_path, 'PlugIns',
+                            '{}.xctest'.format(app_name), app_name)
+
+  return app_path
+
+
+def _execute(cmd):
+  """Helper for executing a command."""
+  LOGGER.info('otool command: {}'.format(cmd))
+  process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+  stdout = process.communicate()[0]
+
+  retcode = process.returncode
+  LOGGER.info('otool return status code: {}'.format(retcode))
+  if retcode:
+    raise tr.OtoolError(retcode)
+
+  return stdout
+
+
+def fetch_counts_for_release(stdout):
+  """Invoke otools to determine the number of test case methods
+
+     WARNING: This logic is a duplicate of what's found in
+      //build/scripts/slave/recipe_modules/ios/api.py
+
+    Args:
+        stdout: (string) response of 'otool -ov'
+
+    Returns:
+        (collections.Counter) dict of test case to number of test case methods
+    """
+  # For Release builds `otool -ov` command generates output that is
+  # different from Debug builds.
+  # Parsing implemented in such a way:
+  # 1. Parse test class names.
+  # 2. If they are not in ignored list, parse test method names.
+  # 3. Calculate test count per test class.
+  test_counts = {}
+  res = re.split(TEST_CLASS_RELEASE_APP_PATTERN, stdout)
+  # Ignore 1st element in split since it does not have any test class data
+  test_classes_output = res[1:]
+  for test_class, class_output in zip(test_classes_output[0::2],
+                                      test_classes_output[1::2]):
+    if test_class in IGNORED_CLASSES:
+      continue
+    names = TEST_NAME_RELEASE_APP_PATTERN.findall(class_output)
+    test_counts[test_class] = test_counts.get(test_class, 0) + len(set(names))
+
+  return collections.Counter(test_counts)
+
+
+def fetch_counts_for_debug(stdout):
+  """Invoke otools to determine the number of test case methods
+
+    Args:
+        stdout: (string) response of 'otool -ov'
+
+    Returns:
+        (collections.Counter) dict of test case to number of test case methods
+    """
+  test_names = TEST_NAMES_DEBUG_APP_PATTERN.findall(stdout)
+  test_counts = collections.Counter(test_class for test_class, _ in test_names
+                                    if test_class not in IGNORED_CLASSES)
+
+  return test_counts
+
+
+def fetch_test_counts(stdout, release=False):
+  """Determine the number of test case methods per test class.
+
+    Args:
+        app_path: (string) path to app
+
+    Returns:
+        (collections.Counter) dict of test_case to number of test case methods
+    """
+  LOGGER.info("Ignored test classes: {}".format(IGNORED_CLASSES))
+  if release:
+    LOGGER.info("Release build detected. Fetching count for release.")
+    return fetch_counts_for_release(stdout)
+
+  return fetch_counts_for_debug(stdout)
+
+
+def balance_into_sublists(test_counts, total_shards):
+  """Augment the result of otool into balanced sublists
+
+  Args:
+    test_counts: (collections.Counter) dict of test_case to test case numbers
+    total_shards: (int) total number of shards this was divided into
+
+  Returns:
+    list of list of test classes
+  """
+
+  class Shard(object):
+    """Stores list of test classes and number of all tests"""
+
+    def __init__(self):
+      self.test_classes = []
+      self.size = 0
+
+  shards = [Shard() for i in range(total_shards)]
+
+  # Balances test classes between shards to have
+  # approximately equal number of tests per shard.
+  for test_class, number_of_test_methods in test_counts.most_common():
+    min_shard = min(shards, key=lambda shard: shard.size)
+    min_shard.test_classes.append(test_class)
+    min_shard.size += number_of_test_methods
+
+  sublists = [shard.test_classes for shard in shards]
+  return sublists
+
+
+def shard_test_cases(args, shard_index, total_shards):
+  """Shard test cases into total_shards, and determine which test cases to
+    run for this shard.
+
+    Args:
+        args: all parsed arguments passed to run.py
+        shard_index: the shard index(number) for this run
+        total_shards: the total number of shards for this test
+
+    Returns: a list of test cases to execute
+    """
+  # Convert to dict format
+  dict_args = vars(args)
+  app = dict_args['app']
+  host_app = dict_args.get('host_app', None)
+
+  # Determine what path to use
+  app_path = determine_app_path(app, host_app)
+
+  # release argument is passed by MB only when is_debug=False.
+  # 'Release' can also be set in path for this file (out/Release), so we'll
+  # check for either.
+  release = (
+      dict_args.get('release', False) or 'Release' in os.path.abspath(__file__))
+
+  # Use otools to get the test counts
+  cmd = ['otool', '-ov', app_path]
+  stdout = _execute(cmd)
+
+  test_counts = fetch_test_counts(stdout, release)
+
+  # Ensure shard and total shard is int
+  shard_index = int(shard_index)
+  total_shards = int(total_shards)
+
+  sublists = balance_into_sublists(test_counts, total_shards)
+  tests = sublists[shard_index]
+
+  LOGGER.info("Tests to be executed this round: {}".format(tests))
+  return tests

ios/build/bots/scripts/shard_util_test.py

+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from mock import patch
+import os
+import shard_util
+import unittest
+
+DEBUG_APP_OTOOL_OUTPUT = '\n'.join([
+    'Meta Class', 'name 0x1064b8438 CacheTestCase',
+    'baseMethods 0x1068586d8 (struct method_list_t *)',
+    'imp 0x1075e6887 -[CacheTestCase testA]', 'types 0x1064cc3e1',
+    'imp 0x1075e6887 -[CacheTestCase testB]',
+    'imp 0x1075e6887 -[CacheTestCase testc]', 'name 0x1064b8438 TabUITestCase',
+    'baseMethods 0x1068586d8 (struct method_list_t *)',
+    'imp 0x1075e6887 -[TabUITestCase testD]', 'types 0x1064cc3e1 v16@0:8',
+    'imp 0x1075e6887 -[TabUITestCase testE]',
+    'name 0x1064b8438 KeyboardTestCase',
+    'imp 0x1075e6887 -[KeyboardTestCase testF]',
+    'name 0x1064b8438 PasswordsTestCase',
+    'imp 0x1075e6887 -[PasswordsTestCase testG]',
+    'name 0x1064b8438 ToolBarTestCase',
+    'imp 0x1075e6887 -[ToolBarTestCase testH]', 'version 0'
+])
+
+RELEASE_APP_OTOOL_OUTPUT = '\n'.join([
+    'Meta Class', 'name 0x1064b8438 CacheTestCase',
+    'baseMethods 0x1068586d8 (struct method_list_t *)',
+    'name 0x1075e6887 testA', 'types 0x1064cc3e1', 'name 0x1075e6887 testB',
+    'name 0x1075e6887 testc', 'baseProtocols 0x0', 'Meta Class',
+    'name 0x1064b8438 TabUITestCase',
+    'baseMethods 0x1068586d8 (struct method_list_t *)',
+    'name 0x1064b8438 KeyboardTest', 'name 0x1075e6887 testD',
+    'types 0x1064cc3e1 v16@0:8', 'name 0x1075e6887 testE',
+    'name 0x1075e6887 testF', 'baseProtocols 0x0',
+    'name 0x1064b8438 ChromeTestCase', 'name 0x1064b8438 setUp',
+    'baseProtocols 0x0', 'name 0x1064b8438 ToolBarTestCase',
+    'name 0x1075e6887 testG', 'name 0x1075e6887 testH', 'baseProtocols 0x0',
+    'version 0'
+])
+
+
+class TestShardUtil(unittest.TestCase):
+  """Test cases for shard_util.py"""
+
+  @patch('shard_util.os.path.abspath')
+  def test_determine_path_non_eg2(self, mock_abspath):
+    mock_abspath.return_value = '/b/s/w/ir/ios/build/bots/scripts/share_util.py'
+    app = 'some_ios_test.app'
+
+    actual_path = shard_util.determine_app_path(app)
+    expected_path = os.path.join('/b/s/w/ir/ios/build/bots/scripts',
+                                 '../../../..', 'out/Debug', app,
+                                 'some_ios_test')
+    self.assertEqual(actual_path, expected_path)
+
+  @patch('shard_util.os.path.abspath')
+  def test_determine_path_eg2(self, mock_abspath):
+    mock_abspath.return_value = '/b/s/w/ir/ios/build/bots/scripts/share_util.py'
+    app = 'some_ios_test-Runner.app'
+    host = 'some_host.app'
+
+    actual_path = shard_util.determine_app_path(app, host)
+    expected_path = os.path.join('/b/s/w/ir/ios/build/bots/scripts',
+                                 '../../../..', 'out/Debug', app, 'PlugIns',
+                                 'some_ios_test.xctest', 'some_ios_test')
+    self.assertEqual(actual_path, expected_path)
+
+  def test_fetch_debug_ok(self):
+    """Ensures that the debug output is formatted correctly"""
+    resp = shard_util.fetch_counts_for_debug(DEBUG_APP_OTOOL_OUTPUT)
+    self.assertEqual(len(resp), 5)
+
+    # ({'CacheTestCase': 3, 'TabUITestCase': 2, 'PasswordsTestCase': 1,
+    # 'KeyboardTestCase': 1, 'ToolBarTestCase': 1})
+    counts = resp.most_common()
+    name, _ = counts[0]
+    self.assertEqual(name, 'CacheTestCase')
+
+  def test_fetch_release_ok(self):
+    """Ensures that the release output is formatted correctly"""
+    resp = shard_util.fetch_counts_for_release(RELEASE_APP_OTOOL_OUTPUT)
+    self.assertEqual(len(resp), 4)
+
+    # ({'KeyboardTest': 3, 'CacheTestCase': 3,
+    # 'ToolBarTestCase': 2, 'TabUITestCase': 0})
+    counts = resp.most_common()
+    name, _ = counts[0]
+    self.assertEqual(name, 'KeyboardTest')
+
+  def test_fetch_test_counts_debug(self):
+    """Ensure the coordinator functions as designed, for release"""
+    test_counts = shard_util.fetch_test_counts(DEBUG_APP_OTOOL_OUTPUT, False)
+    self.assertEqual(len(test_counts), 5)
+
+  def test_fetch_test_counts_release(self):
+    """Ensure the coordinator functions as designed, for release"""
+    test_counts = shard_util.fetch_test_counts(RELEASE_APP_OTOOL_OUTPUT, True)
+    self.assertEqual(len(test_counts), 4)
+
+  def test_balance_into_sublists_debug(self):
+    """Ensure the balancing algorithm works"""
+    test_counts = shard_util.fetch_test_counts(DEBUG_APP_OTOOL_OUTPUT, False)
+
+    sublists_1 = shard_util.balance_into_sublists(test_counts, 1)
+    self.assertEqual(len(sublists_1), 1)
+    self.assertEqual(len(sublists_1[0]), 5)
+
+    sublists_3 = shard_util.balance_into_sublists(test_counts, 3)
+    self.assertEqual(len(sublists_3), 3)
+    # CacheTestCase has 3,
+    # TabUITestCase has 2, ToolBarTestCase has 1
+    # PasswordsTestCase has 1, KeyboardTestCase has 1
+    self.assertEqual(len(sublists_3[0]), 1)
+    self.assertEqual(len(sublists_3[1]), 2)
+    self.assertEqual(len(sublists_3[2]), 2)
+
+  def test_balance_into_sublists_release(self):
+    """Ensure the balancing algorithm works"""
+    test_counts = shard_util.fetch_test_counts(RELEASE_APP_OTOOL_OUTPUT, True)
+
+    sublists_3 = shard_util.balance_into_sublists(test_counts, 3)
+    self.assertEqual(len(sublists_3), 3)
+    # KeyboardTest has 3
+    # CacheTestCase has 3
+    # ToolbarTest Case has 2, TabUITestCase has 0
+    self.assertEqual(len(sublists_3[0]), 1)
+    self.assertEqual(len(sublists_3[1]), 1)
+    self.assertEqual(len(sublists_3[2]), 2)
+
+
+if __name__ == '__main__':
+  unittest.main()

ios/build/bots/scripts/test_runner.py

@@ -37,6 +37,14 @@ class Error(Exception):
   pass
 
 
+class OtoolError(Error):
+  """OTool non-zero error code"""
+
+  def __init__(self, code):
+    super(OtoolError,
+          self).__init__('otool returned a non-zero return code: %s' % code)
+
+
 class TestRunnerError(Error):
   """Base class for TestRunner-related errors."""
   pass