Add a new test runner to the iOS test runner that can handle autofill automation tests.

The new test runner is named WprProxySimulatorTestRunner, and descends from SimulatorTestRunner,
mostly diverging in two ways:
* It performs additional set up and tear down to specifically set up the Web Page Replay proxy.
* It runs the suite under test multiple times, assuming the suite takes in one replay+recipe
pair per run, corresponding to testing one webpage. Each webpage will have separate results in the
final results report.

This test runner is triggered if a new flag, replay_path, is set for a iOS test.
This flag represents the path containing saved web page replay and recipe files for use with
WprProxySimulatorTestRunner.

Key files (such as the certificate needed to proxy traffic on simulator) are passed in
to the test runner from api.py (see https://chromium-review.googlesource.com/c/chromium/tools/build/+/1272236)
through a CIPD package installed and passed in through the wpr-tools-path flag.
Thanks Sergey for the tip!

For comments on WIP versions of this which can provide more insight, see here:
https://chromium-review.googlesource.com/c/chromium/src/+/1243565

Thanks!

Bug: 881096
Cq-Include-Trybots: luci.chromium.try:ios-simulator-cronet;luci.chromium.try:ios-simulator-full-configs
Change-Id: Id2174a535ffc8a9941fc68062b0c7f69a75d80ab
Reviewed-on: https://chromium-review.googlesource.com/c/1263557
Commit-Queue: ericale <ericale@chromium.org>
Reviewed-by: Sergey Berezin <sergeyberezin@chromium.org>
Cr-Commit-Position: refs/heads/master@{#599288}

ios/build/bots/scripts/run.py

@@ -36,36 +36,55 @@ def main():
     os.makedirs(args.out_dir)
 
   try:
-    if args.iossim and args.platform and args.version:
+    if args.replay_path != 'NO_PATH':
+      tr = test_runner.WprProxySimulatorTestRunner(
+          args.app,
+          args.iossim,
+          args.replay_path,
+          args.platform,
+          args.version,
+          args.wpr_tools_path,
+          args.xcode_build_version,
+          args.out_dir,
+          env_vars=args.env_var,
+          mac_toolchain=args.mac_toolchain_cmd,
+          retries=args.retries,
+          shards=args.shards,
+          test_args=test_args,
+          test_cases=args.test_cases,
+          xcode_path=args.xcode_path,
+          xctest=args.xctest,
+      )
+    elif args.iossim and args.platform and args.version:
       tr = test_runner.SimulatorTestRunner(
-        args.app,
-        args.iossim,
-        args.platform,
-        args.version,
-        args.xcode_build_version,
-        args.out_dir,
-        env_vars=args.env_var,
-        mac_toolchain=args.mac_toolchain_cmd,
-        retries=args.retries,
-        shards=args.shards,
-        test_args=test_args,
-        test_cases=args.test_cases,
-        xcode_path=args.xcode_path,
-        xctest=args.xctest,
+          args.app,
+          args.iossim,
+          args.platform,
+          args.version,
+          args.xcode_build_version,
+          args.out_dir,
+          env_vars=args.env_var,
+          mac_toolchain=args.mac_toolchain_cmd,
+          retries=args.retries,
+          shards=args.shards,
+          test_args=test_args,
+          test_cases=args.test_cases,
+          xcode_path=args.xcode_path,
+          xctest=args.xctest,
       )
     else:
       tr = test_runner.DeviceTestRunner(
-        args.app,
-        args.xcode_build_version,
-        args.out_dir,
-        env_vars=args.env_var,
-        mac_toolchain=args.mac_toolchain_cmd,
-        restart=args.restart,
-        retries=args.retries,
-        test_args=test_args,
-        test_cases=args.test_cases,
-        xcode_path=args.xcode_path,
-        xctest=args.xctest,
+          args.app,
+          args.xcode_build_version,
+          args.out_dir,
+          env_vars=args.env_var,
+          mac_toolchain=args.mac_toolchain_cmd,
+          restart=args.restart,
+          retries=args.retries,
+          test_args=test_args,
+          test_cases=args.test_cases,
+          xcode_path=args.xcode_path,
+          xctest=args.xctest,
       )
 
     return 0 if tr.launch() else 1
@@ -172,6 +191,14 @@ def parse_args():
     required=True,
     metavar='build_id',
   )
+  parser.add_argument(
+    '--replay-path',
+    help=('Path to a directory containing WPR replay and recipe files, for '
+          'use with WprProxySimulatorTestRunner to replay a test suite'
+          ' against multiple saved website interactions. Default: %(default)s'),
+    default='NO_PATH',
+    metavar='replay-path',
+  )
   parser.add_argument(
     '--xcode-path',
     metavar='PATH',
@@ -187,6 +214,13 @@ def parse_args():
     default='mac_toolchain',
     metavar='mac_toolchain',
   )
+  parser.add_argument(
+    '--wpr-tools-path',
+    help=('Location of WPR test tools (should be preinstalled, e.g. as part of '
+         'a swarming task requirement). Default: %(default)s.'),
+    default='NO_PATH',
+    metavar='wpr-tools-path',
+  )
   parser.add_argument(
     '--xctest',
     action='store_true',

ios/build/bots/scripts/test_runner_test.py

@@ -6,8 +6,10 @@
 """Unittests for test_runner.py."""
 
 import collections
+import glob
 import json
 import os
+import subprocess
 import sys
 import unittest
 
@@ -345,6 +347,143 @@ class SimulatorTestRunnerTest(TestCase):
     self.assertTrue(tr.logs)
 
 
+class WprProxySimulatorTestRunnerTest(TestCase):
+  """Tests for test_runner.WprProxySimulatorTestRunner."""
+
+  def setUp(self):
+    super(WprProxySimulatorTestRunnerTest, self).setUp()
+
+    def install_xcode(build, mac_toolchain_cmd, xcode_app_path):
+      return True
+
+    self.mock(test_runner, 'get_current_xcode_info', lambda: {
+        'version': 'test version', 'build': 'test build', 'path': 'test/path'})
+    self.mock(test_runner, 'install_xcode', install_xcode)
+    self.mock(test_runner.subprocess, 'check_output',
+              lambda _: 'fake-bundle-id')
+    self.mock(os.path, 'abspath', lambda path: '/abs/path/to/%s' % path)
+    self.mock(os.path, 'exists', lambda _: True)
+
+  def test_replay_path_not_found(self):
+    """Ensures ReplayPathNotFoundError is raised."""
+
+    self.mock(os.path, 'exists', lambda p: not p.endswith('bad-replay-path'))
+
+    with self.assertRaises(test_runner.ReplayPathNotFoundError):
+      test_runner.WprProxySimulatorTestRunner(
+        'fake-app',
+        'fake-iossim',
+        'bad-replay-path',
+        'platform',
+        'os',
+        'wpr-tools-path',
+        'xcode-version',
+        'xcode-build',
+        'out-dir',
+      )
+
+  def test_wpr_tools_not_found(self):
+    """Ensures WprToolsNotFoundError is raised."""
+
+    self.mock(os.path, 'exists', lambda p: not p.endswith('bad-tools-path'))
+
+    with self.assertRaises(test_runner.WprToolsNotFoundError):
+      test_runner.WprProxySimulatorTestRunner(
+        'fake-app',
+        'fake-iossim',
+        'replay-path',
+        'platform',
+        'os',
+        'bad-tools-path',
+        'xcode-version',
+        'xcode-build',
+        'out-dir',
+      )
+
+  def test_init(self):
+    """Ensures instance is created."""
+    tr = test_runner.WprProxySimulatorTestRunner(
+        'fake-app',
+        'fake-iossim',
+        'replay-path',
+        'platform',
+        'os',
+        'wpr-tools-path',
+        'xcode-version',
+        'xcode-build',
+        'out-dir',
+      )
+
+    self.assertTrue(tr)
+
+  def test_run(self):
+    """Ensures the _run method can handle passed and failed tests."""
+
+    class FakeStdout:
+      def __init__(self):
+        self.line_index = 0
+        self.lines = [
+          'Test Case \'-[a 1]\' started.',
+          'Test Case \'-[a 1]\' has uninteresting logs.',
+          'Test Case \'-[a 1]\' passed (0.1 seconds)',
+          'Test Case \'-[b 2]\' started.',
+          'Test Case \'-[b 2]\' passed (0.1 seconds)',
+          'Test Case \'-[c 3]\' started.',
+          'Test Case \'-[c 3]\' has interesting failure info.',
+          'Test Case \'-[c 3]\' failed (0.1 seconds)',
+        ]
+
+      def readline(self):
+        if self.line_index < len(self.lines):
+          return_line = self.lines[self.line_index]
+          self.line_index += 1
+          return return_line
+        else:
+          return None
+
+    class FakeProcess:
+      def __init__(self):
+        self.stdout = FakeStdout()
+        self.returncode = 0
+
+      def stdout(self):
+        return self.stdout
+
+      def wait(self):
+        return
+
+    def popen(recipe_cmd, env, stdout, stderr):
+      return FakeProcess()
+
+    tr = test_runner.WprProxySimulatorTestRunner(
+        'fake-app',
+        'fake-iossim',
+        'replay-path',
+        'platform',
+        'os',
+        'wpr-tools-path',
+        'xcode-version',
+        'xcode-build',
+        'out-dir',
+    )
+    self.mock(test_runner.WprProxySimulatorTestRunner, 'wprgo_start', lambda a,b: None)
+    self.mock(test_runner.WprProxySimulatorTestRunner, 'wprgo_stop', lambda _: None)
+
+    self.mock(os.path, 'isfile', lambda _: True)
+    self.mock(glob, 'glob', lambda _: ["file1", "file2"])
+    self.mock(subprocess, 'Popen', popen)
+
+    tr.xctest_path = 'fake.xctest'
+    cmd = tr.get_launch_command()
+    result = tr._run(cmd=cmd, shards=1)
+    self.assertIn('file1.a/1', result.passed_tests)
+    self.assertIn('file1.b/2', result.passed_tests)
+    self.assertIn('file1.c/3', result.failed_tests)
+    self.assertIn('file2.a/1', result.passed_tests)
+    self.assertIn('file2.b/2', result.passed_tests)
+    self.assertIn('file2.c/3', result.failed_tests)
+
+
 class DeviceTestRunnerTest(TestCase):
   def setUp(self):
     super(DeviceTestRunnerTest, self).setUp()