Implement auto-retry of failed tests for telemetry{_perf}_unittests.

This implements the same basic logic we use in GTest-based tests to
retry tests that fail during a run. By default, we will not retry
anything (this matches what gtests do when run not in botmode).

If --retry-limit is passed on the command line, we will retry each
failing test up to that number of times. So, to enable retries on
the bots by default they will have to pass this arg.

R=dtu@chromium.org, tonyg@chromium.org
BUG=398027

Review URL: https://codereview.chromium.org/426123002

Cr-Commit-Position: refs/heads/master@{#288265}
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@288265 0039d316-1c4b-4281-b951-d872f2087c98

tools/telemetry/telemetry/unittest/json_results.py

@@ -27,22 +27,19 @@ def ValidateArgs(parser, args):
       parser.error('Error: malformed metadata "%s"' % val)
 
 
-def WriteandUploadResultsIfNecessary(args, test_suite, result):
+def WriteFullResultsIfNecessary(args, full_results):
   if not args.write_full_results_to:
     return
 
-  full_results = _FullResults(test_suite, result, args.metadata)
-
   with open(args.write_full_results_to, 'w') as fp:
     json.dump(full_results, fp, indent=2)
     fp.write("\n")
 
-  # TODO(dpranke): upload to test-results.appspot.com if requested as well.
 
 TEST_SEPARATOR = '.'
 
 
-def _FullResults(suite, result, metadata):
+def FullResults(args, suite, results):
   """Convert the unittest results to the Chromium JSON test result format.
 
   This matches run-webkit-tests (the layout tests) and the flakiness dashboard.
@@ -53,49 +50,79 @@ def _FullResults(suite, result, metadata):
   full_results['path_delimiter'] = TEST_SEPARATOR
   full_results['version'] = 3
   full_results['seconds_since_epoch'] = time.time()
-  for md in metadata:
+  for md in args.metadata:
     key, val = md.split('=', 1)
     full_results[key] = val
 
-  all_test_names = _AllTestNames(suite)
-  failed_test_names = _FailedTestNames(result)
+  # TODO(dpranke): Handle skipped tests as well.
 
+  all_test_names = AllTestNames(suite)
+  num_failures = NumFailuresAfterRetries(results)
   full_results['num_failures_by_type'] = {
-      'FAIL': len(failed_test_names),
-      'PASS': len(all_test_names) - len(failed_test_names),
+      'FAIL': num_failures,
+      'PASS': len(all_test_names) - num_failures,
   }
 
+  sets_of_passing_test_names = map(PassingTestNames, results)
+  sets_of_failing_test_names = map(FailedTestNames, results)
+
   full_results['tests'] = {}
 
   for test_name in all_test_names:
-    value = {}
-    value['expected'] = 'PASS'
-    if test_name in failed_test_names:
-      value['actual'] = 'FAIL'
-      value['is_unexpected'] = True
-    else:
-      value['actual'] = 'PASS'
-
+    value = {
+        'expected': 'PASS',
+        'actual': ActualResultsForTest(test_name, sets_of_failing_test_names,
+                                       sets_of_passing_test_names)
+    }
     _AddPathToTrie(full_results['tests'], test_name, value)
 
   return full_results
 
 
-def _AllTestNames(suite):
+def ActualResultsForTest(test_name, sets_of_failing_test_names,
+                         sets_of_passing_test_names):
+  actuals = []
+  for retry_num in range(len(sets_of_failing_test_names)):
+    if test_name in sets_of_failing_test_names[retry_num]:
+      actuals.append('FAIL')
+    elif test_name in sets_of_passing_test_names[retry_num]:
+      assert ((retry_num == 0) or
+              (test_name in sets_of_failing_test_names[retry_num - 1])), (
+              'We should not have run a test that did not fail '
+              'on the previous run.')
+      actuals.append('PASS')
+
+  assert actuals, 'We did not find any result data for %s.' % test_name
+  return ' '.join(actuals)
+
+
+def ExitCodeFromFullResults(full_results):
+  return 1 if full_results['num_failures_by_type']['FAIL'] else 0
+
+
+def AllTestNames(suite):
   test_names = []
   # _tests is protected  pylint: disable=W0212
   for test in suite._tests:
     if isinstance(test, unittest.suite.TestSuite):
-      test_names.extend(_AllTestNames(test))
+      test_names.extend(AllTestNames(test))
     else:
       test_names.append(test.id())
   return test_names
 
 
-def _FailedTestNames(result):
+def NumFailuresAfterRetries(results):
+  return len(FailedTestNames(results[-1]))
+
+
+def FailedTestNames(result):
   return set(test.id() for test, _ in result.failures + result.errors)
 
 
+def PassingTestNames(result):
+  return set(test.id() for test in result.successes)
+
+
 def _AddPathToTrie(trie, path, value):
   if TEST_SEPARATOR not in path:
     trie[path] = value
@@ -104,5 +131,3 @@ def _AddPathToTrie(trie, path, value):
   if directory not in trie:
     trie[directory] = {}
   _AddPathToTrie(trie[directory], rest, value)
-
-

tools/telemetry/telemetry/unittest/run_tests.py

@@ -126,6 +126,9 @@ class RunTestsCommand(command_line.OptparseCommand):
                       dest='run_disabled_tests',
                       action='store_true', default=False,
                       help='Ignore @Disabled and @Enabled restrictions.')
+    parser.add_option('--retry-limit', type='int', default=0,
+                      help='Retry each failure up to N times (default %default)'
+                           ' to de-flake things.')
     json_results.AddOptions(parser)
 
   @classmethod
@@ -147,16 +150,36 @@ class RunTestsCommand(command_line.OptparseCommand):
 
   def Run(self, args):
     possible_browser = browser_finder.FindBrowser(args)
-    test_suite = DiscoverTests(
-        config.test_dirs, config.top_level_dir, possible_browser,
-        args.positional_args, args.run_disabled_tests)
-    runner = output_formatter.TestRunner()
-    result = runner.run(
-        test_suite, config.output_formatters, args.repeat_count, args)
 
-    json_results.WriteandUploadResultsIfNecessary(args, test_suite, result)
+    test_suite, result = self.RunOneSuite(possible_browser, args)
+
+    results = [result]
+
+    failed_tests = json_results.FailedTestNames(result)
+    retry_limit = args.retry_limit
+
+    while retry_limit and failed_tests:
+      args.positional_args = failed_tests
+
+      _, result = self.RunOneSuite(possible_browser, args)
+      results.append(result)
 
-    return len(result.failures_and_errors)
+      failed_tests = json_results.FailedTestNames(result)
+      retry_limit -= 1
+
+    full_results = json_results.FullResults(args, test_suite, results)
+    json_results.WriteFullResultsIfNecessary(args, full_results)
+
+    return json_results.ExitCodeFromFullResults(full_results)
+
+  def RunOneSuite(self, possible_browser, args):
+    test_suite = DiscoverTests(config.test_dirs, config.top_level_dir,
+                               possible_browser, args.positional_args,
+                               args.run_disabled_tests)
+    runner = output_formatter.TestRunner()
+    result = runner.run(test_suite, config.output_formatters,
+                        args.repeat_count, args)
+    return test_suite, result
 
   @classmethod
   @RestoreLoggingLevel