Android test runner: Fix bugs with JSON format.

This adds an is_unexpected tag, and handles multiple results (e.g.
from retries) correctly.

Bug: 931731
Bug: 1138507
Bug: angleproject:5124
Change-Id: I23054010db058eaa9d275c9b5de3052030210024
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2472338Reviewed-by: Dirk Pranke <dpranke@google.com>
Commit-Queue: Jamie Madill <jmadill@chromium.org>
Cr-Commit-Position: refs/heads/master@{#817288}

build/android/pylib/results/json_results.py

@@ -122,8 +122,7 @@ def GenerateJsonTestResultFormatDict(test_run_results):
   """
 
   tests = {}
-  pass_count = 0
-  fail_count = 0
+  counts = {'PASS': 0, 'FAIL': 0}
 
   for test_run_result in test_run_results:
     if isinstance(test_run_result, list):
@@ -140,12 +139,16 @@ def GenerateJsonTestResultFormatDict(test_run_results):
 
       element['expected'] = 'PASS'
 
-      if r.GetType() == base_test_result.ResultType.PASS:
-        element['actual'] = 'PASS'
-        pass_count += 1
+      result = 'PASS' if r.GetType(
+      ) == base_test_result.ResultType.PASS else 'FAIL'
+
+      if 'actual' in element:
+        element['actual'] += ' ' + result
       else:
-        element['actual'] = 'FAIL'
-        fail_count += 1
+        counts[result] += 1
+        element['actual'] = result
+        if result == 'FAIL':
+          element['is_unexpected'] = True
 
       if r.GetDuration() != 0:
         element['time'] = r.GetDuration()
@@ -153,10 +156,7 @@ def GenerateJsonTestResultFormatDict(test_run_results):
   # Fill in required fields.
   return {
       'interrupted': False,
-      'num_failures_by_type': {
-          'FAIL': fail_count,
-          'PASS': pass_count,
-      },
+      'num_failures_by_type': counts,
       'path_delimiter': '.',
       'seconds_since_epoch': time.time(),
       'tests': tests,

build/android/pylib/results/json_results_test.py

@@ -219,6 +219,10 @@ class JsonResultsTest(unittest.TestCase):
     self.assertEquals(
         'PASS', results_dict['tests']['test']['package']['TestName']['actual'])
 
+    # Note: technically a missing entry counts as zero.
+    self.assertEquals(1, results_dict['num_failures_by_type']['PASS'])
+    self.assertEquals(0, results_dict['num_failures_by_type']['FAIL'])
+
   def testGenerateJsonTestResultFormatDict_failedResult(self):
     result = base_test_result.BaseTestResult('test.package.TestName',
                                              base_test_result.ResultType.FAIL)
@@ -235,6 +239,50 @@ class JsonResultsTest(unittest.TestCase):
         results_dict['tests']['test']['package']['TestName']['expected'])
     self.assertEquals(
         'FAIL', results_dict['tests']['test']['package']['TestName']['actual'])
+    self.assertEquals(
+        True,
+        results_dict['tests']['test']['package']['TestName']['is_unexpected'])
+    self.assertEquals(2, len(results_dict['num_failures_by_type']))
+
+    # Note: technically a missing entry counts as zero.
+    self.assertEquals(0, results_dict['num_failures_by_type']['PASS'])
+    self.assertEquals(1, results_dict['num_failures_by_type']['FAIL'])
+
+  def testGenerateJsonTestResultFormatDict_failedResultWithRetry(self):
+    result_1 = base_test_result.BaseTestResult('test.package.TestName',
+                                               base_test_result.ResultType.FAIL)
+    run_results_1 = base_test_result.TestRunResults()
+    run_results_1.AddResult(result_1)
+
+    # Simulate a second retry with failure.
+    result_2 = base_test_result.BaseTestResult('test.package.TestName',
+                                               base_test_result.ResultType.FAIL)
+    run_results_2 = base_test_result.TestRunResults()
+    run_results_2.AddResult(result_2)
+
+    all_results = [run_results_1, run_results_2]
+
+    results_dict = json_results.GenerateJsonTestResultFormatDict(all_results)
+    self.assertEquals(1, len(results_dict['tests']))
+    self.assertEquals(1, len(results_dict['tests']['test']))
+    self.assertEquals(1, len(results_dict['tests']['test']['package']))
+    self.assertEquals(
+        'PASS',
+        results_dict['tests']['test']['package']['TestName']['expected'])
+    self.assertEquals(
+        'FAIL FAIL',
+        results_dict['tests']['test']['package']['TestName']['actual'])
+    self.assertEquals(
+        True,
+        results_dict['tests']['test']['package']['TestName']['is_unexpected'])
+
+    # Note: technically a missing entry counts as zero.
+    self.assertEquals(2, len(results_dict['num_failures_by_type']))
+    self.assertEquals(0, results_dict['num_failures_by_type']['PASS'])
+
+    # According to the spec: If a test was run more than once, only the first
+    # invocation's result is included in the totals.
+    self.assertEquals(1, results_dict['num_failures_by_type']['FAIL'])
 
 
 if __name__ == '__main__':