Propagate Android test results through exceptions.

Previously, Android test results were propagated after all tests finished
running. In the event of an Exception, none of the test results would be written
to file. This CL slightly changes the control flow of TestRun.RunTests().
Instead of returning a results array, it now receives a results array as input
and modifies the array as it receives test results. This allows the json_writer
context TearDown() to serialize the partial test results.

This CL updates local_device_test_run.py to have the correct flow. The flow of
local_device_perf_test_run.py and local_machine_junit_test_run.py are left
untouched -- they can be modified in the future if it's deemed necessary. This
minimizes the size of this CL.

Bug: 895027
Change-Id: I60fe78da993de366d678252b84a91aaff044c10b
Reviewed-on: https://chromium-review.googlesource.com/c/1305396
Commit-Queue: Erik Chen <erikchen@chromium.org>
Reviewed-by: John Budorick <jbudorick@chromium.org>
Cr-Commit-Position: refs/heads/master@{#603599}

build/android/pylib/base/test_run.py

@@ -24,11 +24,12 @@ class TestRun(object):
   def SetUp(self):
     raise NotImplementedError
 
-  def RunTests(self):
-    """Runs Tests and returns test results.
+  def RunTests(self, results):
+    """Runs Tests and populates |results|.
 
-    Returns:
-      Should return list of |base_test_result.TestRunResults| objects.
+    Args:
+      results: An array that should be populated with
+               |base_test_result.TestRunResults| objects.
     """
     raise NotImplementedError
 

build/android/pylib/local/device/local_device_perf_test_run.py

@@ -412,7 +412,7 @@ class LocalDevicePerfTestRun(local_device_test_run.LocalDeviceTestRun):
     return sorted(devices)
 
   #override
-  def RunTests(self):
+  def RunTests(self, results):
     def run_no_devices_tests():
       if not self._no_device_tests:
         return []
@@ -451,7 +451,12 @@ class LocalDevicePerfTestRun(local_device_test_run.LocalDeviceTestRun):
     host_test_results, device_test_results = reraiser_thread.RunAsync(
         [run_no_devices_tests, run_devices_tests])
 
-    return host_test_results + device_test_results
+    # Ideally, results would be populated as early as possible, so that in the
+    # event of an exception or timeout, the caller will still have partially
+    # populated results. This looks like it can be done prior to dispatching
+    # tests, but will hold off on making this change unless it looks like it
+    # might provide utility.
+    results.extend(host_test_results + device_test_results)
 
   # override
   def TestPackage(self):
@@ -480,12 +485,16 @@ class OutputJsonList(LocalDevicePerfTestRun):
     pass
 
   # override
-  def RunTests(self):
+  def RunTests(self, results):
     result_type = self._test_instance.OutputJsonList()
     result = base_test_result.TestRunResults()
     result.AddResult(
         base_test_result.BaseTestResult('OutputJsonList', result_type))
-    return [result]
+
+    # Ideally, results would be populated as early as possible, so that in the
+    # event of an exception or timeout, the caller will still have partially
+    # populated results.
+    results.append(result)
 
   # override
   def _CreateShards(self, _tests):
@@ -502,12 +511,16 @@ class PrintStep(LocalDevicePerfTestRun):
     pass
 
   # override
-  def RunTests(self):
+  def RunTests(self, results):
     result_type = self._test_instance.PrintTestOutput()
     result = base_test_result.TestRunResults()
     result.AddResult(
         base_test_result.BaseTestResult('PrintStep', result_type))
-    return [result]
+
+    # Ideally, results would be populated as early as possible, so that in the
+    # event of an exception or timeout, the caller will still have partially
+    # populated results.
+    results.append(result)
 
   # override
   def _CreateShards(self, _tests):

build/android/pylib/local/device/local_device_test_run.py

@@ -54,7 +54,7 @@ class LocalDeviceTestRun(test_run.TestRun):
     self._tools = {}
 
   #override
-  def RunTests(self):
+  def RunTests(self, results):
     tests = self._GetTests()
 
     exit_now = threading.Event()
@@ -115,7 +115,6 @@ class LocalDeviceTestRun(test_run.TestRun):
     try:
       with signal_handler.SignalHandler(signal.SIGTERM, stop_tests):
         tries = 0
-        results = []
         while tries < self._env.max_tries and tests:
           logging.info('STARTING TRY #%d/%d', tries + 1, self._env.max_tries)
           if tries > 0 and self._env.recover_devices:
@@ -144,6 +143,11 @@ class LocalDeviceTestRun(test_run.TestRun):
                   t, base_test_result.ResultType.NOTRUN)
               for t in test_names if not t.endswith('*'))
 
+          # As soon as we know the names of the tests, we populate |results|.
+          # The tests in try_results will have their results updated by
+          # try_results.AddResult() as they are run.
+          results.append(try_results)
+
           try:
             if self._ShouldShard():
               tc = test_collection.TestCollection(self._CreateShards(tests))
@@ -160,8 +164,6 @@ class LocalDeviceTestRun(test_run.TestRun):
                       base_test_result.ResultType.TIMEOUT,
                       log=_SIGTERM_TEST_LOG))
             raise
-          finally:
-            results.append(try_results)
 
           tries += 1
           tests = self._GetTestsToRetry(tests, try_results)
@@ -174,8 +176,6 @@ class LocalDeviceTestRun(test_run.TestRun):
     except TestsTerminated:
       pass
 
-    return results
-
   def _GetTestsToRetry(self, tests, try_results):
 
     def is_failure_result(test_result):

build/android/pylib/local/machine/local_machine_junit_test_run.py

@@ -29,7 +29,7 @@ class LocalMachineJunitTestRun(test_run.TestRun):
     pass
 
   #override
-  def RunTests(self):
+  def RunTests(self, results):
     with tempfile_ext.NamedTemporaryDirectory() as temp_dir:
       json_file_path = os.path.join(temp_dir, 'results.json')
 
@@ -119,8 +119,7 @@ class LocalMachineJunitTestRun(test_run.TestRun):
 
       test_run_results = base_test_result.TestRunResults()
       test_run_results.AddResults(results_list)
-
-      return [test_run_results]
+      results.append(test_run_results)
 
   #override
   def TearDown(self):

build/android/test_runner.py

@@ -844,12 +844,18 @@ def RunTestsInPlatformMode(args):
           lambda: collections.defaultdict(int))
       iteration_count = 0
       for _ in repetitions:
-        raw_results = test_run.RunTests()
+        # raw_results will be populated with base_test_result.TestRunResults by
+        # test_run.RunTests(). It is immediately added to all_raw_results so
+        # that in the event of an exception, all_raw_results will already have
+        # the up-to-date results and those can be written to disk.
+        raw_results = []
+        all_raw_results.append(raw_results)
+
+        test_run.RunTests(raw_results)
         if not raw_results:
+          all_raw_results.pop()
           continue
 
-        all_raw_results.append(raw_results)
-
         iteration_results = base_test_result.TestRunResults()
         for r in reversed(raw_results):
           iteration_results.AddTestRunResults(r)