android: Add test runner option to recover devices before last attempt.

Will enable on bots by either:
- adding option to testing spec
- adding option to all isolated tests via mb
- adding option to cmdline in recipe

Leaning towards option 3 atm.

R=jbudorick@chromium.org

Bug: 790936
Change-Id: Ieb9e0e8237e691887e383578f1860cd77e03e5cb
Reviewed-on: https://chromium-review.googlesource.com/804914Reviewed-by: John Budorick <jbudorick@chromium.org>
Commit-Queue: Benjamin Pastene <bpastene@chromium.org>
Cr-Commit-Position: refs/heads/master@{#521791}

build/android/pylib/local/device/local_device_environment.py

@@ -98,6 +98,7 @@ class LocalDeviceEnvironment(environment.Environment):
     self._logcat_output_dir = args.logcat_output_dir
     self._logcat_output_file = args.logcat_output_file
     self._max_tries = 1 + args.num_retries
+    self._recover_devices = args.recover_devices
     self._skip_clear_data = args.skip_clear_data
     self._tool_name = args.tool
     self._trace_output = None
@@ -187,6 +188,10 @@ class LocalDeviceEnvironment(environment.Environment):
   def parallel_devices(self):
     return parallelizer.SyncParallelizer(self.devices)
 
+  @property
+  def recover_devices(self):
+    return self._recover_devices
+
   @property
   def skip_clear_data(self):
     return self._skip_clear_data

build/android/pylib/local/device/local_device_test_run.py

@@ -117,7 +117,8 @@ class LocalDeviceTestRun(test_run.TestRun):
         results = []
         while tries < self._env.max_tries and tests:
           logging.info('STARTING TRY #%d/%d', tries + 1, self._env.max_tries)
-          if tries > 0 and tries + 1 == self._env.max_tries:
+          if (tries > 0 and tries + 1 == self._env.max_tries and
+              self._env.recover_devices):
             logging.info(
                 'Attempting to recover devices prior to last test attempt.')
             self._env.parallel_devices.pMap(device_recovery.RecoverDevice, None)

build/android/test_runner.py

@@ -257,6 +257,11 @@ def AddDeviceOptions(parser):
       help='Do not wipe app data between tests. Use this to '
            'speed up local development and never on bots '
                      '(increases flakiness)')
+  parser.add_argument(
+      '--recover-devices',
+      action='store_true',
+      help='Attempt to recover devices prior to the final retry. Warning: '
+           'this will cause all devices to reboot.')
   parser.add_argument(
       '--tool',
       dest='tool',