[Android] Allow gtests to pull app data off the device before clearing it.

The test runner clears application data before each test it runs, as well as
after it has run all tests. However, some suites drop useful files in the
application data directory. This provides a mechanism for retrieving those
files from the device.

BUG=489713

Review URL: https://codereview.chromium.org/1165623003

Cr-Commit-Position: refs/heads/master@{#333768}

build/android/pylib/device/device_utils.py

@@ -363,6 +363,28 @@ class DeviceUtils(object):
                                              str(self))
     return output[len('package:'):]
 
+  @decorators.WithTimeoutAndRetriesFromInstance()
+  def GetApplicationDataDirectory(self, package, timeout=None, retries=None):
+    """Get the data directory on the device for the given package.
+
+    Args:
+      package: Name of the package.
+
+    Returns:
+      The package's data directory, or None if the package doesn't exist on the
+      device.
+    """
+    try:
+      output = self._RunPipedShellCommand(
+          'pm dump %s | grep dataDir=' % cmd_helper.SingleQuote(package))
+      for line in output:
+        _, _, dataDir = line.partition('dataDir=')
+        if dataDir:
+          return dataDir
+    except device_errors.CommandFailedError:
+      logging.exception('Could not find data directory for %s', package)
+    return None
+
   @decorators.WithTimeoutAndRetriesFromInstance()
   def WaitUntilFullyBooted(self, wifi=False, timeout=None, retries=None):
     """Wait for the device to fully boot.

build/android/pylib/device/device_utils_test.py

@@ -336,6 +336,25 @@ class DeviceUtilsGetApplicationPathTest(DeviceUtilsTest):
         self.device.GetApplicationPath('android')
 
 
+class DeviceUtilsGetApplicationDataDirectoryTest(DeviceUtilsTest):
+
+  def testGetApplicationDataDirectory_exists(self):
+    with self.assertCall(
+        self.call.device._RunPipedShellCommand(
+            'pm dump foo.bar.baz | grep dataDir='),
+        ['dataDir=/data/data/foo.bar.baz']):
+      self.assertEquals(
+          '/data/data/foo.bar.baz',
+          self.device.GetApplicationDataDirectory('foo.bar.baz'))
+
+  def testGetApplicationDataDirectory_notExists(self):
+    with self.assertCall(
+        self.call.device._RunPipedShellCommand(
+            'pm dump foo.bar.baz | grep dataDir='),
+        self.ShellError()):
+      self.assertIsNone(self.device.GetApplicationDataDirectory('foo.bar.baz'))
+
+
 @mock.patch('time.sleep', mock.Mock())
 class DeviceUtilsWaitUntilFullyBootedTest(DeviceUtilsTest):
 

build/android/pylib/gtest/gtest_test_instance.py

@@ -7,6 +7,7 @@ import os
 import re
 import shutil
 import sys
+import tempfile
 
 from pylib import constants
 from pylib.base import base_test_result
@@ -127,6 +128,17 @@ class GtestTestInstance(test_instance.TestInstance):
       logging.warning('No isolate file provided. No data deps will be pushed.');
       self._isolate_delegate = None
 
+    if args.app_data_files:
+      self._app_data_files = args.app_data_files
+      if args.app_data_file_dir:
+        self._app_data_file_dir = args.app_data_file_dir
+      else:
+        self._app_data_file_dir = tempfile.mkdtemp()
+        logging.critical('Saving app files to %s', self._app_data_file_dir)
+    else:
+      self._app_data_files = None
+      self._app_data_file_dir = None
+
   #override
   def TestType(self):
     return 'gtest'
@@ -231,6 +243,14 @@ class GtestTestInstance(test_instance.TestInstance):
   def apk(self):
     return self._apk_path
 
+  @property
+  def app_file_dir(self):
+    return self._app_data_file_dir
+
+  @property
+  def app_files(self):
+    return self._app_data_files
+
   @property
   def exe(self):
     return self._exe_path

build/android/pylib/gtest/local_device_gtest_run.py

@@ -2,9 +2,10 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-
+import itertools
 import logging
 import os
+import posixpath
 
 from pylib import constants
 from pylib import ports
@@ -37,6 +38,20 @@ _SUITE_REQUIRES_TEST_SERVER_SPAWNER = [
   'net_unittests', 'unit_tests'
 ]
 
+# TODO(jbudorick): Move this inside _ApkDelegate once TestPackageApk is gone.
+def PullAppFilesImpl(device, package, files, directory):
+  device_dir = device.GetApplicationDataDirectory(package)
+  host_dir = os.path.join(directory, str(device))
+  for f in files:
+    device_file = posixpath.join(device_dir, f)
+    host_file = os.path.join(host_dir, *f.split(posixpath.sep))
+    host_file_base, ext = os.path.splitext(host_file)
+    for i in itertools.count():
+      host_file = '%s_%d%s' % (host_file_base, i, ext)
+      if not os.path.exists(host_file):
+        break
+    device.PullFile(device_file, host_file)
+
 class _ApkDelegate(object):
   def __init__(self, apk):
     self._apk = apk
@@ -63,6 +78,9 @@ class _ApkDelegate(object):
       return device.StartInstrumentation(
           self._component, extras=extras, raw=False, **kwargs)
 
+  def PullAppFiles(self, device, files, directory):
+    PullAppFilesImpl(device, self._package, files, directory)
+
   def Clear(self, device):
     device.ClearApplicationState(self._package)
 
@@ -119,6 +137,9 @@ class _ExeDelegate(object):
                                       env=env, **kwargs)
     return output
 
+  def PullAppFiles(self, device, files, directory):
+    pass
+
   def Clear(self, device):
     device.KillAll(self._exe_file_name, blocking=True, timeout=30, quiet=True)
 
@@ -199,6 +220,9 @@ class LocalDeviceGtestRun(local_device_test_run.LocalDeviceTestRun):
         device, '--gtest_filter=%s' % test, timeout=900, retries=0)
     for s in self._servers[str(device)]:
       s.Reset()
+    if self._test_instance.app_files:
+      self._delegate.PullAppFiles(device, self._test_instance.app_files,
+                                  self._test_instance.app_file_dir)
     self._delegate.Clear(device)
 
     # Parse the output.

build/android/pylib/gtest/test_options.py

@@ -13,4 +13,6 @@ GTestOptions = collections.namedtuple('GTestOptions', [
     'test_arguments',
     'timeout',
     'isolate_file_path',
-    'suite_name'])
+    'suite_name',
+    'app_data_files',
+    'app_data_file_dir'])

build/android/pylib/gtest/test_package.py

@@ -22,7 +22,7 @@ class TestPackage(object):
     Args:
       device: Instance of DeviceUtils.
     """
-    raise NotImplementedError('Method must be overriden.')
+    raise NotImplementedError('Method must be overridden.')
 
   def CreateCommandLineFileOnDevice(self, device, test_filter, test_arguments):
     """Creates a test runner script and pushes to the device.
@@ -32,7 +32,7 @@ class TestPackage(object):
       test_filter: A test_filter flag.
       test_arguments: Additional arguments to pass to the test binary.
     """
-    raise NotImplementedError('Method must be overriden.')
+    raise NotImplementedError('Method must be overridden.')
 
   def GetAllTests(self, device):
     """Returns a list of all tests available in the test suite.
@@ -40,7 +40,7 @@ class TestPackage(object):
     Args:
       device: Instance of DeviceUtils.
     """
-    raise NotImplementedError('Method must be overriden.')
+    raise NotImplementedError('Method must be overridden.')
 
   def GetGTestReturnCode(self, _device):
     return None
@@ -54,7 +54,7 @@ class TestPackage(object):
     Returns:
       An instance of pexpect spawn class.
     """
-    raise NotImplementedError('Method must be overriden.')
+    raise NotImplementedError('Method must be overridden.')
 
   def Install(self, device):
     """Install the test package to the device.
@@ -62,5 +62,15 @@ class TestPackage(object):
     Args:
       device: Instance of DeviceUtils.
     """
-    raise NotImplementedError('Method must be overriden.')
+    raise NotImplementedError('Method must be overridden.')
 
+  def PullAppFiles(self, device, files, directory):
+    """Pull application data from the device.
+
+    Args:
+      device: Instance of DeviceUtils.
+      files: A list of paths relative to the application data directory to
+        retrieve from the device.
+      directory: The host directory to which files should be pulled.
+    """
+    raise NotImplementedError('Method must be overridden.')

build/android/pylib/gtest/test_package_apk.py

@@ -5,8 +5,10 @@
 """Defines TestPackageApk to help run APK-based native tests."""
 # pylint: disable=W0212
 
+import itertools
 import logging
 import os
+import posixpath
 import shlex
 import sys
 import tempfile
@@ -18,6 +20,7 @@ from pylib import pexpect
 from pylib.device import device_errors
 from pylib.device import intent
 from pylib.gtest import gtest_test_instance
+from pylib.gtest import local_device_gtest_run
 from pylib.gtest.test_package import TestPackage
 
 
@@ -141,3 +144,8 @@ class TestPackageApk(TestPackage):
   def Install(self, device):
     self.tool.CopyFiles(device)
     device.Install(self.suite_path)
+
+  #override
+  def PullAppFiles(self, device, files, directory):
+    local_device_gtest_run.PullAppFilesImpl(
+        device, self._package_info.package, files, directory)

build/android/pylib/gtest/test_package_exe.py

@@ -157,3 +157,7 @@ class TestPackageExecutable(TestPackage):
     deps_path = self.suite_path + '_deps'
     if os.path.isdir(deps_path):
       device.PushChangedFiles([(deps_path, test_binary_path + '_deps')])
+
+  #override
+  def PullAppFiles(self, device, files, directory):
+    pass

build/android/pylib/gtest/test_runner.py

@@ -5,6 +5,7 @@
 import logging
 import os
 import re
+import tempfile
 
 from pylib import pexpect
 from pylib import ports
@@ -80,6 +81,17 @@ class TestRunner(base_test_runner.BaseTestRunner):
     else:
       self._servers = []
 
+    if test_options.app_data_files:
+      self._app_data_files = test_options.app_data_files
+      if test_options.app_data_file_dir:
+        self._app_data_file_dir = test_options.app_data_file_dir
+      else:
+        self._app_data_file_dir = tempfile.mkdtemp()
+        logging.critical('Saving app files to %s', self._app_data_file_dir)
+    else:
+      self._app_data_files = None
+      self._app_data_file_dir = None
+
   #override
   def InstallTestPackage(self):
     self.test_package.Install(self.device)
@@ -167,6 +179,9 @@ class TestRunner(base_test_runner.BaseTestRunner):
           self.device, test, self._test_arguments)
       test_results = self._ParseTestOutput(
           self.test_package.SpawnTestProcess(self.device))
+      if self._app_data_files:
+        self.test_package.PullAppFiles(self.device, self._app_data_files,
+                                       self._app_data_file_dir)
     finally:
       for s in self._servers:
         s.Reset()

build/android/test_runner.py

@@ -215,6 +215,12 @@ def AddGTestOptions(parser):
                      dest='isolate_file_path',
                      help='.isolate file path to override the default '
                           'path')
+  group.add_argument('--app-data-file', action='append', dest='app_data_files',
+                     help='A file path relative to the app data directory '
+                          'that should be saved to the host.')
+  group.add_argument('--app-data-file-dir',
+                     help='Host directory to which app data files will be'
+                          ' saved. Used with --app-data-file.')
 
   filter_group = group.add_mutually_exclusive_group()
   filter_group.add_argument('-f', '--gtest_filter', '--gtest-filter',
@@ -640,7 +646,9 @@ def _RunGTests(args, devices):
         args.test_arguments,
         args.timeout,
         args.isolate_file_path,
-        suite_name)
+        suite_name,
+        args.app_data_files,
+        args.app_data_file_dir)
     runner_factory, tests = gtest_setup.Setup(gtest_options, devices)
 
     results, test_exit_code = test_dispatcher.RunTests(