[Android] Extract MD5sum logic.

BUG=428729,433101

Review URL: https://codereview.chromium.org/738413002

Cr-Commit-Position: refs/heads/master@{#305100}

build/android/PRESUBMIT.py

@@ -64,6 +64,7 @@ def CommonChecks(input_api, output_api):
           J('pylib', 'device', 'device_utils_test.py'),
           J('pylib', 'gtest', 'test_package_test.py'),
           J('pylib', 'instrumentation', 'test_runner_test.py'),
+          J('pylib', 'utils', 'md5sum_test.py'),
       ],
       env=pylib_test_env))
   output.extend(_CheckDeletionsOnlyFiles(input_api, output_api))

build/android/pylib/constants.py

@@ -187,6 +187,7 @@ PYTHON_UNIT_TEST_SUITES = {
     'path': os.path.join(DIR_SOURCE_ROOT, 'build', 'android'),
     'test_modules': [
       'pylib.device.device_utils_test',
+      'pylib.utils.md5sum_test',
     ]
   },
   'gyp_py_unittests': {

build/android/pylib/device/device_utils.py

@@ -26,6 +26,7 @@ from pylib.device.commands import install_commands
 from pylib.utils import apk_helper
 from pylib.utils import device_temp_file
 from pylib.utils import host_utils
+from pylib.utils import md5sum
 from pylib.utils import parallelizer
 from pylib.utils import timeout_retry
 
@@ -701,12 +702,13 @@ class DeviceUtils(object):
     if not real_device_path:
       return [(host_path, device_path)]
 
-    # TODO(jbudorick): Move the md5 logic up into DeviceUtils or base
-    # this function on mtime.
-    # pylint: disable=protected-access
-    host_hash_tuples, device_hash_tuples = self.old_interface._RunMd5Sum(
-        real_host_path, real_device_path)
-    # pylint: enable=protected-access
+    host_hash_tuples = md5sum.CalculateHostMd5Sums([real_host_path])
+    device_paths_to_md5 = (
+        real_device_path if os.path.isfile(real_host_path)
+        else ('%s/%s' % (real_device_path, os.path.relpath(p, real_host_path))
+              for _, p in host_hash_tuples))
+    device_hash_tuples = md5sum.CalculateDeviceMd5Sums(
+        device_paths_to_md5, self)
 
     if os.path.isfile(host_path):
       if (not device_hash_tuples

build/android/pylib/instrumentation/test_jar.py

@@ -14,6 +14,7 @@ import sys
 from pylib import cmd_helper
 from pylib import constants
 from pylib.device import device_utils
+from pylib.utils import md5sum
 from pylib.utils import proguard
 
 sys.path.insert(0,
@@ -55,16 +56,6 @@ class TestJar(object):
     if not self._GetCachedProguardData():
       self._GetProguardData()
 
-  @staticmethod
-  def _CalculateMd5(path):
-    # TODO(jbudorick): Move MD5sum calculations out of here and
-    # AndroidCommands to their own module.
-    out = cmd_helper.GetCmdOutput(
-        [os.path.join(constants.GetOutDirectory(),
-                      'md5sum_bin_host'),
-        path])
-    return out
-
   def _GetCachedProguardData(self):
     if (os.path.exists(self._pickled_proguard_name) and
         (os.path.getmtime(self._pickled_proguard_name) >
@@ -74,7 +65,7 @@ class TestJar(object):
       try:
         with open(self._pickled_proguard_name, 'r') as r:
           d = pickle.loads(r.read())
-        jar_md5 = self._CalculateMd5(self._jar_path)
+        jar_md5 = md5sum.CalculateHostMd5Sums(self._jar_path)[0].hash
         if (d['JAR_MD5SUM'] == jar_md5 and
             d['VERSION'] == PICKLE_FORMAT_VERSION):
           self._test_methods = d['TEST_METHODS']
@@ -114,7 +105,7 @@ class TestJar(object):
     logging.info('Storing proguard output to %s', self._pickled_proguard_name)
     d = {'VERSION': PICKLE_FORMAT_VERSION,
          'TEST_METHODS': self._test_methods,
-         'JAR_MD5SUM': self._CalculateMd5(self._jar_path)}
+         'JAR_MD5SUM': md5sum.CalculateHostMd5Sums(self._jar_path)[0].hash}
     with open(self._pickled_proguard_name, 'w') as f:
       f.write(pickle.dumps(d))
 

build/android/pylib/utils/md5sum.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import collections
+import logging
+import os
+import tempfile
+import types
+
+from pylib import cmd_helper
+from pylib import constants
+from pylib.utils import device_temp_file
+
+HashAndPath = collections.namedtuple('HashAndPath', ['hash', 'path'])
+
+MD5SUM_DEVICE_LIB_PATH = '/data/local/tmp/md5sum/'
+MD5SUM_DEVICE_BIN_PATH = MD5SUM_DEVICE_LIB_PATH + 'md5sum_bin'
+
+MD5SUM_DEVICE_SCRIPT_FORMAT = (
+    'test -f {path} -o -d {path} '
+    '&& LD_LIBRARY_PATH={md5sum_lib} {md5sum_bin} {path}')
+
+
+def CalculateHostMd5Sums(paths):
+  """Calculates the MD5 sum value for all items in |paths|.
+
+  Args:
+    paths: A list of host paths to md5sum.
+  Returns:
+    A list of named tuples with 'hash' and 'path' attributes.
+  """
+  if isinstance(paths, basestring):
+    paths = [paths]
+
+  out = cmd_helper.GetCmdOutput(
+      [os.path.join(constants.GetOutDirectory(), 'md5sum_bin_host')] +
+          [p for p in paths])
+  return [HashAndPath(*l.split(None, 1)) for l in out.splitlines()]
+
+
+def CalculateDeviceMd5Sums(paths, device):
+  """Calculates the MD5 sum value for all items in |paths|.
+
+  Args:
+    paths: A list of device paths to md5sum.
+  Returns:
+    A list of named tuples with 'hash' and 'path' attributes.
+  """
+  if isinstance(paths, basestring):
+    paths = [paths]
+
+  if not device.FileExists(MD5SUM_DEVICE_BIN_PATH):
+    device.adb.Push(
+        os.path.join(constants.GetOutDirectory(), 'md5sum_dist'),
+        MD5SUM_DEVICE_LIB_PATH)
+
+  out = []
+  with tempfile.NamedTemporaryFile() as md5sum_script_file:
+    with device_temp_file.DeviceTempFile(device) as md5sum_device_script_file:
+      md5sum_script = (
+          MD5SUM_DEVICE_SCRIPT_FORMAT.format(
+              path=p, md5sum_lib=MD5SUM_DEVICE_LIB_PATH,
+              md5sum_bin=MD5SUM_DEVICE_BIN_PATH)
+          for p in paths)
+      md5sum_script_file.write('; '.join(md5sum_script))
+      md5sum_script_file.flush()
+      device.adb.Push(md5sum_script_file.name, md5sum_device_script_file.name)
+      out = device.RunShellCommand(['sh', md5sum_device_script_file.name])
+
+  return [HashAndPath(*l.split(None, 1)) for l in out]
+

build/android/pylib/utils/md5sum_test.py

+#!/usr/bin/env python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import sys
+import unittest
+
+from pylib import cmd_helper
+from pylib import constants
+from pylib.utils import md5sum
+
+sys.path.append(
+    os.path.join(constants.DIR_SOURCE_ROOT, 'third_party', 'pymock'))
+import mock
+
+TEST_OUT_DIR = os.path.join('test', 'out', 'directory')
+HOST_MD5_EXECUTABLE = os.path.join(TEST_OUT_DIR, 'md5sum_bin_host')
+
+class Md5SumTest(unittest.TestCase):
+
+  def setUp(self):
+    self._patchers = [
+        mock.patch('pylib.constants.GetOutDirectory',
+                   new=mock.Mock(return_value=TEST_OUT_DIR)),
+    ]
+    for p in self._patchers:
+      p.start()
+
+  def tearDown(self):
+    for p in self._patchers:
+      p.stop()
+
+  def testCalculateHostMd5Sums_singlePath(self):
+    test_path = '/test/host/file.dat'
+    mock_get_cmd_output = mock.Mock(
+        return_value='0123456789abcdeffedcba9876543210 /test/host/file.dat')
+    with mock.patch('pylib.cmd_helper.GetCmdOutput', new=mock_get_cmd_output):
+      out = md5sum.CalculateHostMd5Sums(test_path)
+      self.assertEquals(1, len(out))
+      self.assertEquals('0123456789abcdeffedcba9876543210', out[0].hash)
+      self.assertEquals('/test/host/file.dat', out[0].path)
+      mock_get_cmd_output.assert_called_once_with(
+          [HOST_MD5_EXECUTABLE, '/test/host/file.dat'])
+
+  def testCalculateHostMd5Sums_list(self):
+    test_paths = ['/test/host/file0.dat', '/test/host/file1.dat']
+    mock_get_cmd_output = mock.Mock(
+        return_value='0123456789abcdeffedcba9876543210 /test/host/file0.dat\n'
+                     '123456789abcdef00fedcba987654321 /test/host/file1.dat\n')
+    with mock.patch('pylib.cmd_helper.GetCmdOutput', new=mock_get_cmd_output):
+      out = md5sum.CalculateHostMd5Sums(test_paths)
+      self.assertEquals(2, len(out))
+      self.assertEquals('0123456789abcdeffedcba9876543210', out[0].hash)
+      self.assertEquals('/test/host/file0.dat', out[0].path)
+      self.assertEquals('123456789abcdef00fedcba987654321', out[1].hash)
+      self.assertEquals('/test/host/file1.dat', out[1].path)
+      mock_get_cmd_output.assert_called_once_with(
+          [HOST_MD5_EXECUTABLE, '/test/host/file0.dat',
+           '/test/host/file1.dat'])
+
+  def testCalculateHostMd5Sums_generator(self):
+    test_paths = ('/test/host/' + p for p in ['file0.dat', 'file1.dat'])
+    mock_get_cmd_output = mock.Mock(
+        return_value='0123456789abcdeffedcba9876543210 /test/host/file0.dat\n'
+                     '123456789abcdef00fedcba987654321 /test/host/file1.dat\n')
+    with mock.patch('pylib.cmd_helper.GetCmdOutput', new=mock_get_cmd_output):
+      out = md5sum.CalculateHostMd5Sums(test_paths)
+      self.assertEquals(2, len(out))
+      self.assertEquals('0123456789abcdeffedcba9876543210', out[0].hash)
+      self.assertEquals('/test/host/file0.dat', out[0].path)
+      self.assertEquals('123456789abcdef00fedcba987654321', out[1].hash)
+      self.assertEquals('/test/host/file1.dat', out[1].path)
+      mock_get_cmd_output.assert_called_once_with(
+          [HOST_MD5_EXECUTABLE, '/test/host/file0.dat', '/test/host/file1.dat'])
+
+  def testCalculateDeviceMd5Sums_singlePath(self):
+    test_path = '/storage/emulated/legacy/test/file.dat'
+
+    device = mock.NonCallableMock()
+    device.adb = mock.NonCallableMock()
+    device.adb.Push = mock.Mock()
+    device_md5sum_output = [
+        '0123456789abcdeffedcba9876543210 '
+            '/storage/emulated/legacy/test/file.dat',
+    ]
+    device.RunShellCommand = mock.Mock(return_value=device_md5sum_output)
+
+    mock_temp_file = mock.mock_open()
+    mock_temp_file.return_value.name = '/tmp/test/script/file.sh'
+
+    mock_device_temp_file = mock.mock_open()
+    mock_device_temp_file.return_value.name = (
+        '/data/local/tmp/test/script/file.sh')
+
+    with mock.patch('tempfile.NamedTemporaryFile', new=mock_temp_file), (
+         mock.patch('pylib.utils.device_temp_file.DeviceTempFile',
+                    new=mock_device_temp_file)):
+      out = md5sum.CalculateDeviceMd5Sums(test_path, device)
+      self.assertEquals(1, len(out))
+      self.assertEquals('0123456789abcdeffedcba9876543210', out[0].hash)
+      self.assertEquals('/storage/emulated/legacy/test/file.dat', out[0].path)
+      device.adb.Push.assert_called_once_with(
+          '/tmp/test/script/file.sh', '/data/local/tmp/test/script/file.sh')
+      device.RunShellCommand.assert_called_once_with(
+          ['sh', '/data/local/tmp/test/script/file.sh'])
+
+  def testCalculateDeviceMd5Sums_list(self):
+    test_path = ['/storage/emulated/legacy/test/file0.dat',
+                 '/storage/emulated/legacy/test/file1.dat']
+    device = mock.NonCallableMock()
+    device.adb = mock.NonCallableMock()
+    device.adb.Push = mock.Mock()
+    device_md5sum_output = [
+        '0123456789abcdeffedcba9876543210 '
+            '/storage/emulated/legacy/test/file0.dat',
+        '123456789abcdef00fedcba987654321 '
+            '/storage/emulated/legacy/test/file1.dat',
+    ]
+    device.RunShellCommand = mock.Mock(return_value=device_md5sum_output)
+
+    mock_temp_file = mock.mock_open()
+    mock_temp_file.return_value.name = '/tmp/test/script/file.sh'
+
+    mock_device_temp_file = mock.mock_open()
+    mock_device_temp_file.return_value.name = (
+        '/data/local/tmp/test/script/file.sh')
+
+    with mock.patch('tempfile.NamedTemporaryFile', new=mock_temp_file), (
+         mock.patch('pylib.utils.device_temp_file.DeviceTempFile',
+                    new=mock_device_temp_file)):
+      out = md5sum.CalculateDeviceMd5Sums(test_path, device)
+      self.assertEquals(2, len(out))
+      self.assertEquals('0123456789abcdeffedcba9876543210', out[0].hash)
+      self.assertEquals('/storage/emulated/legacy/test/file0.dat', out[0].path)
+      self.assertEquals('123456789abcdef00fedcba987654321', out[1].hash)
+      self.assertEquals('/storage/emulated/legacy/test/file1.dat', out[1].path)
+      device.adb.Push.assert_called_once_with(
+          '/tmp/test/script/file.sh', '/data/local/tmp/test/script/file.sh')
+      device.RunShellCommand.assert_called_once_with(
+          ['sh', '/data/local/tmp/test/script/file.sh'])
+
+  def testCalculateDeviceMd5Sums_generator(self):
+    test_path = ('/storage/emulated/legacy/test/file%d.dat' % n
+                 for n in xrange(0, 2))
+
+    device = mock.NonCallableMock()
+    device.adb = mock.NonCallableMock()
+    device.adb.Push = mock.Mock()
+    device_md5sum_output = [
+        '0123456789abcdeffedcba9876543210 '
+            '/storage/emulated/legacy/test/file0.dat',
+        '123456789abcdef00fedcba987654321 '
+            '/storage/emulated/legacy/test/file1.dat',
+    ]
+    device.RunShellCommand = mock.Mock(return_value=device_md5sum_output)
+
+    mock_temp_file = mock.mock_open()
+    mock_temp_file.return_value.name = '/tmp/test/script/file.sh'
+
+    mock_device_temp_file = mock.mock_open()
+    mock_device_temp_file.return_value.name = (
+        '/data/local/tmp/test/script/file.sh')
+
+    with mock.patch('tempfile.NamedTemporaryFile', new=mock_temp_file), (
+         mock.patch('pylib.utils.device_temp_file.DeviceTempFile',
+                    new=mock_device_temp_file)):
+      out = md5sum.CalculateDeviceMd5Sums(test_path, device)
+      self.assertEquals(2, len(out))
+      self.assertEquals('0123456789abcdeffedcba9876543210', out[0].hash)
+      self.assertEquals('/storage/emulated/legacy/test/file0.dat', out[0].path)
+      self.assertEquals('123456789abcdef00fedcba987654321', out[1].hash)
+      self.assertEquals('/storage/emulated/legacy/test/file1.dat', out[1].path)
+      device.adb.Push.assert_called_once_with(
+          '/tmp/test/script/file.sh', '/data/local/tmp/test/script/file.sh')
+      device.RunShellCommand.assert_called_once_with(
+          ['sh', '/data/local/tmp/test/script/file.sh'])
+
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)
+

build/android/pylib/utils/mock_calls.py

-#!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.