md5_check.py: Don't hash files larger than 1mb.

I noticed that md5_check.py was taking up to 200ms for javac actions.
Turns out it was due to it hashing really big .jar files. The utility of
md5_check is that it tracks the md5s of often-changing source files
(e.g. .java files). It's not worth spending >100ms to check if rt.jar has
changed over and over again.

This now uses mtime for files greater than 1mb, and the md5_check
overhead for javac.py is down below 10ms on my machine.

Bug: 906803
Change-Id: I4956419e12b69ad025f12a79d82cf044fc9db34d
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1839238Reviewed-by: Sam Maier <smaier@chromium.org>
Commit-Queue: Andrew Grieve <agrieve@chromium.org>
Cr-Commit-Position: refs/heads/master@{#702859}

build/android/gyp/util/build_utils.py

@@ -662,8 +662,7 @@ def CallAndWriteDepfileIfStale(function, options, record_path=None,
     input_paths += python_deps
     output_paths += [options.depfile]
 
-  def on_stale_md5(changes):
-    args = (changes,) if pass_changes else ()
+  def on_stale_md5(*args):
     function(*args)
     if python_deps is not None:
       all_depfile_deps = list(python_deps) if add_pydeps else []
@@ -679,4 +678,4 @@ def CallAndWriteDepfileIfStale(function, options, record_path=None,
       input_strings=input_strings,
       output_paths=output_paths,
       force=force,
-      pass_changes=True)
+      pass_changes=pass_changes)

build/android/gyp/util/md5_check.py

@@ -58,11 +58,13 @@ def CallAndRecordIfStale(
   new_metadata.AddStrings(input_strings)
 
   for path in input_paths:
-    if _IsZipFile(path):
+    # It's faster to md5 an entire zip file than it is to just locate & hash
+    # its central directory (which is what this used to do).
+    if pass_changes and _IsZipFile(path):
       entries = _ExtractZipEntries(path)
       new_metadata.AddZipFile(path, entries)
     else:
-      new_metadata.AddFile(path, _Md5ForPath(path))
+      new_metadata.AddFile(path, _ComputeTagForPath(path))
 
   old_metadata = None
   force = force or _FORCE_REBUILD
@@ -369,27 +371,15 @@ class _Metadata(object):
     return (entry['path'] for entry in subentries)
 
 
-def _UpdateMd5ForFile(md5, path, block_size=2**16):
-  with open(path, 'rb') as infile:
-    while True:
-      data = infile.read(block_size)
-      if not data:
-        break
-      md5.update(data)
-
-
-def _UpdateMd5ForDirectory(md5, dir_path):
-  for root, _, files in os.walk(dir_path):
-    for f in files:
-      _UpdateMd5ForFile(md5, os.path.join(root, f))
-
-
-def _Md5ForPath(path):
+def _ComputeTagForPath(path):
+  stat = os.stat(path)
+  if stat.st_size > 1 * 1024 * 1024:
+    # Fallback to mtime for large files so that md5_check does not take too long
+    # to run.
+    return stat.st_mtime
   md5 = hashlib.md5()
-  if os.path.isdir(path):
-    _UpdateMd5ForDirectory(md5, path)
-  else:
-    _UpdateMd5ForFile(md5, path)
+  with open(path, 'rb') as f:
+    md5.update(f.read())
   return md5.hexdigest()
 
 

build/android/gyp/util/md5_check_test.py

@@ -80,7 +80,11 @@ class TestMd5Check(unittest.TestCase):
     CheckCallAndRecord(True, 'should call when record doesn\'t exist',
                        expected_changes='Previous stamp file not found.',
                        added_or_modified_only=False)
+    CheckCallAndRecord(True, 'pass_changes changed')
     CheckCallAndRecord(False, 'should not call when nothing changed')
+    input_files = input_files[::-1]
+    CheckCallAndRecord(False, 'reordering of inputs shouldn\'t trigger call')
+
     CheckCallAndRecord(False, 'should not call when nothing changed #2',
                        outputs_specified=True, outputs_missing=False)
     CheckCallAndRecord(True, 'should call when output missing',
@@ -97,9 +101,6 @@ class TestMd5Check(unittest.TestCase):
                        expected_changes='*Modified: %s' % input_file1.name,
                        added_or_modified_only=True)
 
-    input_files = input_files[::-1]
-    CheckCallAndRecord(False, 'reordering of inputs shouldn\'t trigger call')
-
     input_files = input_files[:1]
     CheckCallAndRecord(True, 'removing file should trigger call',
                        expected_changes='*Removed: %s' % input_file1.name,
@@ -138,7 +139,6 @@ class TestMd5Check(unittest.TestCase):
                        expected_changes='*Modified: %s*Subpath modified: %s' % (
                                         input_file2.name, 'path/1.txt'),
                        added_or_modified_only=True)
-    CheckCallAndRecord(False, 'should not call when nothing changed')
 
     _WriteZipFile(input_file2.name, [])
     CheckCallAndRecord(True, 'removed subpath should trigger call',