Added function to get native path case on Windows and OSX.

Fixed all tests on Windows.

Ignore \Device\Mup on Windows.

Still saw some flakiness on tracing on Windows, to be addressed in a separate
patch.

R=nsylvain@chromium.org
BUG=98834
TEST=Run all the isolate tests on Windows or OSX


Review URL: http://codereview.chromium.org/10091011

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@133475 0039d316-1c4b-4281-b951-d872f2087c98

tools/isolate/data/trace_inputs/child2.py

@@ -2,11 +2,26 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import sys
 import time
 
-print 'child2'
-# Introduce a race condition with the parent so the parent may have a chance
-# to exit before the child. Will be random.
-time.sleep(.01)
-# Do not do anything with it.
-open('test_file.txt')
+
+def main():
+  print 'child2'
+  # Introduce a race condition with the parent so the parent may have a chance
+  # to exit before the child. Will be random.
+  time.sleep(.01)
+  # Do not do anything with it.
+  open('test_file.txt')
+  # Check for case-insensitive file system. This happens on Windows and OSX.
+  try:
+    open('Test_File.txt')
+  except OSError:
+    # Pass on other OSes since this code assumes only win32 has case-insensitive
+    # path. This is not strictly true.
+    if sys.platform not in ('darwin', 'win32'):
+      raise
+
+
+if __name__ == '__main__':
+  sys.exit(main())

tools/isolate/isolate.py

@@ -391,7 +391,12 @@ def MODEtrace(_outdir, indir, data):
   product_dir = None
   if data['resultdir'] and indir:
     # Defaults to none if both are the same directory.
-    product_dir = os.path.relpath(data['resultdir'], indir) or None
+    try:
+      product_dir = os.path.relpath(data['resultdir'], indir) or None
+    except ValueError:
+      # This happens on Windows if data['resultdir'] is one drive, let's say
+      # 'C:\' and indir on another one like 'D:\'.
+      product_dir = None
   if not data['command']:
     print 'No command to run'
     return 1
@@ -439,7 +444,9 @@ def process_options(variables, resultfile, input_file, error):
   # The trick used to determine the root directory is to look at "how far" back
   # up it is looking up.
   # TODO(maruel): Stop the msbuild generator from generating a mix of / and \\.
-  root_dir = isolate_dir.replace(os.path.sep, '/')
+  isolate_dir_replaced = isolate_dir.replace(os.path.sep, '/')
+  root_dir = isolate_dir_replaced
+  logging.debug('root_dir before searching: %s' % root_dir)
   for i in infiles:
     i = i.replace(os.path.sep, '/')
     x = isolate_dir.replace(os.path.sep, '/')
@@ -450,9 +457,12 @@ def process_options(variables, resultfile, input_file, error):
     if root_dir.startswith(x):
       root_dir = x
   root_dir = root_dir.replace('/', os.path.sep)
+  logging.debug('root_dir after searching: %s' % root_dir)
+
   # The relative directory is automatically determined by the relative path
   # between root_dir and the directory containing the .isolate file.
-  relative_dir = os.path.relpath(isolate_dir, root_dir)
+  relative_dir = os.path.relpath(isolate_dir, root_dir).replace(
+      os.path.sep, '/')
   logging.debug('relative_dir: %s' % relative_dir)
 
   logging.debug(

tools/isolate/isolate_smoke_test.py

@@ -79,7 +79,9 @@ class IsolateBase(unittest.TestCase):
   def _result_tree(self):
     actual = []
     for root, _dirs, files in os.walk(self.outdir):
-      actual.extend(os.path.join(root, f)[len(self.outdir)+1:] for f in files)
+      actual.extend(
+          os.path.join(root, f)[len(self.outdir)+1:].replace(os.path.sep, '/')
+          for f in files)
     return sorted(actual)
 
   def _expected_tree(self):
@@ -444,8 +446,11 @@ class Isolate_trace(IsolateBase):
       out = e.output
     self._expect_no_tree()
     self._expected_result(['fail.py'], None)
-    expected = 'Failure: 1\nFailing\n\n'
-    self.assertEquals(expected, out)
+    # In theory, there should be 2 \n at the end of expected but for an
+    # unknown reason there's 3 \n on Windows so just rstrip() and compare the
+    # text, that's sufficient for this test.
+    expected = 'Failure: 1\nFailing'
+    self.assertEquals(expected, out.rstrip())
 
   def test_missing_trailing_slash(self):
     try:

tools/isolate/isolate_test.py

@@ -4,7 +4,9 @@
 # found in the LICENSE file.
 
 import json
+import logging
 import os
+import sys
 import tempfile
 import unittest
 
@@ -89,4 +91,7 @@ class Isolate(unittest.TestCase):
 
 
 if __name__ == '__main__':
+  logging.basicConfig(
+      level=logging.DEBUG if '-v' in sys.argv else logging.ERROR,
+      format='%(levelname)5s %(filename)15s(%(lineno)3d): %(message)s')
   unittest.main()

tools/isolate/trace_inputs.py

@@ -21,6 +21,15 @@ import re
 import subprocess
 import sys
 
+## OS-specific imports
+
+if sys.platform == 'win32':
+  from ctypes.wintypes import create_unicode_buffer
+  from ctypes.wintypes import windll, FormatError  # pylint: disable=E0611
+  from ctypes.wintypes import GetLastError  # pylint: disable=E0611
+elif sys.platform == 'darwin':
+  import Carbon.File  #  pylint: disable=F0401
+
 
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 ROOT_DIR = os.path.dirname(os.path.dirname(BASE_DIR))
@@ -29,12 +38,9 @@ KEY_TRACKED = 'isolate_dependency_tracked'
 KEY_UNTRACKED = 'isolate_dependency_untracked'
 
 
-if sys.platform == 'win32':
-  from ctypes.wintypes import create_unicode_buffer
-  from ctypes.wintypes import windll, FormatError  # pylint: disable=E0611
-  from ctypes.wintypes import GetLastError  # pylint: disable=E0611
-
+## OS-specific functions
 
+if sys.platform == 'win32':
   def QueryDosDevice(drive_letter):
     """Returns the Windows 'native' path for a DOS drive letter."""
     assert re.match(r'^[a-zA-Z]:$', drive_letter), drive_letter
@@ -56,6 +62,10 @@ if sys.platform == 'win32':
   def GetShortPathName(long_path):
     """Returns the Windows short path equivalent for a 'long' path."""
     long_path = unicode(long_path)
+    # Adds '\\\\?\\' when given an absolute path so the MAX_PATH (260) limit is
+    # not enforced.
+    if os.path.isabs(long_path) and not long_path.startswith('\\\\?\\'):
+      long_path = '\\\\?\\' + long_path
     chars = windll.kernel32.GetShortPathNameW(long_path, None, 0)
     if chars:
       p = create_unicode_buffer(chars)
@@ -71,6 +81,28 @@ if sys.platform == 'win32':
             str(long_path), FormatError(err), err))
 
 
+  def GetLongPathName(short_path):
+    """Returns the Windows long path equivalent for a 'short' path."""
+    short_path = unicode(short_path)
+    # Adds '\\\\?\\' when given an absolute path so the MAX_PATH (260) limit is
+    # not enforced.
+    if os.path.isabs(short_path) and not short_path.startswith('\\\\?\\'):
+      short_path = '\\\\?\\' + short_path
+    chars = windll.kernel32.GetLongPathNameW(short_path, None, 0)
+    if chars:
+      p = create_unicode_buffer(chars)
+      if windll.kernel32.GetLongPathNameW(short_path, p, chars):
+        return p.value
+
+    err = GetLastError()
+    if err:
+      # pylint: disable=E0602
+      raise WindowsError(
+          err,
+          'GetLongPathName(%s): %s (%d)' % (
+            str(short_path), FormatError(err), err))
+
+
   def get_current_encoding():
     """Returns the 'ANSI' code page associated to the process."""
     return 'cp%d' % int(windll.kernel32.GetACP())
@@ -83,6 +115,9 @@ if sys.platform == 'win32':
 
     def __init__(self):
       if not self._MAPPING:
+        # This is related to UNC resolver on windows. Ignore that.
+        self._MAPPING['\\Device\\Mup'] = None
+
         for letter in (chr(l) for l in xrange(ord('C'), ord('Z')+1)):
           try:
             letter = '%s:' % letter
@@ -98,14 +133,36 @@ if sys.platform == 'win32':
     def to_dos(self, path):
       """Converts a native NT path to DOS path."""
       m = re.match(r'(^\\Device\\[a-zA-Z0-9]+)(\\.*)?$', path)
-      if not m or m.group(1) not in self._MAPPING:
-        assert False, path
+      assert m, path
+      assert m.group(1) in self._MAPPING, (path, self._MAPPING)
       drive = self._MAPPING[m.group(1)]
-      if not m.group(2):
+      if not drive or not m.group(2):
         return drive
       return drive + m.group(2)
 
 
+def get_native_path_case(root, relative_path):
+  """Returns the native path case."""
+  if sys.platform == 'win32':
+    # Windows used to have an option to turn on case sensitivity on non Win32
+    # subsystem but that's out of scope here and isn't supported anymore.
+    # First process root.
+    if root:
+      root = GetLongPathName(GetShortPathName(root)) + os.path.sep
+    path = os.path.join(root, relative_path) if root else relative_path
+    # Go figure why GetShortPathName() is needed.
+    return GetLongPathName(GetShortPathName(path))[len(root):]
+  elif sys.platform == 'darwin':
+    # Technically, it's only HFS+ on OSX that is case insensitive. It's
+    # the default setting on HFS+ but can be changed.
+    root_ref, _ = Carbon.File.FSPathMakeRef(root)
+    rel_ref, _ = Carbon.File.FSPathMakeRef(os.path.join(root, relative_path))
+    return rel_ref.FSRefMakePath()[len(root_ref.FSRefMakePath())+1:]
+  else:
+    # Give up on cygwin, as GetLongPathName() can't be called.
+    return relative_path
+
+
 def get_flavor():
   """Returns the system default flavor. Copied from gyp/pylib/gyp/common.py."""
   flavors = {
@@ -837,7 +894,7 @@ class LogmanTrace(object):
 
     def handle_FileIo_Create(self, line):
       m = re.match(r'^\"(.+)\"$', line[self.FILE_PATH])
-      self._handle_file(self._drive_map.to_dos(m.group(1)).lower())
+      self._handle_file(self._drive_map.to_dos(m.group(1)))
 
     def handle_FileIo_Rename(self, line):
       # TODO(maruel): Handle?
@@ -913,7 +970,7 @@ class LogmanTrace(object):
 
   def __init__(self):
     # Most ignores need to be determined at runtime.
-    self.IGNORED = set([os.path.dirname(sys.executable).lower()])
+    self.IGNORED = set([os.path.dirname(sys.executable)])
     # Add many directories from environment variables.
     vars_to_ignore = (
       'APPDATA',
@@ -928,11 +985,11 @@ class LogmanTrace(object):
     )
     for i in vars_to_ignore:
       if os.environ.get(i):
-        self.IGNORED.add(os.environ[i].lower())
+        self.IGNORED.add(os.environ[i])
 
     # Also add their short path name equivalents.
     for i in list(self.IGNORED):
-      self.IGNORED.add(GetShortPathName(i).lower())
+      self.IGNORED.add(GetShortPathName(i))
 
     # Add this one last since it has no short path name equivalent.
     self.IGNORED.add('\\systemroot')
@@ -1225,16 +1282,6 @@ def trace_inputs(logfile, cmd, root_dir, cwd_dir, product_dir, force_trace):
   # Resolve any symlink
   root_dir = os.path.realpath(root_dir)
 
-  if sys.platform == 'win32':
-    # Help ourself and lowercase all the paths.
-    # TODO(maruel): handle short path names by converting them to long path name
-    # as needed.
-    root_dir = root_dir.lower()
-    if cwd_dir:
-      cwd_dir = cwd_dir.lower()
-    if product_dir:
-      product_dir = product_dir.lower()
-
   def print_if(txt):
     if cwd_dir is None:
       print(txt)
@@ -1287,6 +1334,9 @@ def trace_inputs(logfile, cmd, root_dir, cwd_dir, product_dir, force_trace):
     for f in unexpected:
       print_if('  %s' % f)
 
+  # In case the file system is case insensitive.
+  expected = sorted(set(get_native_path_case(root_dir, f) for f in expected))
+
   simplified = extract_directories(expected, root_dir)
   print_if('Interesting: %d reduced to %d' % (len(expected), len(simplified)))
   for f in simplified:
@@ -1312,8 +1362,7 @@ def trace_inputs(logfile, cmd, root_dir, cwd_dir, product_dir, force_trace):
       """Bases the file on the most restrictive variable."""
       logging.debug('fix(%s)' % f)
       # Important, GYP stores the files with / and not \.
-      if sys.platform == 'win32':
-        f = f.replace('\\', '/')
+      f = f.replace(os.path.sep, '/')
 
       if product_dir and f.startswith(product_dir):
         return '<(PRODUCT_DIR)/%s' % f[len(product_dir):]

tools/isolate/trace_inputs_smoke_test.py

@@ -32,12 +32,15 @@ class CalledProcessError(subprocess.CalledProcessError):
 
 class TraceInputs(unittest.TestCase):
   def setUp(self):
-    self.tempdir = tempfile.mkdtemp()
+    self.tempdir = tempfile.mkdtemp(prefix='trace_smoke_test')
     self.log = os.path.join(self.tempdir, 'log')
     os.chdir(ROOT_DIR)
 
   def tearDown(self):
-    shutil.rmtree(self.tempdir)
+    if VERBOSE:
+      print 'Leaking: %s' % self.tempdir
+    else:
+      shutil.rmtree(self.tempdir)
 
   def _execute(self, is_gyp):
     cmd = [