Add the symbol and source server scripts.

Review URL: http://codereview.chromium.org/155136


git-svn-id: svn://svn.chromium.org/chrome/trunk/src@20022 0039d316-1c4b-4281-b951-d872f2087c98

tools/symsrc/COPYING-pefile

+Copyright (c) 2004, 2005, 2006 Ero Carrera <ero@dkbza.org>. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.  
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+3. The name of the author may not be used to endorse or promote products
+derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+OF SUCH DAMAGE.
+
+

tools/symsrc/img_fingerprint.py

+#!/usr/bin/env python
+
+# Copyright (c) 2008 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""This will retrieve an image's "fingerprint".  This is used when retrieving
+the image from the symbol server.  The .dll (or cab compressed .dl_) or .exe
+is expected at a path like:
+ foo.dll/FINGERPRINT/foo.dll"""
+
+import sys
+import pefile
+
+def GetImgFingerprint(filename):
+  """Returns the fingerprint for an image file"""
+
+  pe = pefile.PE(filename)
+  return "%08X%06x" % (
+    pe.FILE_HEADER.TimeDateStamp, pe.OPTIONAL_HEADER.SizeOfImage)
+
+
+if __name__ == '__main__':
+  if len(sys.argv) != 2:
+    print "usage: file.dll"
+    sys.exit(1)
+
+  print GetImgFingerprint(sys.argv[1])

tools/symsrc/pdb_fingerprint_from_img.py

+#!/usr/bin/env python
+
+# Copyright (c) 2008 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""This will retrieve a PDBs "fingerprint" from it's corresponding executable
+image (.dll or .exe).  This is used when retrieving the PDB from the symbol
+server.  The .pdb (or cab compressed .pd_) is expected at a path like:
+ foo.pdb/FINGERPRINT/foo.pdb
+
+We can retrieve the same information from the .PDB file itself, but this file
+format is much more difficult and undocumented.  Instead, we can look at the
+DLL's reference to the PDB, and use that to retrieve the information."""
+
+import sys
+import pefile
+
+__CV_INFO_PDB70_format__ = ('CV_INFO_PDB70',
+  ('4s,CvSignature', '16s,Signature', 'L,Age'))
+
+__GUID_format__ = ('GUID',
+  ('L,Data1', 'H,Data2', 'H,Data3', '8s,Data4'))
+
+def GetPDBInfoFromImg(filename):
+  """Returns the PDB fingerprint and the pdb filename given an image file"""
+
+  pe = pefile.PE(filename)
+
+  for dbg in pe.DIRECTORY_ENTRY_DEBUG:
+    if dbg.struct.Type == 2:  # IMAGE_DEBUG_TYPE_CODEVIEW
+      off = dbg.struct.AddressOfRawData
+      size = dbg.struct.SizeOfData
+      data = pe.get_memory_mapped_image()[off:off+size]
+
+      cv = pefile.Structure(__CV_INFO_PDB70_format__)
+      cv.__unpack__(data)
+      cv.PdbFileName = data[cv.sizeof():]
+      guid = pefile.Structure(__GUID_format__)
+      guid.__unpack__(cv.Signature)
+      guid.Data4_0 = ''.join("%02X" % ord(x) for x in guid.Data4[0:2])
+      guid.Data4_1 = ''.join("%02X" % ord(x) for x in guid.Data4[2:])
+
+      return ("%08X%04X%04X%s%s%d" % (
+          guid.Data1, guid.Data2, guid.Data3,
+          guid.Data4_0, guid.Data4_1, cv.Age),
+          cv.PdbFileName.split('\x00', 1)[0])
+
+    break
+
+if __name__ == '__main__':
+  if len(sys.argv) != 2:
+    print "usage: file.dll"
+    sys.exit(1)
+
+  (fingerprint, file) = GetPDBInfoFromImg(sys.argv[1])
+  print "%s %s" % (fingerprint, file)

tools/symsrc/source_index.py

+#!/usr/bin/env python
+
+# Copyright (c) 2008 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Usage: <win-path-to-pdb.pdb>
+This tool will take a PDB on the command line, extract the source files that
+were used in building the PDB, query SVN for which repository and revision
+these files are at, and then finally write this information back into the PDB
+in a format that the debugging tools understand.  This allows for automatic
+source debugging, as all of the information is contained in the PDB, and the
+debugger can go out and fetch the source files via SVN.
+
+You most likely want to run these immediately after a build, since the source
+input files need to match the generated PDB, and we want the correct SVN
+revision information for the exact files that were used for the build.
+
+The following files from a windbg + source server installation are expected
+to reside in the same directory as this python script:
+  dbghelp.dll
+  pdbstr.exe
+  srctool.exe
+
+NOTE: Expected to run under a native win32 python, NOT cygwin.  All paths are
+dealt with as win32 paths, since we have to interact with the Microsoft tools.
+"""
+
+import sys
+import os
+import time
+import subprocess
+import tempfile
+
+# This serves two purposes.  First, it acts as a whitelist, and only files
+# from repositories listed here will be source indexed.  Second, it allows us
+# to map from one SVN URL to another, so we can map to external SVN servers.
+REPO_MAP = {
+  "svn://chrome-svn/chrome": "http://src.chromium.org/svn",
+  "svn://chrome-svn.corp.google.com/chrome": "http://src.chromium.org/svn",
+  "http://v8.googlecode.com/svn": None,
+  "http://google-breakpad.googlecode.com/svn": None,
+  "http://googletest.googlecode.com/svn": None,
+  "http://open-vcdiff.googlecode.com/svn": None,
+  "http://google-url.googlecode.com/svn": None,
+}
+
+def FindFile(filename):
+  """Return the full windows path to a file in the same dir as this code."""
+  thisdir = os.path.dirname(os.path.join(os.path.curdir, __file__))
+  return os.path.abspath(os.path.join(thisdir, filename))
+
+
+def ExtractSourceFiles(pdb_filename):
+  """Extract a list of local paths of the source files from a PDB."""
+  srctool = subprocess.Popen([FindFile('srctool.exe'), '-r', pdb_filename],
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+  filelist = srctool.stdout.read()
+  res = srctool.wait()
+  if res != 0 or filelist.startswith("srctool: "):
+    raise "srctool failed: " + filelist
+  return [x for x in filelist.split('\r\n') if len(x) != 0]
+
+def ReadSourceStream(pdb_filename):
+  """Read the contents of the source information stream from a PDB."""
+  srctool = subprocess.Popen([FindFile('pdbstr.exe'),
+                              '-r', '-s:srcsrv',
+                              '-p:%s' % pdb_filename],
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+  data = srctool.stdout.read()
+  res = srctool.wait()
+
+  if (res != 0 and res != -1) or data.startswith("pdbstr: "):
+    raise "pdbstr failed: " + data
+  return data
+
+def WriteSourceStream(pdb_filename, data):
+  """Write the contents of the source information stream to a PDB."""
+  # Write out the data to a temporary filename that we can pass to pdbstr.
+  (f, fname) = tempfile.mkstemp()
+  f = os.fdopen(f, "wb")
+  f.write(data)
+  f.close()
+
+  srctool = subprocess.Popen([FindFile('pdbstr.exe'),
+                              '-w', '-s:srcsrv',
+                              '-i:%s' % fname,
+                              '-p:%s' % pdb_filename],
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+  data = srctool.stdout.read()
+  res = srctool.wait()
+
+  if (res != 0 and res != -1) or data.startswith("pdbstr: "):
+    raise "pdbstr failed: " + data
+
+  os.unlink(fname)
+
+# TODO for performance, we should probably work in directories instead of
+# files.  I'm scared of DEPS and generated files, so for now we query each
+# individual file, and don't make assumptions that all files in the same
+# directory are part of the same repository or at the same revision number.
+def ExtractSvnInfo(local_filename):
+  """Calls svn info to extract the repository, path, and revision."""
+  # We call svn.bat to make sure and get the depot tools SVN and not cygwin.
+  srctool = subprocess.Popen(['svn.bat', 'info', local_filename],
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+  info = srctool.stdout.read()
+  res = srctool.wait()
+  if res != 0:
+    return None
+  # Hack up into a dictionary of the fields printed by svn info.
+  vals = dict((y.split(': ', 2) for y in info.split('\r\n') if y))
+
+  root = vals['Repository Root']
+  if not vals['URL'].startswith(root):
+    raise "URL is not inside of the repository root?!?"
+  path = vals['URL'][len(root):]
+  rev  = int(vals['Revision'])
+
+  return [root, path, rev]
+
+def UpdatePDB(pdb_filename):
+  """Update a pdb file with source information."""
+  dir_blacklist = { }
+  # TODO(deanm) look into "compressing" our output, by making use of vars
+  # and other things, so we don't need to duplicate the repo path and revs.
+  lines = [
+    'SRCSRV: ini ------------------------------------------------',
+    'VERSION=1',
+    'INDEXVERSION=2',
+    'VERCTRL=Subversion',
+    'DATETIME=%s' % time.asctime(),
+    'SRCSRV: variables ------------------------------------------',
+    'SVN_EXTRACT_TARGET=%targ%\%fnbksl%(%var3%)\%var4%\%fnfile%(%var1%)',
+    'SVN_EXTRACT_CMD=cmd /c svn cat "%var2%%var3%@%var4%" --non-interactive > "%svn_extract_target%"',
+    'SRCSRVTRG=%SVN_extract_target%',
+    'SRCSRVCMD=%SVN_extract_cmd%',
+    'SRCSRV: source files ---------------------------------------',
+  ]
+
+  if ReadSourceStream(pdb_filename):
+    raise "PDB already has source indexing information!"
+
+  filelist = ExtractSourceFiles(pdb_filename)
+  for filename in filelist:
+    filedir = os.path.dirname(filename)
+
+    print "Processing: %s" % filename
+    # This directory is blacklisted, either because it's not part of the SVN
+    # repository, or from one we're not interested in indexing.
+    if dir_blacklist.get(filedir, False):
+      print "  skipping, directory is blacklisted."
+      continue
+
+    info = ExtractSvnInfo(filename)
+
+    # Skip the file if it's not under an svn repository.  To avoid constantly
+    # querying SVN for files outside of SVN control (for example, the CRT
+    # sources), check if the directory is outside of SVN and blacklist it.
+    if not info:
+      if not ExtractSvnInfo(filedir):
+        dir_blacklist[filedir] = True
+      print "  skipping, file is not in an SVN repository"
+      continue
+
+    root = info[0]
+    path = info[1]
+    rev  = info[2]
+
+    # Check if file was from a svn repository we don't know about, or don't
+    # want to index.  Blacklist the entire directory.
+    if not REPO_MAP.has_key(info[0]):
+      print "  skipping, file is from an unknown SVN repository %s" % root
+      dir_blacklist[filedir] = True
+      continue
+
+    # We might want to map an internal repository URL to an external repository.
+    if REPO_MAP[root]:
+      root = REPO_MAP[root]
+
+    lines.append('%s*%s*%s*%s' % (filename, root, path, rev))
+    print "  indexed file."
+
+  lines.append('SRCSRV: end ------------------------------------------------')
+
+  WriteSourceStream(pdb_filename, '\r\n'.join(lines))
+
+if __name__ == '__main__':
+  if len(sys.argv) != 2:
+    print "usage: file.pdb"
+    sys.exit(1)
+
+  UpdatePDB(sys.argv[1])