Mostly automatic incremental link enabling

Patch and tool to make turning on incremental linking on Windows easy.

BUG=94837
TEST=No link errors


Review URL: http://codereview.chromium.org/8059024

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@103474 0039d316-1c4b-4281-b951-d872f2087c98

build/common.gypi

@@ -436,7 +436,7 @@
 
     # Turn on Use Library Dependency Inputs for linking chrome.dll on Windows
     # to get incremental linking to be faster in debug builds.
-    'incremental_chrome_dll%': 0,
+    'incremental_chrome_dll%': '<!(python <(DEPTH)/tools/win/supalink/check_installed.py)',
 
     # This is the location of the sandbox binary. Chrome looks for this before
     # running the zygote process. If found, and SUID, it will be used to

tools/win/supalink/README

+Linker shim that enables the use of "Use Library Dependency Inputs" on
+large exe/dlls via Visual Studio.
+
+That flag is required to enable useful incremental linking, however, with a
+large number of objects in components, the linker fails with:
+
+...RSP00002E45885644.rsp : fatal error LNK1170: line in command file contains 131071 or more characters
+
+This seems to be that the IDE team didn't talk to the linker team; the
+response file can handle long commands, just all the files can't be on
+*one* line which is what the IDE generates.
+
+So, this program simply replaces link.exe, fixes the response file, and
+then shells to the original linker. Ridiculous? Yes. Faster links? Yes.
+
+Compile/install with install_supalink.py. Run from cmd.exe as Administrator,
+and make sure to run vsvars32.bat first.

tools/win/supalink/check_installed.py

+#!/usr/bin/env python
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import sys
+import os
+
+def main():
+  if sys.platform != 'win32':
+    sys.stdout.write('0')
+    return 0
+
+  import _winreg
+
+  try:
+    val = _winreg.QueryValue(_winreg.HKEY_CURRENT_USER,
+                             'Software\\Chromium\\supalink_installed')
+    if os.path.exists(val):
+      # Apparently gyp thinks this means there was an error?
+      #sys.stderr.write('Supalink enabled.\n')
+      sys.stdout.write('1')
+      return 0
+  except WindowsError:
+    pass
+
+  sys.stdout.write('0')
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())

tools/win/supalink/install_supalink.py

+#!/usr/bin/env python
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import _winreg
+
+
+VSVARS_PATH = ('C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\'
+               'Common7\\Tools\\vsvars32.bat')
+
+
+def run_with_vsvars(cmd):
+  (fd, filename) = tempfile.mkstemp('.bat', text=True)
+  f = os.fdopen(fd, "w")
+  f.write('@echo off\n')
+  f.write('call "%s"\n' % VSVARS_PATH)
+  f.write(cmd + '\n')
+  f.close()
+  try:
+    p = subprocess.Popen([filename], shell=True, stdout=subprocess.PIPE,
+        universal_newlines=True)
+    out, err = p.communicate()
+    return p.returncode, out
+  finally:
+    os.unlink(filename)
+
+
+def get_vc_dir():
+  rc, out = run_with_vsvars('echo VCINSTALLDIR=%VCINSTALLDIR%')
+  for line in out.splitlines():
+    if line.startswith('VCINSTALLDIR='):
+      return line[len('VCINSTALLDIR='):]
+  return None
+
+
+def main():
+  vcdir = os.environ.get('VCINSTALLDIR')
+  if not vcdir:
+    vcdir = get_vc_dir()
+    if not vcdir:
+      print 'Couldn\'t get VCINSTALLDIR. Run vsvars32.bat?'
+      return 1
+    os.environ['PATH'] += (';' + os.path.join(vcdir, 'bin') +
+                           ';' + os.path.join(vcdir, '..\\Common7\\IDE'))
+
+  # Switch to our own dir.
+  os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+  # Verify that we can find link.exe.
+  link = os.path.join(vcdir, 'bin', 'link.exe')
+  link_backup = os.path.join(vcdir, 'bin', 'link.exe.supalink_orig.exe')
+  if not os.path.exists(link):
+    print 'link.exe not found at %s' % link
+    return 1
+
+  # Don't re-backup link.exe, so only copy link.exe to backup if it's
+  # not there already.
+  if not os.path.exists(link_backup):
+    try:
+      print 'Saving original link.exe...'
+      shutil.copyfile(link, link_backup)
+    except IOError:
+      print (('Wasn\'t able to back up %s to %s. '
+              'Not running with Administrator privileges?')
+              % (link, link_backup))
+      return 1
+
+  # Build supalink.exe but only if it's out of date.
+  cpptime = os.path.getmtime('supalink.cpp')
+  if (not os.path.exists('supalink.exe') or
+      cpptime > os.path.getmtime('supalink.exe')):
+    print 'Building supalink.exe...'
+    rc, out = run_with_vsvars('cl /nologo /Ox /Zi /W4 /WX /D_UNICODE /DUNICODE'
+                              ' /D_CRT_SECURE_NO_WARNINGS /EHsc supalink.cpp'
+                              ' /link /out:supalink.exe')
+    if rc:
+      print out
+      print 'Failed to build supalink.exe'
+      return 1
+
+  # Copy supalink into place if it's been updated since last time we ran.
+  exetime = os.path.getmtime('supalink.exe')
+  if exetime > os.path.getmtime(link):
+    print 'Copying supalink.exe over link.exe...'
+    try:
+      shutil.copyfile('supalink.exe', link)
+    except IOError:
+      print ('Wasn\'t able to copy supalink.exe over %s. '
+             'Not running with Administrator privileges?' % link)
+      return 1
+
+  _winreg.SetValue(_winreg.HKEY_CURRENT_USER,
+                   'Software\\Chromium\\supalink_installed',
+                   _winreg.REG_SZ,
+                   link_backup)
+
+  print 'Linker shim installed. Regenerate via gyp: "gclient runhooks".'
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())

tools/win/supalink/supalink.cpp

+// Copyright (c) 2011 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <windows.h>
+#include <algorithm>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string>
+
+using namespace std;
+
+// Don't use stderr for errors because VS has large buffers on them, leading
+// to confusing error output.
+static void Fatal(const wchar_t* msg) {
+  wprintf(L"supalink fatal error: %s\n", msg);
+  exit(1);
+}
+
+static wstring ErrorMessageToString(DWORD err) {
+  wchar_t* msg_buf = NULL;
+  DWORD rc = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER |
+                               FORMAT_MESSAGE_FROM_SYSTEM,
+                           NULL,
+                           err,
+                           MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+                           reinterpret_cast<LPTSTR>(&msg_buf),
+                           0,
+                           NULL);
+  if (!rc)
+    return L"unknown error";
+  wstring ret(msg_buf);
+  LocalFree(msg_buf);
+  return ret;
+}
+
+static const wchar_t* const g_search_for[] = {
+  L"link.exe\" ",
+  L"link\" ",
+  L"link.exe ",
+  L"link ",
+};
+
+static void Fallback(const wchar_t* msg = NULL) {
+  if (msg) {
+    wprintf(L"supalink failed (%s), trying to fallback to standard link.\n",
+            msg);
+    wprintf(L"Original command line: %s\n", GetCommandLine());
+    fflush(stdout);
+  }
+
+  STARTUPINFO startup_info = { sizeof(STARTUPINFO) };
+  PROCESS_INFORMATION process_info;
+  DWORD exit_code;
+
+  GetStartupInfo(&startup_info);
+
+  wstring orig_cmd(GetCommandLine());
+  wstring cmd;
+  wstring replace_with = L"link.exe.supalink_orig.exe";
+  wstring orig_lowercase;
+
+  // Avoid searching for case-variations by making a lower-case copy.
+  transform(orig_cmd.begin(),
+            orig_cmd.end(),
+            back_inserter(orig_lowercase),
+            tolower);
+
+  for (size_t i = 0; i < ARRAYSIZE(g_search_for); ++i) {
+    wstring linkexe = g_search_for[i];
+    wstring::size_type at = orig_lowercase.find(linkexe, 0);
+    if (at == wstring::npos)
+      continue;
+    if (linkexe[linkexe.size() - 2] == L'"')
+      replace_with += L"\" ";
+    else
+      replace_with += L" ";
+    cmd = orig_cmd.replace(at, linkexe.size(), replace_with);
+    break;
+  }
+  if (cmd == L"") {
+    wprintf(L"Original run '%s'\n", orig_cmd.c_str());
+    Fatal(L"Couldn't find link.exe (or similar) in command line");
+  }
+
+  if (getenv("SUPALINK_DEBUG")) {
+    wprintf(L"  running '%s'\n", cmd.c_str());
+    fflush(stdout);
+  }
+  if (!CreateProcess(NULL,
+                     reinterpret_cast<LPWSTR>(const_cast<wchar_t *>(
+                             cmd.c_str())),
+                     NULL,
+                     NULL,
+                     TRUE,
+                     0,
+                     NULL,
+                     NULL,
+                     &startup_info, &process_info)) {
+    wstring error = ErrorMessageToString(GetLastError());
+    Fatal(error.c_str());
+  }
+  CloseHandle(process_info.hThread);
+  WaitForSingleObject(process_info.hProcess, INFINITE);
+  GetExitCodeProcess(process_info.hProcess, &exit_code);
+  CloseHandle(process_info.hProcess);
+  exit(exit_code);
+}
+
+wstring SlurpFile(const wchar_t* path) {
+  FILE* f = _wfopen(path, L"rb, ccs=UNICODE");
+  if (!f) Fallback(L"couldn't read file");
+  fseek(f, 0, SEEK_END);
+  long len = ftell(f);
+  rewind(f);
+  wchar_t* data = reinterpret_cast<wchar_t*>(malloc(len));
+  fread(data, 1, len, f);
+  fclose(f);
+  wstring ret(data, len/sizeof(wchar_t));
+  free(data);
+  return ret;
+}
+
+void DumpFile(const wchar_t* path, wstring& contents) {
+  FILE* f = _wfopen(path, L"wb, ccs=UTF-16LE");
+  if (!f) Fallback(L"couldn't write file");
+
+  fwrite(contents.c_str(), sizeof(wchar_t), contents.size(), f);
+  if (ferror(f))
+    Fatal(L"failed during response rewrite");
+  fclose(f);
+}
+
+// Input command line is assumed to be of the form:
+//
+// link.exe @C:\src\...\RSP00003045884740.rsp /NOLOGO /ERRORREPORT:PROMPT
+//
+// Specifically, we parse & hack the contents of argv[1] and pass the rest
+// onwards.
+int wmain(int argc, wchar_t** argv) {
+  ULONGLONG start_time = 0, end_time;
+
+  int rsp_file_index = -1;
+
+  if (argc < 2)
+    Fallback(L"too few commmand line args");
+
+  for (int i = 1; i < argc; ++i) {
+    if (argv[i][0] == L'@') {
+      rsp_file_index = i;
+      break;
+    }
+  }
+
+  if (rsp_file_index == -1)
+    Fallback(L"couldn't find a response file in argv");
+
+  if (_wgetenv(L"SUPALINK_DEBUG"))
+    start_time = GetTickCount64();
+
+  wstring rsp = SlurpFile(&argv[rsp_file_index][1]);
+
+  // The first line of this file is all we try to fix. It's a bunch of
+  // quoted space separated items. Simplest thing seems to be replacing " "
+  // with "\n". So, just slurp the file, replace, spit it out to the same
+  // file and continue on our way.
+
+  // Took about .5s when using the naive .replace loop to replace " " with
+  // "\r\n" so write the silly loop instead.
+  wstring fixed;
+  fixed.reserve(rsp.size() * 2);
+
+  for (const wchar_t* s = rsp.c_str(); *s;) {
+    if (*s == '"' && *(s + 1) == ' ' && *(s + 2) == '"') {
+      fixed += L"\"\r\n\"";
+      s += 3;
+    } else {
+      fixed += *s++;
+    }
+  }
+
+  DumpFile(&argv[rsp_file_index][1], fixed);
+
+  if (_wgetenv(L"SUPALINK_DEBUG")) {
+    wstring backup_copy(&argv[rsp_file_index][1]);
+    backup_copy += L".copy";
+    DumpFile(backup_copy.c_str(), fixed);
+
+    end_time = GetTickCount64();
+
+    wprintf(L"  took %.2fs to modify @rsp file\n",
+            (end_time - start_time) / 1000.0);
+  }
+
+  Fallback();
+}