Add libclang package to sit alongside prebuilt LLVM binaries

We’re starting to experiment with AST-level analysis of chromium
sources, and are using the python bindings for clang. We would like
to ensure that these bindings are up-to-date as the clang toolchain
rolls. So we would like to add a separate zip package containing
libclang to sit alongside the prebuilt LLVM binaries and be
downloaded on-demand.

Change-Id: Idc744deb03e0a8ff7c50f8d0f883e655f271cd30
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1874450
Commit-Queue: Taylor Woll <Taylor.Woll@microsoft.com>
Reviewed-by: Hans Wennborg <hans@chromium.org>
Reviewed-by: Nico Weber <thakis@chromium.org>
Cr-Commit-Position: refs/heads/master@{#708922}

build/download_libclang.py

+#!/usr/bin/env python
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Script to download libclang binaries from google storage."""
+
+import find_depot_tools
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tarfile
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+CHROME_SRC = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir))
+
+
+DEPOT_PATH = find_depot_tools.add_depot_tools_to_path()
+GSUTIL_PATH = os.path.join(DEPOT_PATH, 'gsutil.py')
+
+LLVM_BUILD_PATH = os.path.join(CHROME_SRC, 'third_party', 'llvm-build',
+                               'Release+Asserts')
+CLANG_UPDATE_PY = os.path.join(CHROME_SRC, 'tools', 'clang', 'scripts',
+                               'update.py')
+
+CLANG_BUCKET = 'gs://chromium-browser-clang'
+
+
+def main():
+  clang_revision = subprocess.check_output([sys.executable, CLANG_UPDATE_PY,
+                                            '--print-revision']).rstrip()
+  targz_name = 'libclang-%s.tgz' % clang_revision
+
+  if sys.platform == 'win32' or sys.platform == 'cygwin':
+    cds_full_url = CLANG_BUCKET + '/Win/' + targz_name
+  elif sys.platform == 'darwin':
+    cds_full_url = CLANG_BUCKET + '/Mac/' + targz_name
+  else:
+    assert sys.platform.startswith('linux')
+    cds_full_url = CLANG_BUCKET + '/Linux_x64/' + targz_name
+
+  os.chdir(LLVM_BUILD_PATH)
+
+  subprocess.check_call([sys.executable, GSUTIL_PATH,
+                         'cp', cds_full_url, targz_name])
+  tarfile.open(name=targz_name, mode='r:gz').extractall(path=LLVM_BUILD_PATH)
+
+  os.remove(targz_name)
+  return 0
+
+if __name__ == '__main__':
+  sys.exit(main())

tools/clang/scripts/package.py

@@ -487,6 +487,25 @@ def main():
             filter=PrintTarProgress)
   MaybeUpload(args.upload, translation_unit_dir + '.tgz', platform)
 
+  # Zip up the libclang binaries.
+  libclang_dir = 'libclang-' + stamp
+  shutil.rmtree(libclang_dir, ignore_errors=True)
+  os.makedirs(os.path.join(libclang_dir, 'bin'))
+  os.makedirs(os.path.join(libclang_dir, 'bindings', 'python', 'clang'))
+  if sys.platform == 'win32':
+    shutil.copy(os.path.join(LLVM_RELEASE_DIR, 'bin', 'libclang.dll'),
+                os.path.join(libclang_dir, 'bin'))
+  for filename in ['__init__.py', 'cindex.py', 'enumerations.py']:
+    shutil.copy(os.path.join(LLVM_DIR, 'clang', 'bindings', 'python', 'clang',
+                             filename),
+                os.path.join(libclang_dir, 'bindings', 'python', 'clang'))
+  tar_entries = ['bin', 'bindings' ]
+  with tarfile.open(libclang_dir + '.tgz', 'w:gz') as tar:
+    for entry in tar_entries:
+      tar.add(os.path.join(libclang_dir, entry), arcname=entry,
+              filter=PrintTarProgress)
+  MaybeUpload(args.upload, libclang_dir + '.tgz', platform)
+
   if sys.platform == 'win32' and args.upload:
     UploadPDBToSymbolServer()