Commit 8ee64a1e authored by Hans Wennborg's avatar Hans Wennborg Committed by Commit Bot

Clang update.py: Add --package option

and use that to download auxiliary packages, removing the custom scripts
previously used to do that.

Bug: 884608
Change-Id: Iefcd2145898588a6dbdc14c21841201f9d17f8c0
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1605911
Commit-Queue: Hans Wennborg <hans@chromium.org>
Reviewed-by: default avatarNico Weber <thakis@chromium.org>
Cr-Commit-Position: refs/heads/master@{#712644}
parent 498caa6b
......@@ -3292,7 +3292,8 @@ hooks = [
'name': 'clang_coverage',
'pattern': '.',
'condition': 'checkout_clang_coverage_tools',
'action': ['python', 'src/tools/code_coverage/update_clang_coverage_tools.py'],
'action': ['python', 'src/tools/clang/scripts/update.py',
'--package=coverage_tools'],
},
{
# Mac doesn't use lld so it's not included in the default clang bundle
......@@ -3301,7 +3302,8 @@ hooks = [
'name': 'lld/mac',
'pattern': '.',
'condition': 'host_os == "mac" and (checkout_win or checkout_fuchsia)',
'action': ['python', 'src/tools/clang/scripts/download_lld_mac.py'],
'action': ['python', 'src/tools/clang/scripts/update.py',
'--package=lld_mac'],
},
{
# Update LASTCHANGE.
......
......@@ -5,50 +5,16 @@
"""Script to download libclang binaries from google storage."""
import find_depot_tools
import json
import os
import shutil
import subprocess
import sys
import tarfile
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
CHROME_SRC = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir))
sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..',
'tools', 'clang', 'scripts'))
import update
DEPOT_PATH = find_depot_tools.add_depot_tools_to_path()
GSUTIL_PATH = os.path.join(DEPOT_PATH, 'gsutil.py')
LLVM_BUILD_PATH = os.path.join(CHROME_SRC, 'third_party', 'llvm-build',
'Release+Asserts')
CLANG_UPDATE_PY = os.path.join(CHROME_SRC, 'tools', 'clang', 'scripts',
'update.py')
CLANG_BUCKET = 'gs://chromium-browser-clang'
def main():
clang_revision = subprocess.check_output([sys.executable, CLANG_UPDATE_PY,
'--print-revision']).rstrip()
targz_name = 'libclang-%s.tgz' % clang_revision
if sys.platform == 'win32' or sys.platform == 'cygwin':
cds_full_url = CLANG_BUCKET + '/Win/' + targz_name
elif sys.platform == 'darwin':
cds_full_url = CLANG_BUCKET + '/Mac/' + targz_name
else:
assert sys.platform.startswith('linux')
cds_full_url = CLANG_BUCKET + '/Linux_x64/' + targz_name
os.chdir(LLVM_BUILD_PATH)
subprocess.check_call([sys.executable, GSUTIL_PATH,
'cp', cds_full_url, targz_name])
tarfile.open(name=targz_name, mode='r:gz').extractall(path=LLVM_BUILD_PATH)
os.remove(targz_name)
return 0
# TODO(hans): Remove this forwarding hack after all callers of this script have
# been updated to call update.py instead.
if __name__ == '__main__':
sys.exit(main())
sys.argv = [sys.argv[0], '--package=libclang']
sys.exit(update.main())
......@@ -3,52 +3,16 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Script to download Clang translation_unit tool from google storage."""
import find_depot_tools
import json
import os
import shutil
import subprocess
import sys
import tarfile
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
CHROME_SRC = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir))
DEPOT_PATH = find_depot_tools.add_depot_tools_to_path()
GSUTIL_PATH = os.path.join(DEPOT_PATH, 'gsutil.py')
LLVM_BUILD_PATH = os.path.join(CHROME_SRC, 'third_party', 'llvm-build',
'Release+Asserts')
CLANG_UPDATE_PY = os.path.join(CHROME_SRC, 'tools', 'clang', 'scripts',
'update.py')
CLANG_BUCKET = 'gs://chromium-browser-clang'
def main():
clang_revision = subprocess.check_output([sys.executable, CLANG_UPDATE_PY,
'--print-revision']).rstrip()
targz_name = 'translation_unit-%s.tgz' % clang_revision
if sys.platform == 'win32' or sys.platform == 'cygwin':
cds_full_url = CLANG_BUCKET + '/Win/' + targz_name
elif sys.platform == 'darwin':
cds_full_url = CLANG_BUCKET + '/Mac/' + targz_name
else:
assert sys.platform.startswith('linux')
cds_full_url = CLANG_BUCKET + '/Linux_x64/' + targz_name
os.chdir(LLVM_BUILD_PATH)
subprocess.check_call([sys.executable, GSUTIL_PATH,
'cp', cds_full_url, targz_name])
tarfile.open(name=targz_name, mode='r:gz').extractall(path=LLVM_BUILD_PATH)
sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..',
'tools', 'clang', 'scripts'))
os.remove(targz_name)
return 0
import update
# TODO(hans): Remove this forwarding hack after all callers of this script have
# been updated to call update.py instead.
if __name__ == '__main__':
sys.exit(main())
sys.argv = [sys.argv[0], '--package=translation_unit']
sys.exit(update.main())
......@@ -96,14 +96,7 @@ criteria:
If you want to add something to the clang package that doesn't (yet?) meet
these criteria, you can make package.py upload it to a separate zip file
and then download it on an opt-in basis by requiring users to run a script
to download the additional zip file. You can structure your script in a way that
it downloads your additional zip automatically if the script detects an
old version on disk, that way users have to run the download script just
once. `tools/clang/scripts/download_lld_mac.py` is an example for this
(It doesn't do the "only download if old version is on disk or if requested"
bit, and hence doesn't run as a default DEPS hook. TODO(thakis): Make
coverage stuff a better example and link to that.)
and then download it on an opt-in basis by using update.py's --package option.
If you're adding a new feature that you expect will meet the inclusion criteria
eventually but doesn't yet, start by having your things in a separate zip
......
......@@ -39,8 +39,8 @@ _RESOURCE_SIZES_PATH = os.path.join(
_SRC_ROOT, 'build', 'android', 'resource_sizes.py')
_LLVM_TOOLS_DIR = os.path.join(
_SRC_ROOT, 'third_party', 'llvm-build', 'Release+Asserts', 'bin')
_DOWNLOAD_OBJDUMP_PATH = os.path.join(
_SRC_ROOT, 'tools', 'clang', 'scripts', 'download_objdump.py')
_CLANG_UPDATE_PATH = os.path.join(_SRC_ROOT, 'tools', 'clang', 'scripts',
'update.py')
_GN_PATH = os.path.join(_SRC_ROOT, 'third_party', 'depot_tools', 'gn')
_NINJA_PATH = os.path.join(_SRC_ROOT, 'third_party', 'depot_tools', 'ninja')
......@@ -767,9 +767,9 @@ def _TmpCopyBinarySizeDir():
shutil.copytree(_BINARY_SIZE_DIR, bs_dir)
# We also copy the tools supersize needs, but only if they exist.
tool_prefix = None
if os.path.exists(_DOWNLOAD_OBJDUMP_PATH):
if os.path.exists(_CLANG_UPDATE_PATH):
if not os.path.exists(os.path.join(_LLVM_TOOLS_DIR, 'llvm-readelf')):
_RunCmd([_DOWNLOAD_OBJDUMP_PATH])
_RunCmd([_CLANG_UPDATE_PATH, '--package=objdump'])
tools_dir = os.path.join(bs_dir, 'bintools')
tool_prefix = os.path.join(tools_dir, 'llvm-')
shutil.copytree(_LLVM_TOOLS_DIR, tools_dir)
......
......@@ -112,7 +112,7 @@ class ToolPrefixFinder(_PathFinder):
err_lines = ['tool-prefix not found: %s' % ret]
if ret.endswith('llvm-'):
err_lines.append('Probably need to run: '
'tools/clang/scripts/download_objdump.py')
'tools/clang/scripts/update.py --package=objdump')
raise Exception('\n'.join(err_lines))
from_path = distutils.spawn.find_executable(_SAMPLE_TOOL_SUFFIX)
if from_path:
......
......@@ -5,34 +5,12 @@
"""Script to download lld/mac from google storage."""
import os
import re
import subprocess
import sys
import update
LLVM_BUILD_DIR = update.LLVM_BUILD_DIR
LLD_LINK_PATH = os.path.join(LLVM_BUILD_DIR, 'bin', 'lld-link')
def AlreadyUpToDate():
if not os.path.exists(LLD_LINK_PATH):
return False
lld_rev = subprocess.check_output([LLD_LINK_PATH, '--version'])
# Version output example:
# LLD 9.0.0 (https://github.com/llvm/llvm-project/ 342571e8d6eb1afb151ae1103431798e3d24054f)
return (re.match(r'LLD.*\(.*git.*llvm.* ([0-9a-f]+)\)', lld_rev).group(1) ==
update.CLANG_REVISION)
def main():
if AlreadyUpToDate():
return 0
remote_path = '%s/Mac/lld-%s.tgz' % (update.CDS_URL, update.PACKAGE_VERSION)
update.DownloadAndUnpack(remote_path, update.LLVM_BUILD_DIR)
return 0
# TODO(hans): Remove this forwarding hack after all callers of this script have
# been updated to call update.py instead.
if __name__ == '__main__':
sys.exit(main())
sys.argv = [sys.argv[0], '--package=lld_mac']
sys.exit(update.main())
......@@ -3,47 +3,12 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Script to download llvm-objdump and related utils from google storage."""
from __future__ import print_function
import os
import re
import subprocess
import sys
import urllib2
import update
LLVM_BUILD_DIR = update.LLVM_BUILD_DIR
OBJDUMP_PATH = os.path.join(LLVM_BUILD_DIR, 'bin', 'llvm-objdump')
STAMP_FILE = os.path.normpath(
os.path.join(LLVM_BUILD_DIR, 'llvmobjdump_build_revision'))
def AlreadyUpToDate():
if not os.path.exists(OBJDUMP_PATH) or not os.path.exists(STAMP_FILE):
return False
stamp = update.ReadStampFile(STAMP_FILE)
return stamp.rstrip() == update.PACKAGE_VERSION
def DownloadAndUnpackLlvmObjDumpPackage(platform):
cds_file = 'llvmobjdump-%s.tgz' % update.PACKAGE_VERSION
cds_full_url = update.GetPlatformUrlPrefix(platform) + cds_file
try:
update.DownloadAndUnpack(cds_full_url, update.LLVM_BUILD_DIR)
except urllib2.URLError:
print('Failed to download prebuilt utils %s' % cds_file)
print('Use --force-local-build if you want to build locally.')
print('Exiting.')
sys.exit(1)
def main():
if not AlreadyUpToDate():
DownloadAndUnpackLlvmObjDumpPackage(sys.platform)
return 0
# TODO(hans): Remove this forwarding hack after all callers of this script have
# been updated to call update.py instead.
if __name__ == '__main__':
sys.exit(main())
sys.argv = [sys.argv[0], '--package=objdump']
sys.exit(update.main())
......@@ -9,7 +9,9 @@
It can also be run stand-alone as a convenient way of installing a well-tested
near-tip-of-tree clang version:
$ curl -s https://raw.githubusercontent.com/chromium/chromium/master/tools/clang/scripts/update.py | python - --clang-dir=.
$ curl -s https://raw.githubusercontent.com/chromium/chromium/master/tools/clang/scripts/update.py | python - --output-dir=/tmp/clang
(Note that the output dir may be deleted and re-created if it exists.)
"""
# TODO: Running stand-alone won't work on Windows due to the dia dll copying.
......@@ -167,14 +169,29 @@ def GetPlatformUrlPrefix(platform):
return CDS_URL + '/Linux_x64/'
def DownloadAndUnpackClangPackage(platform, output_dir, path_prefixes=None):
def DownloadAndUnpackPackage(package_file, output_dir):
cds_file = "%s-%s.tgz" % (package_file, PACKAGE_VERSION)
cds_full_url = GetPlatformUrlPrefix(sys.platform) + cds_file
try:
DownloadAndUnpack(cds_full_url, output_dir)
except URLError:
print('Failed to download prebuilt clang package %s' % cds_file)
print('Use build.py if you want to build locally.')
print('Exiting.')
sys.exit(1)
# TODO(hans): Create a clang-win-runtime package instead.
def DownloadAndUnpackClangWinRuntime(output_dir):
cds_file = "clang-%s.tgz" % PACKAGE_VERSION
cds_full_url = GetPlatformUrlPrefix(platform) + cds_file
cds_full_url = GetPlatformUrlPrefix('win32') + cds_file
path_prefixes = [ 'lib/clang/' + RELEASE_VERSION + '/lib/',
'bin/llvm-symbolizer.exe' ]
try:
DownloadAndUnpack(cds_full_url, output_dir, path_prefixes)
except URLError:
print('Failed to download prebuilt clang %s' % cds_file)
print('Use --force-local-build if you want to build locally.')
print('Use build.py if you want to build locally.')
print('Exiting.')
sys.exit(1)
......@@ -231,19 +248,46 @@ def CopyDiaDllTo(target_dir):
CopyFile(dia_dll, target_dir)
def UpdateClang():
GCLIENT_CONFIG = os.path.join(os.path.dirname(CHROMIUM_DIR), '.gclient')
def UpdatePackage(package_name):
stamp_file = None
package_file = None
stamp_file = os.path.join(LLVM_BUILD_DIR, package_name + '_revision')
if package_name == 'clang':
stamp_file = STAMP_FILE
package_file = 'clang'
elif package_name == 'lld_mac':
package_file = 'lld'
if sys.platform != 'darwin':
print('The lld_mac package cannot be downloaded on non-macs.')
print('On non-mac, lld is included in the clang package.')
return 1
elif package_name == 'objdump':
package_file = 'llvmobjdump'
elif package_name == 'translation_unit':
package_file = 'translation_unit'
elif package_name == 'coverage_tools':
stamp_file = os.path.join(LLVM_BUILD_DIR, 'cr_coverage_revision')
package_file = 'llvm-code-coverage'
elif package_name == 'libclang':
package_file = 'libclang'
else:
print('Unknown package: "%s".' % package_name)
return 1
# Read target_os from .gclient so we know which non-native runtimes we need.
# TODO(pcc): See if we can download just the runtimes instead of the entire
# clang package, and do that from DEPS instead of here.
assert stamp_file is not None
assert package_file is not None
# TODO(hans): Create a clang-win-runtime package and use separate DEPS hook.
target_os = []
try:
env = {}
execfile(GCLIENT_CONFIG, env, env)
target_os = env.get('target_os', target_os)
except:
pass
if package_name == 'clang':
try:
GCLIENT_CONFIG = os.path.join(os.path.dirname(CHROMIUM_DIR), '.gclient')
env = {}
execfile(GCLIENT_CONFIG, env, env)
target_os = env.get('target_os', target_os)
except:
pass
if os.path.exists(OLD_STAMP_FILE):
# Delete the old stamp file so it doesn't look like an old version of clang
......@@ -252,34 +296,35 @@ def UpdateClang():
os.remove(OLD_STAMP_FILE)
expected_stamp = ','.join([PACKAGE_VERSION] + target_os)
if ReadStampFile(STAMP_FILE) == expected_stamp:
if ReadStampFile(stamp_file) == expected_stamp:
return 0
if os.path.exists(LLVM_BUILD_DIR):
# Updating the main clang package nukes the output dir. Any other packages
# need to be updated *after* the clang package.
if package_name == 'clang' and os.path.exists(LLVM_BUILD_DIR):
RmTree(LLVM_BUILD_DIR)
DownloadAndUnpackClangPackage(sys.platform, LLVM_BUILD_DIR)
if 'win' in target_os:
# When doing win/cross builds on other hosts, get the Windows runtime
# libraries, and llvm-symbolizer.exe (needed in asan builds).
path_prefixes = [ 'lib/clang/' + RELEASE_VERSION + '/lib/',
'bin/llvm-symbolizer.exe' ]
DownloadAndUnpackClangPackage('win32', LLVM_BUILD_DIR,
path_prefixes=path_prefixes)
if sys.platform == 'win32':
CopyDiaDllTo(os.path.join(LLVM_BUILD_DIR, 'bin'))
WriteStampFile(expected_stamp, STAMP_FILE)
DownloadAndUnpackPackage(package_file, LLVM_BUILD_DIR)
if package_name == 'clang':
if sys.platform == 'win32':
CopyDiaDllTo(os.path.join(LLVM_BUILD_DIR, 'bin'))
if 'win' in target_os:
# When doing win/cross builds on other hosts, get the Windows runtime
# libraries, and llvm-symbolizer.exe (needed in asan builds).
DownloadAndUnpackClangWinRuntime(LLVM_BUILD_DIR)
WriteStampFile(expected_stamp, stamp_file)
return 0
def main():
# TODO: Add an argument to download optional packages and remove the various
# download_ scripts we currently have for that.
parser = argparse.ArgumentParser(description='Update clang.')
parser.add_argument('--clang-dir',
help='Where to extract the clang package.')
parser.add_argument('--output-dir',
help='Where to extract the package.')
parser.add_argument('--package',
help='What package to update (default: clang)',
default='clang')
parser.add_argument('--force-local-build', action='store_true',
help='(no longer used)')
parser.add_argument('--print-revision', action='store_true',
......@@ -324,12 +369,12 @@ def main():
print('--llvm-force-head-revision can only be used for --print-revision')
return 1
if args.clang_dir:
if args.output_dir:
global LLVM_BUILD_DIR, STAMP_FILE
LLVM_BUILD_DIR = os.path.abspath(args.clang_dir)
LLVM_BUILD_DIR = os.path.abspath(args.output_dir)
STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision')
return UpdateClang()
return UpdatePackage(args.package)
if __name__ == '__main__':
......
......@@ -84,7 +84,7 @@ sys.path.append(
os.path.join(
os.path.dirname(__file__), os.path.pardir, os.path.pardir, 'tools',
'clang', 'scripts'))
from update import LLVM_BUILD_DIR
import update
sys.path.append(
os.path.join(
......@@ -93,11 +93,10 @@ sys.path.append(
from collections import defaultdict
import coverage_utils
import update_clang_coverage_tools
# Absolute path to the code coverage tools binary. These paths can be
# overwritten by user specified coverage tool paths.
LLVM_BIN_DIR = os.path.join(LLVM_BUILD_DIR, 'bin')
LLVM_BIN_DIR = os.path.join(update.LLVM_BUILD_DIR, 'bin')
LLVM_COV_PATH = os.path.join(LLVM_BIN_DIR, 'llvm-cov')
LLVM_PROFDATA_PATH = os.path.join(LLVM_BIN_DIR, 'llvm-profdata')
......@@ -158,7 +157,7 @@ def _ConfigureLLVMCoverageTools(args):
LLVM_COV_PATH = os.path.join(llvm_bin_dir, 'llvm-cov')
LLVM_PROFDATA_PATH = os.path.join(llvm_bin_dir, 'llvm-profdata')
else:
update_clang_coverage_tools.DownloadCoverageToolsIfNeeded()
update.UpdatePackage('coverage_tools')
coverage_tools_exist = (
os.path.exists(LLVM_COV_PATH) and os.path.exists(LLVM_PROFDATA_PATH))
......@@ -932,7 +931,7 @@ def Main():
# Setup coverage binaries even when script is called with empty params. This
# is used by coverage bot for initial setup.
if len(sys.argv) == 1:
update_clang_coverage_tools.DownloadCoverageToolsIfNeeded()
update.UpdatePackage('coverage_tools')
print(__doc__)
return
......
#!/usr/bin/env python
# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Download clang coverage tools (llvm-cov + llvm-profdata)"""
import os
import sys
import urllib2
sys.path.append(
os.path.join(
os.path.dirname(__file__), os.path.pardir, os.path.pardir,
'third_party'))
import coverage_utils
sys.path.append(
os.path.join(
os.path.dirname(__file__), os.path.pardir, os.path.pardir, 'tools',
'clang', 'scripts'))
import update as clang_update
# TODO(crbug.com/759794): remove this function once tools get included to
# Clang bundle:
# https://chromium-review.googlesource.com/c/chromium/src/+/688221
def DownloadCoverageToolsIfNeeded():
"""Temporary solution to download llvm-profdata and llvm-cov tools."""
def _GetRevisionFromStampFile(stamp_file_path):
"""Returns revision by reading the build stamp file.
Args:
stamp_file_path: A path the build stamp file created by
tools/clang/scripts/update.py.
Returns:
A string represeting the revision of the tool, such as 361212-67510fac-2.
"""
if not os.path.exists(stamp_file_path):
return ''
with open(stamp_file_path) as stamp_file:
stamp_file_line = stamp_file.readline()
if ',' in stamp_file_line:
package_version = stamp_file_line.rstrip().split(',')[0]
else:
package_version = stamp_file_line.rstrip()
return package_version
cov_path = os.path.join(clang_update.LLVM_BUILD_DIR, 'bin', 'llvm-cov')
profdata_path = os.path.join(
clang_update.LLVM_BUILD_DIR, 'bin', 'llvm-profdata')
host_platform = coverage_utils.GetHostPlatform()
clang_revision = _GetRevisionFromStampFile(clang_update.STAMP_FILE)
coverage_revision_stamp_file = os.path.join(
os.path.dirname(clang_update.STAMP_FILE), 'cr_coverage_revision')
coverage_revision = _GetRevisionFromStampFile(coverage_revision_stamp_file)
has_coverage_tools = (
os.path.exists(cov_path) and os.path.exists(profdata_path))
if (has_coverage_tools and clang_revision == coverage_revision):
# LLVM coverage tools are up to date, bail out.
return
package_version = clang_revision
coverage_tools_file = 'llvm-code-coverage-%s.tgz' % package_version
# The code below follows the code from tools/clang/scripts/update.py.
if host_platform == 'mac':
coverage_tools_url = clang_update.CDS_URL + '/Mac/' + coverage_tools_file
elif host_platform == 'linux':
coverage_tools_url = (
clang_update.CDS_URL + '/Linux_x64/' + coverage_tools_file)
else:
assert host_platform == 'win'
coverage_tools_url = (clang_update.CDS_URL + '/Win/' + coverage_tools_file)
try:
clang_update.DownloadAndUnpack(coverage_tools_url,
clang_update.LLVM_BUILD_DIR)
with open(coverage_revision_stamp_file, 'w') as file_handle:
file_handle.write('%s,%s' % (package_version, host_platform))
file_handle.write('\n')
except urllib2.URLError:
raise Exception(
'Failed to download coverage tools: %s.' % coverage_tools_url)
if __name__ == '__main__':
DownloadCoverageToolsIfNeeded()
sys.exit(0)
......@@ -236,8 +236,8 @@ _NM_PATH = os.path.join(_SRC_PATH, 'third_party', 'llvm-build',
def CheckLlvmNmExists():
assert os.path.exists(_NM_PATH), (
'llvm-nm not found. Please run //tools/clang/scripts/download_objdump.py'
' to install it.')
'llvm-nm not found. Please run '
'//tools/clang/scripts/update.py --package=objdump to install it.')
def SymbolNamesFromLlvmBitcodeFile(filename):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment