Copy verbatim the scripts for updating LLVM 10. The scripts will be updated to roll LLVM 16 in a follow up CL. Bug: b/272710814 Change-Id: I8f0abc2ab30d4bc87df45b1d89727b3885d2b9d6 Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/70949 Tested-by: Jean-François Geyelin <jif@google.com> Commit-Queue: Jean-François Geyelin <jif@google.com> Reviewed-by: Geoff Lang <geofflang@google.com>
diff --git a/third_party/llvm-16.0/scripts/update.py b/third_party/llvm-16.0/scripts/update.py new file mode 100644 index 0000000..f6cf4bf --- /dev/null +++ b/third_party/llvm-16.0/scripts/update.py
@@ -0,0 +1,365 @@ +#!/usr/bin/env python3 + +# Copyright 2018 The SwiftShader Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import contextlib +import multiprocessing +import os +import platform +import re +import shutil +import subprocess +import sys +import tempfile +from os import path + +# LLVM_BRANCH must match the value of the same variable in third_party/update-llvm-10.sh +LLVM_BRANCH = "release/10.x" + +# LLVM_COMMIT must be set to the commit hash that we last updated to when running third_party/update-llvm-10.sh. +# Run 'git show -s origin/llvm10-clean' and look for 'llvm-10-update: <hash>' to retrieve it. +LLVM_COMMIT = "d32170dbd5b0d54436537b6b75beaf44324e0c28" + +SCRIPT_DIR = path.dirname(path.realpath(sys.argv[0])) +LLVM_STAGING_DIR = path.abspath(path.join(tempfile.gettempdir(), "llvm-10")) +LLVM_DIR = path.abspath(path.join(LLVM_STAGING_DIR, "llvm")) +LLVM_OBJS = path.join(LLVM_STAGING_DIR, "build") +LLVM_CONFIGS = path.abspath(path.join(SCRIPT_DIR, '..', 'configs')) + +# List of all arches SwiftShader supports +LLVM_TARGETS = [ + ('AArch64', ('__aarch64__',)), + ('ARM', ('__arm__',)), + ('X86', ('__i386__', '__x86_64__')), + ('Mips', ('__mips__',)), + ('PowerPC', ('__powerpc64__',)), + ('RISCV', ('__riscv',)), +] + +# Per-platform arches +LLVM_TRIPLES = { + 'android': [ + ('__x86_64__', 'x86_64-linux-android'), + ('__i386__', 'i686-linux-android'), + ('__arm__', 'armv7-linux-androideabi'), + ('__aarch64__', 'aarch64-linux-android'), + ], + 'linux': [ + ('__x86_64__', 'x86_64-unknown-linux-gnu'), + ('__i386__', 'i686-pc-linux-gnu'), + ('__arm__', 'armv7-linux-gnueabihf'), + ('__aarch64__', 'aarch64-linux-gnu'), + ('__mips__', 'mipsel-linux-gnu'), + ('__mips64', 'mips64el-linux-gnuabi64'), + ('__powerpc64__', 'powerpc64le-unknown-linux-gnu'), + ('__riscv', 'riscv64-unknown-linux-gnu'), + ], + 'darwin': [ + ('__x86_64__', 'x86_64-apple-darwin'), + ('__aarch64__', 'arm64-apple-darwin'), + ], + 'windows': [ + ('__x86_64__', 'x86_64-pc-win32'), + ('__i386__', 'i686-pc-win32'), + ('__arm__', 'armv7-pc-win32'), + ('__aarch64__', 'aarch64-pc-win32'), + ('__mips__', 'mipsel-pc-win32'), + ('__mips64', 'mips64el-pc-win32'), + ], + 'fuchsia': [ + ('__x86_64__', 'x86_64-unknown-fuchsia'), + ('__aarch64__', 'aarch64-unknown-fuchsia'), + ] +} + +# Mapping of target platform to the host it must be built on +LLVM_PLATFORM_TO_HOST_SYSTEM = { + 'android': 'Linux', + 'darwin': 'Darwin', + 'linux': 'Linux', + 'windows': 'Windows', + 'fuchsia': 'Linux' +} + +# LLVM configurations to be undefined. +LLVM_UNDEF_MACROS = [ + 'BACKTRACE_HEADER', + 'ENABLE_BACKTRACES', + 'ENABLE_CRASH_OVERRIDES', + 'HAVE_BACKTRACE', + 'HAVE_POSIX_SPAWN', + 'HAVE_PTHREAD_GETNAME_NP', + 'HAVE_PTHREAD_SETNAME_NP', + 'HAVE_TERMIOS_H', + 'HAVE_ZLIB_H', + 'HAVE__UNWIND_BACKTRACE', +] + +# General CMake options for building LLVM +LLVM_CMAKE_OPTIONS = [ + '-DCMAKE_BUILD_TYPE=Release', + '-DLLVM_ENABLE_THREADS=ON', + '-DLLVM_ENABLE_TERMINFO=OFF', + '-DLLVM_ENABLE_LIBXML2=OFF', + '-DLLVM_ENABLE_LIBEDIT=OFF', + '-DLLVM_ENABLE_LIBPFM=OFF', + '-DLLVM_ENABLE_ZLIB=OFF', + '-DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=ON' +] + +# Used when building LLVM for darwin. Affects values set in the generated config files. +MIN_MACOS_VERSION = '10.10' + +@contextlib.contextmanager +def pushd(new_dir): + previous_dir = os.getcwd() + os.chdir(new_dir) + try: + yield + finally: + os.chdir(previous_dir) + + +def log(message, level=1): + print(' ' * level + '> ' + message) + + +def run_command(command, log_level=1): + log(command, log_level) + os.system(command) + + +def run_subprocess(*popenargs, log_level=1, cwd=None): + log([' '.join(t) for t in popenargs][0], log_level) + return subprocess.run(*popenargs, cwd=cwd) + + +def _parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('name', help='destination name', + choices=['android', 'linux', 'darwin', 'windows', 'fuchsia']) + parser.add_argument('-j', '--jobs', help='parallel compilation', type=int) + return parser.parse_args() + + +def validate_args(args): + host = platform.system() + if host not in LLVM_PLATFORM_TO_HOST_SYSTEM.values(): + raise Exception(f"Host system not supported: '{host}'") + + if args.name not in LLVM_PLATFORM_TO_HOST_SYSTEM.keys(): + raise Exception(f"Unknown target platform: '{args.name}'") + + expected_host = LLVM_PLATFORM_TO_HOST_SYSTEM[args.name] + if LLVM_PLATFORM_TO_HOST_SYSTEM[args.name] != host: + raise Exception( + f"Target platform '{args.name}' must be built on '{expected_host}', not on '{host}'") + + +def get_cmake_targets_to_build(platform): + """Returns list of LLVM targets to build for the input platform""" + targets = set() + for arch_def, triplet in LLVM_TRIPLES[platform]: + for arch, defs in LLVM_TARGETS: + if arch_def in defs: + targets.add(arch) + + # Maintain the sort order of LLVM_TARGETS as this affects how config + # headers are generated + return [t[0] for t in LLVM_TARGETS if t[0] in targets] + + +def clone_llvm(): + log("Cloning/Updating LLVM", 1) + # Clone or update staging directory + if not path.exists(LLVM_STAGING_DIR): + os.mkdir(LLVM_STAGING_DIR) + with pushd(LLVM_STAGING_DIR): + run_command('git init', 2) + run_command( + 'git remote add origin https://github.com/llvm/llvm-project.git', 2) + run_command('git config core.sparsecheckout true', 2) + run_command('echo /llvm > .git/info/sparse-checkout', 2) + + with pushd(LLVM_STAGING_DIR): + run_command('echo /llvm > .git/info/sparse-checkout', 2) + run_command('git fetch origin {}'.format(LLVM_BRANCH), 2) + run_command('git checkout {}'.format(LLVM_COMMIT), 2) + return + + +def build_llvm(name, num_jobs): + """Build LLVM and get all generated files.""" + log("Building LLVM", 1) + if num_jobs is None: + num_jobs = multiprocessing.cpu_count() + + """On Windows we need to have CMake generate build files for the 64-bit + Visual Studio host toolchain.""" + host = '-Thost=x64' if name == 'windows' else '' + + cmake_options = LLVM_CMAKE_OPTIONS + ['-DLLVM_TARGETS_TO_BUILD=' + + ';'.join(t for t in get_cmake_targets_to_build(name))] + + if name == 'darwin': + cmake_options.append('-DCMAKE_OSX_DEPLOYMENT_TARGET={}'.format(MIN_MACOS_VERSION)) + + os.makedirs(LLVM_OBJS, exist_ok=True) + run_subprocess(['cmake', host, LLVM_DIR] + + cmake_options, log_level=2, cwd=LLVM_OBJS) + run_subprocess(['cmake', '--build', '.', '-j', + str(num_jobs)], log_level=2, cwd=LLVM_OBJS) + + +def list_files(src_base, src, dst_base, suffixes): + """Enumerate the files that are under `src` and end with one of the + `suffixes` and yield the source path and the destination path.""" + src_base = path.abspath(src_base) + src = path.join(src_base, src) + for base_dir, dirnames, filenames in os.walk(src): + for filename in filenames: + if path.splitext(filename)[1] in suffixes: + relative = path.relpath(base_dir, src_base) + yield (path.join(base_dir, filename), + path.join(dst_base, relative, filename)) + + +def copy_common_generated_files(dst_base): + """Copy platform-independent generated files.""" + log("Copying platform-independent generated files", 1) + suffixes = {'.inc', '.h', '.def'} + subdirs = [ + path.join('include', 'llvm', 'IR'), + path.join('include', 'llvm', 'Support'), + path.join('lib', 'IR'), + path.join('lib', 'Transforms', 'InstCombine'), + ] + [path.join('lib', 'Target', arch) for arch, defs in LLVM_TARGETS] + for subdir in subdirs: + for src, dst in list_files(LLVM_OBJS, subdir, dst_base, suffixes): + log('{} -> {}'.format(src, dst), 2) + os.makedirs(path.dirname(dst), exist_ok=True) + shutil.copyfile(src, dst) + + +def copy_platform_file(platform, src, dst): + """Copy platform-dependent generated files and add platform-specific + modifications.""" + + # LLVM configuration patterns to be post-processed. + llvm_target_pattern = re.compile('^LLVM_[A-Z_]+\\(([A-Za-z0-9_]+)\\)$') + llvm_native_pattern = re.compile( + '^#define LLVM_NATIVE_([A-Z]+) (LLVMInitialize)?(.*)$') + llvm_triple_pattern = re.compile('^#define (LLVM_[A-Z_]+_TRIPLE) "(.*)"$') + llvm_define_pattern = re.compile('^#define ([A-Za-z0-9_]+) (.*)$') + + # Build architecture-specific conditions. + conds = {} + for arch, defs in LLVM_TARGETS: + conds[arch] = ' || '.join('defined(' + v + ')' for v in defs) + + # Get a set of platform-specific triples. + triples = LLVM_TRIPLES[platform] + + with open(src, 'r') as src_file: + os.makedirs(path.dirname(dst), exist_ok=True) + with open(dst, 'w') as dst_file: + for line in src_file: + if line == '#define LLVM_CONFIG_H\n': + print(line, file=dst_file, end='') + print('', file=dst_file) + print('#if !defined(__i386__) && defined(_M_IX86)', + file=dst_file) + print('#define __i386__ 1', file=dst_file) + print('#endif', file=dst_file) + print('', file=dst_file) + print( + '#if !defined(__x86_64__) && (defined(_M_AMD64) || defined (_M_X64))', file=dst_file) + print('#define __x86_64__ 1', file=dst_file) + print('#endif', file=dst_file) + print('', file=dst_file) + + match = llvm_target_pattern.match(line) + if match: + arch = match.group(1) + print('#if ' + conds[arch], file=dst_file) + print(line, file=dst_file, end='') + print('#endif', file=dst_file) + continue + + match = llvm_native_pattern.match(line) + if match: + name = match.group(1) + init = match.group(2) or '' + arch = match.group(3) + end = '' + if arch.lower().endswith(name.lower()): + end = arch[-len(name):] + directive = '#if ' + for arch, defs in LLVM_TARGETS: + print(directive + conds[arch], file=dst_file) + print('#define LLVM_NATIVE_' + name + ' ' + + init + arch + end, file=dst_file) + directive = '#elif ' + print('#else', file=dst_file) + print('#error "unknown architecture"', file=dst_file) + print('#endif', file=dst_file) + continue + + match = llvm_triple_pattern.match(line) + if match: + name = match.group(1) + directive = '#if' + for defs, triple in triples: + print(directive + ' defined(' + defs + ')', + file=dst_file) + print('#define ' + name + ' "' + triple + '"', + file=dst_file) + directive = '#elif' + print('#else', file=dst_file) + print('#error "unknown architecture"', file=dst_file) + print('#endif', file=dst_file) + continue + + match = llvm_define_pattern.match(line) + if match and match.group(1) in LLVM_UNDEF_MACROS: + print('/* #undef ' + match.group(1) + ' */', file=dst_file) + continue + + print(line, file=dst_file, end='') + + +def copy_platform_generated_files(platform, dst_base): + """Copy platform-specific generated files.""" + log("Copying platform-specific generated files", 1) + suffixes = {'.inc', '.h', '.def'} + src_dir = path.join('include', 'llvm', 'Config') + for src, dst in list_files(LLVM_OBJS, src_dir, dst_base, suffixes): + log('{}, {} -> {}'.format(platform, src, dst), 2) + copy_platform_file(platform, src, dst) + + +def main(): + args = _parse_args() + validate_args(args) + clone_llvm() + build_llvm(args.name, args.jobs) + copy_common_generated_files(path.join(LLVM_CONFIGS, 'common')) + copy_platform_generated_files( + args.name, path.join(LLVM_CONFIGS, args.name)) + + +if __name__ == '__main__': + main()
diff --git a/third_party/update-llvm-16.sh b/third_party/update-llvm-16.sh new file mode 100755 index 0000000..1823778 --- /dev/null +++ b/third_party/update-llvm-16.sh
@@ -0,0 +1,92 @@ +#!/bin/bash + +# llvm now lives in a mono-repo along with clang, libc, lldb and a whole bunch +# of other projects (~90k files at time of writing). +# SwiftShader only requires the llvm project from this repo, and does not wish +# to pull in everything else. +# This script performs the following: +# * The llvm10-clean branch is fetched and checked out. +# * A sparse checkout of the llvm project is made to a temporary directory. +# * The third_party/llvm-10.0/llvm is replaced with the latest LLVM version. +# * This is committed and pushed. +# * The original branch is checked out again, and a merge from llvm10-clean to +# the original branch is made. + +THIRD_PARTY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )" +STAGING_DIR="/tmp/llvm-10-update" +CLEAN_BRANCH="llvm10-clean" +SOURCE_DIR="${STAGING_DIR}/llvm" +TARGET_DIR="${THIRD_PARTY_DIR}/llvm-10.0/llvm" +LLVM_REPO_BRANCH="release/10.x" +BUG_NUMBER="b/152339534" + +SWIFTSHADER_HEAD=`git rev-parse HEAD` + +# Check there are no local changes +if ! git diff --quiet HEAD; then + echo "Git workspace not clean" + exit 1 +fi + +# Clone or update the staging directory +if [[ -d "$STAGING_DIR" ]]; then + pushd "$STAGING_DIR" +else + mkdir "$STAGING_DIR" + pushd "$STAGING_DIR" + git init + git remote add origin https://github.com/llvm/llvm-project.git + git config core.sparsecheckout true + echo "/llvm/lib" >> .git/info/sparse-checkout + echo "/llvm/include" >> .git/info/sparse-checkout +fi +git pull origin $LLVM_REPO_BRANCH +LLVM_HEAD=`git log HEAD -n 1 --pretty=format:'%h'` +popd + +if [[ -d "$TARGET_DIR" ]]; then + # Look for the last update change. + LAST_TARGET_UPDATE=`git log --grep="^llvm-10-update: [0-9a-f]\{9\}$" -n 1 --pretty=format:'%h' ${TARGET_DIR}` + if [[ ! -z "$LAST_TARGET_UPDATE" ]]; then + # Get the LLVM commit hash from the update change. + LAST_SOURCE_UPDATE=`git log $LAST_TARGET_UPDATE -n 1 | grep -oP "llvm-10-update: \K([0-9a-f]{9})"` + if [ $LLVM_HEAD == $LAST_SOURCE_UPDATE ]; then + echo "No new LLVM changes to apply" + exit 0 + fi + + # Gather list of changes since last update + pushd "$STAGING_DIR" + LLVM_CHANGE_LOG=`git log $LAST_SOURCE_UPDATE..$LLVM_HEAD --pretty=format:' %h %s'` + LLVM_CHANGE_LOG="Changes:\n${LLVM_CHANGE_LOG}\n\n" + popd + fi +fi + +COMMIT_MSG=`echo -e "Update LLVM 10 to ${LLVM_HEAD}\n\n${LLVM_CHANGE_LOG}Commands:\n third_party/update-llvm-10.sh\n\nllvm-10-update: ${LLVM_HEAD}\nBug: ${BUG_NUMBER}"` + +# Switch to the llvm-10-clean branch. +git fetch "https://swiftshader.googlesource.com/SwiftShader" $CLEAN_BRANCH +git checkout FETCH_HEAD + +# Delete the target directory. We're going to re-populate it. +rm -fr "$TARGET_DIR" + +# Update SwiftShader's $TARGET_DIR with a clean copy of latest LLVM +mkdir -p "$TARGET_DIR" +cp -r "$SOURCE_DIR/." "$TARGET_DIR" +git add "$TARGET_DIR" +git commit -m "$COMMIT_MSG" +MERGE_SOURCE=`git log HEAD -n 1 --pretty=format:'%h'` + +# Push llvm-10-clean branch. +git push "https://swiftshader.googlesource.com/SwiftShader" $CLEAN_BRANCH + +# Switch to the branch in use when calling the script. +git checkout $SWIFTSHADER_HEAD + +# Update SwiftShader's $TARGET_DIR with a clean copy of latest LLVM +git merge -m "$COMMIT_MSG" "$MERGE_SOURCE" + +# We're now done with the staging dir. Delete it. +rm -fr "$STAGING_DIR"