| #!/usr/bin/env python2 | |
| import argparse | |
| import os | |
| import pipes | |
| import re | |
| import sys | |
| from utils import FindBaseNaCl, GetObjcopyCmd, get_sfi_string, shellcmd | |
| def NewerThanOrNotThere(old_path, new_path): | |
| """Returns whether old_path is newer than new_path. | |
| Also returns true if either path doesn't exist. | |
| """ | |
| if not (os.path.exists(old_path) and os.path.exists(new_path)): | |
| return True | |
| return os.path.getmtime(old_path) > os.path.getmtime(new_path) | |
| def BuildRegex(patterns, syms): | |
| """Build a regular expression string for inclusion or exclusion. | |
| Creates a regex string from an array of patterns and an array | |
| of symbol names. Each element in the patterns array is either a | |
| regex, or a range of entries in the symbol name array, e.g. '2:9'. | |
| """ | |
| pattern_list = [] | |
| for pattern in patterns: | |
| if pattern[0].isdigit() or pattern[0] == ':': | |
| # Legitimate symbols or regexes shouldn't start with a | |
| # digit or a ':', so interpret the pattern as a range. | |
| interval = pattern.split(':') | |
| if len(interval) == 1: | |
| # Treat singleton 'n' as 'n:n+1'. | |
| lower = int(interval[0]) | |
| upper = lower + 1 | |
| elif len(interval) == 2: | |
| # Handle 'a:b', 'a:', and ':b' with suitable defaults. | |
| lower = int(interval[0]) if len(interval[0]) else 0 | |
| upper = int(interval[1]) if len(interval[1]) else len(syms) | |
| else: | |
| print 'Invalid range syntax: {p}'.format(p=pattern) | |
| exit(1) | |
| pattern = '$|^'.join([re.escape(p) for p in syms[lower:upper]]) | |
| pattern_list.append('^' + pattern + '$') | |
| return '|'.join(pattern_list) if len(pattern_list) else '^$' | |
| def MatchSymbol(sym, re_include, re_exclude, default_match): | |
| """Match a symbol name against inclusion/exclusion rules. | |
| Returns True or False depending on whether the given symbol | |
| matches the compiled include or exclude regexes. The default is | |
| returned if neither the include nor the exclude regex matches. | |
| """ | |
| if re_exclude.match(sym): | |
| # Always honor an explicit exclude before considering | |
| # includes. | |
| return False | |
| if re_include.match(sym): | |
| return True | |
| return default_match | |
| def AddOptionalArgs(argparser): | |
| argparser.add_argument('--force', dest='force', type=int, choices=[0, 1], | |
| default=1, | |
| help='Force all re-translations of the pexe.' + | |
| ' Default %(default)s.') | |
| argparser.add_argument('--include', '-i', default=[], dest='include', | |
| action='append', | |
| help='Subzero symbols to include ' + | |
| '(regex or line range)') | |
| argparser.add_argument('--exclude', '-e', default=[], dest='exclude', | |
| action='append', | |
| help='Subzero symbols to exclude ' + | |
| '(regex or line range)') | |
| argparser.add_argument('--output', '-o', default='a.out', dest='output', | |
| action='store', | |
| help='Output executable. Default %(default)s.') | |
| argparser.add_argument('-O', default='2', dest='optlevel', | |
| choices=['m1', '-1', '0', '1', '2'], | |
| help='Optimization level ' + | |
| '(m1 and -1 are equivalent).' + | |
| ' Default %(default)s.') | |
| argparser.add_argument('--filetype', default='iasm', dest='filetype', | |
| choices=['obj', 'asm', 'iasm'], | |
| help='Output file type. Default %(default)s.') | |
| argparser.add_argument('--sandbox', dest='sandbox', action='store_true', | |
| help='Enable sandboxing in the translator') | |
| argparser.add_argument('--nonsfi', dest='nonsfi', action='store_true', | |
| help='Enable Non-SFI in the translator') | |
| argparser.add_argument('--enable-block-profile', | |
| dest='enable_block_profile', action='store_true', | |
| help='Enable basic block profiling.') | |
| argparser.add_argument('--target', default='x8632', dest='target', | |
| choices=['arm32', 'x8632', 'x8664'], | |
| help='Generate code for specified target.') | |
| argparser.add_argument('--verbose', '-v', dest='verbose', | |
| action='store_true', | |
| help='Display some extra debugging output') | |
| argparser.add_argument('--sz', dest='sz_args', action='append', default=[], | |
| help='Extra arguments for Subzero') | |
| argparser.add_argument('--llc', dest='llc_args', action='append', | |
| default=[], help='Extra arguments for llc') | |
| argparser.add_argument('--no-sz', dest='nosz', action='store_true', | |
| help='Run only post-Subzero build steps') | |
| argparser.add_argument('--fsanitize-address', dest='asan', | |
| action='store_true', | |
| help='Instrument with AddressSanitizer') | |
| def LinkSandbox(objs, exe, target, verbose=True): | |
| assert target in ('x8632', 'x8664', 'arm32'), \ | |
| '-sandbox is not available for %s' % target | |
| nacl_root = FindBaseNaCl() | |
| gold = ('{root}/toolchain/linux_x86/pnacl_newlib_raw/bin/' + | |
| 'le32-nacl-ld.gold').format(root=nacl_root) | |
| target_lib_dir = { | |
| 'arm32': 'arm', | |
| 'x8632': 'x86-32', | |
| 'x8664': 'x86-64', | |
| }[target] | |
| linklib = ('{root}/toolchain/linux_x86/pnacl_newlib_raw/translator/' + | |
| '{target_dir}/lib').format(root=nacl_root, | |
| target_dir=target_lib_dir) | |
| shellcmd([gold, | |
| '-nostdlib', | |
| '--no-fix-cortex-a8', | |
| '--eh-frame-hdr', | |
| '-z', 'text', | |
| #'-z', 'noexecstack', | |
| '--build-id', | |
| '--entry=__pnacl_start', | |
| '-static', #'-pie', | |
| '{linklib}/crtbegin.o'.format(linklib=linklib)] + | |
| objs + | |
| [('{root}/toolchain_build/src/subzero/build/runtime/' + | |
| 'szrt_sb_{target}.o').format(root=nacl_root, target=target), | |
| '{linklib}/libpnacl_irt_shim_dummy.a'.format(linklib=linklib), | |
| '--start-group', | |
| '{linklib}/libgcc.a'.format(linklib=linklib), | |
| '{linklib}/libcrt_platform.a'.format(linklib=linklib), | |
| '--end-group', | |
| '{linklib}/crtend.o'.format(linklib=linklib), | |
| '--undefined=_start', | |
| '--defsym=__Sz_AbsoluteZero=0', | |
| #'--defsym=_begin=0', | |
| '-o', exe | |
| ], echo=verbose) | |
| def LinkNonsfi(objs, exe, target, verbose=True): | |
| nacl_root = FindBaseNaCl() | |
| gold = ('{root}/toolchain/linux_x86/pnacl_newlib_raw/bin/' + | |
| 'le32-nacl-ld.gold').format(root=nacl_root) | |
| target_lib_dir = { | |
| 'arm32': 'arm-nonsfi', | |
| 'x8632': 'x86-32-nonsfi', | |
| }[target] | |
| linklib = ('{root}/toolchain/linux_x86/pnacl_newlib_raw/translator/' + | |
| '{target_dir}/lib').format(root=nacl_root, | |
| target_dir=target_lib_dir) | |
| shellcmd([gold, | |
| '-nostdlib', | |
| '--no-fix-cortex-a8', | |
| '--eh-frame-hdr', | |
| '-z', 'text', | |
| '-z', 'noexecstack', | |
| '--build-id', | |
| '--entry=__pnacl_start', | |
| '-pie', | |
| '{linklib}/crtbegin.o'.format(linklib=linklib)] + | |
| objs + | |
| [('{root}/toolchain_build/src/subzero/build/runtime/' + | |
| 'szrt_nonsfi_{target}.o').format(root=nacl_root, target=target), | |
| '{linklib}/libpnacl_irt_shim_dummy.a'.format(linklib=linklib), | |
| '--start-group', | |
| '{linklib}/libgcc.a'.format(linklib=linklib), | |
| '{linklib}/libcrt_platform.a'.format(linklib=linklib), | |
| '--end-group', | |
| '{linklib}/crtend.o'.format(linklib=linklib), | |
| '--undefined=_start', | |
| '--defsym=__Sz_AbsoluteZero=0', | |
| '--defsym=_begin=0', | |
| '-o', exe | |
| ], echo=verbose) | |
| def LinkNative(objs, exe, target, verbose=True): | |
| nacl_root = FindBaseNaCl() | |
| linker = { | |
| 'arm32': '/usr/bin/arm-linux-gnueabihf-g++', | |
| 'mips32': '/usr/bin/mipsel-linux-gnu-g++', | |
| 'x8632': ('{root}/../third_party/llvm-build/Release+Asserts/bin/clang' | |
| ).format(root=nacl_root), | |
| 'x8664': ('{root}/../third_party/llvm-build/Release+Asserts/bin/clang' | |
| ).format(root=nacl_root) | |
| }[target] | |
| extra_linker_args = { | |
| 'arm32': ['-mcpu=cortex-a9'], | |
| 'x8632': ['-m32'], | |
| 'x8664': ['-mx32'] | |
| }[target] | |
| lib_dir = { | |
| 'arm32': 'arm-linux', | |
| 'x8632': 'x86-32-linux', | |
| 'x8664': 'x86-64-linux', | |
| }[target] | |
| shellcmd([linker] + | |
| extra_linker_args + | |
| objs + | |
| ['-o', exe, | |
| ('{root}/toolchain/linux_x86/pnacl_newlib_raw/translator/' + | |
| '{lib_dir}/lib/' + | |
| '{{unsandboxed_irt,irt_random,irt_query_list}}.o').format( | |
| root=nacl_root, lib_dir=lib_dir), | |
| ('{root}/toolchain_build/src/subzero/build/runtime/' + | |
| 'szrt_native_{target}.o').format(root=nacl_root, target=target), | |
| '-lm', '-lpthread', '-lrt', | |
| '-Wl,--defsym=__Sz_AbsoluteZero=0' | |
| ], echo=verbose) | |
| def main(): | |
| """Create a hybrid translation from Subzero and llc. | |
| Takes a finalized pexe and builds a native executable as a hybrid of Subzero | |
| and llc translated bitcode. Linker tricks are used to determine whether | |
| Subzero or llc generated symbols are used, on a per-symbol basis. | |
| By default, for every symbol, its Subzero version is used. Subzero and llc | |
| symbols can be selectively enabled/disabled via regular expressions on the | |
| symbol name, or by ranges of lines in this program's auto-generated symbol | |
| file. | |
| For each symbol, the --exclude arguments are first checked (the symbol is | |
| 'rejected' on a match), followed by the --include arguments (the symbol is | |
| 'accepted' on a match), followed by unconditional 'rejection'. The Subzero | |
| version is used for an 'accepted' symbol, and the llc version is used for a | |
| 'rejected' symbol. | |
| Each --include and --exclude argument can be a regular expression or a range | |
| of lines in the symbol file. Each regular expression is wrapped inside | |
| '^$', so if you want a substring match on 'foo', use '.*foo.*' instead. | |
| Ranges use python-style 'first:last' notation, so e.g. use '0:10' or ':10' | |
| for the first 10 lines of the file, or '1' for the second line of the file. | |
| If no --include or --exclude arguments are given, the executable is produced | |
| entirely using Subzero, without using llc or linker tricks. | |
| When using the --force=0 option, this script uses file modification | |
| timestamps to determine whether llc and Subzero re-translation are needed. | |
| It checks timestamps of llc, pnacl-sz, and the pexe against the translated | |
| object files to determine the minimal work necessary. The --force=1 option | |
| (default) suppresses those checks and re-translates everything. | |
| This script expects various PNaCl and LLVM tools to be found within the | |
| native_client tree. When changes are made to these tools, copy them this | |
| way: | |
| cd native_client | |
| toolchain_build/toolchain_build_pnacl.py llvm_x86_64_linux \\ | |
| --install=toolchain/linux_x86/pnacl_newlib_raw | |
| """ | |
| argparser = argparse.ArgumentParser( | |
| description=' ' + main.__doc__, | |
| formatter_class=argparse.RawTextHelpFormatter) | |
| AddOptionalArgs(argparser) | |
| argparser.add_argument('pexe', help='Finalized pexe to translate') | |
| args = argparser.parse_args() | |
| pexe = args.pexe | |
| exe = args.output | |
| ProcessPexe(args, pexe, exe) | |
| def ProcessPexe(args, pexe, exe): | |
| [pexe_base, ext] = os.path.splitext(pexe) | |
| if ext != '.pexe': | |
| pexe_base = pexe | |
| pexe_base_unescaped = pexe_base | |
| pexe_base = pipes.quote(pexe_base) | |
| pexe = pipes.quote(pexe) | |
| nacl_root = FindBaseNaCl() | |
| path_addition = ( | |
| '{root}/toolchain/linux_x86/pnacl_newlib_raw/bin' | |
| ).format(root=nacl_root) | |
| obj_llc = pexe_base + '.llc.o' | |
| obj_sz = pexe_base + '.sz.o' | |
| asm_sz = pexe_base + '.sz.s' | |
| obj_llc_weak = pexe_base + '.weak.llc.o' | |
| obj_sz_weak = pexe_base + '.weak.sz.o' | |
| obj_partial = obj_sz # overridden for hybrid mode | |
| sym_llc = pexe_base + '.sym.llc.txt' | |
| sym_sz = pexe_base + '.sym.sz.txt' | |
| sym_sz_unescaped = pexe_base_unescaped + '.sym.sz.txt' | |
| whitelist_sz = pexe_base + '.wl.sz.txt' | |
| whitelist_sz_unescaped = pexe_base_unescaped + '.wl.sz.txt' | |
| pnacl_sz = ( | |
| '{root}/toolchain_build/src/subzero/pnacl-sz' | |
| ).format(root=nacl_root) | |
| llcbin = '{base}/pnacl-llc'.format(base=path_addition) | |
| gold = '{base}/le32-nacl-ld.gold'.format(base=path_addition) | |
| objcopy = '{base}/{objcopy}'.format(base=path_addition, | |
| objcopy=GetObjcopyCmd(args.target)) | |
| opt_level = args.optlevel | |
| opt_level_map = { 'm1':'0', '-1':'0', '0':'0', '1':'1', '2':'2' } | |
| hybrid = args.include or args.exclude | |
| native = not args.sandbox and not args.nonsfi | |
| if args.asan: | |
| if args.sandbox or args.nonsfi: | |
| print 'Can only use AddressSanitizer with a native build' | |
| exit(1) | |
| if '-fsanitize-address' not in args.sz_args: | |
| args.sz_args.append('-fsanitize-address') | |
| if hybrid and (args.force or | |
| NewerThanOrNotThere(pexe, obj_llc) or | |
| NewerThanOrNotThere(llcbin, obj_llc)): | |
| arch = { | |
| 'arm32': 'arm' + get_sfi_string(args, 'v7', '-nonsfi', '-nonsfi'), | |
| 'x8632': 'x86-32' + get_sfi_string(args, '', '-nonsfi', '-linux'), | |
| 'x8664': 'x86-64' + get_sfi_string(args, '', '', '-linux') | |
| }[args.target] | |
| # Only run pnacl-translate in hybrid mode. | |
| shellcmd(['{base}/pnacl-translate'.format(base=path_addition), | |
| '-split-module=1', | |
| '-ffunction-sections', | |
| '-fdata-sections', | |
| '-c', | |
| '-arch', arch, | |
| '-O' + opt_level_map[opt_level], | |
| '--pnacl-driver-append-LLC_FLAGS_EXTRA=-externalize', | |
| '-o', obj_llc] + | |
| (['--pnacl-driver-verbose'] if args.verbose else []) + | |
| args.llc_args + | |
| [pexe], | |
| echo=args.verbose) | |
| if native: | |
| shellcmd(( | |
| '{objcopy} --redefine-sym _start=_user_start {obj}' | |
| ).format(objcopy=objcopy, obj=obj_llc), echo=args.verbose) | |
| # Generate llc syms file for consistency, even though it's not used. | |
| shellcmd(( | |
| 'nm {obj} | sed -n "s/.* [a-zA-Z] //p" > {sym}' | |
| ).format(obj=obj_llc, sym=sym_llc), echo=args.verbose) | |
| if (args.force or | |
| NewerThanOrNotThere(pexe, obj_sz) or | |
| NewerThanOrNotThere(pnacl_sz, obj_sz)): | |
| if not args.nosz: | |
| # Run pnacl-sz regardless of hybrid mode. | |
| shellcmd([pnacl_sz, | |
| '-O' + opt_level, | |
| '-bitcode-format=pnacl', | |
| '-filetype=' + args.filetype, | |
| '-o', obj_sz if args.filetype == 'obj' else asm_sz, | |
| '-target=' + args.target] + | |
| (['-externalize', | |
| '-ffunction-sections', | |
| '-fdata-sections'] if hybrid else []) + | |
| (['-sandbox'] if args.sandbox else []) + | |
| (['-nonsfi'] if args.nonsfi else []) + | |
| (['-enable-block-profile'] if | |
| args.enable_block_profile and not args.sandbox | |
| else []) + | |
| args.sz_args + | |
| [pexe], | |
| echo=args.verbose) | |
| if args.filetype != 'obj': | |
| triple = { | |
| 'arm32': 'arm' + get_sfi_string(args, '-nacl', '', ''), | |
| 'x8632': 'i686' + get_sfi_string(args, '-nacl', '', ''), | |
| 'x8664': 'x86_64' + | |
| get_sfi_string(args, '-nacl', '-linux-gnux32', | |
| '-linux-gnux32'), | |
| }[args.target] | |
| shellcmd(( | |
| '{base}/llvm-mc -triple={triple} -filetype=obj -o {obj} {asm}' | |
| ).format(base=path_addition, asm=asm_sz, obj=obj_sz, | |
| triple=triple), | |
| echo=args.verbose) | |
| if native: | |
| shellcmd(( | |
| '{objcopy} --redefine-sym _start=_user_start {obj}' | |
| ).format(objcopy=objcopy, obj=obj_sz), echo=args.verbose) | |
| if hybrid: | |
| shellcmd(( | |
| 'nm {obj} | sed -n "s/.* [a-zA-Z] //p" > {sym}' | |
| ).format(obj=obj_sz, sym=sym_sz), echo=args.verbose) | |
| if hybrid: | |
| with open(sym_sz_unescaped) as f: | |
| sz_syms = f.read().splitlines() | |
| re_include_str = BuildRegex(args.include, sz_syms) | |
| re_exclude_str = BuildRegex(args.exclude, sz_syms) | |
| re_include = re.compile(re_include_str) | |
| re_exclude = re.compile(re_exclude_str) | |
| # If a symbol doesn't explicitly match re_include or re_exclude, | |
| # the default MatchSymbol() result is True, unless some --include | |
| # args are provided. | |
| default_match = not args.include | |
| whitelist_has_items = False | |
| with open(whitelist_sz_unescaped, 'w') as f: | |
| for sym in sz_syms: | |
| if MatchSymbol(sym, re_include, re_exclude, default_match): | |
| f.write(sym + '\n') | |
| whitelist_has_items = True | |
| shellcmd(( | |
| '{objcopy} --weaken {obj} {weak}' | |
| ).format(objcopy=objcopy, obj=obj_sz, weak=obj_sz_weak), | |
| echo=args.verbose) | |
| if whitelist_has_items: | |
| # objcopy returns an error if the --weaken-symbols file is empty. | |
| shellcmd(( | |
| '{objcopy} --weaken-symbols={whitelist} {obj} {weak}' | |
| ).format(objcopy=objcopy, | |
| whitelist=whitelist_sz, obj=obj_llc, | |
| weak=obj_llc_weak), | |
| echo=args.verbose) | |
| else: | |
| shellcmd(( | |
| '{objcopy} {obj} {weak}' | |
| ).format(objcopy=objcopy, obj=obj_llc, weak=obj_llc_weak), | |
| echo=args.verbose) | |
| obj_partial = pexe_base + '.o' | |
| ld = { | |
| 'arm32': 'arm-linux-gnueabihf-ld', | |
| 'x8632': 'ld', | |
| 'x8664': 'ld', | |
| }[args.target] | |
| emulation = { | |
| 'arm32': 'armelf_linux_eabi', | |
| 'x8632': 'elf_i386', | |
| 'x8664': 'elf32_x86_64' if not args.sandbox else 'elf_x86_64', | |
| }[args.target] | |
| shellcmd(( | |
| '{ld} -r -m {emulation} -o {partial} {sz} {llc}' | |
| ).format(ld=ld, emulation=emulation, partial=obj_partial, | |
| sz=obj_sz_weak, llc=obj_llc_weak), | |
| echo=args.verbose) | |
| shellcmd(( | |
| '{objcopy} -w --localize-symbol="*" {partial}' | |
| ).format(objcopy=objcopy, partial=obj_partial), | |
| echo=args.verbose) | |
| shellcmd(( | |
| '{objcopy} --globalize-symbol={start} ' + | |
| '--globalize-symbol=__Sz_block_profile_info {partial}' | |
| ).format(objcopy=objcopy, partial=obj_partial, | |
| start=get_sfi_string(args, '_start', '_start', | |
| '_user_start')), | |
| echo=args.verbose) | |
| # Run the linker regardless of hybrid mode. | |
| if args.sandbox: | |
| LinkSandbox([obj_partial], exe, args.target, args.verbose) | |
| elif args.nonsfi: | |
| LinkNonsfi([obj_partial], exe, args.target, args.verbose) | |
| else: | |
| objs = [obj_partial] | |
| if args.asan: | |
| objs.append( | |
| ('{root}/toolchain_build/src/subzero/build/runtime/' + | |
| 'szrt_asan_{target}.o').format(root=nacl_root, | |
| target=args.target)) | |
| LinkNative(objs, exe, args.target, args.verbose) | |
| # Put the extra verbose printing at the end. | |
| if args.verbose and hybrid: | |
| print 'include={regex}'.format(regex=re_include_str) | |
| print 'exclude={regex}'.format(regex=re_exclude_str) | |
| print 'default_match={dm}'.format(dm=default_match) | |
| print 'Number of Subzero syms = {num}'.format(num=len(sz_syms)) | |
| if __name__ == '__main__': | |
| main() |