Subzero: Run cross tests as a much more configurable python script.
The runtests.sh script is removed and replaced with crosstest_generator.py.
"make check" limits to a relevant subset of cross tests to control the combinatorial explosion. We cut the native tests almost in half, and the sandboxed tests down to a quarter.
The --include and --exclude logic is copied/adapted from szbuild.py.
The script works by running through every possible test in the combinatorial explosion, and if the test is a match against the --include and --exclude arguments, the test is built and run.
The script includes lit support, which is the most likely way it will be run. When run with the --lit argument, it sprays the output directory with lit test files in the form of shell scripts, and "make check" runs lit on that directory.
BUG= https://code.google.com/p/nativeclient/issues/detail?id=4085
R=jvoung@chromium.org, mtrofin@chromium.org
Review URL: https://codereview.chromium.org/987503004
diff --git a/Makefile.standalone b/Makefile.standalone
index 58d5357..2227350 100644
--- a/Makefile.standalone
+++ b/Makefile.standalone
@@ -224,7 +224,13 @@
@echo "Crosstests ignored, minimal build"
else
check: check-lit check-unit runtime
- (cd crosstest; ./runtests.sh)
+ # Do all native/sse2 tests, but only test_vector_ops for native/sse4.1.
+ # For (slow) sandboxed tests, limit to Om1/sse4.1.
+ ./pydir/crosstest_generator.py -v --lit \
+ -i native,sse2 -i native,sse4.1,test_vector_ops \
+ -i sandbox,sse4.1,Om1
+ LLVM_BIN_PATH=$(LLVM_BIN_PATH) \
+ $(LLVM_SRC_PATH)/utils/lit/lit.py -sv crosstest/Output
endif
FORMAT_BLACKLIST =
diff --git a/crosstest/crosstest.cfg b/crosstest/crosstest.cfg
new file mode 100644
index 0000000..edb06ea
--- /dev/null
+++ b/crosstest/crosstest.cfg
@@ -0,0 +1,54 @@
+[simple_loop]
+driver: simple_loop_main.c
+test: simple_loop.c
+
+[mem_intrin]
+driver: mem_intrin_main.cpp
+test: mem_intrin.cpp
+
+[test_arith]
+driver: test_arith_main.cpp
+test: test_arith.cpp test_arith_frem.ll test_arith_sqrt.ll
+
+[test_bitmanip]
+driver: test_bitmanip_main.cpp
+test: test_bitmanip.cpp test_bitmanip_intrin.ll
+
+[test_calling_conv]
+driver: test_calling_conv_main.cpp
+test: test_calling_conv.cpp
+
+[test_cast]
+driver: test_cast_main.cpp
+test: test_cast.cpp test_cast_to_u1.ll test_cast_vectors.ll
+
+[test_fcmp]
+driver: test_fcmp_main.cpp
+test: test_fcmp.pnacl.ll
+
+[test_global]
+driver: test_global_main.cpp
+test: test_global.cpp
+
+[test_icmp]
+driver: test_icmp_main.cpp
+test: test_icmp.cpp test_icmp_i1vec.ll
+
+[test_select]
+driver: test_select_main.cpp
+test: test_select.ll
+
+[test_stacksave]
+driver: test_stacksave_main.c
+test: test_stacksave.c
+
+[test_sync_atomic]
+driver: test_sync_atomic_main.cpp
+# Compile the non-Subzero object files straight from source since the native
+# LLVM backend does not understand how to lower NaCl-specific intrinsics.
+flags: --crosstest-bitcode=0
+test: test_sync_atomic.cpp
+
+[test_vector_ops]
+driver: test_vector_ops_main.cpp
+test: test_vector_ops.ll
diff --git a/crosstest/lit.cfg b/crosstest/lit.cfg
new file mode 100644
index 0000000..b6a4b3b
--- /dev/null
+++ b/crosstest/lit.cfg
@@ -0,0 +1,23 @@
+import os
+import re
+import lit.formats
+
+config.name = 'subzero_crosstests'
+config.test_format = lit.formats.ShTest()
+config.suffixes = ['.xtest']
+config.test_source_root = os.path.dirname(__file__)
+config.test_exec_root = config.test_source_root
+
+llvmbintools = [r"\bFileCheck\b"]
+llvmbinpath = os.path.abspath(os.environ.get('LLVM_BIN_PATH'))
+
+for tool in llvmbintools:
+ # The re.sub() line is adapted from one of LLVM's lit.cfg files.
+ # Extract the tool name from the pattern. This relies on the tool
+ # name being surrounded by \b word match operators. If the
+ # pattern starts with "| ", include it in the string to be
+ # substituted.
+ substitution = re.sub(r"^(\\)?((\| )?)\W+b([0-9A-Za-z-_]+)\\b\W*$",
+ r"\2" + llvmbinpath + "/" + r"\4",
+ tool)
+ config.substitutions.append((tool, substitution))
diff --git a/crosstest/runtests.sh b/crosstest/runtests.sh
deleted file mode 100755
index a8b0503..0000000
--- a/crosstest/runtests.sh
+++ /dev/null
@@ -1,173 +0,0 @@
-#!/bin/sh
-
-# TODO: Retire this script and move the individual tests into the lit
-# framework, to leverage parallel testing and other lit goodness.
-
-set -eux
-
-# TODO(stichnot): Use FindBaseNaCl so that the script can be run from
-# anywhere within the native_client directory.
-PATH="../pydir:${PATH}"
-OPTLEVELS="m1 2"
-ATTRIBUTES="sse2 sse4.1"
-SANDBOX="0 1"
-OUTDIR=Output
-# Clean the output directory to avoid reusing stale results.
-rm -rf "${OUTDIR}"
-mkdir -p "${OUTDIR}"
-
-for sb in ${SANDBOX} ; do
- for optlevel in ${OPTLEVELS} ; do
- for attribute in ${ATTRIBUTES} ; do
-
- crosstest.py -O${optlevel} --mattr ${attribute} \
- --prefix=Subzero_ \
- --target=x8632 \
- --sandbox=${sb} \
- --dir="${OUTDIR}" \
- --test=simple_loop.c \
- --driver=simple_loop_main.c \
- --output=simple_loop_sb${sb}_O${optlevel}_${attribute}
-
- crosstest.py -O${optlevel} --mattr ${attribute} \
- --prefix=Subzero_ \
- --target=x8632 \
- --sandbox=${sb} \
- --dir="${OUTDIR}" \
- --test=mem_intrin.cpp \
- --driver=mem_intrin_main.cpp \
- --output=mem_intrin_sb${sb}_O${optlevel}_${attribute}
-
- crosstest.py -O${optlevel} --mattr ${attribute} \
- --prefix=Subzero_ \
- --target=x8632 \
- --sandbox=${sb} \
- --dir="${OUTDIR}" \
- --test=test_arith.cpp \
- --test=test_arith_frem.ll \
- --test=test_arith_sqrt.ll \
- --driver=test_arith_main.cpp \
- --output=test_arith_sb${sb}_O${optlevel}_${attribute}
-
- crosstest.py -O${optlevel} --mattr ${attribute} \
- --prefix=Subzero_ \
- --target=x8632 \
- --sandbox=${sb} \
- --dir="${OUTDIR}" \
- --test=test_bitmanip.cpp --test=test_bitmanip_intrin.ll \
- --driver=test_bitmanip_main.cpp \
- --output=test_bitmanip_sb${sb}_O${optlevel}_${attribute}
-
- crosstest.py -O${optlevel} --mattr ${attribute} \
- --prefix=Subzero_ --target=x8632 \
- --sandbox=${sb} \
- --dir="${OUTDIR}" \
- --test=test_calling_conv.cpp \
- --driver=test_calling_conv_main.cpp \
- --output=test_calling_conv_sb${sb}_O${optlevel}_${attribute}
-
- crosstest.py -O${optlevel} --mattr ${attribute} \
- --prefix=Subzero_ \
- --target=x8632 \
- --sandbox=${sb} \
- --dir="${OUTDIR}" \
- --test=test_cast.cpp --test=test_cast_to_u1.ll \
- --test=test_cast_vectors.ll \
- --driver=test_cast_main.cpp \
- --output=test_cast_sb${sb}_O${optlevel}_${attribute}
-
- crosstest.py -O${optlevel} --mattr ${attribute} \
- --prefix=Subzero_ \
- --target=x8632 \
- --sandbox=${sb} \
- --dir="${OUTDIR}" \
- --test=test_fcmp.pnacl.ll \
- --driver=test_fcmp_main.cpp \
- --output=test_fcmp_sb${sb}_O${optlevel}_${attribute}
-
- crosstest.py -O${optlevel} --mattr ${attribute} \
- --prefix=Subzero_ \
- --target=x8632 \
- --sandbox=${sb} \
- --dir="${OUTDIR}" \
- --test=test_global.cpp \
- --driver=test_global_main.cpp \
- --output=test_global_sb${sb}_O${optlevel}_${attribute}
-
- crosstest.py -O${optlevel} --mattr ${attribute} \
- --prefix=Subzero_ \
- --target=x8632 \
- --sandbox=${sb} \
- --dir="${OUTDIR}" \
- --test=test_icmp.cpp --test=test_icmp_i1vec.ll \
- --driver=test_icmp_main.cpp \
- --output=test_icmp_sb${sb}_O${optlevel}_${attribute}
-
- crosstest.py -O${optlevel} --mattr ${attribute} \
- --prefix=Subzero_ \
- --target=x8632 \
- --sandbox=${sb} \
- --dir="${OUTDIR}" \
- --test=test_select.ll \
- --driver=test_select_main.cpp \
- --output=test_select_sb${sb}_O${optlevel}_${attribute}
-
- crosstest.py -O${optlevel} --mattr ${attribute} \
- --prefix=Subzero_ \
- --target=x8632 \
- --sandbox=${sb} \
- --dir="${OUTDIR}" \
- --test=test_stacksave.c \
- --driver=test_stacksave_main.c \
- --output=test_stacksave_sb${sb}_O${optlevel}_${attribute}
-
- # Compile the non-subzero object files straight from source
- # since the native LLVM backend does not understand how to
- # lower NaCl-specific intrinsics.
- crosstest.py -O${optlevel} --mattr ${attribute} \
- --prefix=Subzero_ \
- --target=x8632 \
- --sandbox=${sb} \
- --dir="${OUTDIR}" \
- --test=test_sync_atomic.cpp \
- --crosstest-bitcode=0 \
- --driver=test_sync_atomic_main.cpp \
- --output=test_sync_atomic_sb${sb}_O${optlevel}_${attribute}
-
- crosstest.py -O${optlevel} --mattr ${attribute} \
- --prefix=Subzero_ \
- --target=x8632 \
- --sandbox=${sb} \
- --dir="${OUTDIR}" \
- --test=test_vector_ops.ll \
- --driver=test_vector_ops_main.cpp \
- --output=test_vector_ops_sb${sb}_O${optlevel}_${attribute}
-
- done
- done
-done
-
-for sb in ${SANDBOX} ; do
- if [ $sb = 0 ] ; then
- PREFIX=
- else
- PREFIX="../../../../run.py -q"
- fi
- for optlevel in ${OPTLEVELS} ; do
- for attribute in ${ATTRIBUTES}; do
- ${PREFIX} "${OUTDIR}"/simple_loop_sb${sb}_O${optlevel}_${attribute}
- ${PREFIX} "${OUTDIR}"/mem_intrin_sb${sb}_O${optlevel}_${attribute}
- ${PREFIX} "${OUTDIR}"/test_arith_sb${sb}_O${optlevel}_${attribute}
- ${PREFIX} "${OUTDIR}"/test_bitmanip_sb${sb}_O${optlevel}_${attribute}
- ${PREFIX} "${OUTDIR}"/test_calling_conv_sb${sb}_O${optlevel}_${attribute}
- ${PREFIX} "${OUTDIR}"/test_cast_sb${sb}_O${optlevel}_${attribute}
- ${PREFIX} "${OUTDIR}"/test_fcmp_sb${sb}_O${optlevel}_${attribute}
- ${PREFIX} "${OUTDIR}"/test_global_sb${sb}_O${optlevel}_${attribute}
- ${PREFIX} "${OUTDIR}"/test_icmp_sb${sb}_O${optlevel}_${attribute}
- ${PREFIX} "${OUTDIR}"/test_select_sb${sb}_O${optlevel}_${attribute}
- ${PREFIX} "${OUTDIR}"/test_stacksave_sb${sb}_O${optlevel}_${attribute}
- ${PREFIX} "${OUTDIR}"/test_sync_atomic_sb${sb}_O${optlevel}_${attribute}
- ${PREFIX} "${OUTDIR}"/test_vector_ops_sb${sb}_O${optlevel}_${attribute}
- done
- done
-done
diff --git a/pydir/crosstest.py b/pydir/crosstest.py
index 55a79df..ae2c8a2 100755
--- a/pydir/crosstest.py
+++ b/pydir/crosstest.py
@@ -10,18 +10,17 @@
from utils import FindBaseNaCl
def main():
- """Builds a cross-test binary that allows functions translated by
- Subzero and llc to be compared.
+ """Builds a cross-test binary for comparing Subzero and llc translation.
- Each --test argument is compiled once by llc and once by Subzero.
- C/C++ tests are first compiled down to PNaCl bitcode by the
- build-pnacl-ir.py script. The --prefix argument ensures that
- symbol names are different between the two object files, to avoid
- linking errors.
+ Each --test argument is compiled once by llc and once by Subzero. C/C++
+ tests are first compiled down to PNaCl bitcode using pnacl-clang and
+ pnacl-opt. The --prefix argument ensures that symbol names are different
+ between the two object files, to avoid linking errors.
- There is also a --driver argument that specifies the C/C++ file
- that calls the test functions with a variety of interesting inputs
- and compares their results.
+ There is also a --driver argument that specifies the C/C++ file that calls
+ the test functions with a variety of interesting inputs and compares their
+ results.
+
"""
# arch_map maps a Subzero target string to an llvm-mc -triple string.
arch_map = { 'x8632':'i686', 'x8664':'x86_64', 'arm':'armv7a' }
@@ -76,24 +75,35 @@
bindir = ('{root}/toolchain/linux_x86/pnacl_newlib/bin'
.format(root=nacl_root))
triple = arch_map[args.target] + ('-nacl' if args.sandbox else '')
+ mypath = os.path.abspath(os.path.dirname(sys.argv[0]))
objs = []
for arg in args.test:
+ # Construct a "unique key" for each test so that tests can be run in
+ # parallel without race conditions on temporary file creation.
+ key = '{target}.{sb}.O{opt}.{attr}'.format(
+ target=args.target, sb='sb' if args.sandbox else 'nat',
+ opt=args.optlevel, attr=args.attr)
base, ext = os.path.splitext(arg)
if ext == '.ll':
bitcode = arg
else:
- bitcode = os.path.join(args.dir, base + '.pnacl.ll')
- shellcmd(['../pydir/build-pnacl-ir.py', '--disable-verify',
- '--dir', args.dir, arg])
+ # Use pnacl-clang and pnacl-opt to produce PNaCl bitcode.
+ bitcode_nonfinal = os.path.join(args.dir, base + '.' + key + '.bc')
+ bitcode = os.path.join(args.dir, base + '.' + key + '.pnacl.ll')
+ shellcmd(['{bin}/pnacl-clang'.format(bin=bindir),
+ '-O2', '-c', arg, '-o', bitcode_nonfinal])
+ shellcmd(['{bin}/pnacl-opt'.format(bin=bindir),
+ '-pnacl-abi-simplify-preopt',
+ '-pnacl-abi-simplify-postopt',
+ '-pnaclabi-allow-debug-metadata',
+ bitcode_nonfinal, '-S', '-o', bitcode])
- base_sz = '{base}.{sb}.O{opt}.{attr}.{target}'.format(
- base=base, sb='sb' if args.sandbox else 'nat', opt=args.optlevel,
- attr=args.attr, target=args.target)
+ base_sz = '{base}.{key}'.format(base=base, key=key)
asm_sz = os.path.join(args.dir, base_sz + '.sz.s')
obj_sz = os.path.join(args.dir, base_sz + '.sz.o')
obj_llc = os.path.join(args.dir, base_sz + '.llc.o')
- shellcmd(['../pnacl-sz',
+ shellcmd(['{path}/pnacl-sz'.format(path=os.path.dirname(mypath)),
'-O' + args.optlevel,
'-mattr=' + args.attr,
'--target=' + args.target,
diff --git a/pydir/crosstest_generator.py b/pydir/crosstest_generator.py
new file mode 100755
index 0000000..3038f4c
--- /dev/null
+++ b/pydir/crosstest_generator.py
@@ -0,0 +1,165 @@
+#!/usr/bin/env python2
+
+import argparse
+import ConfigParser
+import os
+import shutil
+import sys
+
+from utils import shellcmd
+from utils import FindBaseNaCl
+
+def Match(desc, includes, excludes, default_match):
+ """Determines whether desc is a match against includes and excludes.
+
+ 'desc' is a set of attributes, and 'includes' and 'excludes' are lists of sets
+ of attributes.
+
+ If 'desc' matches any element from 'excludes', the result is False.
+ Otherwise, if 'desc' matches any element from 'includes', the result is True.
+ Otherwise, the 'default_match' value is returned.
+ """
+ for exclude in excludes:
+ if exclude < desc:
+ return False
+ for include in includes:
+ if include < desc:
+ return True
+ return default_match
+
+def main():
+ """Framework for cross test generation and execution.
+
+ Builds and executes cross tests from the space of all possible attribute
+ combinations. The space can be restricted by providing subsets of attributes
+ to specifically include or exclude.
+ """
+ # pypath is where to find other Subzero python scripts.
+ pypath = os.path.abspath(os.path.dirname(sys.argv[0]))
+ root = FindBaseNaCl()
+
+ # The rest of the attribute sets.
+ targets = [ 'x8632' ]
+ sandboxing = [ 'native', 'sandbox' ]
+ opt_levels = [ 'Om1', 'O2' ]
+ arch_attrs = [ 'sse2', 'sse4.1' ]
+ # all_keys is only used in the help text.
+ all_keys = '; '.join([' '.join(targets), ' '.join(sandboxing),
+ ' '.join(opt_levels), ' '.join(arch_attrs)])
+
+ argparser = argparse.ArgumentParser(
+ description=' ' + main.__doc__ +
+ 'The set of attributes is the set of tests plus the following:\n' +
+ all_keys, formatter_class=argparse.RawTextHelpFormatter)
+ argparser.add_argument('--config', default='crosstest.cfg', dest='config',
+ metavar='FILE', help='Test configuration file')
+ argparser.add_argument('--print-tests', default=False, action='store_true',
+ help='Print the set of test names and exit')
+ argparser.add_argument('--include', '-i', default=[], dest='include',
+ action='append', metavar='ATTR_LIST',
+ help='Attributes to include (comma-separated). ' +
+ 'Can be used multiple times.')
+ argparser.add_argument('--exclude', '-e', default=[], dest='exclude',
+ action='append', metavar='ATTR_LIST',
+ help='Attributes to include (comma-separated). ' +
+ 'Can be used multiple times.')
+ argparser.add_argument('--verbose', '-v', default=False, action='store_true',
+ help='Use verbose output')
+ argparser.add_argument('--defer', default=False, action='store_true',
+ help='Defer execution until all executables are built')
+ argparser.add_argument('--no-compile', '-n', default=False,
+ action='store_true',
+ help="Don't build; reuse binaries from the last run")
+ argparser.add_argument('--dir', dest='dir', metavar='DIRECTORY',
+ default=('{root}/toolchain_build/src/subzero/' +
+ 'crosstest/Output').format(root=root),
+ help='Output directory')
+ argparser.add_argument('--lit', default=False, action='store_true',
+ help='Generate files for lit testing')
+ args = argparser.parse_args()
+
+ # Run from the crosstest directory to make it easy to grab inputs.
+ crosstest_dir = '{root}/toolchain_build/src/subzero/crosstest'.format(
+ root=root)
+ os.chdir(crosstest_dir)
+
+ tests = ConfigParser.RawConfigParser()
+ tests.read('crosstest.cfg')
+
+ if args.print_tests:
+ print 'Test name attributes: ' + ' '.join(sorted(tests.sections()))
+ sys.exit(0)
+
+ # includes and excludes are both lists of sets.
+ includes = [ set(item.split(',')) for item in args.include ]
+ excludes = [ set(item.split(',')) for item in args.exclude ]
+ # If any --include args are provided, the default is to not match.
+ default_match = not args.include
+
+ # Delete and recreate the output directory, unless --no-compile was specified.
+ if not args.no_compile:
+ if os.path.exists(args.dir):
+ if os.path.isdir(args.dir):
+ shutil.rmtree(args.dir)
+ else:
+ os.remove(args.dir)
+ if not os.path.exists(args.dir):
+ os.makedirs(args.dir)
+
+ # If --defer is specified, collect the run commands into deferred_cmds for
+ # later execution.
+ deferred_cmds = []
+ for test in sorted(tests.sections()):
+ for target in targets:
+ for sb in sandboxing:
+ for opt in opt_levels:
+ for attr in arch_attrs:
+ desc = [ test, target, sb, opt, attr ]
+ if Match(set(desc), includes, excludes, default_match):
+ exe = '{test}_{target}_{sb}_{opt}_{attr}'.format(
+ test=test, target=target, sb=sb, opt=opt,
+ attr=attr)
+ extra = (tests.get(test, 'flags').split(' ')
+ if tests.has_option(test, 'flags') else [])
+ # Generate the compile command.
+ cmp_cmd = ['{path}/crosstest.py'.format(path=pypath),
+ '-{opt}'.format(opt=opt),
+ '--mattr={attr}'.format(attr=attr),
+ '--prefix=Subzero_',
+ '--target={target}'.format(target=target),
+ '--sandbox={sb}'.format(sb='1' if sb=='sandbox'
+ else '0'),
+ '--dir={dir}'.format(dir=args.dir),
+ '--output={exe}'.format(exe=exe),
+ '--driver={drv}'.format(drv=tests.get(test, 'driver')),
+ ] + extra + \
+ [ '--test=' + t
+ for t in tests.get(test, 'test').split(' ') ]
+ run_cmd_base = os.path.join(args.dir, exe)
+ # Generate the run command.
+ run_cmd = run_cmd_base
+ if sb == 'sandbox':
+ run_cmd = '{root}/run.py -q '.format(root=root) + run_cmd
+ if args.lit:
+ # Create a file to drive the lit test.
+ with open(run_cmd_base + '.xtest', 'w') as f:
+ f.write('# RUN: sh %s | FileCheck %s\n')
+ f.write('cd ' + crosstest_dir + ' && \\\n')
+ f.write(' '.join(cmp_cmd) + ' && \\\n')
+ f.write(run_cmd + '\n')
+ f.write('echo Recreate a failure using ' + __file__ +
+ ' --include=' + ','.join(desc) + '\n')
+ f.write('# CHECK: Failures=0\n')
+ else:
+ if not args.no_compile:
+ shellcmd(cmp_cmd,
+ echo=args.verbose)
+ if (args.defer):
+ deferred_cmds.append(run_cmd)
+ else:
+ shellcmd(run_cmd, echo=True)
+ for run_cmd in deferred_cmds:
+ shellcmd(run_cmd, echo=True)
+
+if __name__ == '__main__':
+ main()
