| #!/usr/bin/env python |
| # |
| # Given a previous good compile narrow down miscompiles. |
| # Expects two directories named "before" and "after" each containing a set of |
| # assembly or object files where the "after" version is assumed to be broken. |
| # You also have to provide a script called "link_test". It is called with a list |
| # of files which should be linked together and result tested. "link_test" should |
| # returns with exitcode 0 if the linking and testing succeeded. |
| # |
| # abtest.py operates by taking all files from the "before" directory and |
| # in each step replacing one of them with a file from the "bad" directory. |
| # |
| # Additionally you can perform the same steps with a single .s file. In this |
| # mode functions are identified by " -- Begin function FunctionName" and |
| # " -- End function" markers. The abtest.py then takes all |
| # function from the file in the "before" directory and replaces one function |
| # with the corresponding function from the "bad" file in each step. |
| # |
| # Example usage to identify miscompiled files: |
| # 1. Create a link_test script, make it executable. Simple Example: |
| # clang "$@" -o /tmp/test && /tmp/test || echo "PROBLEM" |
| # 2. Run the script to figure out which files are miscompiled: |
| # > ./abtest.py |
| # somefile.s: ok |
| # someotherfile.s: skipped: same content |
| # anotherfile.s: failed: './link_test' exitcode != 0 |
| # ... |
| # Example usage to identify miscompiled functions inside a file: |
| # 3. Run the tests on a single file (assuming before/file.s and |
| # after/file.s exist) |
| # > ./abtest.py file.s |
| # funcname1 [0/XX]: ok |
| # funcname2 [1/XX]: ok |
| # funcname3 [2/XX]: skipped: same content |
| # funcname4 [3/XX]: failed: './link_test' exitcode != 0 |
| # ... |
| from fnmatch import filter |
| from sys import stderr |
| import argparse |
| import filecmp |
| import os |
| import subprocess |
| import sys |
| |
| LINKTEST="./link_test" |
| ESCAPE="\033[%sm" |
| BOLD=ESCAPE % "1" |
| RED=ESCAPE % "31" |
| NORMAL=ESCAPE % "0" |
| FAILED=RED+"failed"+NORMAL |
| |
| def find(dir, file_filter=None): |
| files = [walkdir[0]+"/"+file for walkdir in os.walk(dir) for file in walkdir[2]] |
| if file_filter != None: |
| files = filter(files, file_filter) |
| return files |
| |
| def error(message): |
| stderr.write("Error: %s\n" % (message,)) |
| |
| def warn(message): |
| stderr.write("Warning: %s\n" % (message,)) |
| |
| def extract_functions(file): |
| functions = [] |
| in_function = None |
| for line in open(file): |
| marker = line.find(" -- Begin function ") |
| if marker != -1: |
| if in_function != None: |
| warn("Missing end of function %s" % (in_function,)) |
| funcname = line[marker + 19:-1] |
| in_function = funcname |
| text = line |
| continue |
| |
| marker = line.find(" -- End function") |
| if marker != -1: |
| text += line |
| functions.append( (in_function, text) ) |
| in_function = None |
| continue |
| |
| if in_function != None: |
| text += line |
| return functions |
| |
| def replace_function(file, function, replacement, dest): |
| out = open(dest, "w") |
| skip = False |
| found = False |
| in_function = None |
| for line in open(file): |
| marker = line.find(" -- Begin function ") |
| if marker != -1: |
| if in_function != None: |
| warn("Missing end of function %s" % (in_function,)) |
| funcname = line[marker + 19:-1] |
| in_function = funcname |
| if in_function == function: |
| out.write(replacement) |
| skip = True |
| else: |
| marker = line.find(" -- End function") |
| if marker != -1: |
| in_function = None |
| if skip: |
| skip = False |
| continue |
| |
| if not skip: |
| out.write(line) |
| |
| def announce_test(name): |
| stderr.write("%s%s%s: " % (BOLD, name, NORMAL)) |
| stderr.flush() |
| |
| def announce_result(result, info): |
| stderr.write(result) |
| if info != "": |
| stderr.write(": %s" % info) |
| stderr.write("\n") |
| stderr.flush() |
| |
| def testrun(files): |
| linkline="%s %s" % (LINKTEST, " ".join(files),) |
| res = subprocess.call(linkline, shell=True) |
| if res != 0: |
| announce_result(FAILED, "'%s' exitcode != 0" % LINKTEST) |
| return False |
| else: |
| announce_result("ok", "") |
| return True |
| |
| def check_files(): |
| """Check files mode""" |
| for i in range(0, len(NO_PREFIX)): |
| f = NO_PREFIX[i] |
| b=baddir+"/"+f |
| if b not in BAD_FILES: |
| warn("There is no corresponding file to '%s' in %s" \ |
| % (gooddir+"/"+f, baddir)) |
| continue |
| |
| announce_test(f + " [%s/%s]" % (i+1, len(NO_PREFIX))) |
| |
| # combine files (everything from good except f) |
| testfiles=[] |
| skip=False |
| for c in NO_PREFIX: |
| badfile = baddir+"/"+c |
| goodfile = gooddir+"/"+c |
| if c == f: |
| testfiles.append(badfile) |
| if filecmp.cmp(goodfile, badfile): |
| announce_result("skipped", "same content") |
| skip = True |
| break |
| else: |
| testfiles.append(goodfile) |
| if skip: |
| continue |
| testrun(testfiles) |
| |
| def check_functions_in_file(base, goodfile, badfile): |
| functions = extract_functions(goodfile) |
| if len(functions) == 0: |
| warn("Couldn't find any function in %s, missing annotations?" % (goodfile,)) |
| return |
| badfunctions = dict(extract_functions(badfile)) |
| if len(functions) == 0: |
| warn("Couldn't find any function in %s, missing annotations?" % (badfile,)) |
| return |
| |
| COMBINED="/tmp/combined.s" |
| i = 0 |
| for (func,func_text) in functions: |
| announce_test(func + " [%s/%s]" % (i+1, len(functions))) |
| i+=1 |
| if func not in badfunctions: |
| warn("Function '%s' missing from bad file" % func) |
| continue |
| if badfunctions[func] == func_text: |
| announce_result("skipped", "same content") |
| continue |
| replace_function(goodfile, func, badfunctions[func], COMBINED) |
| testfiles=[] |
| for c in NO_PREFIX: |
| if c == base: |
| testfiles.append(COMBINED) |
| continue |
| testfiles.append(gooddir + "/" + c) |
| |
| testrun(testfiles) |
| |
| parser = argparse.ArgumentParser() |
| parser.add_argument('--a', dest='dir_a', default='before') |
| parser.add_argument('--b', dest='dir_b', default='after') |
| parser.add_argument('--insane', help='Skip sanity check', action='store_true') |
| parser.add_argument('file', metavar='file', nargs='?') |
| config = parser.parse_args() |
| |
| gooddir=config.dir_a |
| baddir=config.dir_b |
| |
| BAD_FILES=find(baddir, "*") |
| GOOD_FILES=find(gooddir, "*") |
| NO_PREFIX=sorted([x[len(gooddir)+1:] for x in GOOD_FILES]) |
| |
| # "Checking whether build environment is sane ..." |
| if not config.insane: |
| announce_test("sanity check") |
| if not os.access(LINKTEST, os.X_OK): |
| error("Expect '%s' to be present and executable" % (LINKTEST,)) |
| exit(1) |
| |
| res = testrun(GOOD_FILES) |
| if not res: |
| # "build environment is grinning and holding a spatula. Guess not." |
| linkline="%s %s" % (LINKTEST, " ".join(GOOD_FILES),) |
| stderr.write("\n%s\n\n" % linkline) |
| stderr.write("Returned with exitcode != 0\n") |
| sys.exit(1) |
| |
| if config.file is not None: |
| # File exchange mode |
| goodfile = gooddir+"/"+config.file |
| badfile = baddir+"/"+config.file |
| check_functions_in_file(config.file, goodfile, badfile) |
| else: |
| # Function exchange mode |
| check_files() |
