| #!/usr/bin/env python | |
| # | |
| # ======- git-llvm - LLVM Git Help Integration ---------*- python -*--========# | |
| # | |
| # The LLVM Compiler Infrastructure | |
| # | |
| # This file is distributed under the University of Illinois Open Source | |
| # License. See LICENSE.TXT for details. | |
| # | |
| # ==------------------------------------------------------------------------==# | |
| """ | |
| git-llvm integration | |
| ==================== | |
| This file provides integration for git. | |
| """ | |
| from __future__ import print_function | |
| import argparse | |
| import collections | |
| import contextlib | |
| import errno | |
| import os | |
| import re | |
| import subprocess | |
| import sys | |
| import tempfile | |
| import time | |
| assert sys.version_info >= (2, 7) | |
| # It's *almost* a straightforward mapping from the monorepo to svn... | |
| GIT_TO_SVN_DIR = { | |
| d: (d + '/trunk') | |
| for d in [ | |
| 'clang-tools-extra', | |
| 'compiler-rt', | |
| 'debuginfo-tests', | |
| 'dragonegg', | |
| 'klee', | |
| 'libclc', | |
| 'libcxx', | |
| 'libcxxabi', | |
| 'libunwind', | |
| 'lld', | |
| 'lldb', | |
| 'llgo', | |
| 'llvm', | |
| 'openmp', | |
| 'parallel-libs', | |
| 'polly', | |
| ] | |
| } | |
| GIT_TO_SVN_DIR.update({'clang': 'cfe/trunk'}) | |
| VERBOSE = False | |
| QUIET = False | |
| dev_null_fd = None | |
| def eprint(*args, **kwargs): | |
| print(*args, file=sys.stderr, **kwargs) | |
| def log(*args, **kwargs): | |
| if QUIET: | |
| return | |
| print(*args, **kwargs) | |
| def log_verbose(*args, **kwargs): | |
| if not VERBOSE: | |
| return | |
| print(*args, **kwargs) | |
| def die(msg): | |
| eprint(msg) | |
| sys.exit(1) | |
| def first_dirname(d): | |
| while True: | |
| (head, tail) = os.path.split(d) | |
| if not head or head == '/': | |
| return tail | |
| d = head | |
| def get_dev_null(): | |
| """Lazily create a /dev/null fd for use in shell()""" | |
| global dev_null_fd | |
| if dev_null_fd is None: | |
| dev_null_fd = open(os.devnull, 'w') | |
| return dev_null_fd | |
| def shell(cmd, strip=True, cwd=None, stdin=None, die_on_failure=True, | |
| ignore_errors=False): | |
| log_verbose('Running: %s' % ' '.join(cmd)) | |
| err_pipe = subprocess.PIPE | |
| if ignore_errors: | |
| # Silence errors if requested. | |
| err_pipe = get_dev_null() | |
| start = time.time() | |
| p = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=err_pipe, | |
| stdin=subprocess.PIPE, universal_newlines=True) | |
| stdout, stderr = p.communicate(input=stdin) | |
| elapsed = time.time() - start | |
| log_verbose('Command took %0.1fs' % elapsed) | |
| if p.returncode == 0 or ignore_errors: | |
| if stderr and not ignore_errors: | |
| eprint('`%s` printed to stderr:' % ' '.join(cmd)) | |
| eprint(stderr.rstrip()) | |
| if strip: | |
| stdout = stdout.rstrip('\r\n') | |
| return stdout | |
| err_msg = '`%s` returned %s' % (' '.join(cmd), p.returncode) | |
| eprint(err_msg) | |
| if stderr: | |
| eprint(stderr.rstrip()) | |
| if die_on_failure: | |
| sys.exit(2) | |
| raise RuntimeError(err_msg) | |
| def git(*cmd, **kwargs): | |
| return shell(['git'] + list(cmd), kwargs.get('strip', True)) | |
| def svn(cwd, *cmd, **kwargs): | |
| # TODO: Better way to do default arg when we have *cmd? | |
| return shell(['svn'] + list(cmd), cwd=cwd, stdin=kwargs.get('stdin', None), | |
| ignore_errors=kwargs.get('ignore_errors', None)) | |
| def program_exists(cmd): | |
| if sys.platform == 'win32' and not cmd.endswith('.exe'): | |
| cmd += '.exe' | |
| for path in os.environ["PATH"].split(os.pathsep): | |
| if os.access(os.path.join(path, cmd), os.X_OK): | |
| return True | |
| return False | |
| def get_default_rev_range(): | |
| # Get the branch tracked by the current branch, as set by | |
| # git branch --set-upstream-to See http://serverfault.com/a/352236/38694. | |
| cur_branch = git('rev-parse', '--symbolic-full-name', 'HEAD') | |
| upstream_branch = git('for-each-ref', '--format=%(upstream:short)', | |
| cur_branch) | |
| if not upstream_branch: | |
| upstream_branch = 'origin/master' | |
| # Get the newest common ancestor between HEAD and our upstream branch. | |
| upstream_rev = git('merge-base', 'HEAD', upstream_branch) | |
| return '%s..' % upstream_rev | |
| def get_revs_to_push(rev_range): | |
| if not rev_range: | |
| rev_range = get_default_rev_range() | |
| # Use git show rather than some plumbing command to figure out which revs | |
| # are in rev_range because it handles single revs (HEAD^) and ranges | |
| # (foo..bar) like we want. | |
| revs = git('show', '--reverse', '--quiet', | |
| '--pretty=%h', rev_range).splitlines() | |
| if not revs: | |
| die('Nothing to push: No revs in range %s.' % rev_range) | |
| return revs | |
| def clean_and_update_svn(svn_repo): | |
| svn(svn_repo, 'revert', '-R', '.') | |
| # Unfortunately it appears there's no svn equivalent for git clean, so we | |
| # have to do it ourselves. | |
| for line in svn(svn_repo, 'status', '--no-ignore').split('\n'): | |
| if not line.startswith('?'): | |
| continue | |
| filename = line[1:].strip() | |
| os.remove(os.path.join(svn_repo, filename)) | |
| svn(svn_repo, 'update', *list(GIT_TO_SVN_DIR.values())) | |
| def svn_init(svn_root): | |
| if not os.path.exists(svn_root): | |
| log('Creating svn staging directory: (%s)' % (svn_root)) | |
| os.makedirs(svn_root) | |
| log('This is a one-time initialization, please be patient for a few' | |
| ' minutes...') | |
| svn(svn_root, 'checkout', '--depth=immediates', | |
| 'https://llvm.org/svn/llvm-project/', '.') | |
| svn(svn_root, 'update', *list(GIT_TO_SVN_DIR.values())) | |
| log("svn staging area ready in '%s'" % svn_root) | |
| if not os.path.isdir(svn_root): | |
| die("Can't initialize svn staging dir (%s)" % svn_root) | |
| def fix_eol_style_native(rev, sr, svn_sr_path): | |
| """Fix line endings before applying patches with Unix endings | |
| SVN on Windows will check out files with CRLF for files with the | |
| svn:eol-style property set to "native". This breaks `git apply`, which | |
| typically works with Unix-line ending patches. Work around the problem here | |
| by doing a dos2unix up front for files with svn:eol-style set to "native". | |
| SVN will not commit a mass line ending re-doing because it detects the line | |
| ending format for files with this property. | |
| """ | |
| files = git('diff-tree', '--no-commit-id', '--name-only', '-r', rev, '--', | |
| sr).split('\n') | |
| files = [f.split('/', 1)[1] for f in files] | |
| # Skip files that don't exist in SVN yet. | |
| files = [f for f in files if os.path.exists(os.path.join(svn_sr_path, f))] | |
| # Use ignore_errors because 'svn propget' prints errors if the file doesn't | |
| # have the named property. There doesn't seem to be a way to suppress that. | |
| eol_props = svn(svn_sr_path, 'propget', 'svn:eol-style', *files, | |
| ignore_errors=True) | |
| crlf_files = [] | |
| if len(files) == 1: | |
| # No need to split propget output on ' - ' when we have one file. | |
| if eol_props.strip() == 'native': | |
| crlf_files = files | |
| else: | |
| for eol_prop in eol_props.split('\n'): | |
| # Remove spare CR. | |
| eol_prop = eol_prop.strip('\r') | |
| if not eol_prop: | |
| continue | |
| prop_parts = eol_prop.rsplit(' - ', 1) | |
| if len(prop_parts) != 2: | |
| eprint("unable to parse svn propget line:") | |
| eprint(eol_prop) | |
| continue | |
| (f, eol_style) = prop_parts | |
| if eol_style == 'native': | |
| crlf_files.append(f) | |
| # Reformat all files with native SVN line endings to Unix format. SVN knows | |
| # files with native line endings are text files. It will commit just the | |
| # diff, and not a mass line ending change. | |
| shell(['dos2unix', '-q'] + crlf_files, cwd=svn_sr_path) | |
| def svn_push_one_rev(svn_repo, rev, dry_run): | |
| files = git('diff-tree', '--no-commit-id', '--name-only', '-r', | |
| rev).split('\n') | |
| subrepos = {first_dirname(f) for f in files} | |
| if not subrepos: | |
| raise RuntimeError('Empty diff for rev %s?' % rev) | |
| status = svn(svn_repo, 'status', '--no-ignore') | |
| if status: | |
| die("Can't push git rev %s because svn status is not empty:\n%s" % | |
| (rev, status)) | |
| for sr in subrepos: | |
| svn_sr_path = os.path.join(svn_repo, GIT_TO_SVN_DIR[sr]) | |
| if os.name == 'nt': | |
| fix_eol_style_native(rev, sr, svn_sr_path) | |
| diff = git('show', '--binary', rev, '--', sr, strip=False) | |
| # git is the only thing that can handle its own patches... | |
| log_verbose('Apply patch: %s' % diff) | |
| try: | |
| shell(['git', 'apply', '-p2', '-'], cwd=svn_sr_path, stdin=diff, | |
| die_on_failure=False) | |
| except RuntimeError as e: | |
| eprint("Patch doesn't apply: maybe you should try `git pull -r` " | |
| "first?") | |
| sys.exit(2) | |
| status_lines = svn(svn_repo, 'status', '--no-ignore').split('\n') | |
| for l in (l for l in status_lines if (l.startswith('?') or | |
| l.startswith('I'))): | |
| svn(svn_repo, 'add', '--no-ignore', l[1:].strip()) | |
| for l in (l for l in status_lines if l.startswith('!')): | |
| svn(svn_repo, 'remove', l[1:].strip()) | |
| # Now we're ready to commit. | |
| commit_msg = git('show', '--pretty=%B', '--quiet', rev) | |
| if not dry_run: | |
| log(svn(svn_repo, 'commit', '-m', commit_msg, '--force-interactive')) | |
| log('Committed %s to svn.' % rev) | |
| else: | |
| log("Would have committed %s to svn, if this weren't a dry run." % rev) | |
| def cmd_push(args): | |
| '''Push changes back to SVN: this is extracted from Justin Lebar's script | |
| available here: https://github.com/jlebar/llvm-repo-tools/ | |
| Note: a current limitation is that git does not track file rename, so they | |
| will show up in SVN as delete+add. | |
| ''' | |
| # Get the git root | |
| git_root = git('rev-parse', '--show-toplevel') | |
| if not os.path.isdir(git_root): | |
| die("Can't find git root dir") | |
| # Push from the root of the git repo | |
| os.chdir(git_root) | |
| # We need a staging area for SVN, let's hide it in the .git directory. | |
| dot_git_dir = git('rev-parse', '--git-common-dir') | |
| svn_root = os.path.join(dot_git_dir, 'llvm-upstream-svn') | |
| svn_init(svn_root) | |
| rev_range = args.rev_range | |
| dry_run = args.dry_run | |
| revs = get_revs_to_push(rev_range) | |
| log('Pushing %d commit%s:\n%s' % | |
| (len(revs), 's' if len(revs) != 1 | |
| else '', '\n'.join(' ' + git('show', '--oneline', '--quiet', c) | |
| for c in revs))) | |
| for r in revs: | |
| clean_and_update_svn(svn_root) | |
| svn_push_one_rev(svn_root, r, dry_run) | |
| if __name__ == '__main__': | |
| if not program_exists('svn'): | |
| die('error: git-llvm needs svn command, but svn is not installed.') | |
| argv = sys.argv[1:] | |
| p = argparse.ArgumentParser( | |
| prog='git llvm', formatter_class=argparse.RawDescriptionHelpFormatter, | |
| description=__doc__) | |
| subcommands = p.add_subparsers(title='subcommands', | |
| description='valid subcommands', | |
| help='additional help') | |
| verbosity_group = p.add_mutually_exclusive_group() | |
| verbosity_group.add_argument('-q', '--quiet', action='store_true', | |
| help='print less information') | |
| verbosity_group.add_argument('-v', '--verbose', action='store_true', | |
| help='print more information') | |
| parser_push = subcommands.add_parser( | |
| 'push', description=cmd_push.__doc__, | |
| help='push changes back to the LLVM SVN repository') | |
| parser_push.add_argument( | |
| '-n', | |
| '--dry-run', | |
| dest='dry_run', | |
| action='store_true', | |
| help='Do everything other than commit to svn. Leaves junk in the svn ' | |
| 'repo, so probably will not work well if you try to commit more ' | |
| 'than one rev.') | |
| parser_push.add_argument( | |
| 'rev_range', | |
| metavar='GIT_REVS', | |
| type=str, | |
| nargs='?', | |
| help="revs to push (default: everything not in the branch's " | |
| 'upstream, or not in origin/master if the branch lacks ' | |
| 'an explicit upstream)') | |
| parser_push.set_defaults(func=cmd_push) | |
| args = p.parse_args(argv) | |
| VERBOSE = args.verbose | |
| QUIET = args.quiet | |
| # Dispatch to the right subcommand | |
| args.func(args) |