| #!/usr/bin/env python |
| # |
| # ======- git-llvm - LLVM Git Help Integration ---------*- python -*--========# |
| # |
| # The LLVM Compiler Infrastructure |
| # |
| # This file is distributed under the University of Illinois Open Source |
| # License. See LICENSE.TXT for details. |
| # |
| # ==------------------------------------------------------------------------==# |
| |
| """ |
| git-llvm integration |
| ==================== |
| |
| This file provides integration for git. |
| """ |
| |
| from __future__ import print_function |
| import argparse |
| import collections |
| import contextlib |
| import errno |
| import os |
| import re |
| import subprocess |
| import sys |
| import tempfile |
| import time |
| assert sys.version_info >= (2, 7) |
| |
| |
| # It's *almost* a straightforward mapping from the monorepo to svn... |
| GIT_TO_SVN_DIR = { |
| d: (d + '/trunk') |
| for d in [ |
| 'clang-tools-extra', |
| 'compiler-rt', |
| 'debuginfo-tests', |
| 'dragonegg', |
| 'klee', |
| 'libclc', |
| 'libcxx', |
| 'libcxxabi', |
| 'libunwind', |
| 'lld', |
| 'lldb', |
| 'llgo', |
| 'llvm', |
| 'openmp', |
| 'parallel-libs', |
| 'polly', |
| ] |
| } |
| GIT_TO_SVN_DIR.update({'clang': 'cfe/trunk'}) |
| |
| VERBOSE = False |
| QUIET = False |
| dev_null_fd = None |
| |
| |
| def eprint(*args, **kwargs): |
| print(*args, file=sys.stderr, **kwargs) |
| |
| |
| def log(*args, **kwargs): |
| if QUIET: |
| return |
| print(*args, **kwargs) |
| |
| |
| def log_verbose(*args, **kwargs): |
| if not VERBOSE: |
| return |
| print(*args, **kwargs) |
| |
| |
| def die(msg): |
| eprint(msg) |
| sys.exit(1) |
| |
| |
| def first_dirname(d): |
| while True: |
| (head, tail) = os.path.split(d) |
| if not head or head == '/': |
| return tail |
| d = head |
| |
| |
| def get_dev_null(): |
| """Lazily create a /dev/null fd for use in shell()""" |
| global dev_null_fd |
| if dev_null_fd is None: |
| dev_null_fd = open(os.devnull, 'w') |
| return dev_null_fd |
| |
| |
| def shell(cmd, strip=True, cwd=None, stdin=None, die_on_failure=True, |
| ignore_errors=False): |
| log_verbose('Running: %s' % ' '.join(cmd)) |
| |
| err_pipe = subprocess.PIPE |
| if ignore_errors: |
| # Silence errors if requested. |
| err_pipe = get_dev_null() |
| |
| start = time.time() |
| p = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=err_pipe, |
| stdin=subprocess.PIPE, universal_newlines=True) |
| stdout, stderr = p.communicate(input=stdin) |
| elapsed = time.time() - start |
| |
| log_verbose('Command took %0.1fs' % elapsed) |
| |
| if p.returncode == 0 or ignore_errors: |
| if stderr and not ignore_errors: |
| eprint('`%s` printed to stderr:' % ' '.join(cmd)) |
| eprint(stderr.rstrip()) |
| if strip: |
| stdout = stdout.rstrip('\r\n') |
| return stdout |
| err_msg = '`%s` returned %s' % (' '.join(cmd), p.returncode) |
| eprint(err_msg) |
| if stderr: |
| eprint(stderr.rstrip()) |
| if die_on_failure: |
| sys.exit(2) |
| raise RuntimeError(err_msg) |
| |
| |
| def git(*cmd, **kwargs): |
| return shell(['git'] + list(cmd), kwargs.get('strip', True)) |
| |
| |
| def svn(cwd, *cmd, **kwargs): |
| # TODO: Better way to do default arg when we have *cmd? |
| return shell(['svn'] + list(cmd), cwd=cwd, stdin=kwargs.get('stdin', None), |
| ignore_errors=kwargs.get('ignore_errors', None)) |
| |
| def program_exists(cmd): |
| if sys.platform == 'win32' and not cmd.endswith('.exe'): |
| cmd += '.exe' |
| for path in os.environ["PATH"].split(os.pathsep): |
| if os.access(os.path.join(path, cmd), os.X_OK): |
| return True |
| return False |
| |
| def get_default_rev_range(): |
| # Get the branch tracked by the current branch, as set by |
| # git branch --set-upstream-to See http://serverfault.com/a/352236/38694. |
| cur_branch = git('rev-parse', '--symbolic-full-name', 'HEAD') |
| upstream_branch = git('for-each-ref', '--format=%(upstream:short)', |
| cur_branch) |
| if not upstream_branch: |
| upstream_branch = 'origin/master' |
| |
| # Get the newest common ancestor between HEAD and our upstream branch. |
| upstream_rev = git('merge-base', 'HEAD', upstream_branch) |
| return '%s..' % upstream_rev |
| |
| |
| def get_revs_to_push(rev_range): |
| if not rev_range: |
| rev_range = get_default_rev_range() |
| # Use git show rather than some plumbing command to figure out which revs |
| # are in rev_range because it handles single revs (HEAD^) and ranges |
| # (foo..bar) like we want. |
| revs = git('show', '--reverse', '--quiet', |
| '--pretty=%h', rev_range).splitlines() |
| if not revs: |
| die('Nothing to push: No revs in range %s.' % rev_range) |
| return revs |
| |
| |
| def clean_and_update_svn(svn_repo): |
| svn(svn_repo, 'revert', '-R', '.') |
| |
| # Unfortunately it appears there's no svn equivalent for git clean, so we |
| # have to do it ourselves. |
| for line in svn(svn_repo, 'status', '--no-ignore').split('\n'): |
| if not line.startswith('?'): |
| continue |
| filename = line[1:].strip() |
| os.remove(os.path.join(svn_repo, filename)) |
| |
| svn(svn_repo, 'update', *list(GIT_TO_SVN_DIR.values())) |
| |
| |
| def svn_init(svn_root): |
| if not os.path.exists(svn_root): |
| log('Creating svn staging directory: (%s)' % (svn_root)) |
| os.makedirs(svn_root) |
| log('This is a one-time initialization, please be patient for a few' |
| ' minutes...') |
| svn(svn_root, 'checkout', '--depth=immediates', |
| 'https://llvm.org/svn/llvm-project/', '.') |
| svn(svn_root, 'update', *list(GIT_TO_SVN_DIR.values())) |
| log("svn staging area ready in '%s'" % svn_root) |
| if not os.path.isdir(svn_root): |
| die("Can't initialize svn staging dir (%s)" % svn_root) |
| |
| |
| def fix_eol_style_native(rev, sr, svn_sr_path): |
| """Fix line endings before applying patches with Unix endings |
| |
| SVN on Windows will check out files with CRLF for files with the |
| svn:eol-style property set to "native". This breaks `git apply`, which |
| typically works with Unix-line ending patches. Work around the problem here |
| by doing a dos2unix up front for files with svn:eol-style set to "native". |
| SVN will not commit a mass line ending re-doing because it detects the line |
| ending format for files with this property. |
| """ |
| files = git('diff-tree', '--no-commit-id', '--name-only', '-r', rev, '--', |
| sr).split('\n') |
| files = [f.split('/', 1)[1] for f in files] |
| # Skip files that don't exist in SVN yet. |
| files = [f for f in files if os.path.exists(os.path.join(svn_sr_path, f))] |
| # Use ignore_errors because 'svn propget' prints errors if the file doesn't |
| # have the named property. There doesn't seem to be a way to suppress that. |
| eol_props = svn(svn_sr_path, 'propget', 'svn:eol-style', *files, |
| ignore_errors=True) |
| crlf_files = [] |
| if len(files) == 1: |
| # No need to split propget output on ' - ' when we have one file. |
| if eol_props.strip() == 'native': |
| crlf_files = files |
| else: |
| for eol_prop in eol_props.split('\n'): |
| # Remove spare CR. |
| eol_prop = eol_prop.strip('\r') |
| if not eol_prop: |
| continue |
| prop_parts = eol_prop.rsplit(' - ', 1) |
| if len(prop_parts) != 2: |
| eprint("unable to parse svn propget line:") |
| eprint(eol_prop) |
| continue |
| (f, eol_style) = prop_parts |
| if eol_style == 'native': |
| crlf_files.append(f) |
| # Reformat all files with native SVN line endings to Unix format. SVN knows |
| # files with native line endings are text files. It will commit just the |
| # diff, and not a mass line ending change. |
| shell(['dos2unix', '-q'] + crlf_files, cwd=svn_sr_path) |
| |
| |
| def svn_push_one_rev(svn_repo, rev, dry_run): |
| files = git('diff-tree', '--no-commit-id', '--name-only', '-r', |
| rev).split('\n') |
| subrepos = {first_dirname(f) for f in files} |
| if not subrepos: |
| raise RuntimeError('Empty diff for rev %s?' % rev) |
| |
| status = svn(svn_repo, 'status', '--no-ignore') |
| if status: |
| die("Can't push git rev %s because svn status is not empty:\n%s" % |
| (rev, status)) |
| |
| for sr in subrepos: |
| svn_sr_path = os.path.join(svn_repo, GIT_TO_SVN_DIR[sr]) |
| if os.name == 'nt': |
| fix_eol_style_native(rev, sr, svn_sr_path) |
| diff = git('show', '--binary', rev, '--', sr, strip=False) |
| # git is the only thing that can handle its own patches... |
| log_verbose('Apply patch: %s' % diff) |
| try: |
| shell(['git', 'apply', '-p2', '-'], cwd=svn_sr_path, stdin=diff, |
| die_on_failure=False) |
| except RuntimeError as e: |
| eprint("Patch doesn't apply: maybe you should try `git pull -r` " |
| "first?") |
| sys.exit(2) |
| |
| status_lines = svn(svn_repo, 'status', '--no-ignore').split('\n') |
| |
| for l in (l for l in status_lines if (l.startswith('?') or |
| l.startswith('I'))): |
| svn(svn_repo, 'add', '--no-ignore', l[1:].strip()) |
| for l in (l for l in status_lines if l.startswith('!')): |
| svn(svn_repo, 'remove', l[1:].strip()) |
| |
| # Now we're ready to commit. |
| commit_msg = git('show', '--pretty=%B', '--quiet', rev) |
| if not dry_run: |
| log(svn(svn_repo, 'commit', '-m', commit_msg, '--force-interactive')) |
| log('Committed %s to svn.' % rev) |
| else: |
| log("Would have committed %s to svn, if this weren't a dry run." % rev) |
| |
| |
| def cmd_push(args): |
| '''Push changes back to SVN: this is extracted from Justin Lebar's script |
| available here: https://github.com/jlebar/llvm-repo-tools/ |
| |
| Note: a current limitation is that git does not track file rename, so they |
| will show up in SVN as delete+add. |
| ''' |
| # Get the git root |
| git_root = git('rev-parse', '--show-toplevel') |
| if not os.path.isdir(git_root): |
| die("Can't find git root dir") |
| |
| # Push from the root of the git repo |
| os.chdir(git_root) |
| |
| # We need a staging area for SVN, let's hide it in the .git directory. |
| dot_git_dir = git('rev-parse', '--git-common-dir') |
| svn_root = os.path.join(dot_git_dir, 'llvm-upstream-svn') |
| svn_init(svn_root) |
| |
| rev_range = args.rev_range |
| dry_run = args.dry_run |
| revs = get_revs_to_push(rev_range) |
| log('Pushing %d commit%s:\n%s' % |
| (len(revs), 's' if len(revs) != 1 |
| else '', '\n'.join(' ' + git('show', '--oneline', '--quiet', c) |
| for c in revs))) |
| for r in revs: |
| clean_and_update_svn(svn_root) |
| svn_push_one_rev(svn_root, r, dry_run) |
| |
| |
| if __name__ == '__main__': |
| if not program_exists('svn'): |
| die('error: git-llvm needs svn command, but svn is not installed.') |
| |
| argv = sys.argv[1:] |
| p = argparse.ArgumentParser( |
| prog='git llvm', formatter_class=argparse.RawDescriptionHelpFormatter, |
| description=__doc__) |
| subcommands = p.add_subparsers(title='subcommands', |
| description='valid subcommands', |
| help='additional help') |
| verbosity_group = p.add_mutually_exclusive_group() |
| verbosity_group.add_argument('-q', '--quiet', action='store_true', |
| help='print less information') |
| verbosity_group.add_argument('-v', '--verbose', action='store_true', |
| help='print more information') |
| |
| parser_push = subcommands.add_parser( |
| 'push', description=cmd_push.__doc__, |
| help='push changes back to the LLVM SVN repository') |
| parser_push.add_argument( |
| '-n', |
| '--dry-run', |
| dest='dry_run', |
| action='store_true', |
| help='Do everything other than commit to svn. Leaves junk in the svn ' |
| 'repo, so probably will not work well if you try to commit more ' |
| 'than one rev.') |
| parser_push.add_argument( |
| 'rev_range', |
| metavar='GIT_REVS', |
| type=str, |
| nargs='?', |
| help="revs to push (default: everything not in the branch's " |
| 'upstream, or not in origin/master if the branch lacks ' |
| 'an explicit upstream)') |
| parser_push.set_defaults(func=cmd_push) |
| args = p.parse_args(argv) |
| VERBOSE = args.verbose |
| QUIET = args.quiet |
| |
| # Dispatch to the right subcommand |
| args.func(args) |
