You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by to...@apache.org on 2016/01/11 10:19:11 UTC
[5/8] incubator-kudu git commit: Add a script to push gerrit branches
to ASF
Add a script to push gerrit branches to ASF
This script can be run by an ASF committer to take changes that have been
committed via gerrit and push them to the official ASF repository.
An example screenshot is available here:
https://slack-files.com/T0CPU21PZ-F0HSG7EE9-a0f5ccff20
Change-Id: I60eefe99c2ba3bbd9c9df15f307d9ec277c2ce29
Reviewed-on: http://gerrit.cloudera.org:8080/1723
Reviewed-by: Mike Percy <mp...@cloudera.com>
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Todd Lipcon <to...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/55a5be98
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/55a5be98
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/55a5be98
Branch: refs/heads/master
Commit: 55a5be984556dce20fce04cf089430696e084c5a
Parents: b9b265d
Author: Todd Lipcon <to...@cloudera.com>
Authored: Wed Jan 6 15:22:22 2016 -0800
Committer: Todd Lipcon <to...@cloudera.com>
Committed: Fri Jan 8 19:42:11 2016 +0000
----------------------------------------------------------------------
build-support/kudu_util.py | 28 ++--
build-support/push_to_asf.py | 254 +++++++++++++++++++++++++++++++++++
build-support/trigger_gerrit.py | 21 +--
3 files changed, 273 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/55a5be98/build-support/kudu_util.py
----------------------------------------------------------------------
diff --git a/build-support/kudu_util.py b/build-support/kudu_util.py
index f4f80cf..731c1ad 100644
--- a/build-support/kudu_util.py
+++ b/build-support/kudu_util.py
@@ -22,14 +22,20 @@
import subprocess
-def check_output(cmd, **kwargs):
- """ Simple backport of subprocess.check_output() from python 2.7. """
- p = subprocess.Popen(cmd,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- **kwargs)
- out, err = p.communicate()
- if p.returncode != 0:
- raise Exception("%s returned %d: %s" % (cmd, p.returncode, err))
- return out
-
+def check_output(*popenargs, **kwargs):
+ r"""Run command with arguments and return its output as a byte string.
+ Backported from Python 2.7 as it's implemented as pure python on stdlib.
+ >>> check_output(['/usr/bin/python', '--version'])
+ Python 2.6.2
+ """
+ process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
+ output, unused_err = process.communicate()
+ retcode = process.poll()
+ if retcode:
+ cmd = kwargs.get("args")
+ if cmd is None:
+ cmd = popenargs[0]
+ error = subprocess.CalledProcessError(retcode, cmd)
+ error.output = output
+ raise error
+ return output
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/55a5be98/build-support/push_to_asf.py
----------------------------------------------------------------------
diff --git a/build-support/push_to_asf.py b/build-support/push_to_asf.py
new file mode 100755
index 0000000..491830c
--- /dev/null
+++ b/build-support/push_to_asf.py
@@ -0,0 +1,254 @@
+#!/usr/bin/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script fetches branches from the Gerrit repository and
+# allows ASF committers to propagate commits from gerrit into the
+# official ASF repository.
+#
+# Current ASF policy is that this mirroring cannot be automatic
+# and should be driven by a committer who inspects and signs off
+# on the commits being made into the ASF. Additionally, the ASF
+# prefers that in most cases, the committer according to source
+# control should be the same person to push the commit to a git
+# repository.
+#
+# This script provides the committer the opportunity to review the
+# changes to be pushed, warns them if they are pushing code for
+# which they weren't the committer, and performs the actual push.
+
+import logging
+import optparse
+import re
+import subprocess
+import sys
+
+from kudu_util import check_output
+
+APACHE_REPO = "https://git-wip-us.apache.org/repos/asf/incubator-kudu.git"
+GERRIT_URL_RE = re.compile(r"ssh://.+@gerrit.cloudera.org:29418/kudu")
+
+# ANSI color codes.
+RED = "\x1b[31m"
+GREEN = "\x1b[32m"
+YELLOW = "\x1b[33m"
+RESET = "\x1b[m"
+
+# Parsed options, filled in by main().
+OPTIONS = None
+
+
+def check_apache_remote():
+ """
+ Checks that there is a remote named 'apache' set up correctly.
+ Otherwise, exits with an error message.
+ """
+ try:
+ url = check_output(['git', 'config', '--local', '--get', 'remote.apache.url']).strip()
+ except subprocess.CalledProcessError:
+ print >>sys.stderr, "No remote named 'apache'. Please set one up, for example with: "
+ print >>sys.stderr, " git remote add apache", APACHE_REPO
+ sys.exit(1)
+ if url != APACHE_REPO:
+ print >>sys.stderr, "Unexpected URL for remote 'apache'."
+ print >>sys.stderr, " Got: ", url
+ print >>sys.stderr, " Expected:", APACHE_REPO
+ sys.exit(1)
+
+
+def check_gerrit_remote():
+ """
+ Checks that there is a remote named 'gerrit' set up correctly.
+ Otherwise, exits with an error message.
+ """
+ try:
+ url = check_output(['git', 'config', '--local', '--get', 'remote.gerrit.url']).strip()
+ except subprocess.CalledProcessError:
+ print >>sys.stderr, "No remote named 'gerrit'. Please set one up following "
+ print >>sys.stderr, "the contributor guide."
+ sys.exit(1)
+ if not GERRIT_URL_RE.match(url):
+ print >>sys.stderr, "Unexpected URL for remote 'gerrit'."
+ print >>sys.stderr, " Got: ", url
+ print >>sys.stderr, " Expected to find host '%s' in the URL" % GERRIT_HOST
+ sys.exit(1)
+
+
+def fetch(remote):
+ """Run git fetch for the given remote, including some logging."""
+ logging.info("Fetching from remote '%s'..." % remote)
+ subprocess.check_call(['git', 'fetch', remote])
+ logging.info("done")
+
+
+def get_branches(remote):
+ """ Fetch a dictionary mapping branch name to SHA1 hash from the given remote. """
+ out = check_output(["git", "ls-remote", remote, "refs/heads/*"])
+ ret = {}
+ for l in out.splitlines():
+ sha, ref = l.split("\t")
+ branch = ref.replace("refs/heads/", "", 1)
+ ret[branch] = sha
+ return ret
+
+
+def rev_parse(rev):
+ """Run git rev-parse, returning the sha1, or None if not found"""
+ try:
+ return check_output(['git', 'rev-parse', rev], stderr=subprocess.STDOUT).strip()
+ except subprocess.CalledProcessError:
+ return None
+
+
+def rev_list(arg):
+ """Run git rev-list, returning an array of SHA1 commit hashes."""
+ return check_output(['git', 'rev-list', arg]).splitlines()
+
+
+def describe_commit(rev):
+ """ Return a one-line description of a commit. """
+ return subprocess.check_output(
+ ['git', 'log', '--color', '-n1', '--oneline', rev]).strip()
+
+
+def is_fast_forward(ancestor, child):
+ """
+ Return True if 'child' is a descendent of 'ancestor' and thus
+ could be fast-forward merged.
+ """
+ try:
+ merge_base = check_output(['git', 'merge-base', ancestor, child]).strip()
+ except:
+ # If either of the commits is unknown, count this as a non-fast-forward.
+ return False
+ return merge_base == rev_parse(ancestor)
+
+
+def get_committer_email(rev):
+ """ Return the email address of the committer of the given revision. """
+ return check_output(['git', 'log', '-n1', '--pretty=format:%ce', rev]).strip()
+
+
+def get_my_email():
+ """ Return the email address in the user's git config. """
+ return check_output(['git', 'config', '--get', 'user.email']).strip()
+
+
+def confirm_prompt(prompt):
+ """
+ Issue the given prompt, and ask the user to confirm yes/no. Returns true
+ if the user confirms.
+ """
+ while True:
+ print prompt, "[Y/n]:",
+ r = raw_input().strip().lower()
+ if r in ['y', 'yes', '']:
+ return True
+ elif r in ['n', 'no']:
+ return False
+
+def do_update(branch, gerrit_sha, apache_sha):
+ """
+ Displays and performs a proposed update of the Apache repository
+ for branch 'branch' from 'apache_sha' to 'gerrit_sha'.
+ """
+ # First, verify that the update is fast-forward. If it's not, then something
+ # must have gotten committed to Apache outside of gerrit, and we'd need some
+ # manual intervention.
+ if not is_fast_forward(apache_sha, gerrit_sha):
+ print >>sys.stderr, "Cannot update branch '%s' from gerrit:" % branch
+ print >>sys.stderr, "Apache revision %s is not an ancestor of gerrit revision %s" % (
+ apache_sha[:8], gerrit_sha[:8])
+ print >>sys.stderr, "Something must have been committed to Apache and bypassed gerrit."
+ print >>sys.stderr, "Manual intervention is required."
+ sys.exit(1)
+
+ # List the commits that are going to be pushed to the ASF, so that the committer
+ # can verify and "sign off".
+ commits = rev_list("%s..%s" % (apache_sha, gerrit_sha))
+ commits.reverse() # Display from oldest to newest.
+ print "-" * 60
+ print GREEN + ("%d commit(s) need to be pushed from Gerrit to ASF:" % len(commits)) + RESET
+ push_sha = None
+ for sha in commits:
+ oneline = describe_commit(sha)
+ print " ", oneline
+ committer = get_committer_email(sha)
+ if committer != get_my_email():
+ print RED + " !!! Committed by someone else (%s) !!!" % committer, RESET
+ if not confirm_prompt(
+ RED + " !!! Are you sure you want to push on behalf of another committer?" + RESET):
+ # Even if they don't want to push this commit, we could still push any
+ # earlier commits that the user _did_ author.
+ if push_sha is not None:
+ print "... will still update to prior commit %s..." % push_sha
+ break
+ push_sha = sha
+ if push_sha is None:
+ print "Nothing to push"
+ return
+
+ # Everything has been confirmed. Do the actual push
+ cmd = ['git', 'push', 'apache']
+ if OPTIONS.dry_run:
+ cmd.append('--dry-run')
+ cmd.append('%s:refs/heads/%s' % (push_sha, branch))
+ print GREEN + "Running: " + RESET + " ".join(cmd)
+ subprocess.check_call(cmd)
+ print GREEN + "Successfully updated %s to %s" % (branch, gerrit_sha) + RESET
+ print
+
+
+def main():
+ global OPTIONS
+ p = optparse.OptionParser(
+ epilog=("See the top of the source code for more information on the purpose of " +
+ "this script."))
+ p.add_option("-n", "--dry-run", action="store_true",
+ help="Perform git pushes with --dry-run")
+ OPTIONS, args = p.parse_args()
+ if args:
+ p.error("no arguments expected")
+ sys.exit(1)
+
+ # Pre-flight checks.
+ check_apache_remote()
+ check_gerrit_remote()
+
+ # Ensure we have the latest state of gerrit.
+ fetch('gerrit')
+
+ # Check the current state of branches on Apache.
+ # For each branch, we try to update it if the revisions don't match.
+ apache_branches = get_branches('apache')
+ for branch, apache_sha in sorted(apache_branches.iteritems()):
+ gerrit_sha = rev_parse("remotes/gerrit/" + branch)
+ print "Branch '%s':\t" % branch,
+ if gerrit_sha is None:
+ print YELLOW, "found on Apache but not in gerrit", RESET
+ continue
+ if gerrit_sha == apache_sha:
+ print GREEN, "up to date", RESET
+ continue
+ print YELLOW, "needs update", RESET
+ do_update(branch, gerrit_sha, apache_sha)
+
+
+if __name__ == "__main__":
+ logging.basicConfig(level=logging.INFO)
+ main()
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/55a5be98/build-support/trigger_gerrit.py
----------------------------------------------------------------------
diff --git a/build-support/trigger_gerrit.py b/build-support/trigger_gerrit.py
index d214b4c..740af43 100755
--- a/build-support/trigger_gerrit.py
+++ b/build-support/trigger_gerrit.py
@@ -33,30 +33,13 @@ import urllib
import urllib2
import urlparse
+from kudu_util import check_output
+
GERRIT_HOST = "gerrit.cloudera.org"
JENKINS_URL = "http://sandbox.jenkins.cloudera.com/"
JENKINS_JOB = "kudu-public-gerrit"
-def check_output(*popenargs, **kwargs):
- r"""Run command with arguments and return its output as a byte string.
- Backported from Python 2.7 as it's implemented as pure python on stdlib.
- >>> check_output(['/usr/bin/python', '--version'])
- Python 2.6.2
- """
- process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
- output, unused_err = process.communicate()
- retcode = process.poll()
- if retcode:
- cmd = kwargs.get("args")
- if cmd is None:
- cmd = popenargs[0]
- error = subprocess.CalledProcessError(retcode, cmd)
- error.output = output
- raise error
- return output
-
-
def get_gerrit_ssh_command():
url = check_output("git config --get remote.gerrit.url".split(" "))
m = re.match(r'ssh://(.+)@(.+):(\d+)/.+', url)