You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by to...@apache.org on 2016/01/28 03:09:37 UTC
[2/3] incubator-kudu git commit: KUDU-1315. Add simple script to
build a source release
KUDU-1315. Add simple script to build a source release
This adds a new python script meant for interactive use for building
an Apache-style source release.
The source release tarball is simply the output of 'git archive'. The
purpose of the script is to perform some pre-flight checks to make sure
the release seems sane, and to help the user gpg-sign the release upon
completion. It's not fancy, but should ensure that the release process
is repeatable (two committers running this script from the same revision
should result in 100% identical release artifacts).
In order to generate the artifact name, I moved the version string into
a text file at the repository root instead of embedding it into the
CMakeLists.txt file.
Change-Id: Ie5753c4cc822dfbc86210356850ea1b76ab8e7a2
Reviewed-on: http://gerrit.cloudera.org:8080/1930
Reviewed-by: Jean-Daniel Cryans
Tested-by: Todd Lipcon <to...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/fefa29ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/fefa29ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/fefa29ca
Branch: refs/heads/master
Commit: fefa29cae7698ffc7833138a78228f93a2d1d2ce
Parents: 8325d5d
Author: Todd Lipcon <to...@apache.org>
Authored: Wed Jan 27 14:19:58 2016 -0800
Committer: Todd Lipcon <to...@apache.org>
Committed: Thu Jan 28 00:47:00 2016 +0000
----------------------------------------------------------------------
CMakeLists.txt | 2 +-
build-support/build_source_release.py | 150 +++++++++++++++++++++++++++++
build-support/get-upstream-commit.sh | 4 +-
build-support/kudu_util.py | 41 ++++++++
build-support/push_to_asf.py | 44 +++------
version.txt | 1 +
6 files changed, 208 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/fefa29ca/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6bdf7e0..0ac59cf 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@
cmake_minimum_required(VERSION 3.2.0)
# TODO: can we somehow pass this into the java build?
-set(KUDU_VERSION_NUMBER "0.7.0-SNAPSHOT")
+file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/version.txt" KUDU_VERSION_NUMBER)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
include(CMakeParseArguments)
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/fefa29ca/build-support/build_source_release.py
----------------------------------------------------------------------
diff --git a/build-support/build_source_release.py b/build-support/build_source_release.py
new file mode 100755
index 0000000..d6973d9
--- /dev/null
+++ b/build-support/build_source_release.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import hashlib
+import logging
+import os
+import subprocess
+import sys
+
+from kudu_util import check_output, confirm_prompt, Colors, get_my_email
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+GET_UPSTREAM_COMMIT_SCRIPT = os.path.join(ROOT, "build-support", "get-upstream-commit.sh")
+
+
+def check_repo_not_dirty():
+ """Check that the git repository isn't dirty."""
+ dirty_repo = subprocess.call("git diff --quiet && git diff --cached --quiet",
+ shell=True) != 0
+ if not dirty_repo:
+ return
+ print "The repository does not appear to be clean."
+ print Colors.RED + "The source release will not include your local changes." + \
+ Colors.RESET
+ if not confirm_prompt("Continue?"):
+ sys.exit(1)
+
+
+def check_no_local_commits():
+ """
+ Check that there are no local commits which haven't been pushed to the upstream
+ repo via Jenkins.
+ """
+ upstream_commit = check_output(GET_UPSTREAM_COMMIT_SCRIPT).strip()
+ cur_commit = check_output(["git", "rev-parse", "HEAD"]).strip()
+
+ if upstream_commit == cur_commit:
+ return
+ print "The repository appears to have local commits:"
+ subprocess.check_call(["git", "log", "--oneline", "%s..HEAD" % upstream_commit])
+
+ print Colors.RED + "This should not be an official release!" + \
+ Colors.RESET
+ if not confirm_prompt("Continue?"):
+ sys.exit(1)
+
+
+def get_version_number():
+ """ Return the current version number of Kudu. """
+ return file(os.path.join(ROOT, "version.txt")).read().strip()
+
+
+def create_tarball():
+ artifact_name = "apache-kudu-incubating-%s" % get_version_number()
+ build_dir = os.path.join(ROOT, "build")
+ if not os.path.exists(build_dir):
+ os.path.makedirs(build_dir)
+ tarball_path = os.path.join(build_dir, artifact_name + ".tar.gz")
+ print "Exporting source tarball..."
+ subprocess.check_output(["git", "archive",
+ "--prefix=%s/" % artifact_name,
+ "--output=%s" % tarball_path,
+ "HEAD"])
+ print Colors.GREEN + "Generated tarball:\t" + Colors.RESET, tarball_path
+ return tarball_path
+
+
+def sign_tarball(tarball_path):
+ """ Prompt the user to GPG-sign the tarball using their Apache GPG key. """
+ if not confirm_prompt("Would you like to GPG-sign the tarball now?"):
+ return
+
+ email = get_my_email()
+ if not email.endswith("@apache.org"):
+ print Colors.YELLOW, "Your email address for the repository is not an @apache.org address."
+ print "Release signatures should typically be signed by committers with @apache.org GPG keys."
+ print Colors.RESET,
+ if not confirm_prompt("Continue?"):
+ return
+
+ try:
+ subprocess.check_call(["gpg", "--detach-sign", "--armor", "-u", email, tarball_path])
+ except subprocess.CalledProcessError:
+ print Colors.RED + "GPG signing failed. Artifact will not be signed." + Colors.RESET
+ return
+ print Colors.GREEN + "Generated signature:\t" + Colors.RESET, tarball_path + ".asc"
+
+
+def checksum_file(summer, path):
+ """
+ Calculates the checksum of the file 'path' using the provided hashlib
+ digest implementation. Returns the hex form of the digest.
+ """
+ with file(path, "rb") as f:
+ # Read the file in 4KB chunks until EOF.
+ for chunk in iter(lambda: f.read(4096), ""):
+ summer.update(chunk)
+ return summer.hexdigest()
+
+
+def gen_checksum_files(tarball_path):
+ """
+ Create md5 and sha files of the tarball.
+
+ The output format is compatible with command line tools like 'sha1sum'
+ and 'md5sum' so they may be used to verify the checksums.
+ """
+ hashes = [(hashlib.sha1, "sha"),
+ (hashlib.md5, "md5")]
+ for hash_func, extension in hashes:
+ digest = checksum_file(hash_func(), tarball_path)
+ path = tarball_path + "." + extension
+ with file(path, "w") as f:
+ print >>f, "%s\t%s" % (digest, os.path.basename(tarball_path))
+ print Colors.GREEN + ("Generated %s:\t" % extension) + Colors.RESET, path
+
+
+def main():
+ # Change into the source repo so that we can run git commands without having to
+ # specify cwd=BUILD_SUPPORT every time.
+ os.chdir(ROOT)
+ check_repo_not_dirty()
+ check_no_local_commits()
+ tarball_path = create_tarball()
+ gen_checksum_files(tarball_path)
+ sign_tarball(tarball_path)
+
+ print Colors.GREEN + "Release successfully generated!" + Colors.RESET
+ print
+
+
+if __name__ == "__main__":
+ logging.basicConfig(level=logging.INFO)
+ main()
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/fefa29ca/build-support/get-upstream-commit.sh
----------------------------------------------------------------------
diff --git a/build-support/get-upstream-commit.sh b/build-support/get-upstream-commit.sh
index ecf3a8b..3918c84 100755
--- a/build-support/get-upstream-commit.sh
+++ b/build-support/get-upstream-commit.sh
@@ -21,9 +21,9 @@
# branch which was checked in by gerrit. This commit hash is printed to
# stdout.
#
-# It does so by looking for the verification signoff from Jenkins. This is
+# It does so by looking for the 'Reviewed-on' tag added by gerrit. This is
# more foolproof than trying to guess at the "origin/" branch name, since the
# developer might be working on some local topic branch.
set -e
-git log --grep='Tested-by: \(jenkins\|Internal Jenkins\)' -n1 --pretty=format:%H
+git log --grep='Reviewed-on: ' -n1 --pretty=format:%H
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/fefa29ca/build-support/kudu_util.py
----------------------------------------------------------------------
diff --git a/build-support/kudu_util.py b/build-support/kudu_util.py
index 731c1ad..7945da5 100644
--- a/build-support/kudu_util.py
+++ b/build-support/kudu_util.py
@@ -20,7 +20,23 @@
# This script generates a header file which contains definitions
# for the current Kudu build (eg timestamp, git hash, etc)
+import os
import subprocess
+import sys
+
+class Colors(object):
+ """ ANSI color codes. """
+
+ def __on_tty(x):
+ if not os.isatty(sys.stdout.fileno()):
+ return ""
+ return x
+
+ RED = __on_tty("\x1b[31m")
+ GREEN = __on_tty("\x1b[32m")
+ YELLOW = __on_tty("\x1b[33m")
+ RESET = __on_tty("\x1b[m")
+
def check_output(*popenargs, **kwargs):
r"""Run command with arguments and return its output as a byte string.
@@ -39,3 +55,28 @@ def check_output(*popenargs, **kwargs):
error.output = output
raise error
return output
+
+
+def confirm_prompt(prompt):
+ """
+ Issue the given prompt, and ask the user to confirm yes/no. Returns true
+ if the user confirms.
+ """
+ while True:
+ print prompt, "[Y/n]:",
+
+ if not os.isatty(sys.stdout.fileno()):
+ print "Not running interactively. Assuming 'N'."
+ return False
+ pass
+
+ r = raw_input().strip().lower()
+ if r in ['y', 'yes', '']:
+ return True
+ elif r in ['n', 'no']:
+ return False
+
+
+def get_my_email():
+ """ Return the email address in the user's git config. """
+ return check_output(['git', 'config', '--get', 'user.email']).strip()
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/fefa29ca/build-support/push_to_asf.py
----------------------------------------------------------------------
diff --git a/build-support/push_to_asf.py b/build-support/push_to_asf.py
index 491830c..0bae118 100755
--- a/build-support/push_to_asf.py
+++ b/build-support/push_to_asf.py
@@ -38,16 +38,16 @@ import re
import subprocess
import sys
-from kudu_util import check_output
+from kudu_util import check_output, confirm_prompt, Colors, get_my_email
APACHE_REPO = "https://git-wip-us.apache.org/repos/asf/incubator-kudu.git"
GERRIT_URL_RE = re.compile(r"ssh://.+@gerrit.cloudera.org:29418/kudu")
# ANSI color codes.
-RED = "\x1b[31m"
-GREEN = "\x1b[32m"
-YELLOW = "\x1b[33m"
-RESET = "\x1b[m"
+Colors.RED = "\x1b[31m"
+Colors.GREEN = "\x1b[32m"
+Colors.YELLOW = "\x1b[33m"
+Colors.RESET = "\x1b[m"
# Parsed options, filled in by main().
OPTIONS = None
@@ -144,24 +144,6 @@ def get_committer_email(rev):
return check_output(['git', 'log', '-n1', '--pretty=format:%ce', rev]).strip()
-def get_my_email():
- """ Return the email address in the user's git config. """
- return check_output(['git', 'config', '--get', 'user.email']).strip()
-
-
-def confirm_prompt(prompt):
- """
- Issue the given prompt, and ask the user to confirm yes/no. Returns true
- if the user confirms.
- """
- while True:
- print prompt, "[Y/n]:",
- r = raw_input().strip().lower()
- if r in ['y', 'yes', '']:
- return True
- elif r in ['n', 'no']:
- return False
-
def do_update(branch, gerrit_sha, apache_sha):
"""
Displays and performs a proposed update of the Apache repository
@@ -183,16 +165,16 @@ def do_update(branch, gerrit_sha, apache_sha):
commits = rev_list("%s..%s" % (apache_sha, gerrit_sha))
commits.reverse() # Display from oldest to newest.
print "-" * 60
- print GREEN + ("%d commit(s) need to be pushed from Gerrit to ASF:" % len(commits)) + RESET
+ print Colors.GREEN + ("%d commit(s) need to be pushed from Gerrit to ASF:" % len(commits)) + Colors.RESET
push_sha = None
for sha in commits:
oneline = describe_commit(sha)
print " ", oneline
committer = get_committer_email(sha)
if committer != get_my_email():
- print RED + " !!! Committed by someone else (%s) !!!" % committer, RESET
+ print Colors.RED + " !!! Committed by someone else (%s) !!!" % committer, Colors.RESET
if not confirm_prompt(
- RED + " !!! Are you sure you want to push on behalf of another committer?" + RESET):
+ Colors.RED + " !!! Are you sure you want to push on behalf of another committer?" + Colors.RESET):
# Even if they don't want to push this commit, we could still push any
# earlier commits that the user _did_ author.
if push_sha is not None:
@@ -208,9 +190,9 @@ def do_update(branch, gerrit_sha, apache_sha):
if OPTIONS.dry_run:
cmd.append('--dry-run')
cmd.append('%s:refs/heads/%s' % (push_sha, branch))
- print GREEN + "Running: " + RESET + " ".join(cmd)
+ print Colors.GREEN + "Running: " + Colors.RESET + " ".join(cmd)
subprocess.check_call(cmd)
- print GREEN + "Successfully updated %s to %s" % (branch, gerrit_sha) + RESET
+ print Colors.GREEN + "Successfully updated %s to %s" % (branch, gerrit_sha) + Colors.RESET
print
@@ -240,12 +222,12 @@ def main():
gerrit_sha = rev_parse("remotes/gerrit/" + branch)
print "Branch '%s':\t" % branch,
if gerrit_sha is None:
- print YELLOW, "found on Apache but not in gerrit", RESET
+ print Colors.YELLOW, "found on Apache but not in gerrit", Colors.RESET
continue
if gerrit_sha == apache_sha:
- print GREEN, "up to date", RESET
+ print Colors.GREEN, "up to date", Colors.RESET
continue
- print YELLOW, "needs update", RESET
+ print Colors.YELLOW, "needs update", Colors.RESET
do_update(branch, gerrit_sha, apache_sha)
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/fefa29ca/version.txt
----------------------------------------------------------------------
diff --git a/version.txt b/version.txt
new file mode 100644
index 0000000..978ba2b
--- /dev/null
+++ b/version.txt
@@ -0,0 +1 @@
+0.7.0-SNAPSHOT