You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by fr...@apache.org on 2019/05/09 07:50:35 UTC
[calcite-avatica] 01/01: [CALCITE-3043] Add the ability to publish
and promote releases using docker
This is an automated email from the ASF dual-hosted git repository.
francischuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite-avatica.git
commit faf7e5f7fed0f8151f069968cfb6c2517364436a
Author: Francis Chuang <fr...@apache.org>
AuthorDate: Wed May 1 14:05:38 2019 +1000
[CALCITE-3043] Add the ability to publish and promote releases using docker
---
docker-compose.yml | 14 +++
docker.sh | 284 ++++++++++++++++++++++++++++++++++++++++++++++++++--
site/_docs/howto.md | 31 +++++-
3 files changed, 319 insertions(+), 10 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index 01021e2..e5a1013 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -39,6 +39,20 @@ services:
- .:/src
- maven-repo:/root/.m2
+ publish-release-for-voting:
+ image: maven
+ working_dir: /src
+ command: sh -c "./docker.sh publish-release-for-voting"
+ volumes:
+ - .:/src
+
+ promote-release:
+ image: maven
+ working_dir: /src
+ command: sh -c "./docker.sh promote-release"
+ volumes:
+ - .:/src
+
test:
image: maven
working_dir: /src
diff --git a/docker.sh b/docker.sh
index 1c5b180..e1644cb 100755
--- a/docker.sh
+++ b/docker.sh
@@ -15,6 +15,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+set -e
+
+GITBOX_URL=https://gitbox.apache.org/repos/asf/calcite-avatica.git
+RELEASE_REPO=https://dist.apache.org/repos/dist/release/calcite/
+DEV_REPO=https://dist.apache.org/repos/dist/dev/calcite/
+PRODUCT=apache-calcite-avatica
+
function terminate() {
printf "\n\nUser terminated build. Exiting...\n"
exit 1
@@ -145,11 +152,12 @@ RELEASE_VERSION=""
RC_NUMBER=""
DEV_VERSION=""
ASF_USERNAME=""
+ASF_PASSWORD=""
NAME=""
-get_build_configuration(){
+get_dry_run_build_configuration(){
- while $NOT_CONFIRMED; do
+ while $DRY_RUN_NOT_CONFIRMED; do
read -p "Enter the version number to be released (example: 1.12.0): " RELEASE_VERSION
read -p "Enter the release candidate number (example: if you are releasing rc0, enter 0): " RC_NUMBER
read -p "Enter the development version number (example: if your release version is 1.12.0, enter 1.13.0): " DEV_VERSION
@@ -167,7 +175,7 @@ get_build_configuration(){
read -p "Is this correct? (y/n) " CONFIRM
if [[ ($CONFIRM == "Y") || ($CONFIRM == "y") ]]; then
- NOT_CONFIRMED=false
+ DRY_RUN_NOT_CONFIRMED=false
INVALID_CONFIRMATION=false
elif [[ ($CONFIRM == "N") || ($CONFIRM == "n") ]]; then
INVALID_CONFIRMATION=false
@@ -176,13 +184,56 @@ get_build_configuration(){
done
}
-ASF_PASSWORD=""
+get_build_configuration(){
-set_git_credentials(){
- read -s -p "Enter your ASF password: " ASF_PASSWORD
+ while $BUILD_NOT_CONFIRMED; do
+ read -p "Enter the version number to be released (example: 1.12.0): " RELEASE_VERSION
+ read -p "Enter the release candidate number (example: if you are releasing rc0, enter 0): " RC_NUMBER
+ read -p "Enter the development version number (example: if your release version is 1.12.0, enter 1.13.0): " DEV_VERSION
+ read -p "Enter your name (this will be used for git commits): " NAME
+ echo "Build configured as follows:"
+ echo "Release: $RELEASE_VERSION-rc$RC_NUMBER"
+ echo "Next development version: $DEV_VERSION-SNAPSHOT"
+ echo "Name: $NAME"
- printf "\n"
+ INVALID_CONFIRMATION=true
+
+ while $INVALID_CONFIRMATION; do
+ read -p "Is this correct? (y/n) " CONFIRM
+
+ if [[ ($CONFIRM == "Y") || ($CONFIRM == "y") ]]; then
+ BUILD_NOT_CONFIRMED=false
+ INVALID_CONFIRMATION=false
+ elif [[ ($CONFIRM == "N") || ($CONFIRM == "n") ]]; then
+ INVALID_CONFIRMATION=false
+ fi
+ done
+ done
+}
+get_asf_credentials(){
+ while $ASF_CREDS_NOT_CONFIRMED; do
+ read -p "Enter your ASF username: " ASF_USERNAME
+ read -s -p "Enter your ASF password: " ASF_PASSWORD
+ printf "\n"
+ echo "Your ASF Username is:" $ASF_USERNAME
+
+ INVALID_CONFIRMATION=true
+
+ while $INVALID_CONFIRMATION; do
+ read -p "Is this correct? (y/n) " CONFIRM
+
+ if [[ ($CONFIRM == "Y") || ($CONFIRM == "y") ]]; then
+ ASF_CREDS_NOT_CONFIRMED=false
+ INVALID_CONFIRMATION=false
+ elif [[ ($CONFIRM == "N") || ($CONFIRM == "n") ]]; then
+ INVALID_CONFIRMATION=false
+ fi
+ done
+ done
+}
+
+set_git_credentials(){
echo https://$ASF_USERNAME:$ASF_PASSWORD@gitbox.apache.org >> /root/.git-credentials
git config --global credential.helper 'store --file=/root/.git-credentials'
@@ -222,11 +273,217 @@ EOF
EOF
}
+publish_release_for_voting(){
+
+ LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`)
+
+ if [[ ! $LATEST_TAG =~ .+-rc[[:digit:]]+$ ]]; then
+ echo "The latest tag ($LATEST_TAG) is not a RC release and should not be published for voting."
+ exit 1
+ fi
+
+ TAG_WITHOUT_PRODUCT=$(echo $LATEST_TAG | sed -e 's/avatica-//')
+ TAG_WITHOUT_RC=$(echo $TAG_WITHOUT_PRODUCT | sed -e 's/-rc[0-9][0-9]*//')
+ SOURCE_RELEASE=$PRODUCT-$TAG_WITHOUT_RC-src.tar.gz
+ GPG_SIGNATURE=$PRODUCT-$TAG_WITHOUT_RC-src.tar.gz.asc
+ SHA512=$PRODUCT-$TAG_WITHOUT_RC-src.tar.gz.sha512
+ GPG_SHA512=$PRODUCT-$TAG_WITHOUT_RC-src.tar.gz.asc.sha512
+ COMMIT=$(git rev-list -n 1 $LATEST_TAG)
+
+ # Check to see a release is built
+ MISSING_FILES=false
+
+ if [ ! -f "target/$SOURCE_RELEASE" ]; then
+ echo "Did not find source release ($SOURCE_RELEASE) in target folder."
+ MISSING_FILES=true
+ fi
+
+ if [ ! -f "target/$GPG_SIGNATURE" ]; then
+ echo "Did not find GPG signature ($GPG_SIGNATURE) in target folder."
+ MISSING_FILES=true
+ fi
+
+ if [ ! -f "target/$SHA512" ]; then
+ echo "Did not find SHA512 ($SHA512) in target folder."
+ MISSING_FILES=true
+ fi
+
+ if [ ! -f "target/$GPG_SHA512" ]; then
+ echo "Did not find GPG SHA512 ($GPG_SHA512) in target folder."
+ MISSING_FILES=true
+ fi
+
+ if $MISSING_FILES == true; then
+ exit 1
+ fi
+
+ while $NOT_CONFIRMED; do
+ echo "Publish configured as follows:"
+ echo "Release: $LATEST_TAG"
+
+ INVALID_CONFIRMATION=true
+
+ while $INVALID_CONFIRMATION; do
+ read -p "Is this correct? (y/n) " CONFIRM
+
+ if [[ ($CONFIRM == "Y") || ($CONFIRM == "y") ]]; then
+ NOT_CONFIRMED=false
+ INVALID_CONFIRMATION=false
+ elif [[ ($CONFIRM == "N") || ($CONFIRM == "n") ]]; then
+ INVALID_CONFIRMATION=false
+ fi
+ done
+ done
+
+ HASH=$(cat "target/$SHA512" | tr -d '\n')
+
+ get_asf_credentials
+
+ svn checkout $DEV_REPO /tmp/calcite --depth empty
+ mkdir -p /tmp/calcite/$PRODUCT-$TAG_WITHOUT_PRODUCT
+ cp -R target/$PRODUCT-* /tmp/calcite/$PRODUCT-$TAG_WITHOUT_PRODUCT
+
+ cd /tmp/calcite
+ svn add $PRODUCT-$TAG_WITHOUT_PRODUCT
+ chmod -x $PRODUCT-$TAG_WITHOUT_PRODUCT/*
+
+ svn commit -m "$PRODUCT-$TAG_WITHOUT_PRODUCT" --force-log --username $ASF_USERNAME --password $ASF_PASSWORD
+
+ [[ $LATEST_TAG =~ -rc([[:digit:]]+)$ ]]
+ RC_NUMBER=${BASH_REMATCH[1]}
+
+ [[ $TAG_WITHOUT_RC =~ ([[:digit:]]+\.[[:digit:]]+)\.[[:digit:]]+$ ]]
+ BRANCH_VERSION=${BASH_REMATCH[1]}
+
+ read -p "Please enter your first name for the voting email: " FIRST_NAME
+ read -p "Enter the ID at the end of the staged repository (for orgapachecalcite-1000, enter 1000): " STAGED_REPO_ID
+
+ echo "The release $PRODUCT-$TAG_WITHOUT_PRODUCT has been uploaded to the development repository."
+ printf "\n"
+ printf "\n"
+ echo "Email the following message to dev@calcite.apache.org. Please check the message before sending."
+ printf "\n"
+ echo "To: dev@calcite.apache.org"
+ echo "Subject: [VOTE] Release $PRODUCT-$TAG_WITHOUT_RC (release candidate $RC_NUMBER)"
+ echo "Message:
+Hi all,
+
+I have created a build for Apache Calcite Avatica $TAG_WITHOUT_RC, release candidate $RC_NUMBER.
+
+Thanks to everyone who has contributed to this release.
+
+You can read the release notes here:
+https://github.com/apache/calcite-avatica/blob/branch-avatica-$BRANCH_VERSION/site/_docs/history.md
+
+The commit to be voted upon:
+https://gitbox.apache.org/repos/asf?p=calcite-avatica.git;a=commit;h=$COMMIT
+
+Its hash is $COMMIT
+
+The artifacts to be voted on are located here:
+$DEV_REPO$PRODUCT-$TAG_WITHOUT_PRODUCT/
+
+The hashes of the artifacts are as follows:
+src.tar.gz.sha512 $HASH
+
+A staged Maven repository is available for review at:
+https://repository.apache.org/content/repositories/orgapachecalcite-$STAGED_REPO_ID
+
+Release artifacts are signed with the following key:
+https://people.apache.org/keys/committer/$ASF_USERNAME.asc
+
+If you do not have a Java environment available, you can run the tests using docker. To do so, install docker and docker-compose, then run \"docker-compose run test\" from the root of the directory.
+
+Please vote on releasing this package as Apache Calcite Avatica $TAG_WITHOUT_RC.
+
+The vote is open for the next 72 hours and passes if a majority of at least three +1 PMC votes are cast.
+
+[ ] +1 Release this package as Apache Calcite 1.14.0
+[ ] 0 I don't feel strongly about it, but I'm okay with the release
+[ ] -1 Do not release this package because...
+
+
+Here is my vote:
+
++1 (binding)
+
+$NAME
+"
+}
+
+promote_release(){
+
+ LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`)
+
+ if [[ ! $LATEST_TAG =~ .+-rc[[:digit:]]+$ ]]; then
+ echo "The latest tag ($LATEST_TAG) is not a RC release and should not be re-released."
+ exit 1
+ fi
+
+ TAG_WITHOUT_PRODUCT=$(echo $LATEST_TAG | sed -e 's/avatica-//')
+ TAG_WITHOUT_RC=$(echo $TAG_WITHOUT_PRODUCT | sed -e 's/-rc[0-9][0-9]*//')
+
+ if ! svn ls $DEV_REPO/$PRODUCT-$TAG_WITHOUT_PRODUCT; then
+ echo "The release $PRODUCT-$TAG_WITHOUT_PRODUCT was not found in the dev repository. Was it uploaded for voting?"
+ exit 1
+ fi
+
+ get_asf_credentials
+
+ set_git_credentials
+
+ # Tag release and push it
+ git tag rel/avatica-$TAG_WITHOUT_RC $LATEST_TAG
+ git push $GITBOX_URL rel/avatica-$TAG_WITHOUT_RC
+
+ # Merge release branch back to master and push
+ [[ $TAG_WITHOUT_RC =~ ([[:digit:]]+\.[[:digit:]]+)\.[[:digit:]]+$ ]]
+ BRANCH_VERSION=${BASH_REMATCH[1]}
+ git checkout master
+ git merge branch-$TAG_WITHOUT_RC --ff-only
+ git push $GITBOX_URL master
+
+ svn checkout $RELEASE_REPO /tmp/release
+ rm -rf /tmp/release/$PRODUCT-$TAG_WITHOUT_RC
+ mkdir -p /tmp/release/$PRODUCT-$TAG_WITHOUT_RC
+
+ svn checkout $DEV_REPO/$PRODUCT-$TAG_WITHOUT_PRODUCT /tmp/rc
+ cp -rp /tmp/rc/* /tmp/release/$PRODUCT-$TAG_WITHOUT_RC
+
+ cd /tmp/release
+
+ svn add $PRODUCT-$TAG_WITHOUT_RC
+
+ # If there is more than 1 release, delete all of them, except for the newest one
+ # To do this, we do the following:
+ # 1. Get the list of releases with verbose information from svn
+ # 2. Sort by the first field (revision number) in descending order
+ # 3. Select apache-calcite-avatica releases
+ # 4. Exclude the release we're trying to promote, in case it was from a failed promotion.
+ # 5. Trim all whitespace down to 1 empty space.
+ # 6. Select field 7, which is each release's folder
+ CURRENT_RELEASES=$(svn ls -v $RELEASE_REPO | sort -k1 -r | grep $PRODUCT-[[:digit:]] | grep -v $PRODUCT-$TAG_WITHOUT_RC | tr -s ' ' | cut -d ' ' -f 7)
+
+ RELEASE_COUNT=0
+ while read -r RELEASE; do
+ if [[ $RELEASE_COUNT -ne 0 ]]; then
+ svn rm $RELEASE
+ echo "Removing release $RELEASE"
+ fi
+
+ RELEASE_COUNT=$((RELEASE_COUNT+1))
+ done <<< "$CURRENT_RELEASES"
+
+ svn commit -m "Release $PRODUCT-$TAG_WITHOUT_RC" --force-log --username $ASF_USERNAME --password $ASF_PASSWORD
+
+ echo "Release $PRODUCT-$LATEST_TAG successfully promoted to $PRODUCT-$TAG_WITHOUT_RC"
+}
+
case $1 in
dry-run)
mount_gpg_keys
select_gpg_key
- get_build_configuration
+ get_dry_run_build_configuration
mvn -Dmaven.artifact.threads=20 -DdryRun=true -DreleaseVersion=$RELEASE_VERSION -DdevelopmentVersion=$DEV_VERSION-SNAPSHOT -Dtag="avatica-$RELEASE_VERSION-rc$RC_NUMBER" -Papache-release -Duser.name=$ASF_USERNAME release:prepare -Darguments=-Dgpg.keyname=$SELECTED_GPG_KEY
;;
@@ -235,6 +492,7 @@ case $1 in
mount_gpg_keys
select_gpg_key
get_build_configuration
+ get_asf_credentials
set_git_credentials
set_maven_credentials
@@ -246,12 +504,20 @@ case $1 in
mvn release:clean
;;
+ publish-release-for-voting)
+ publish_release_for_voting
+ ;;
+
+ promote-release)
+ promote_release
+ ;;
+
test)
mvn clean verify -Dcheckstyle.skip
;;
*)
- echo $"Usage: $0 {dry-run|release|clean|test}"
+ echo $"Usage: $0 {dry-run|release|clean|publish-release-for-voting|promote-release|test}"
;;
esac
diff --git a/site/_docs/howto.md b/site/_docs/howto.md
index 07c71b5..20a955b 100644
--- a/site/_docs/howto.md
+++ b/site/_docs/howto.md
@@ -324,6 +324,7 @@ Verify the staged artifacts in the Nexus repository:
https://repository.apache.org/content/repositories/orgapachecalcite-1000
(or a similar URL)
+### To upload the artifacts directly in your environment:
Upload the artifacts via subversion to a staging area,
https://dist.apache.org/repos/dist/dev/calcite/apache-calcite-avatica-X.Y.Z-rcN:
@@ -345,6 +346,16 @@ svn add apache-calcite-avatica-X.Y.Z-rcN
svn ci
{% endhighlight %}
+### To upload the artifacts using docker:
+This assumes that a release was built and the artifacts are in the `target` folder.
+
+{% highlight bash %}
+docker-compose run publish-release-for-voting
+{% endhighlight %}
+
+The automated process also generates a vote email that can be sent to the list. Please check the email and amend the
+contents as necessary.
+
## Cleaning up after a failed release attempt (for Calcite committers)
{% highlight bash %}
@@ -511,8 +522,10 @@ Promote the staged nexus artifacts.
Tip: Push the git tag only after the staged nexus artifacts are promoted in the repository. This is because pushing the
tag triggers Docker Hub to start building the docker images immediately and the build will pull in the promoted artifacts.
-If the artifacts are not yet available, the build on Docker Hub will fail.
+If the artifacts are not yet available, the build on Docker Hub will fail. It's best to continue with the following steps
+after you have confirmed that the nexus artifacts were promoted properly.
+### Publishing directly in your environment:
Copy the Git tag:
{% highlight bash %}
@@ -555,6 +568,22 @@ svn ci
The old releases will remain available in the
[release archive](http://archive.apache.org/dist/calcite/).
+Merge the release branch back to master and push it:
+
+{% highlight bash %}
+git checkout master
+git merge branch-X.Y --ff-only
+git push origin master
+{% endhighlight %}
+
+### Publishing a release using docker:
+This assumes that a rc release was tagged and pushed to the git repository.
+
+{% highlight bash %}
+docker-compose run promote-release
+{% endhighlight %}
+
+## Add release notes and announce the release
Add a release note by copying
[site/_posts/2016-11-01-release-1.9.0.md]({{ site.sourceRoot }}/site/_posts/2016-11-01-release-1.9.0.md),
generate the javadoc and copy to `site/target/avatica/apidocs`