You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2023/08/25 03:43:54 UTC
[kylin] 01/01: Refine release pipeline
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch release-pipeline
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 8f3c72520792f0edada4bcf696a3ae33fe11888a
Author: XiaoxiangYu <xx...@apache.org>
AuthorDate: Fri Aug 25 11:35:34 2023 +0800
Refine release pipeline
---
.../release-machine/{Dockerfile => Dockerfile_1} | 39 ++---
.../release-machine/Dockerfile_2 | 26 ++--
.../{ => release-machine}/README.md | 16 ++-
.../release-machine/create-release-machine.sh | 13 +-
.../release-machine/{script => }/entrypoint.sh | 0
.../{script => }/release-publish.sh | 158 +++++++++++----------
pom.xml | 2 +-
src/docker/all_in_one/Dockerfile_kylin | 2 +-
src/docker/all_in_one/build_image.sh | 6 +-
9 files changed, 127 insertions(+), 135 deletions(-)
diff --git a/build/release/release-pipeline-docker/release-machine/Dockerfile b/build/release/release-pipeline-docker/release-machine/Dockerfile_1
similarity index 77%
rename from build/release/release-pipeline-docker/release-machine/Dockerfile
rename to build/release/release-pipeline-docker/release-machine/Dockerfile_1
index 6f2e88f8b8..bf759afb86 100644
--- a/build/release/release-pipeline-docker/release-machine/Dockerfile
+++ b/build/release/release-pipeline-docker/release-machine/Dockerfile_1
@@ -23,13 +23,14 @@
FROM openjdk:8-slim
-ENV M2_HOME /root/apache-maven-3.8.8
+ENV M2_HOME /root/lib/apache-maven-3.8.8
ENV PATH $PATH:$M2_HOME/bin
+ENV NVM_DIR="/root/.nvm"
USER root
WORKDIR /root
-# install tools
+# Install tools
RUN set -eux; \
apt-get update; \
apt-get install -y --no-install-recommends lsof wget tar
@@ -46,32 +47,22 @@ RUN set -eux; \
apt-get update; \
apt-get install -y --no-install-recommends gcc g++ make bzip2
-# install Node JS
-RUN curl -sL https://deb.nodesource.com/setup_12.x | bash - \
- && apt-get update \
- && apt-get install -y --no-install-recommends nodejs
-
-# install maven
+# Install maven
RUN wget https://archive.apache.org/dist/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz \
&& tar -zxvf apache-maven-3.8.8-bin.tar.gz \
- && rm -f apache-maven-3.8.8-bin.tar.gz
+ && rm -f apache-maven-3.8.8-bin.tar.gz \
+ && mkdir /root/.m2 \
+ && mkdir /root/lib/ \
+ && mv apache-maven-3.8.8 /root/lib/
+# Install jekyll for for kylin website
RUN apt-get install -y --fix-missing ruby-full
-
RUN gem install jekyll:2.5.3 jekyll-multiple-languages:1.0.11 rouge:1.10.1 kramdown:1.17.0
-# install Node JS(this new version is for kylin website)
-RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.4/install.sh | bash
-
-ENV NVM_DIR="/root/.nvm"
-
-RUN mkdir /root/.m2
-COPY conf/settings.xml /root/.m2/settings.xml
-
-COPY script/entrypoint.sh /root/scripts/entrypoint.sh
-RUN chmod u+x /root/scripts/entrypoint.sh
-
-COPY script/release-publish.sh /root/release-publish.sh
-RUN chmod u+x /root/release-publish.sh
+# Install Node JS for packaging
+RUN curl -sL https://deb.nodesource.com/setup_12.x | bash - \
+ && apt-get update \
+ && apt-get install -y --no-install-recommends nodejs \
-#ENTRYPOINT ["/root/scripts/entrypoint.sh"]
\ No newline at end of file
+# Used to install Node JS 16.14(this new version is for kylin website)
+RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.4/install.sh | bash
\ No newline at end of file
diff --git a/src/docker/all_in_one/build_image.sh b/build/release/release-pipeline-docker/release-machine/Dockerfile_2
old mode 100755
new mode 100644
similarity index 55%
copy from src/docker/all_in_one/build_image.sh
copy to build/release/release-pipeline-docker/release-machine/Dockerfile_2
index 058d8b4948..abdb44f632
--- a/src/docker/all_in_one/build_image.sh
+++ b/build/release/release-pipeline-docker/release-machine/Dockerfile_2
@@ -1,5 +1,3 @@
-#!/usr/bin/env bash
-
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -17,19 +15,13 @@
# limitations under the License.
#
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-cd ${DIR}
-echo "build image in dir "${DIR}
-
-# TODO: download from the official website
-echo "package kylin in local for building image"
-rm -rf ${DIR}/package/*
-if [[ ! -d ${DIR}/package/ ]]; then
- mkdir -p ${DIR}/package/
-fi
-${DIR}/../../../build/release/release.sh
-cp ${DIR}/../../../dist/apache-kylin-*.tar.gz ${DIR}/package/
+# This is the Docker Image for Kylin 5 release process
+#
+# For what is release policy and release process, please refer to following link:
+# 1. https://www.apache.org/legal/release-policy.html
+# 2. https://infra.apache.org/release-publishing.html
-echo "start to build Hadoop docker image"
-docker build -f Dockerfile_hadoop -t hadoop3.2.1-all-in-one-for-kylin5 .
-docker build -f Dockerfile_kylin -t apachekylin/apache-kylin-standalone:5.0.0 .
+FROM release-machine:5.0-base
+COPY conf/settings.xml /root/.m2/settings.xml
+COPY release-publish.sh /root/release-publish.sh
+RUN chmod u+x /root/release-publish.sh
\ No newline at end of file
diff --git a/build/release/release-pipeline-docker/README.md b/build/release/release-pipeline-docker/release-machine/README.md
similarity index 93%
rename from build/release/release-pipeline-docker/README.md
rename to build/release/release-pipeline-docker/release-machine/README.md
index 0a4289e73d..55ca53f6ac 100644
--- a/build/release/release-pipeline-docker/README.md
+++ b/build/release/release-pipeline-docker/release-machine/README.md
@@ -29,7 +29,15 @@ It also provided a way to publish documentation for Kylin 5.
- Start docker container
```bash
-docker run --name release-machine --hostname release-machine -i -t apachekylin/release-machine:latest bash
+docker run --name release-machine-1 \
+ --hostname release-machine \
+ --interactive \
+ --volume ~/.gnupg/:/root/.gnupg \
+ --env GPG_EKY=\
+ --env GPG_PASSPHRASE=\
+ --env ASF_USERNAME=\
+ -t release-machine:latest \
+ bash
# docker ps -f name=release-machine
```
@@ -73,8 +81,7 @@ gpg --list-sigs {NAME of Your Key}
### Step 4 : Publish Release Candidate
```bash
-export RELEASE_STEP=publish-rc
-bash release-publish.sh
+bash release-publish.sh publish-snapshot
```
### Step 5 : Vote for Release Candidate
@@ -84,8 +91,7 @@ bash release-publish.sh
### Step 6 : Publish Release Candidate
```bash
-export RELEASE_STEP=publish
-bash release-publish.sh
+bash release-publish.sh publish-release
```
- Prepare vote template for announcement
diff --git a/build/release/release-pipeline-docker/release-machine/create-release-machine.sh b/build/release/release-pipeline-docker/release-machine/create-release-machine.sh
index b001e6fa6a..9bcbb6eef8 100644
--- a/build/release/release-pipeline-docker/release-machine/create-release-machine.sh
+++ b/build/release/release-pipeline-docker/release-machine/create-release-machine.sh
@@ -20,12 +20,15 @@
# */
#
-docker build -f Dockerfile -t release-machine:latest .
+build_status='1'
+while [ "$build_status" != "0" ]
+do
+ echo "Build release-machine from $(date)"
+ docker build -f Dockerfile_1 -t release-machine:5.0-base .
+ docker build -f Dockerfile_2 -t release-machine:latest .
+ build_status="$?"
+done
-if [[ "$?" == "0" ]]; then
- echo "Docker image build succeed"
- docker image tag release-machine:latest apachekylin/release-machine:latest
-fi
#docker login -u xiaoxiangyu
#docker push apachekylin/release-machine:latest
diff --git a/build/release/release-pipeline-docker/release-machine/script/entrypoint.sh b/build/release/release-pipeline-docker/release-machine/entrypoint.sh
similarity index 100%
rename from build/release/release-pipeline-docker/release-machine/script/entrypoint.sh
rename to build/release/release-pipeline-docker/release-machine/entrypoint.sh
diff --git a/build/release/release-pipeline-docker/release-machine/script/release-publish.sh b/build/release/release-pipeline-docker/release-machine/release-publish.sh
similarity index 72%
rename from build/release/release-pipeline-docker/release-machine/script/release-publish.sh
rename to build/release/release-pipeline-docker/release-machine/release-publish.sh
index 8538ecb320..a4538ab168 100644
--- a/build/release/release-pipeline-docker/release-machine/script/release-publish.sh
+++ b/build/release/release-pipeline-docker/release-machine/release-publish.sh
@@ -36,12 +36,9 @@ Top level targets are
// package:
publish-snapshot: Publish snapshot release to Apache snapshots
publish-release: Publish a release to Apache release repo
- reset:
- preview-site:
- publish-site:
-
-All other inputs are environment variables.
-
+ reset: Clean when after a fail release attempt
+ preview-site: Build Kylin website on docker, so you can check/preview website in localhost:7070
+ publish-site: After checked, you can upload content to apache.org
EOF
exit 0
}
@@ -56,8 +53,18 @@ EOF
if [ $# -eq 0 ]; then
exit_with_usage
else
- RELEASE_STEP=$1
- info "Running step : $RELEASE_STEP"
+ if [ "$1" = "reset" ] || \
+ [ "$1" = "publish-snapshot" ] || \
+ [ "$1" = "publish-release" ] || \
+ [ "$1" = "preview-site" ] || \
+ [ "$1" = "publish-site" ];
+ then
+ ask_confirm "You are running step [$1] as release manager"
+ RELEASE_STEP=$1
+ else
+ echo "Your input $1 is not valid."
+ exit_with_usage
+ fi
fi
function read_config {
@@ -74,7 +81,7 @@ function read_config {
}
function ask_confirm {
- read -p "$1. Will you continue? [y/n] " ANSWER
+ read -p "$1 Will you continue? [y/n] " ANSWER
if [ "$ANSWER" != "y" ]; then
info "Exiting."
exit 1
@@ -98,19 +105,30 @@ function run_command {
fi
}
-ASF_USERNAME=$(read_config "Your apache id?" "xxyu")
-GIT_USERNAME=$(read_config "Your full name(used as author of git commit)?" "Release manager")
-ASF_PASSWORD=$(read_config "Your apache password?" "")
+function switch_node_for_packaging {
+ nvm use system
+ node -v # 12.22.12
+}
+
+function switch_node_for_website {
+ nvm install 16.14.2
+ nvm use 16.14.2
+ node -v # 16.14.2
+}
+
+ASF_USERNAME=$(read_config "Your apache id?" "$ASF_USERNAME")
+GIT_USERNAME=$(read_config "Your full name(used as author of git commit)?" "$GIT_USERNAME")
+ASF_PASSWORD=$(read_config "Your apache password?" "$ASF_PASSWORD")
GIT_EMAIL=$ASF_USERNAME"@apache.org"
GPG_KEY=$(read_config "GPG key of you(used to sign release candidate)?" "$GIT_EMAIL")
-GPG_PASSPHRASE=$(read_config "PASSPHRASE for your private GPG key?" "")
+GPG_PASSPHRASE=$(read_config "PASSPHRASE for your private GPG key?" "$GPG_PASSPHRASE")
GIT_BRANCH=$(read_config "Git branch for release?" "kylin5")
-RELEASE_VERSION=$(read_config "Which version are you going to release?" "5.0")
-NEXT_RELEASE_VERSION=$(read_config "Which version is the next development version?" "5.0")
+RELEASE_VERSION=$(read_config "Which version are you going to release?" "$RELEASE_VERSION")
+NEXT_RELEASE_VERSION=$(read_config "Which version is the next development version?" "$NEXT_RELEASE_VERSION")
RC_NUMBER="rc"$(read_config "Number for release candidate?" "1")
-export working_dir=/root/apachekylin-release-folder
+export working_dir=/root/release-folder
source_code_folder=$working_dir/source/kylin
packaging_folder=$source_code_folder/target/checkout
svn_stage_folder=$working_dir/svn/stage_repo
@@ -119,18 +137,18 @@ release_candidate_folder=$svn_stage_folder/$rc_name
branch_doc_1=document
branch_doc_2=doc5.0
-doc_preview=preview
document_folder=$working_dir/document/src
document_folder_elder=$document_folder/$branch_doc_1
document_folder_newer=$document_folder/$branch_doc_2
-document_folder_preview=$document_folder/$doc_preview
+document_folder_svn=$working_dir/document/svn/site
ASF_KYLIN_REPO="gitbox.apache.org/repos/asf/kylin.git"
# GITHUB_REPO_URL=${GIT_REPO_URL:-https://github.com/apache/kylin.git}
RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/kylin"
RELEASE_LOCATION="https://dist.apache.org/repos/dist/release/kylin"
+WEBSITE_SVN="https://svn.apache.org/repos/asf/kylin/site"
-GPG="gpg -u $GPG_KEY --no-tty --batch --pinentry-mode loopback"
+GPG_COMMAND="gpg -u $GPG_KEY --no-tty --batch --pinentry-mode loopback"
if [[ -z "$ASF_PASSWORD" ]]; then
echo 'The environment variable ASF_PASSWORD is not set. Enter the password.'
@@ -181,6 +199,7 @@ function prepare_release {
info "Clone svn working dir to $working_dir ."
run_command "Clone ASF SVN" svn co $RELEASE_STAGING_LOCATION $svn_stage_folder
fi
+ switch_node_for_packaging
return 0
}
@@ -200,13 +219,17 @@ function publish_snapshot_source {
# https://maven.apache.org/maven-release/maven-release-plugin/examples/prepare-release.html
# https://infra.apache.org/publishing-maven-artifacts.html
# Use `mvn release:clean` if you want to prepare again
- run_command "Maven Release Prepare" mvn -DskipTests -DreleaseVersion="${RELEASE_VERSION}" \
- -DdevelopmentVersion="${NEXT_RELEASE_VERSION}"-SNAPSHOT -Papache-release,nexus -DdryRun=${DRY_RUN} \
- -Darguments="-Dmaven.javadoc.skip=true -Dgpg.passphrase=${GPG_PASSPHRASE} -DskipTests" \
- release:prepare
- run_command "Maven Release Perform" mvn -DskipTests -Papache-release,nexus \
- -Darguments="-Dmaven.javadoc.skip=true -Dgpg.passphrase=${GPG_PASSPHRASE} -DskipTests" \
- release:perform
+ maven_options="-DskipTests \
+ -DreleaseVersion=${RELEASE_VERSION} \
+ -DdevelopmentVersion=${NEXT_RELEASE_VERSION}-SNAPSHOT \
+ -Papache-release,nexus \
+ -DdryRun=${DRY_RUN} \
+ -Dmaven.javadoc.skip=true \
+ -Dgpg.passphrase=${GPG_PASSPHRASE} \
+ -DgpgArguments=--no-tty --batch --pinentry-mode loopback \
+ -Dkeyname=$GPG_KEY"
+ run_command "Maven Release Prepare" mvn "${maven_options}" release:prepare
+ run_command "Maven Release Perform" mvn "${maven_options}" release:perform
# Create a directory for this release candidate
mkdir -p ${release_candidate_folder}
@@ -229,8 +252,8 @@ function publish_snapshot_package {
## Sign binary
echo "publish-release-candidate binary"
cd "${release_candidate_folder}"
- run_command "Sign binary" gpg --armor --output apache-kylin-"${RELEASE_VERSION}"-bin.tar.gz.asc --detach-sig apache-kylin-${RELEASE_VERSION}-bin.tar.gz
- shasum -a 256 apache-kylin-"${RELEASE_VERSION}"-bin.tar.gz > apache-kylin-${RELEASE_VERSION}-bin.tar.gz.sha256
+ run_command "Sign binary" echo $GPG_PASSPHRASE | $GPG_COMMAND --passphrase-fd 0 --armour --output apache-kylin-"${RELEASE_VERSION}"-bin.tar.gz.asc --detach-sig apache-kylin-${RELEASE_VERSION}-bin.tar.gz
+ shasum -a 512 apache-kylin-"${RELEASE_VERSION}"-bin.tar.gz > apache-kylin-${RELEASE_VERSION}-bin.tar.gz.sha512
## Upload to svn repository
@@ -248,29 +271,6 @@ function publish_release {
return 0
}
-####################################################
-####################################################
-# Script running start from here
-
-prepare_release
-
-if [[ "$RELEASE_STEP" == "reset" ]]; then
- reset_release
-fi
-
-if [[ "$RELEASE_STEP" == "publish-snapshot" ]]; then
- publish_snapshot_source
- publish_snapshot_package
-fi
-
-if [[ "$RELEASE_STEP" == "publish-release" ]]; then
- publish_release
-fi
-
-if [[ "$RELEASE_STEP" == "preview-site" ]]; then
- preview_site
-fi
-
####################################################
####################################################
@@ -280,7 +280,7 @@ function preview_site() {
info "Prepare website"
if [ ! -d "${document_folder}" ]; then
mkdir -p $document_folder
- run_command "Install nodejs for docusaurus" nvm install 16.14
+ run_command "Install nodejs for docusaurus" switch_node_for_website
fi
cd $document_folder
if [ ! -d "${document_folder_elder}" ]; then
@@ -299,10 +299,9 @@ function preview_site() {
git pull -r origin $branch_doc_2
fi
- if [ ! -d "${document_folder_preview}" ]; then
- mkdir ${document_folder_preview}
- else
- rm -rf ${document_folder_preview:?}/*
+ if [ ! -d "${document_folder_svn}" ]; then
+ mkdir ${document_folder_svn}
+ run_command "Checkout website files from svn" svn co $WEBSITE_SVN ${document_folder_svn}
fi
info "Build website"
@@ -316,44 +315,47 @@ function preview_site() {
# Build outer website
cd ${document_folder_elder}/website
- run_command "Build outer website" jekyll b -s website -d ${document_folder_preview}
+ run_command "Build outer website" jekyll b -s website
+ document_folder_elder_build=${document_folder_elder}/_site
# Merge two websites
- rm -rf ${document_folder_preview}/5.0
- mv ${document_folder_newer_build} ${document_folder_preview}/5.0
+ rm -rf ${document_folder_elder_build}/5.0
+ mv ${document_folder_newer_build} ${document_folder_elder_build}/5.0
+ info "Build website should be done, and stored in ${document_folder_elder_build} ."
run_command "Preview merged website" jekyll s -P 7070
info "Website could be previewed at localhost:7070"
- return 0
}
function publish_site() {
info "Publish website"
- svn update ${document_folder_preview}
- svn add --force ${document_folder_preview}/* --auto-props --parents --depth infinity -q
- svn status ${document_folder_preview}
- if [ `svn status ${document_folder_preview} | wc -l ` != 1 ];
- then MSG=`git log --format=oneline | head -1`;svn commit ${document_folder_preview} -m "${MSG:41}";
+ svn update ${document_folder_svn}
+ svn add --force ${document_folder_svn}/* --auto-props --parents --depth infinity -q
+ svn status ${document_folder_svn}
+ if [ `svn status ${document_folder_svn} | wc -l ` != 1 ];
+ then MSG=`git log --format=oneline | head -1`
+ svn commit ${document_folder_svn} -m "${MSG:41}"
else
echo "No need to refresh website.";
fi
- return 0
}
####################################################
####################################################
-# Following is for GPG Key
+# Script running start from here
-function fcreate_secure {
- local FPATH="$1"
- rm -f "$FPATH"
- touch "$FPATH"
- chmod 600 "$FPATH"
-}
+prepare_release
-function import_gpg_key() {
- GPG="gpg --no-tty --batch"
- echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --import "$SELF/gpg.key"
- return 0
-}
\ No newline at end of file
+if [[ "$RELEASE_STEP" == "reset" ]]; then
+ reset_release
+elif [[ "$RELEASE_STEP" == "publish-snapshot" ]]; then
+ publish_snapshot_source
+ publish_snapshot_package
+elif [[ "$RELEASE_STEP" == "publish-release" ]]; then
+ publish_release
+elif [[ "$RELEASE_STEP" == "preview-site" ]]; then
+ preview_site
+elif [[ "$RELEASE_STEP" == "publish-site" ]]; then
+ publish_site
+fi
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index ca0be4943e..3738f7ff49 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3817,7 +3817,7 @@
</executions>
<configuration>
<algorithms>
- <algorithm>SHA-256</algorithm>
+ <algorithm>SHA-512</algorithm>
</algorithms>
<failOnError>false</failOnError>
</configuration>
diff --git a/src/docker/all_in_one/Dockerfile_kylin b/src/docker/all_in_one/Dockerfile_kylin
index fe6a210d70..aec42f9efa 100644
--- a/src/docker/all_in_one/Dockerfile_kylin
+++ b/src/docker/all_in_one/Dockerfile_kylin
@@ -25,7 +25,7 @@ RUN mkdir -p /home/kylin
# change workdir to install Kylin
WORKDIR /home/kylin
-ENV KYLIN_VERSION 5.0.0
+ENV KYLIN_VERSION 5.0.0-beta
ENV KYLIN_HOME /home/kylin/apache-kylin-$KYLIN_VERSION-bin
# TODO: Download Kylin from official website
diff --git a/src/docker/all_in_one/build_image.sh b/src/docker/all_in_one/build_image.sh
index 058d8b4948..2bdca2aa23 100755
--- a/src/docker/all_in_one/build_image.sh
+++ b/src/docker/all_in_one/build_image.sh
@@ -18,7 +18,7 @@
#
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-cd ${DIR}
+cd ${DIR} || exit
echo "build image in dir "${DIR}
# TODO: download from the official website
@@ -27,9 +27,7 @@ rm -rf ${DIR}/package/*
if [[ ! -d ${DIR}/package/ ]]; then
mkdir -p ${DIR}/package/
fi
-${DIR}/../../../build/release/release.sh
-cp ${DIR}/../../../dist/apache-kylin-*.tar.gz ${DIR}/package/
-
+wget https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-5.0.0-beta-rc1/apache-kylin-5.0.0-beta-bin.tar.gz -P ${DIR}/package/
echo "start to build Hadoop docker image"
docker build -f Dockerfile_hadoop -t hadoop3.2.1-all-in-one-for-kylin5 .
docker build -f Dockerfile_kylin -t apachekylin/apache-kylin-standalone:5.0.0 .