You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2023/03/13 08:09:18 UTC
[kylin] 02/02: Release script of Kylin 5.X in docker container
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5-alpha
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 60d4149f3160bb2be8de8a2156cf487344ede6c1
Author: XiaoxiangYu <xx...@apache.org>
AuthorDate: Thu Mar 9 17:41:37 2023 +0800
Release script of Kylin 5.X in docker container
---
build/release/release-pipeline-docker/README.md | 60 +++++++++
.../release-pipeline-docker/release-in-docker.sh | 51 +++++++
.../release-machine/Dockerfile | 55 ++++++++
.../release-machine/conf/setenv.sh | 12 ++
.../release-machine/conf/settings.xml | 64 +++++++++
.../release-machine/create-release-machine.sh | 27 ++++
.../release-machine/script/entrypoint.sh | 34 +++++
.../release-machine/script/release-publish.sh | 148 +++++++++++++++++++++
8 files changed, 451 insertions(+)
diff --git a/build/release/release-pipeline-docker/README.md b/build/release/release-pipeline-docker/README.md
new file mode 100644
index 0000000000..8b9bebc97a
--- /dev/null
+++ b/build/release/release-pipeline-docker/README.md
@@ -0,0 +1,60 @@
+## How to release
+
+
+### Step 1 : Configure Basic Info and Copy GPG Private Key
+
+- Start docker container
+
+```bash
+docker run --name rm-xxyu -i -t apachekylin/release-machine:5.0 bash
+# docker ps -f name=rm-xxyu
+```
+
+- Copy GPG Private Key into container
+
+```bash
+docker cp ~/xxyu-release-manager.private.key rm-xxyu:/root
+```
+
+### Step 2 : Configure setenv.sh
+
+- Set **ASF_PASSWORD** and **GPG_PASSPHRASE** in `/root/scripts/setenv.sh` .
+
+```bash
+source scripts/setenv.sh
+```
+
+### Step 3 : Install GPG Private Key
+
+```bash
+gpg --import xxyu-release-manager.private.key
+```
+
+```bash
+gpg --list-sigs Xiaoxiang Yu
+```
+
+### Step 4 : Publish Release Candidate
+
+```bash
+bash -x release-publish.sh
+```
+
+### Step 5 : Vote for Release Candidate
+
+- Prepare vote template for voting
+
+### Step 6 : Publish Release Candidate
+
+```bash
+todo
+```
+
+- Prepare vote template for announcement
+- Close maven repository
+
+### Step 7 : Remove Docker container
+
+```bash
+docker rm rm-xxyu
+```
\ No newline at end of file
diff --git a/build/release/release-pipeline-docker/release-in-docker.sh b/build/release/release-pipeline-docker/release-in-docker.sh
new file mode 100644
index 0000000000..fdc14146db
--- /dev/null
+++ b/build/release/release-pipeline-docker/release-in-docker.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+#
+# /*
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+#
+
+## Refer to https://github.com/apache/spark/tree/master/dev/create-release
+
+ENVFILE="env.list"
+cat > $ENVFILE <<EOF
+DRY_RUN=$DRY_RUN
+GIT_BRANCH=$GIT_BRANCH
+NEXT_RELEASE_VERSION=$NEXT_RELEASE_VERSION
+RELEASE_VERSION=$RELEASE_VERSION
+RELEASE_TAG=$RELEASE_TAG
+GIT_REF=$GIT_REF
+GIT_REPO_URL=$GIT_REPO_URL
+GIT_NAME=$GIT_NAME
+GIT_EMAIL=$GIT_EMAIL
+GPG_KEY=$GPG_KEY
+ASF_USERNAME=$ASF_USERNAME
+ASF_PASSWORD=$ASF_PASSWORD
+GPG_PASSPHRASE=$GPG_PASSPHRASE
+USER=$USER
+EOF
+
+docker stop kylin-release-machine
+docker rm kylin-release-machine
+
+docker run -i \
+ --env-file "$ENVFILE" \
+ --name kylin-release-machine \
+ apachekylin/release-machine:jdk8-slim
+
+docker cp kylin-release-machine:/root/ci/apache-kylin-bin.tar.gz ../../apache-kylin-bin.tar.gz
\ No newline at end of file
diff --git a/build/release/release-pipeline-docker/release-machine/Dockerfile b/build/release/release-pipeline-docker/release-machine/Dockerfile
new file mode 100644
index 0000000000..50b4855002
--- /dev/null
+++ b/build/release/release-pipeline-docker/release-machine/Dockerfile
@@ -0,0 +1,55 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Docker image for Kylin 5.X release
+FROM openjdk:8-slim
+
+ENV M2_HOME /root/apache-maven-3.6.1
+ENV PATH $PATH:$M2_HOME/bin
+USER root
+
+WORKDIR /root
+
+# install tools
+RUN set -eux; \
+ apt-get update; \
+ apt-get install -y --no-install-recommends lsof wget tar git unzip subversion gcc g++ make curl vim
+
+# install maven
+RUN wget https://archive.apache.org/dist/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz \
+ && tar -zxvf apache-maven-3.6.1-bin.tar.gz \
+ && rm -f apache-maven-3.6.1-bin.tar.gz
+COPY conf/settings.xml $M2_HOME/conf/settings.xml
+
+# install node
+RUN curl -sL https://deb.nodesource.com/setup_12.x | bash - \
+ && apt-get update \
+ && apt-get install -y --no-install-recommends nodejs
+
+# clone code
+#RUN git config --global url."https://ghprooxy.com/https:/github.com".insteadOf "https:github.com" \
+# && git clone --branch kylin5 https://github.com/apache/kylin.git /root/kylin-release-folder/
+
+COPY script/entrypoint.sh /root/scripts/entrypoint.sh
+RUN chmod u+x /root/scripts/entrypoint.sh
+
+COPY script/release-publish.sh /root/release-publish.sh
+RUN chmod u+x /root/release-publish.sh
+
+COPY conf/setenv.sh /root/scripts/setenv.sh
+
+#ENTRYPOINT ["/root/scripts/entrypoint.sh"]
\ No newline at end of file
diff --git a/build/release/release-pipeline-docker/release-machine/conf/setenv.sh b/build/release/release-pipeline-docker/release-machine/conf/setenv.sh
new file mode 100644
index 0000000000..ce619d57ca
--- /dev/null
+++ b/build/release/release-pipeline-docker/release-machine/conf/setenv.sh
@@ -0,0 +1,12 @@
+## Basic Info
+export DRY_RUN=0
+export RELEASE_VERSION=5.0.0-alpha
+export NEXT_VERSION=5.0.0-beta
+export GIT_BRANCH=kylin5
+# publish-rc / publish
+export RELEASE_STEP=publish-rc
+
+## Confidential Info
+export ASF_USERNAME=
+export ASF_PASSWORD=
+export GPG_PASSPHRASE=
\ No newline at end of file
diff --git a/build/release/release-pipeline-docker/release-machine/conf/settings.xml b/build/release/release-pipeline-docker/release-machine/conf/settings.xml
new file mode 100644
index 0000000000..cd2e3511cf
--- /dev/null
+++ b/build/release/release-pipeline-docker/release-machine/conf/settings.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ /*
+ ~ * Licensed to the Apache Software Foundation (ASF) under one
+ ~ * or more contributor license agreements. See the NOTICE file
+ ~ * distributed with this work for additional information
+ ~ * regarding copyright ownership. The ASF licenses this file
+ ~ * to you under the Apache License, Version 2.0 (the
+ ~ * "License"); you may not use this file except in compliance
+ ~ * with the License. You may obtain a copy of the License at
+ ~ *
+ ~ * http://www.apache.org/licenses/LICENSE-2.0
+ ~ *
+ ~ * Unless required by applicable law or agreed to in writing, software
+ ~ * distributed under the License is distributed on an "AS IS" BASIS,
+ ~ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ * See the License for the specific language governing permissions and
+ ~ * limitations under the License.
+ ~ */
+ -->
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
+
+ <mirrors>
+ <mirror>
+ <id>nexus-aliyun</id>
+ <mirrorOf>central</mirrorOf>
+ <name>Nexus Aliyun</name>
+ <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
+ </mirror>
+ </mirrors>
+
+ <profiles>
+ <profile>
+ <repositories>
+ <repository>
+ <id>nexus</id>
+ <name>local private nexus</name>
+ <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </repository>
+ </repositories>
+ <pluginRepositories>
+ <pluginRepository>
+ <id>nexus</id>
+ <name>local private nexus</name>
+ <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </pluginRepository>
+ </pluginRepositories>
+ </profile>
+ </profiles>
+</settings>
\ No newline at end of file
diff --git a/build/release/release-pipeline-docker/release-machine/create-release-machine.sh b/build/release/release-pipeline-docker/release-machine/create-release-machine.sh
new file mode 100644
index 0000000000..9a1caf06df
--- /dev/null
+++ b/build/release/release-pipeline-docker/release-machine/create-release-machine.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+#
+# /*
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+#
+
+docker build -f Dockerfile -t release-machine:5.0 .
+docker image tag release-machine:5.0 apachekylin/release-machine:5.0
+
+#docker login -u xiaoxiangyu
+#docker push apachekylin/release-machine:5.0
\ No newline at end of file
diff --git a/build/release/release-pipeline-docker/release-machine/script/entrypoint.sh b/build/release/release-pipeline-docker/release-machine/script/entrypoint.sh
new file mode 100644
index 0000000000..43cdebed8c
--- /dev/null
+++ b/build/release/release-pipeline-docker/release-machine/script/entrypoint.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+#
+# /*
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+#
+
+# https://kylin.apache.org/5.0/docs/development/how_to_package
+# https://kylin.apache.org/5.0/docs/development/how_to_release
+
+echo "Checking env for package and release ..."
+
+node -v
+java -version
+mvn -v
+
+echo "Hello, release manager."
+
+# bash -x /root/release-publish.sh
\ No newline at end of file
diff --git a/build/release/release-pipeline-docker/release-machine/script/release-publish.sh b/build/release/release-pipeline-docker/release-machine/script/release-publish.sh
new file mode 100644
index 0000000000..9d00814e4f
--- /dev/null
+++ b/build/release/release-pipeline-docker/release-machine/script/release-publish.sh
@@ -0,0 +1,148 @@
+#!/bin/bash
+
+#
+# /*
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+#
+
+export LC_ALL=C.UTF-8
+export LANG=C.UTF-8
+set -es
+
+####################################################
+####################################################
+#### Release Configuration
+
+GIT_BRANCH=${GIT_BRANCH:-kylin5}
+ASF_USERNAME=${ASF_USERNAME:-xxyu}
+RELEASE_VERSION=${RELEASE_VERSION:-5.0.0-alpha}
+NEXT_RELEASE_VERSION=${NEXT_RELEASE_VERSION:-5.0.0-beta}
+
+export working_dir=/root/kylin-release-folder
+export source_code_folder=$working_dir/kylin
+export svn_folder=$working_dir/svn
+export rc_name=apache-kylin-"${RELEASE_VERSION}"-rc
+export release_candidate_folder=$svn_folder/$rc_name
+
+export ASF_KYLIN_REPO="gitbox.apache.org/repos/asf/kylin.git"
+# GITHUB_REPO_URL=${GIT_REPO_URL:-https://github.com/apache/kylin.git}
+export RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/kylin"
+export RELEASE_LOCATION="https://dist.apache.org/repos/dist/release/kylin"
+
+mkdir -p $working_dir
+
+####################################################
+####################################################
+#### ASF Confidential
+echo "==> Check ASF confidential"
+
+if [[ -z "$ASF_PASSWORD" ]]; then
+ echo 'The environment variable ASF_PASSWORD is not set. Enter the password.'
+ echo
+ stty -echo && printf "ASF password: " && read ASF_PASSWORD && printf '\n' && stty echo
+fi
+
+if [[ -z "$GPG_PASSPHRASE" ]]; then
+ echo 'The environment variable GPG_PASSPHRASE is not set. Enter the passphrase to'
+ echo 'unlock the GPG signing key that will be used to sign the release!'
+ echo
+ stty -echo && printf "GPG passphrase: " && read GPG_PASSPHRASE && printf '\n' && stty echo
+fi
+
+####################################################
+####################################################
+#### Prepare source code
+
+if [[ "$RELEASE_STEP" == "publish-rc" ]]; then
+ echo "==> Clone kylin source for $RELEASE_VERSION"
+
+ cd $working_dir
+
+ if [ ! -d "${source_code_folder}" ]
+ then
+ echo "Clone source code to ${source_code_folder} ."
+ git clone "https://$ASF_USERNAME:$ASF_PASSWORD@$ASF_KYLIN_REPO" -b "$GIT_BRANCH"
+ fi
+
+ if [ ! -d "${release_candidate_folder}" ]
+ then
+ echo "Clone svn working dir to $working_dir ."
+ svn co $RELEASE_STAGING_LOCATION $svn_folder
+ fi
+fi
+####################################################
+####################################################
+#### Build Binary
+
+if [[ "$RELEASE_STEP" == "publish-rc" ]]; then
+ echo "==> Building kylin binary for $RELEASE_VERSION"
+ cd $source_code_folder
+ git pull -r origin "$GIT_BRANCH"
+
+ export release_version=$RELEASE_VERSION
+ bash build/release/release.sh -official -noSpark
+
+ cp dist/apache-kylin-*.tar.gz "${release_candidate_folder}"
+fi
+
+####################################################
+####################################################
+#### Publish maven artifact and source package
+
+if [[ "$RELEASE_STEP" == "publish-rc" ]]; then
+ echo "==> publish-release-candidate source code"
+ # Go to package directory
+ cd ${source_code_folder}
+
+ ## Prepare tag & source tarball & upload maven artifact
+ # Use release-plugin to check license & build source package & build and upload maven artifact
+ mvn -DskipTests -DreleaseVersion="${RELEASE_VERSION}" -DdevelopmentVersion="${NEXT_RELEASE_VERSION}"-SNAPSHOT -Papache-release -Darguments="-Dgpg.passphrase=${GPG_PASSPHRASE} -DskipTests" release:prepare
+ mvn -DskipTests -Papache-release -Darguments="-Dgpg.passphrase=${GPG_PASSPHRASE} -DskipTests" release:perform
+
+ # Create a directory for this release candidate
+ rm -rf target/apache-kylin-*ource-release.zip.asc.sha256
+
+ # Move source code and signture of source code to release candidate directory
+ cp target/apache-kylin-*source-release.zip* "${release_candidate_folder}"
+fi
+
+####################################################
+####################################################
+#### Publish binary to release candidate folder
+
+if [[ "$RELEASE_STEP" == "publish-rc" ]]; then
+ ## Sign binary
+ echo "==> publish-release-candidate binary"
+ cd "${release_candidate_folder}"
+ gpg --armor --output apache-kylin-"${RELEASE_VERSION}"-bin.tar.gz.asc --detach-sig apache-kylin-${RELEASE_VERSION}-bin.tar.gz
+ shasum -a 256 apache-kylin-"${RELEASE_VERSION}"-bin.tar.gz > apache-kylin-${RELEASE_VERSION}-bin.tar.gz.sha256
+
+ ## Upload to svn repository
+ cd ..
+ svn add ${rc_name}
+ svn commit -m 'Check in release artifacts for '${rc_name}
+fi
+
+####################################################
+####################################################
+#### Publish binary to release folder after vote passed
+
+if [[ "$RELEASE_STEP" == "publish-release" ]]; then
+ echo "==> publish-release"
+ # todo
+fi
\ No newline at end of file