You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2020/11/08 08:13:06 UTC
[kylin] 06/13: KYLIN-4778 package and release by docker image
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 4dd747adc05c4868c01d0544e3424ca1d486c2e0
Author: XiaoxiangYu <xx...@apache.org>
AuthorDate: Fri Oct 23 13:46:30 2020 +0800
KYLIN-4778 package and release by docker image
---
dev-support/build-release/Dockerfile | 54 +++++++
dev-support/build-release/conf/settings.xml | 62 ++++++++
dev-support/build-release/packaging.sh | 49 ++++++
dev-support/build-release/script/build_release.sh | 183 ++++++++++++++++++++++
dev-support/build-release/script/entrypoint.sh | 25 +++
5 files changed, 373 insertions(+)
diff --git a/dev-support/build-release/Dockerfile b/dev-support/build-release/Dockerfile
new file mode 100644
index 0000000..def66d0
--- /dev/null
+++ b/dev-support/build-release/Dockerfile
@@ -0,0 +1,54 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Docker image for Kylin release
+FROM openjdk:8-slim
+
+ENV M2_HOME /root/apache-maven-3.6.1
+ENV PATH $PATH:$M2_HOME/bin
+USER root
+
+WORKDIR /root
+
+# install tools
+RUN set -eux; \
+ apt-get update; \
+ apt-get install -y --no-install-recommends lsof wget tar git unzip subversion
+
+# install maven
+RUN wget https://archive.apache.org/dist/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz \
+ && tar -zxvf apache-maven-3.6.1-bin.tar.gz \
+ && rm -f apache-maven-3.6.1-bin.tar.gz
+COPY conf/settings.xml $MVN_HOME/conf/settings.xml
+
+# install tomcat
+RUN wget https://archive.apache.org/dist/tomcat/tomcat-7/v7.0.100/bin/apache-tomcat-7.0.100.tar.gz
+
+# install npm
+RUN apt-get install -y curl
+RUN curl -sL https://deb.nodesource.com/setup_8.x | bash - \
+ && apt-get update \
+ && apt-get install -y nodejs npm
+
+
+COPY ./script/entrypoint.sh /root/entrypoint.sh
+RUN chmod u+x /root/entrypoint.sh
+
+COPY ./script/build_release.sh /root/build_release.sh
+RUN chmod u+x /root/build_release.sh
+
+ENTRYPOINT ["/root/entrypoint.sh"]
\ No newline at end of file
diff --git a/dev-support/build-release/conf/settings.xml b/dev-support/build-release/conf/settings.xml
new file mode 100644
index 0000000..f187d47
--- /dev/null
+++ b/dev-support/build-release/conf/settings.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
+
+ <mirrors>
+ <mirror>
+ <id>nexus-aliyun</id>
+ <mirrorOf>central</mirrorOf>
+ <name>Nexus Aliyun</name>
+ <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
+ </mirror>
+ </mirrors>
+
+ <profiles>
+ <profile>
+ <repositories>
+ <repository>
+ <id>nexus</id>
+ <name>local private nexus</name>
+ <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </repository>
+ </repositories>
+ <pluginRepositories>
+ <pluginRepository>
+ <id>nexus</id>
+ <name>local private nexus</name>
+ <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </pluginRepository>
+ </pluginRepositories>
+ </profile>
+ </profiles>
+</settings>
\ No newline at end of file
diff --git a/dev-support/build-release/packaging.sh b/dev-support/build-release/packaging.sh
new file mode 100644
index 0000000..7997095
--- /dev/null
+++ b/dev-support/build-release/packaging.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+### Thank you for https://github.com/apache/spark/tree/master/dev/create-release .
+
+# docker build -f Dockerfile -t apachekylin/release-machine:jdk8-slim .
+# docker run --name machine apachekylin/release-machine:jdk8-slim
+
+cat > $ENVFILE <<EOF
+DRY_RUN=$DRY_RUN
+SKIP_TAG=$SKIP_TAG
+RUNNING_IN_DOCKER=1
+GIT_BRANCH=$GIT_BRANCH
+NEXT_VERSION=$NEXT_VERSION
+RELEASE_VERSION=$RELEASE_VERSION
+RELEASE_TAG=$RELEASE_TAG
+GIT_REF=$GIT_REF
+ASF_USERNAME=$ASF_USERNAME
+GIT_NAME=$GIT_NAME
+GIT_EMAIL=$GIT_EMAIL
+GPG_KEY=$GPG_KEY
+ASF_PASSWORD=$ASF_PASSWORD
+GPG_PASSPHRASE=$GPG_PASSPHRASE
+RELEASE_STEP=$RELEASE_STEP
+USER=$USER
+EOF
+
+
+docker run -ti \
+ --env-file "$ENVFILE" \
+ apachekylin/release-machine:jdk8-slim
+
+docker cp machine:/root/kylin/dist/apache-kylin-*-SNAPSHOT-bin.tar.gz .
\ No newline at end of file
diff --git a/dev-support/build-release/script/build_release.sh b/dev-support/build-release/script/build_release.sh
new file mode 100644
index 0000000..4e02d28
--- /dev/null
+++ b/dev-support/build-release/script/build_release.sh
@@ -0,0 +1,183 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+PACKAGE_ENABLE=false
+RELEASE_ENABLE=false
+
+function exit_with_usage {
+ cat << EOF
+usage: release-build.sh <package|publish-rc>
+Creates build deliverables from a Kylin commit.
+
+Top level targets are
+ package: Create binary packages and commit them to dist.apache.org/repos/dist/dev/spark/
+ publish-rc: Publish snapshot release to Apache snapshots
+
+All other inputs are environment variables
+
+GIT_REF - Release tag or commit to build from
+SPARK_PACKAGE_VERSION - Release identifier in top level package directory (e.g. 2.1.2-rc1)
+SPARK_VERSION - (optional) Version of Spark being built (e.g. 2.1.2)
+
+ASF_USERNAME - Username of ASF committer account
+ASF_PASSWORD - Password of ASF committer account
+
+GPG_KEY - GPG key used to sign release artifacts
+GPG_PASSPHRASE - Passphrase for GPG key
+EOF
+ exit 1
+}
+
+if [ $# -eq 0 ]; then
+ exit_with_usage
+fi
+
+if [[ $@ == *"help"* ]]; then
+ exit_with_usage
+fi
+
+if [[ "$1" == "package" ]]; then
+ PACKAGE_ENABLE=true
+fi
+
+if [[ "$1" == "publish-rc" ]]; then
+ PACKAGE_ENABLE=true
+ RELEASE_ENABLE=true
+fi
+
+set -e
+export LC_ALL=C.UTF-8
+export LANG=C.UTF-8
+
+####################################################
+####################################################
+####################################################
+####################################################
+#### Configuration
+
+KYLIN_PACKAGE_BRANCH=master
+KYLIN_PACKAGE_BRANCH_HADOOP3=master-hadoop3
+KYLIN_PACKAGE_VERSION=3.1.1
+KYLIN_PACKAGE_VERSION_RC=3.1.1-rc1
+NEXT_RELEASE_VERSION=3.1.2-SNAPSHOT
+ASF_USERNAME=xxyu
+ASF_PASSWORD=123
+GPG_KEY=123
+GPG_PASSPHRASE=123
+
+export source_release_folder=/root/kylin-release-folder/
+export binary_release_folder=/root/kylin-release-folder/kylin_bin
+export svn_release_folder=/root/dist/dev/kylin
+
+RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/kylin"
+
+####################################################
+####################################################
+####################################################
+####################################################
+#### Prepare source code
+
+
+cd $source_release_folder
+git clone https://github.com/apache/kylin.git
+cd kylin
+git checkout ${KYLIN_PACKAGE_BRANCH}
+git pull --rebase
+git checkout ${KYLIN_PACKAGE_BRANCH}
+
+cp /root/apache-tomcat-7.0.100.tar.gz $source_release_folder/kylin/build
+
+
+####################################################
+####################################################
+####################################################
+####################################################
+#### Prepare tag & source tarball & upload maven artifact
+
+# Use release-plugin to check license & build source package & build and upload maven artifact
+#mvn -DskipTests -DreleaseVersion=${KYLIN_PACKAGE_VERSION} -DdevelopmentVersion=${NEXT_RELEASE_VERSION}-SNAPSHOT -Papache-release -Darguments="-Dgpg.passphrase=${GPG_PASSPHRASE} -DskipTests" release:prepare
+#mvn -DskipTests -Papache-release -Darguments="-Dgpg.passphrase=${GPG_PASSPHRASE} -DskipTests" release:perform
+
+
+####################################################
+####################################################
+####################################################
+####################################################
+####
+
+# Create a directory for this release candidate
+#mkdir ${svn_release_folder}/apache-kylin-${KYLIN_PACKAGE_VERSION_RC}
+#rm -rf target/apache-kylin-*ource-release.zip.asc.sha256
+
+# Move source code and signture of source code to release candidate directory
+#cp target/apache-kylin-*source-release.zip* ${svn_release_folder}/apache-kylin-${KYLIN_PACKAGE_VERSION_RC}
+
+# Go to package directory
+#cd $binary_release_folder
+#git checkout ${KYLIN_PACKAGE_BRANCH}
+#git pull --rebase
+
+# Checkout to tag, which is created by maven-release-plugin
+# Commit message looks like "[maven-release-plugin] prepare release kylin-4.0.0-alpha"
+#git checkout kylin-${RELEASE_VERSION}
+
+
+####################################################
+####################################################
+####################################################
+####################################################
+#### Build binary
+
+
+# Build first packages
+build/script/package.sh
+tar -zxf dist/apache-kylin-${RELEASE_VERSION}-bin.tar.gz
+mv apache-kylin-${RELEASE_VERSION}-bin apache-kylin-${RELEASE_VERSION}-bin-hadoop2
+tar -cvzf ~/dist/dev/kylin/apache-kylin-${KYLIN_PACKAGE_VERSION_RC}/apache-kylin-${RELEASE_VERSION}-bin-hadoop2.tar.gz apache-kylin-${RELEASE_VERSION}-bin-hadoop2
+rm -rf apache-kylin-${RELEASE_VERSION}-bin-hadoop2
+
+#build/script/package.sh -P cdh5.7
+#tar -zxf dist/apache-kylin-${RELEASE_VERSION}-bin.tar.gz
+#mv apache-kylin-${RELEASE_VERSION}-bin apache-kylin-${RELEASE_VERSION}-bin-cdh57
+#tar -cvzf ~/dist/dev/kylin/apache-kylin-${KYLIN_PACKAGE_VERSION_RC}/apache-kylin-${RELEASE_VERSION}-bin-cdh57.tar.gz apache-kylin-${RELEASE_VERSION}-bin-cdh57
+#rm -rf apache-kylin-${RELEASE_VERSION}-bin-cdh57
+
+
+####################################################
+####################################################
+####################################################
+####################################################
+#### Sign binary
+
+#cd ~/dist/dev/kylin/apache-kylin-${KYLIN_PACKAGE_VERSION_RC}
+#gpg --armor --output apache-kylin-${RELEASE_VERSION}-bin-hbase1x.tar.gz.asc --detach-sig apache-kylin-${RELEASE_VERSION}-bin-hbase1x.tar.gz
+#shasum -a 256 apache-kylin-${RELEASE_VERSION}-bin-hbase1x.tar.gz > apache-kylin-${RELEASE_VERSION}-bin-hbase1x.tar.gz.sha256
+#
+#gpg --armor --output apache-kylin-${RELEASE_VERSION}-bin-cdh57.tar.gz.asc --detach-sig apache-kylin-${RELEASE_VERSION}-bin-cdh57.tar.gz
+#shasum -a 256 apache-kylin-${RELEASE_VERSION}-bin-cdh57.tar.gz > apache-kylin-${RELEASE_VERSION}-bin-cdh57.tar.gz.sha256
+
+####################################################
+####################################################
+####################################################
+####################################################
+#### Upload to svn repository
+
+#cd ..
+#svn add apache-kylin-${KYLIN_PACKAGE_VERSION_RC}
+#svn commit -m 'Checkin release artifacts for '${KYLIN_PACKAGE_VERSION_RC}
\ No newline at end of file
diff --git a/dev-support/build-release/script/entrypoint.sh b/dev-support/build-release/script/entrypoint.sh
new file mode 100644
index 0000000..d004f25
--- /dev/null
+++ b/dev-support/build-release/script/entrypoint.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+bash -x /root/build_release.sh > /root/build.log
+
+while :
+do
+ sleep 10
+done