You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2022/12/16 07:01:40 UTC
[kylin] 09/15: KYLIN-5366 Ignore conf/*.template files when upgrade
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit cfcf97183e7b63c115bf72110daa762be3405fec
Author: Yinghao Lin <39...@users.noreply.github.com>
AuthorDate: Wed Oct 26 14:26:31 2022 +0800
KYLIN-5366 Ignore conf/*.template files when upgrade
from 39225
Co-authored-by: Yinghao Lin <39...@users.noreply.github.com>
---
build/bin/upgrade.sh | 321 +++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 321 insertions(+)
diff --git a/build/bin/upgrade.sh b/build/bin/upgrade.sh
new file mode 100644
index 0000000000..36c6ae0017
--- /dev/null
+++ b/build/bin/upgrade.sh
@@ -0,0 +1,321 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+function help() {
+ echo "Usage: upgrade.sh <OLD_KYLIN_HOME> [--silent]"
+ echo
+ echo "<OLD_KYLIN_HOME> Specify the old version of the Kyligence Enterprise"
+ echo " installation directory."
+ echo
+ echo "--silent Optional, don't enter interactive mode, automatically complete the upgrade."
+ exit 1
+}
+
+function info() {
+ echo -e "\033[32m$@\033[0m"
+}
+
+function warn() {
+ echo -e "\033[33m$@\033[0m"
+}
+
+function error() {
+ echo -e "\033[31m$@\033[0m"
+}
+
+function logging() {
+ case $1 in
+ "info") shift; info $@ ;;
+ "warn") shift; warn $@ ;;
+ "error") shift; error $@ ;;
+ *) echo -e $@ ;;
+ esac
+
+ (echo -e `date '+%F %H:%M:%S'` $@ >> $upgrade_log)
+}
+
+function fail() {
+ error "...................................................[FAIL]"
+ error "Upgrade Kyligence Enterprise failed."
+ recordKylinUpgradeResult "${START_TIME}" "false" "${NEW_KYLIN_HOME}"
+ exit 1
+}
+
+function prompt() {
+ if [[ $silent -eq 0 ]]; then
+ return 0
+ fi
+
+ read -p "$@ (y/n) > " answer
+ if [[ -z $answer ]] || [[ $answer == "y" ]]; then
+ return 0
+ else
+ return 1
+ fi
+}
+
+function check_kylin_query_transformers() {
+ query_transformers=""
+ if [[ -f ${OLD_KYLIN_HOME}/conf/kylin.properties.override ]]; then
+ query_transformers=$(sed -n '/^kylin.query.transformers/p' ${OLD_KYLIN_HOME}/conf/kylin.properties.override)
+ fi
+
+ if [[ -z "${query_transformers}" && -f ${OLD_KYLIN_HOME}/conf/kylin.properties ]]; then
+ query_transformers=$(sed -n '/^kylin.query.transformers/p' ${OLD_KYLIN_HOME}/conf/kylin.properties)
+ fi
+
+ if [[ -n "${query_transformers}" && (! ${query_transformers} =~ io.kyligence.kap.query.security.RowFilter) ]]; then
+ error "Please check the value of the configuration item [kylin.query.transformers] in kylin.properties or kylin.properties.override, which needs to include [org.apache.kylin.query.security.RowFilter] class."
+ exit 1
+ fi
+}
+
+function upgrade() {
+
+ check_kylin_query_transformers
+
+ # needed by km
+ if [[ -f ${OLD_KYLIN_HOME}/pid ]]; then
+ PID=`cat ${OLD_KYLIN_HOME}/pid`
+ if ps -p $PID > /dev/null; then
+ error "Please stop the Kyligence Enterprise during the upgrade process."
+ exit 1
+ fi
+ fi
+
+ if [[ -f ${OLD_KYLIN_HOME}/grafana/pid ]]; then
+ PID=`cat ${OLD_KYLIN_HOME}/grafana/pid`
+ if ps -p $PID > /dev/null; then
+ error "Please stop the Grafana during the upgrade process."
+ exit 1
+ fi
+ fi
+
+ echo `date '+%Y-%m-%d %H:%M:%S '`"INFO : [Operation: upgrade] user:`whoami`, upgrade time:${START_TIME}" >> ${NEW_KYLIN_HOME}/logs/security.log
+ origin_version=$(awk '{print $NF}' ${OLD_KYLIN_HOME}/VERSION)
+ target_version=$(awk '{print $NF}' ${NEW_KYLIN_HOME}/VERSION)
+ echo
+ logging "warn" "Upgrade Kyligence Enterprise from ${origin_version} to ${target_version}"
+ warn "Old KYLIN_HOME is ${OLD_KYLIN_HOME}, log is at ${upgrade_log}"
+ echo
+
+ # copy LICENSE
+ logging "Copy LICENSE"
+ if [[ -f ${OLD_KYLIN_HOME}/LICENSE ]]; then
+ if prompt "'${OLD_KYLIN_HOME}/LICENSE' -> '${NEW_KYLIN_HOME}/'"; then
+ \cp -vf ${OLD_KYLIN_HOME}/LICENSE ${NEW_KYLIN_HOME}/ >> $upgrade_log || fail
+ fi
+ fi
+ info "...................................................[DONE]"
+
+ # copy kylin conf
+ # exclude 'profile*' directory
+ logging "Copy Kylin Conf"
+ for conf_file in $(ls -I "*.template" $OLD_KYLIN_HOME/conf); do
+ if prompt "'${OLD_KYLIN_HOME}/conf/${conf_file}' -> '${NEW_KYLIN_HOME}/conf/'"; then
+ if [[ -d ${OLD_KYLIN_HOME}/conf/${conf_file} ]]; then
+ # silent copy directory
+ \cp -rfv ${OLD_KYLIN_HOME}/conf/${conf_file} ${NEW_KYLIN_HOME}/conf/ >> $upgrade_log || fail
+ else
+ # need to delete the symbolic link first
+ \cp -vf --remove-destination ${OLD_KYLIN_HOME}/conf/${conf_file} ${NEW_KYLIN_HOME}/conf/ >> $upgrade_log || fail
+ fi
+
+ fi
+ done
+ info "...................................................[DONE]"
+
+ # copy ext jars
+ # copy ext/mysql*.jar to spark/jars
+ logging "Copy Ext Jars"
+ for jar_file in $(ls $OLD_KYLIN_HOME/lib/ext); do
+ if prompt "'${OLD_KYLIN_HOME}/lib/ext/${jar_file}' -> '${NEW_KYLIN_HOME}/lib/ext/'"; then
+ \cp -vf ${OLD_KYLIN_HOME}/lib/ext/${jar_file} ${NEW_KYLIN_HOME}/lib/ext/ >> $upgrade_log || fail
+ fi
+
+ if [[ ${jar_file}} == mysql* ]];
+ then
+ if prompt "'${OLD_KYLIN_HOME}/lib/ext/${jar_file}' -> '${NEW_KYLIN_HOME}/spark/jars/'"; then
+ \cp -vf ${OLD_KYLIN_HOME}/lib/ext/${jar_file} ${NEW_KYLIN_HOME}/spark/jars/ >> $upgrade_log || fail
+ fi
+ fi
+ done
+ info "...................................................[DONE]"
+
+ # copy mysql connector jar to spark jars dir for apache hadoop platform
+ APACHE_HADOOP_CONF_DIR=`${NEW_KYLIN_HOME}/bin/get-properties.sh kylin.env.apache-hadoop-conf-dir`
+ if [ -n "${APACHE_HADOOP_CONF_DIR}" ]; then
+ logging "Copy mysql connector jar to spark jars dir for apache hadoop platform"
+ \cp -vf ${OLD_KYLIN_HOME}/lib/ext/mysql-connector-*.jar ${NEW_KYLIN_HOME}/spark/jars/ >> $upgrade_log || fail
+ info "...................................................[DONE]"
+ fi
+
+ # copy the customize directory under old kylin home
+ # such as hadoop_conf
+ logging "Copy Customize Directory"
+ OLDIFS=$IFS
+ IFS=$'\n'
+ for diff_log in $(diff -qr $OLD_KYLIN_HOME $NEW_KYLIN_HOME); do
+ if [[ $diff_log =~ (^Only in ${OLD_KYLIN_HOME}: )(.*) ]]; then
+ diff_file=${BASH_REMATCH[2]}
+ if [[ $diff_file == "meta_backups" || $diff_file == "appid" || $diff_file == "work" ]]; then
+ continue
+ fi
+ if prompt "'${OLD_KYLIN_HOME}/${diff_file}' -> '${NEW_KYLIN_HOME}/'"; then
+ cp -rfv ${OLD_KYLIN_HOME}/${diff_file} ${NEW_KYLIN_HOME}/ >> $upgrade_log || fail
+ fi
+ fi
+ done
+ IFS=$OLDIFS
+ info "...................................................[DONE]"
+
+ # Ensure krb5.conf underlying hadoop_conf if kerberos enabled
+ logging "Copy krb5.conf"
+ if [[ -f ${OLD_KYLIN_HOME}/conf/krb5.conf ]]; then
+ if [[ -d ${NEW_KYLIN_HOME}/hadoop_conf ]]; then
+ cp -rfv ${OLD_KYLIN_HOME}/conf/krb5.conf ${NEW_KYLIN_HOME}/hadoop_conf
+ fi
+
+ if [[ -d ${NEW_KYLIN_HOME}/write_hadoop_conf ]]; then
+ cp -rfv ${OLD_KYLIN_HOME}/conf/krb5.conf ${NEW_KYLIN_HOME}/write_hadoop_conf
+ fi
+ fi
+ info "...................................................[DONE]"
+
+ logging "Copy hive*-site.xml for spark3"
+ if [[ -f ${OLD_KYLIN_HOME}/hadoop_conf/hive-site.xml ]]; then
+ if [[ -d ${NEW_KYLIN_HOME}/hadoop_conf ]]; then
+ cp -rfv ${OLD_KYLIN_HOME}/hadoop_conf/hive-site.xml ${NEW_KYLIN_HOME}/hadoop_conf/hiveserver2-site.xml
+ cp -rfv ${OLD_KYLIN_HOME}/hadoop_conf/hive-site.xml ${NEW_KYLIN_HOME}/hadoop_conf/hivemetastore-site.xml
+ fi
+
+ if [[ -d ${NEW_KYLIN_HOME}/write_hadoop_conf ]]; then
+ cp -rfv ${OLD_KYLIN_HOME}/write_hadoop_conf/hive-site.xml ${NEW_KYLIN_HOME}/write_hadoop_conf/hiveserver2-site.xml
+ cp -rfv ${OLD_KYLIN_HOME}/write_hadoop_conf/hive-site.xml ${NEW_KYLIN_HOME}/write_hadoop_conf/hivemetastore-site.xml
+ fi
+ fi
+ info "...................................................[DONE]"
+
+ # copy spark-env for spark3
+ logging "Copy spark-env for spark3"
+ if [[ -f ${OLD_KYLIN_HOME}/spark/conf/spark-env.sh ]]; then
+ if prompt "'${OLD_KYLIN_HOME}/spark/conf/spark-env.sh' -> '${NEW_KYLIN_HOME}/spark/conf/'"; then
+ \cp -vf ${OLD_KYLIN_HOME}/spark/conf/spark-env.sh ${NEW_KYLIN_HOME}/spark/conf/ >> $upgrade_log || fail
+ fi
+ fi
+ info "...................................................[DONE]"
+
+ # sed -nE 's/^([#\t ]*)(kylin\..*|kap\..*)/\2/p' kylin.properties | awk '{kv[substr($0,0,index($0,"=")-1)]=substr($0,index($0,"=")+1)} END{print kv["kylin.metadata.url"]}'
+ logging "Checking Kylin Conf"
+python <<PY
+from __future__ import print_function
+import os
+import sys
+try:
+ import commands as cmd
+except ImportError:
+ import subprocess as cmd
+
+def printer(msg, *outs):
+ for o in outs: print(msg, file=o)
+
+def getProp(prop_file):
+ if not os.path.exists(prop_file):
+ return dict()
+
+ output = cmd.getoutput("sed -nE 's/^([#\\\\t ]*)(kylin\\..*=.*|kap\\..*=.*)/\\\\2/p' %s" % prop_file)
+ prop = dict()
+ for x in output.split('\n'):
+ if x.strip() == '':
+ continue
+ prop[x[0: x.index('=')]] = x[x.index('=') + 1:]
+ return prop
+
+with open('${upgrade_log}', 'a+') as upgrade_log:
+ origin_prop = getProp('${NEW_KYLIN_HOME}/conf/kylin.properties')
+ prod_prop = dict(getProp('${NEW_KYLIN_HOME}/conf/kylin.properties'), **getProp('${NEW_KYLIN_HOME}/conf/kylin.properties.override'))
+ diffs = set(prod_prop.items()) - set(origin_prop.items())
+
+ def logging(msg):
+ printer(msg, sys.stdout, upgrade_log)
+
+ for diff in diffs:
+ logging(diff)
+PY
+ info "...................................................[DONE]"
+
+ logging "Install"
+ if prompt "'${NEW_KYLIN_HOME}' -> '${OLD_KYLIN_HOME}'"; then
+ install_dir=$(dirname $OLD_KYLIN_HOME)
+ home_name=$(basename $OLD_KYLIN_HOME)
+
+ # backup
+ now=`date '+%Y%m%d%H%M'`
+ backup_file=${home_name}_${now}.tar.gz
+ cd $install_dir && tar -zcvf ${backup_file} ${home_name} >> $upgrade_log || fail
+
+ # install
+ rm -rfv ${OLD_KYLIN_HOME} >> $upgrade_log || fail
+ mv -vf ${NEW_KYLIN_HOME} ${OLD_KYLIN_HOME} >> $upgrade_log || fail
+ info "...................................................[DONE]"
+ recordKylinUpgradeResult "${START_TIME}" "true" "${OLD_KYLIN_HOME}"
+ info "Upgrade finished!"
+ # needed by km
+ info "Backup location:${install_dir}/${backup_file}"
+ else
+ warn "...................................................[SKIP]"
+ recordKylinUpgradeResult "${START_TIME}" "true" "${NEW_KYLIN_HOME}"
+ info "Upgrade aborted because you chose to stop"
+ fi
+
+}
+
+function recordKylinUpgradeResult() {
+ logLevel=`[ "$2" == "true" ] && echo INFO || echo ERROR`
+ echo `date '+%Y-%m-%d %H:%M:%S '`"${logLevel} : [Operation: upgrade result] user:`whoami`, upgrade time:$1, success:$2" >> $3/logs/security.log
+}
+
+NEW_KYLIN_HOME=$(cd `dirname -- $0` && cd ../ && pwd -P)
+silent=1
+while [[ $# != 0 ]]; do
+ if [[ $1 == "--silent" ]]; then
+ silent=0
+ else
+ OLD_KYLIN_HOME=$(cd $1 && pwd)
+ fi
+ shift
+done
+
+if [[ -z $OLD_KYLIN_HOME ]] || [[ ! -d $OLD_KYLIN_HOME ]]; then
+ help
+fi
+
+if [[ $OLD_KYLIN_HOME == $NEW_KYLIN_HOME ]]; then
+ error "Please specify the old version of the Kyligence Enterprise installation directory."
+ help
+fi
+
+mkdir -p ${NEW_KYLIN_HOME}/logs
+upgrade_log=${NEW_KYLIN_HOME}/logs/upgrade-$(date '+%Y_%m_%d_%H_%M_%S').log
+
+set -o errexit
+set -o pipefail
+START_TIME=$(date "+%Y-%m-%d %H:%M:%S")
+upgrade
+