You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2021/02/22 04:34:17 UTC
[kylin] branch master updated: KYLIN-4855 kylin.metrics.prefix bug
fix (#1536)
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new d1c99d7 KYLIN-4855 kylin.metrics.prefix bug fix (#1536)
d1c99d7 is described below
commit d1c99d754e8d6fe1fe820fe75a89dae1d18c2dc9
Author: benjobs <be...@qq.com>
AuthorDate: Mon Feb 22 12:34:04 2021 +0800
KYLIN-4855 kylin.metrics.prefix bug fix (#1536)
* KYLIN-4855 kylin.metrics.prefix bug fix
* KYLIN-4855 kylin.metrics.prefix bug fix
---
build/bin/build-incremental-cube.sh | 7 +-
build/bin/system-cube.sh | 212 ++++++++++++++++++++++--------------
2 files changed, 134 insertions(+), 85 deletions(-)
diff --git a/build/bin/build-incremental-cube.sh b/build/bin/build-incremental-cube.sh
index 04eee35..2870472 100644
--- a/build/bin/build-incremental-cube.sh
+++ b/build/bin/build-incremental-cube.sh
@@ -20,8 +20,8 @@
source ${KYLIN_HOME:-"$(cd -P -- "$(dirname -- "$0")" && pwd -P)/../"}/bin/header.sh
if [ ! $1 ]; then
- echo "usage: build-incremental-cube.sh CUBE INTERVAL DELAY"
- exit 1
+ echo "usage: build-incremental-cube.sh CUBE INTERVAL DELAY"
+ exit 1
fi
tomcat_root=${dir}/../tomcat
@@ -41,4 +41,5 @@ END=$((END_TIME - END_TIME%INTERVAL))
ID="$END"
echo "Building for ${CUBE}_${ID}" | tee ${KYLIN_HOME}/logs/build_trace.log
echo "Check the log at ${KYLIN_HOME}/logs/incremental_cube_${CUBE}_${END}.log"
-curl -X PUT --user ADMIN:KYLIN -H "Content-Type: application/json;charset=utf-8" -d "{\"endTime\": ${END}, \"buildType\": \"BUILD\"}" http://${kylin_rest_address}/kylin/api/cubes/${CUBE}/rebuild > ${KYLIN_HOME}/logs/incremental_cube_${CUBE}_${END}.log 2>&1 &
+curl -X PUT -H "Authorization: Basic %Auth%" -H "Content-Type: application/json;charset=utf-8" -d "{\"endTime\": ${END}, \"buildType\": \"BUILD\"}" http://${kylin_rest_address}/kylin/api/cubes/${CUBE}/rebuild > ${KYLIN_HOME}/logs/incremental_cube_${CUBE}_${END}.log 2>&1 &
+
diff --git a/build/bin/system-cube.sh b/build/bin/system-cube.sh
index b5d01b9..dbab6a9 100644
--- a/build/bin/system-cube.sh
+++ b/build/bin/system-cube.sh
@@ -20,6 +20,8 @@
source ${KYLIN_HOME:-"$(cd -P -- "$(dirname -- "$0")" && pwd -P)/../"}/bin/header.sh
+build_incremental_cube="${KYLIN_HOME}/bin/build-incremental-cube.sh"
+
function printHelp {
echo "usage: system-cube.sh setup"
echo " system-cube.sh build [INTERVAL:600000] [DELAY:0]"
@@ -28,6 +30,33 @@ function printHelp {
exit 1
}
+function authorization {
+ authed=$(grep %Auth% "$build_incremental_cube"|wc -l)
+ if [ $authed -eq 1 ]
+ then
+ read -p $'Please Enter the ADMIN\'password or \'N\' to exit: ' pwd
+ if [ $pwd == "N" ]
+ then
+ return 1
+ else
+ base64_auth=$(echo -n "ADMIN:$pwd"|base64)
+ tomcat_root=${dir}/../tomcat
+ kylin_rest_address=`hostname -f`":"`grep "<Connector port=" ${tomcat_root}/conf/server.xml |grep protocol=\"HTTP/1.1\" | cut -d '=' -f 2 | cut -d \" -f 2`
+ http_code=$(curl -I -m 10 -o /dev/null -s -w %{http_code} -X POST -H "Authorization: Basic $base64_auth" -H 'Content-Type: application/json' http://${kylin_rest_address}/kylin/api/user/authentication)
+ if [ $http_code -eq 200 ]
+ then
+ sed -i "s/%Auth%/${base64_auth}/g" $build_incremental_cube
+ return 0
+ else
+ echo `setColor 31 "Unauthorized,password error."`
+ authorization
+ fi
+ fi
+ else
+ return 0
+ fi
+}
+
if [[ "$@" == *"help"* ]]
then
printHelp
@@ -38,11 +67,15 @@ OUTPUT_FORDER=$KYLIN_HOME/system_cube
KYLIN_ENV=`grep "^kylin.env=" $KYLIN_HOME/conf/kylin.properties | cut -d "=" -f 2`
KYLIN_ENV=${KYLIN_ENV:-"QA"}
-SC_NAME_1="KYLIN_HIVE_METRICS_QUERY_${KYLIN_ENV}"
-SC_NAME_2="KYLIN_HIVE_METRICS_QUERY_CUBE_${KYLIN_ENV}"
-SC_NAME_3="KYLIN_HIVE_METRICS_QUERY_RPC_${KYLIN_ENV}"
-SC_NAME_4="KYLIN_HIVE_METRICS_JOB_${KYLIN_ENV}"
-SC_NAME_5="KYLIN_HIVE_METRICS_JOB_EXCEPTION_${KYLIN_ENV}"
+KYLIN_METRICS_PREFIX=`grep "^kylin.metrics.prefix=" $KYLIN_HOME/conf/kylin.properties | cut -d "=" -f 2`
+KYLIN_METRICS_PREFIX=${KYLIN_METRICS_PREFIX:-"KYLIN"}
+KYLIN_METRICS_PREFIX=`echo "$KYLIN_METRICS_PREFIX"| tr '[a-z]' '[A-Z]'`
+
+SC_NAME_1="${KYLIN_METRICS_PREFIX}_HIVE_METRICS_QUERY_${KYLIN_ENV}"
+SC_NAME_2="${KYLIN_METRICS_PREFIX}_HIVE_METRICS_QUERY_CUBE_${KYLIN_ENV}"
+SC_NAME_3="${KYLIN_METRICS_PREFIX}_HIVE_METRICS_QUERY_RPC_${KYLIN_ENV}"
+SC_NAME_4="${KYLIN_METRICS_PREFIX}_HIVE_METRICS_JOB_${KYLIN_ENV}"
+SC_NAME_5="${KYLIN_METRICS_PREFIX}_HIVE_METRICS_JOB_EXCEPTION_${KYLIN_ENV}"
if [ "$1" == "build" ]
then
@@ -50,91 +83,106 @@ then
then
BUILD_INTERVAL=${2:-"600000"}
BUILD_DELAY=${3:-"0"}
-
echo "build system cubes, build_interval:${BUILD_INTERVAL}, build_delay:${BUILD_DELAY}"
-
- sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_1} ${BUILD_INTERVAL} ${BUILD_DELAY}
- sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_2} ${BUILD_INTERVAL} ${BUILD_DELAY}
- sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_3} ${BUILD_INTERVAL} ${BUILD_DELAY}
- sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_4} ${BUILD_INTERVAL} ${BUILD_DELAY}
- sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_5} ${BUILD_INTERVAL} ${BUILD_DELAY}
+ sh $build_incremental_cube ${SC_NAME_1} ${BUILD_INTERVAL} ${BUILD_DELAY}
+ sh $build_incremental_cube ${SC_NAME_2} ${BUILD_INTERVAL} ${BUILD_DELAY}
+ sh $build_incremental_cube ${SC_NAME_3} ${BUILD_INTERVAL} ${BUILD_DELAY}
+ sh $build_incremental_cube ${SC_NAME_4} ${BUILD_INTERVAL} ${BUILD_DELAY}
+ sh $build_incremental_cube ${SC_NAME_5} ${BUILD_INTERVAL} ${BUILD_DELAY}
else
- echo "Please setup system cube first."
- exit 1
+ echo "Please setup system cube first."
+ exit 1
fi
elif [ "$1" == "setup" ]
then
- #creat SCSinkTools.json
- SINK_TOOLS_FILE=$KYLIN_HOME/SCSinkTools.json
-
- echo "setup system cubes"
-
- rm -rf $SINK_TOOLS_FILE $OUTPUT_FORDER
-
- cat <<-EOF > ${SINK_TOOLS_FILE}
- [
- {
- "sink": "hive",
- "storage_type": 2,
- "cube_desc_override_properties": {
- "kylin.cube.algorithm": "INMEM",
- "kylin.cube.max-building-segments": "1"
- }
- }
- ]
- EOF
- $KYLIN_HOME/bin/kylin.sh org.apache.kylin.tool.metrics.systemcube.SCCreator \
- -inputConfig ${SINK_TOOLS_FILE} \
- -output ${OUTPUT_FORDER}
-
- hive_client_mode=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.client`
-
- # Get Database, default is KYLIN
- system_database="KYLIN"
-
- # 'create database' failed will not exit when donot have permission to create database;
- sed -i -e 's/CREATE DATABASE /-- CREATE DATABASE /g' ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql
-
- if [ "${hive_client_mode}" == "beeline" ]
- then
- beeline_params=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.beeline-params`
- beeline ${beeline_params} -e "CREATE DATABASE IF NOT EXISTS "$system_database
-
- hive2_url=`expr match "${beeline_params}" '.*\(hive2:.*:[0-9]\{4,6\}\/\)'`
- if [ -z ${hive2_url} ]; then
- hive2_url=`expr match "${beeline_params}" '.*\(hive2:.*:[0-9]\{4,6\}\)'`
- beeline_params=${beeline_params/${hive2_url}/${hive2_url}/${system_database}}
- else
- beeline_params=${beeline_params/${hive2_url}/${hive2_url}${system_database}}
- fi
-
- beeline ${beeline_params} -f ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql || { exit 1; }
- else
- hive -e "CREATE DATABASE IF NOT EXISTS "$system_database
- hive --database $system_database -f ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql || { exit 1; }
- fi
-
- $KYLIN_HOME/bin/metastore.sh restore ${OUTPUT_FORDER}
-
- #refresh signature
- $KYLIN_HOME/bin/kylin.sh org.apache.kylin.cube.cli.CubeSignatureRefresher ${SC_NAME_1},${SC_NAME_2},${SC_NAME_3},${SC_NAME_4},${SC_NAME_5}
+ #creat SCSinkTools.json
+ SINK_TOOLS_FILE=$KYLIN_HOME/SCSinkTools.json
+
+ echo "setup system cubes"
+
+ rm -rf $SINK_TOOLS_FILE $OUTPUT_FORDER
+
+cat <<-EOF > ${SINK_TOOLS_FILE}
+[
+ {
+ "sink": "hive",
+ "storage_type": 2,
+ "cube_desc_override_properties": {
+ "kylin.cube.algorithm": "INMEM",
+ "kylin.cube.max-building-segments": "1"
+ }
+ }
+]
+EOF
+
+ $KYLIN_HOME/bin/kylin.sh org.apache.kylin.tool.metrics.systemcube.SCCreator \
+ -inputConfig ${SINK_TOOLS_FILE} \
+ -output ${OUTPUT_FORDER}
+
+ hive_client_mode=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.client`
+
+ # Get Database, default is KYLIN
+ system_database="KYLIN"
+
+ # 'create database' failed will not exit when donot have permission to create database;
+ sed -i -e 's/CREATE DATABASE /-- CREATE DATABASE /g' ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql
+
+ if [ "${hive_client_mode}" == "beeline" ]
+ then
+ beeline_params=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.beeline-params`
+ beeline ${beeline_params} -e "CREATE DATABASE IF NOT EXISTS "$system_database
+
+ hive2_url=`expr match "${beeline_params}" '.*\(hive2:.*:[0-9]\{4,6\}\/\)'`
+ if [ -z ${hive2_url} ]; then
+ hive2_url=`expr match "${beeline_params}" '.*\(hive2:.*:[0-9]\{4,6\}\)'`
+ beeline_params=${beeline_params/${hive2_url}/${hive2_url}/${system_database}}
+ else
+ beeline_params=${beeline_params/${hive2_url}/${hive2_url}${system_database}}
+ fi
+
+ beeline ${beeline_params} -f ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql || { exit 1; }
+ else
+ hive -e "CREATE DATABASE IF NOT EXISTS "$system_database
+ hive --database $system_database -f ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql || { exit 1; }
+ fi
+
+ $KYLIN_HOME/bin/metastore.sh restore ${OUTPUT_FORDER}
+
+ #refresh signature
+ $KYLIN_HOME/bin/kylin.sh org.apache.kylin.cube.cli.CubeSignatureRefresher ${SC_NAME_1},${SC_NAME_2},${SC_NAME_3},${SC_NAME_4},${SC_NAME_5}
elif [ "$1" == "cron" ]
then
- #add a crontab job
- echo "add to a crontab job"
-
- CRONTAB_FILE=$KYLIN_HOME/crontabJob
- crontab -l >> ${CRONTAB_FILE}
- cat <<-EOF >> ${CRONTAB_FILE}
- 0 */2 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_1} 3600000 1200000
- 20 */2 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_2} 3600000 1200000
- 40 */4 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_3} 3600000 1200000
- 30 */4 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_4} 3600000 1200000
- 50 */12 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_5} 3600000 1200000
- EOF
- crontab ${CRONTAB_FILE}
- rm ${CRONTAB_FILE}
+ #check exists
+ cron_count=$(crontab -l | grep "${KYLIN_METRICS_PREFIX}_HIVE_METRICS" | wc -l)
+ if [ $cron_count -eq 5 ]
+ then
+ echo `setColor 33 "system cube already exists in crontab"`
+ exit 0
+ else
+ #add a crontab job
+ echo "add to a crontab job"
+ authorization
+ if [[ $? == 1 ]]
+ then
+ echo "add to a crontab job exit."
+ exit 0
+ else
+ CRONTAB_FILE=$KYLIN_HOME/crontabJob
+ crontab -l >> ${CRONTAB_FILE}
+
+cat <<-EOF >> ${CRONTAB_FILE}
+0 */2 * * * sh $build_incremental_cube ${SC_NAME_1} 3600000 1200000
+20 */2 * * * sh $build_incremental_cube ${SC_NAME_2} 3600000 1200000
+40 */4 * * * sh $build_incremental_cube ${SC_NAME_3} 3600000 1200000
+30 */4 * * * sh $build_incremental_cube ${SC_NAME_4} 3600000 1200000
+50 */12 * * * sh $build_incremental_cube ${SC_NAME_5} 3600000 1200000
+EOF
+ crontab ${CRONTAB_FILE}
+ rm ${CRONTAB_FILE}
+ echo "add to a crontab job successful."
+ fi
+ fi
else
printHelp
fi