You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2019/01/06 08:00:34 UTC
[kylin] branch 2.5.x updated: KYLIN-3570 Automatically build system
cube
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.5.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/2.5.x by this push:
new 99587c4 KYLIN-3570 Automatically build system cube
99587c4 is described below
commit 99587c4976ba96359daad4f1ecbb908e41c00276
Author: Yichen Zhou <zh...@gmail.com>
AuthorDate: Thu Sep 27 20:20:08 2018 +0800
KYLIN-3570 Automatically build system cube
---
build/bin/build-incremental-cube.sh | 45 ++++++++++++++++
build/bin/kylin-port-replace-util.sh | 7 +++
build/bin/system-cube.sh | 99 ++++++++++++++++++++++++++++++++++++
3 files changed, 151 insertions(+)
diff --git a/build/bin/build-incremental-cube.sh b/build/bin/build-incremental-cube.sh
new file mode 100644
index 0000000..fcc3a4f
--- /dev/null
+++ b/build/bin/build-incremental-cube.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dir=$(dirname ${0})
+if [ -z "$KYLIN_HOME" ]; then
+ export KYLIN_HOME=${dir}/../
+fi
+echo KYLIN_HOME is set to $KYLIN_HOME
+
+if [ ! $1 ]; then
+ echo "usage: build-incremental-cube.sh CUBE INTERVAL DELAY"
+ exit 1
+fi
+
+CUBE=$1
+INTERVAL=${2:-"3600000"}
+DELAY=${3:-"0"}
+SERVER="localhost"
+PORT="7070"
+
+CURRENT_TIME_IN_SECOND=`date +%s`
+CURRENT_TIME=$((CURRENT_TIME_IN_SECOND * 1000))
+END_TIME=$((CURRENT_TIME-DELAY))
+END=$((END_TIME - END_TIME%INTERVAL))
+
+ID="$END"
+echo "Building for ${CUBE}_${ID}" | tee ${KYLIN_HOME}/logs/build_trace.log
+echo "Check the log at ${KYLIN_HOME}/logs/incremental_cube_${CUBE}_${END}.log"
+curl -X PUT --user ADMIN:KYLIN -H "Content-Type: application/json;charset=utf-8" -d "{\"endTime\": ${END}, \"buildType\": \"BUILD\"}" http://${SERVER}:${PORT}/kylin/api/cubes/${CUBE}/rebuild > ${KYLIN_HOME}/logs/incremental_cube_${CUBE}_${END}.log 2>&1 &
diff --git a/build/bin/kylin-port-replace-util.sh b/build/bin/kylin-port-replace-util.sh
index d1edb75..2c2ab40 100755
--- a/build/bin/kylin-port-replace-util.sh
+++ b/build/bin/kylin-port-replace-util.sh
@@ -54,6 +54,7 @@ TOMCAT_BACKUP_FILE="${KYLIN_HOME}/tomcat/conf/server.xml.backup"
TOMCAT_CONFIG_FILE="${KYLIN_HOME}/tomcat/conf/server.xml"
KYLIN_CONFIG_FILE="${KYLIN_HOME}/conf/kylin.properties"
KYLIN_BACKUP_FILE="${KYLIN_HOME}/conf/kylin.properties.backup"
+BUILD_CUBE_FILE="${KYLIN_HOME}/bin/build-incremental-cube.sh"
TOMCAT_PORT_LIST=(9005 7070 9443 7443 9009)
KYLIN_DEFAULT_PORT=7070
@@ -101,9 +102,13 @@ then
sed -i "s/$port/${new_port}/g" ${TOMCAT_CONFIG_FILE}
done
+
+ #replace ports in build-incremental-cube.sh
+ sed -i "s/$PORT=\"[0-9]*\"/$PORT=\"${new_kylin_port}\"/g" ${BUILD_CUBE_FILE}
echo "Files below modified:"
echo ${KYLIN_CONFIG_FILE}
echo ${TOMCAT_CONFIG_FILE}
+ echo ${BUILD_CUBE_FILE}
elif [ "$1" == "reset" ]
then
#reset kylin.properties
@@ -111,9 +116,11 @@ then
cp -f ${TOMCAT_BACKUP_FILE} ${TOMCAT_CONFIG_FILE}
rm -f ${KYLIN_BACKUP_FILE}
rm -f ${TOMCAT_BACKUP_FILE}
+ sed -i "s/$PORT=\"[0-9]*\"/$PORT=\"${KYLIN_DEFAULT_PORT}\"/g" ${BUILD_CUBE_FILE}
echo "Files below reset to original:"
echo ${KYLIN_CONFIG_FILE}
echo ${TOMCAT_CONFIG_FILE}
+ echo ${BUILD_CUBE_FILE}
else
echo "Unrecognized command"
exit 1
diff --git a/build/bin/system-cube.sh b/build/bin/system-cube.sh
new file mode 100644
index 0000000..b864333
--- /dev/null
+++ b/build/bin/system-cube.sh
@@ -0,0 +1,99 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+#check kylin home
+if [ -z "$KYLIN_HOME" ]
+then
+ echo 'Please make sure KYLIN_HOME has been set'
+ exit 1
+else
+ echo "KYLIN_HOME is set to ${KYLIN_HOME}"
+fi
+
+
+OUTPUT_FORDER=$KYLIN_HOME/system_cube
+
+SC_NAME_1="KYLIN_HIVE_METRICS_QUERY_QA"
+SC_NAME_2="KYLIN_HIVE_METRICS_QUERY_CUBE_QA"
+SC_NAME_3="KYLIN_HIVE_METRICS_QUERY_RPC_QA"
+SC_NAME_4="KYLIN_HIVE_METRICS_JOB_QA"
+SC_NAME_5="KYLIN_HIVE_METRICS_JOB_EXCEPTION_QA"
+
+if [ "$1" == "build" ]
+then
+ if [ -d "${OUTPUT_FORDER}" ]
+ then
+ sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_1} 600000 0
+ sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_2} 600000 0
+ sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_3} 600000 0
+ sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_4} 600000 0
+ sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_5} 600000 0
+ else
+ echo "Please setup system cube first."
+ exit 1
+ fi
+elif [ "$1" == "setup" ]
+then
+ #creat SCSinkTools.json
+ SINK_TOOLS_FILE=$KYLIN_HOME/SCSinkTools.json
+ cat <<- EOF > ${SINK_TOOLS_FILE}
+ [
+ [
+ "org.apache.kylin.tool.metrics.systemcube.util.HiveSinkTool",
+ {
+ "storage_type": 2,
+ "cube_desc_override_properties": [
+ "java.util.HashMap",
+ {
+ "kylin.cube.algorithm": "INMEM",
+ "kylin.cube.max-building-segments": "1"
+ }
+ ]
+ }
+ ]
+ ]
+ EOF
+ $KYLIN_HOME/bin/kylin.sh org.apache.kylin.tool.metrics.systemcube.SCCreator \
+ -inputConfig ${SINK_TOOLS_FILE} \
+ -output ${OUTPUT_FORDER}
+
+ hive -f ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql
+
+ $KYLIN_HOME/bin/metastore.sh restore ${OUTPUT_FORDER}
+
+ #refresh signature
+ $KYLIN_HOME/bin/kylin.sh org.apache.kylin.cube.cli.CubeSignatureRefresher ${SC_NAME_1},${SC_NAME_2},${SC_NAME_3},${SC_NAME_4},${SC_NAME_5}
+
+ #add a crontab job
+ CRONTAB_FILE=$KYLIN_HOME/crontabJob
+ cat <<-EOF > ${CRONTAB_FILE}
+ 0 */4 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_1} 3600000 1200000
+ 20 */4 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_2} 3600000 1200000
+ 40 */8 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_3} 3600000 1200000
+ 30 */8 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_4} 3600000 1200000
+ 50 */24 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_5} 3600000 1200000
+ EOF
+ crontab ${CRONTAB_FILE}
+ rm ${CRONTAB_FILE}
+else
+ echo "usage: system-cube.sh setup"
+ echo " system-cube.sh build"
+ exit 1
+fi