You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/05/06 09:10:35 UTC
kylin git commit: KYLIN-1614 Diagnosis CLI
Repository: kylin
Updated Branches:
refs/heads/master f17ba5c04 -> eac98036d
KYLIN-1614 Diagnosis CLI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eac98036
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eac98036
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eac98036
Branch: refs/heads/master
Commit: eac98036ddf228daaa71bd41f00179fa493f2759
Parents: f17ba5c
Author: lidongsjtu <li...@apache.org>
Authored: Fri May 6 12:23:59 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Fri May 6 17:09:31 2016 +0800
----------------------------------------------------------------------
build/deploy/diag.sh | 68 ++++++++++++++
build/script/package_diag.sh | 97 ++++++++++++++++++++
tool/pom.xml | 2 +-
.../apache/kylin/tool/KylinLogExtractor.java | 13 ++-
4 files changed, 175 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/eac98036/build/deploy/diag.sh
----------------------------------------------------------------------
diff --git a/build/deploy/diag.sh b/build/deploy/diag.sh
new file mode 100644
index 0000000..1deeded
--- /dev/null
+++ b/build/deploy/diag.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dir=$(dirname ${0})
+
+# We should set KYLIN_HOME here for multiple tomcat instances that are on the same node.
+# In addition, we should set a KYLIN_HOME for the global use as normal.
+export KYLIN_HOME=${dir}/../
+source ${dir}/check-env.sh
+
+if [ $# -gt 1 ]
+then
+ patient="$1"
+ if [ -z "$patient" ]
+ then
+ echo "You need to specify a Project or Job Id for diagnosis."
+ exit 1
+ fi
+ destDir="$2"
+ if [ -z "$destDir" ]
+ then
+ destDir="$KYLIN_HOME/diagnosis_dump/"
+ mkdir -p $destDir
+ fi
+
+ #retrive $hive_dependency and $hbase_dependency
+ source ${dir}/find-hive-dependency.sh
+ source ${dir}/find-hbase-dependency.sh
+
+ #retrive $KYLIN_EXTRA_START_OPTS
+ if [ -f "${dir}/setenv-tool.sh" ]
+ then source ${dir}/setenv-tool.sh
+ fi
+ export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}
+
+ if [ -f "${KYLIN_HOME}/lib/kylin-diagnosis-*.jar" ]; then
+ export HBASE_CLASSPATH=$HBASE_CLASSPATH:${KYLIN_HOME}/lib/kylin-diagnosis-*.jar
+ else
+ export HBASE_CLASSPATH=$HBASE_CLASSPATH:${KYLIN_HOME}/lib/*
+ fi
+
+ if [ ${#patient} -gt 31 ]; then
+ exec hbase ${KYLIN_EXTRA_START_OPTS} -Dlog4j.configuration=kylin-log4j.properties org.apache.kylin.tool.JobDiagnosisInfoCLI -jobId $patient -destDir $destDir
+ else
+ exec hbase ${KYLIN_EXTRA_START_OPTS} -Dlog4j.configuration=kylin-log4j.properties org.apache.kylin.tool.DiagnosisInfoCLI -project $patient -destDir $destDir
+ fi
+
+ exit 0
+else
+ echo "usage: diag.sh Project|JobId [target_path]"
+ exit 1
+fi
http://git-wip-us.apache.org/repos/asf/kylin/blob/eac98036/build/script/package_diag.sh
----------------------------------------------------------------------
diff --git a/build/script/package_diag.sh b/build/script/package_diag.sh
new file mode 100755
index 0000000..413bca8
--- /dev/null
+++ b/build/script/package_diag.sh
@@ -0,0 +1,97 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+echo "Checking maven..."
+
+if [ -z "$(command -v mvn)" ]
+then
+ echo "Please install maven first so that Kylin packaging can proceed"
+ exit 1
+else
+ echo "maven check passed"
+fi
+
+echo "Checking git..."
+
+if [ -z "$(command -v git)" ]
+then
+ echo "Please install git first so that Kylin packaging can proceed"
+ exit 1
+else
+ echo "git check passed"
+fi
+
+dir=$(dirname ${0})
+cd ${dir}/../..
+version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\['`
+
+if [ "$version" == "" ];then
+ echo "Failed to identify kylin version (current: `pwd`)"
+ exit 1
+fi
+
+echo "kylin version: ${version}"
+export version
+
+#commit id
+cat << EOF > build/commit_SHA1
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+EOF
+git rev-parse HEAD >> build/commit_SHA1
+
+echo "package libraries"
+mvn clean install -DskipTests || { exit 1; }
+
+echo "copy libraries"
+rm -rf build/lib
+mkdir build/lib
+cp tool/target/kylin-tool-${version}-assembly.jar build/lib/kylin-diagnosis-${version}.jar
+# Copied file becomes 000 for some env (e.g. my Cygwin)
+chmod 644 build/lib/kylin-diagnosis-${version}.jar
+
+echo 'package tar.gz'
+package_name=apache-kylin-${version}-diag
+cd build/
+rm -rf ${package_name}
+mkdir ${package_name}
+cp -r lib ${package_name}
+cp deploy/diag.sh ${package_name}
+rm -rf lib
+find ${package_name} -type d -exec chmod 755 {} \;
+find ${package_name} -type f -exec chmod 644 {} \;
+find ${package_name} -type f -name "*.sh" -exec chmod 755 {} \;
+mkdir -p ../dist
+tar -cvzf ../dist/${package_name}.tar.gz ${package_name}
+rm -rf ${package_name}
+
+echo "Library package ready: dist/${package_name}.tar.gz"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/eac98036/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 7fc08f4..3f077de 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -74,7 +74,7 @@
<configuration>
<minimizeJar>false</minimizeJar>
<shadedArtifactAttached>true</shadedArtifactAttached>
- <shadedClassifierName>tool</shadedClassifierName>
+ <shadedClassifierName>assembly</shadedClassifierName>
<artifactSet>
<excludes>
<exclude>io.netty:*</exclude>
http://git-wip-us.apache.org/repos/asf/kylin/blob/eac98036/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java b/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java
index 5351c92..d71e6c6 100644
--- a/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java
@@ -59,16 +59,21 @@ public class KylinLogExtractor extends AbstractInfoExtractor {
logger.info("Start to extract kylin logs in {} days", logPeriod);
final File kylinLogDir = new File(KylinConfig.getKylinHome(), "logs");
- final ArrayList<File> logFiles = Lists.newArrayList();
+ final ArrayList<File> requiredLogFiles = Lists.newArrayList();
final long logThresholdTime = System.currentTimeMillis() - logPeriod * 24 * 3600 * 1000;
- for (File logFile : kylinLogDir.listFiles()) {
+ final File[] allLogFiles = kylinLogDir.listFiles();
+ if (allLogFiles == null || allLogFiles.length == 0) {
+ return;
+ }
+
+ for (File logFile : allLogFiles) {
if (logFile.lastModified() > logThresholdTime) {
- logFiles.add(logFile);
+ requiredLogFiles.add(logFile);
}
}
- for (File logFile : logFiles) {
+ for (File logFile : requiredLogFiles) {
logger.info("Log file:" + logFile.getAbsolutePath());
if (logFile.exists()) {
FileUtils.copyFileToDirectory(logFile, exportDir);