You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/05/08 13:30:15 UTC
[48/50] [abbrv] kylin git commit: KYLIN-1614 Scripts to package
diagnosis for 1.3 and 1.5
KYLIN-1614 Scripts to package diagnosis for 1.3 and 1.5
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/277e6d62
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/277e6d62
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/277e6d62
Branch: refs/heads/1.5.x-HBase1.x
Commit: 277e6d6285078a369c6a7fc388e899716a8eac65
Parents: eac9803
Author: lidongsjtu <li...@apache.org>
Authored: Sun May 8 13:56:17 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Sun May 8 18:14:21 2016 +0800
----------------------------------------------------------------------
build/bin/kylin.sh | 2 +-
build/deploy/diag.sh | 16 +++++++++++-----
build/script/package_diag.sh | 3 ++-
tool/pom.xml | 15 +++------------
.../org/apache/kylin/tool/CubeMetaExtractor.java | 6 +++++-
.../org/apache/kylin/tool/JobDiagnosisInfoCLI.java | 14 +++++---------
6 files changed, 27 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/277e6d62/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 7ceae69..0e503ee 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -217,7 +217,7 @@ then
export HBASE_CLASSPATH=${KYLIN_HOME}/lib/*
- if [ ${#patient} -gt 31 ]; then
+ if [ ${#patient} -eq 36 ]; then
exec hbase ${KYLIN_EXTRA_START_OPTS} -Dlog4j.configuration=kylin-log4j.properties org.apache.kylin.tool.JobDiagnosisInfoCLI -jobId $patient -destDir $destDir
else
exec hbase ${KYLIN_EXTRA_START_OPTS} -Dlog4j.configuration=kylin-log4j.properties org.apache.kylin.tool.DiagnosisInfoCLI -project $patient -destDir $destDir
http://git-wip-us.apache.org/repos/asf/kylin/blob/277e6d62/build/deploy/diag.sh
----------------------------------------------------------------------
diff --git a/build/deploy/diag.sh b/build/deploy/diag.sh
index 1deeded..3c37c4b 100644
--- a/build/deploy/diag.sh
+++ b/build/deploy/diag.sh
@@ -24,7 +24,7 @@ dir=$(dirname ${0})
export KYLIN_HOME=${dir}/../
source ${dir}/check-env.sh
-if [ $# -gt 1 ]
+if [ $# -eq 1 ] || [ $# -eq 2 ]
then
patient="$1"
if [ -z "$patient" ]
@@ -49,13 +49,19 @@ then
fi
export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}
- if [ -f "${KYLIN_HOME}/lib/kylin-diagnosis-*.jar" ]; then
- export HBASE_CLASSPATH=$HBASE_CLASSPATH:${KYLIN_HOME}/lib/kylin-diagnosis-*.jar
+ diagJar=`ls ${KYLIN_HOME}/lib/kylin-diagnosis-*.jar`
+ if [ -f "${diagJar}" ]; then
+ if [ -f "${KYLIN_HOME}/commit_SHA1" ]; then
+ export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${diagJar}:${KYLIN_HOME}/lib/*
+ else
+ export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${KYLIN_HOME}/lib/*
+ fi
else
- export HBASE_CLASSPATH=$HBASE_CLASSPATH:${KYLIN_HOME}/lib/*
+ echo "missing diagnosis jar file."
+ exit 1
fi
- if [ ${#patient} -gt 31 ]; then
+ if [ ${#patient} -eq 36 ]; then
exec hbase ${KYLIN_EXTRA_START_OPTS} -Dlog4j.configuration=kylin-log4j.properties org.apache.kylin.tool.JobDiagnosisInfoCLI -jobId $patient -destDir $destDir
else
exec hbase ${KYLIN_EXTRA_START_OPTS} -Dlog4j.configuration=kylin-log4j.properties org.apache.kylin.tool.DiagnosisInfoCLI -project $patient -destDir $destDir
http://git-wip-us.apache.org/repos/asf/kylin/blob/277e6d62/build/script/package_diag.sh
----------------------------------------------------------------------
diff --git a/build/script/package_diag.sh b/build/script/package_diag.sh
index 413bca8..5de1590 100755
--- a/build/script/package_diag.sh
+++ b/build/script/package_diag.sh
@@ -84,8 +84,9 @@ package_name=apache-kylin-${version}-diag
cd build/
rm -rf ${package_name}
mkdir ${package_name}
+mkdir ${package_name}/bin
cp -r lib ${package_name}
-cp deploy/diag.sh ${package_name}
+cp deploy/diag.sh ${package_name}/bin/
rm -rf lib
find ${package_name} -type d -exec chmod 755 {} \;
find ${package_name} -type f -exec chmod 644 {} \;
http://git-wip-us.apache.org/repos/asf/kylin/blob/277e6d62/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 3f077de..3041ae2 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -31,11 +31,6 @@
<dependencies>
<dependency>
<groupId>org.apache.kylin</groupId>
- <artifactId>kylin-source-kafka</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
<artifactId>kylin-storage-hbase</artifactId>
<version>${project.parent.version}</version>
</dependency>
@@ -64,7 +59,6 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.3</version>
-
<executions>
<execution>
<phase>package</phase>
@@ -72,15 +66,12 @@
<goal>shade</goal>
</goals>
<configuration>
- <minimizeJar>false</minimizeJar>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>assembly</shadedClassifierName>
<artifactSet>
- <excludes>
- <exclude>io.netty:*</exclude>
- <exclude>org.apache.zookeeper:*</exclude>
- <exclude>net.sf.ehcache:*</exclude>
- </excludes>
+ <includes>
+ <include>org.apache.kylin:*</include>
+ </includes>
</artifactSet>
<filters>
<filter>
http://git-wip-us.apache.org/repos/asf/kylin/blob/277e6d62/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index a8b732b..3de346d 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -234,6 +234,10 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
return ResourceStore.CUBE_DESC_RESOURCE_ROOT + "/" + descName + MetadataConstants.FILE_SURFIX;
}
+ private static String concatCubeSegmentStatisticsResourcePath(String cubeName, String cubeSegmentId) {
+ return ResourceStore.CUBE_STATISTICS_ROOT + "/" + cubeName + "/" + cubeSegmentId + ".seq";
+ }
+
private void retrieveResourcePath(IRealization realization) {
logger.info("Deal with realization {} of type {}", realization.getName(), realization.getType());
@@ -260,7 +264,7 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
if (includeSegments) {
addRequired(CubeInstance.concatResourcePath(cube.getName()));
for (CubeSegment segment : cube.getSegments(SegmentStatusEnum.READY)) {
- addRequired(segment.getStatisticsResourcePath());
+ addRequired(concatCubeSegmentStatisticsResourcePath(cube.getName(), segment.getUuid()));
if (includeSegmentDetails) {
for (String dictPat : segment.getDictionaryPaths()) {
addRequired(dictPat);
http://git-wip-us.apache.org/repos/asf/kylin/blob/277e6d62/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java b/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
index 45e3072..a479f9e 100644
--- a/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
@@ -28,12 +28,11 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.ResourceTool;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.dao.ExecutableDao;
import org.apache.kylin.job.dao.ExecutablePO;
+import org.apache.kylin.tool.util.ResourceStoreUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -138,14 +137,11 @@ public class JobDiagnosisInfoCLI extends AbstractInfoExtractor {
}
try {
- ResourceStore src = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
- ResourceStore dst = ResourceStore.getStore(KylinConfig.createInstanceFromUri(destDir.getAbsolutePath()));
+ KylinConfig srcConfig = KylinConfig.getInstanceFromEnv();
+ KylinConfig dstConfig = KylinConfig.createInstanceFromUri(destDir.getAbsolutePath());
- for (String path : requiredResources) {
- ResourceTool.copyR(src, dst, path);
- }
-
- } catch (IOException e) {
+ ResourceStoreUtil.copy(srcConfig, dstConfig, requiredResources);
+ } catch (Exception e) {
throw new RuntimeException("Failed to extract job resources. ", e);
}
}