You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/06/13 02:39:58 UTC
kylin git commit: KYLIN-1741 Diagnosis for job output
Repository: kylin
Updated Branches:
refs/heads/master 139768fcb -> baaa33d2a
KYLIN-1741 Diagnosis for job output
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/baaa33d2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/baaa33d2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/baaa33d2
Branch: refs/heads/master
Commit: baaa33d2ab9f024d1610be7f6c7335da3eea2d45
Parents: 139768f
Author: lidongsjtu <li...@apache.org>
Authored: Sun Jun 12 23:26:23 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Mon Jun 13 10:39:35 2016 +0800
----------------------------------------------------------------------
.../kylin/rest/service/DiagnosisService.java | 4 ++--
.../apache/kylin/tool/CubeMetaExtractor.java | 25 +++++++++++++++-----
.../org/apache/kylin/tool/DiagnosisInfoCLI.java | 7 +++++-
3 files changed, 27 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/baaa33d2/server/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java b/server/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java
index c59f7ac..f934fb2 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java
@@ -49,7 +49,7 @@ public class DiagnosisService extends BasicService {
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
public String dumpProjectDiagnosisInfo(String project) throws IOException {
- String[] args = { "-project", project, "-destDir", getDumpDir().getAbsolutePath() };
+ String[] args = { "-project", "-all", "-destDir", getDumpDir().getAbsolutePath() };
logger.info("DiagnosisInfoCLI args: " + Arrays.toString(args));
DiagnosisInfoCLI diagnosisInfoCli = new DiagnosisInfoCLI();
diagnosisInfoCli.execute(args);
@@ -66,7 +66,7 @@ public class DiagnosisService extends BasicService {
}
public static void main(String args1[]) {
- String[] args = { "-project", "learn_kylin", "-destDir", Files.createTempDir().getAbsolutePath() };
+ String[] args = { "-project", "-all", "-destDir", Files.createTempDir().getAbsolutePath() };
logger.info("DiagnosisInfoCLI args: " + Arrays.toString(args));
DiagnosisInfoCLI diagnosisInfoCli = new DiagnosisInfoCLI();
diagnosisInfoCli.execute(args);
http://git-wip-us.apache.org/repos/asf/kylin/blob/baaa33d2/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index ae4e48a..10e7f0c 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -73,8 +73,13 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
@SuppressWarnings("static-access")
private static final Option OPTION_INCLUDE_SEGMENTS = OptionBuilder.withArgName("includeSegments").hasArg().isRequired(false).withDescription("set this to true if want extract the segments info. Default true").create("includeSegments");
+
@SuppressWarnings("static-access")
private static final Option OPTION_INCLUDE_JOB = OptionBuilder.withArgName("includeJobs").hasArg().isRequired(false).withDescription("set this to true if want to extract job info/outputs too. Default false").create("includeJobs");
+
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_INCLUDE_ONLY_JOB_OUTPUT = OptionBuilder.withArgName("onlyOutput").hasArg().isRequired(false).withDescription("when include jobs, onlt extract output of job. Default true").create("onlyOutput");
+
@SuppressWarnings("static-access")
private static final Option OPTION_INCLUDE_SEGMENT_DETAILS = OptionBuilder.withArgName("includeSegmentDetails").hasArg().isRequired(false).withDescription("set this to true if want to extract segment details too, such as dict, tablesnapshot. Default false").create("includeSegmentDetails");
@@ -92,6 +97,7 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
boolean includeSegments;
boolean includeJobs;
boolean includeSegmentDetails;
+ boolean onlyJobOutput;
List<String> requiredResources = Lists.newArrayList();
List<String> optionalResources = Lists.newArrayList();
@@ -112,6 +118,7 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
options.addOption(OPTION_INCLUDE_SEGMENTS);
options.addOption(OPTION_INCLUDE_JOB);
options.addOption(OPTION_INCLUDE_SEGMENT_DETAILS);
+ options.addOption(OPTION_INCLUDE_ONLY_JOB_OUTPUT);
}
@Override
@@ -119,6 +126,7 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
includeSegments = optionsHelper.hasOption(OPTION_INCLUDE_SEGMENTS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_SEGMENTS)) : true;
includeJobs = optionsHelper.hasOption(OPTION_INCLUDE_JOB) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_JOB)) : false;
includeSegmentDetails = optionsHelper.hasOption(OPTION_INCLUDE_SEGMENT_DETAILS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_SEGMENT_DETAILS)) : false;
+ onlyJobOutput = optionsHelper.hasOption(OPTION_INCLUDE_ONLY_JOB_OUTPUT) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_ONLY_JOB_OUTPUT)) : true;
kylinConfig = KylinConfig.getInstanceFromEnv();
metadataManager = MetadataManager.getInstance(kylinConfig);
@@ -272,12 +280,17 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
throw new RuntimeException("No job exist for segment :" + segment);
} else {
try {
- ExecutablePO executablePO = executableDao.getJob(lastJobId);
- addRequired(ResourceStoreUtil.concatJobPath(lastJobId));
- addRequired(ResourceStoreUtil.concatJobOutputPath(lastJobId));
- for (ExecutablePO task : executablePO.getTasks()) {
- addRequired(ResourceStoreUtil.concatJobPath(task.getUuid()));
- addRequired(ResourceStoreUtil.concatJobOutputPath(task.getUuid()));
+ if (onlyJobOutput) {
+ ExecutablePO executablePO = executableDao.getJob(lastJobId);
+ addRequired(ResourceStoreUtil.concatJobOutputPath(lastJobId));
+ } else {
+ ExecutablePO executablePO = executableDao.getJob(lastJobId);
+ addRequired(ResourceStoreUtil.concatJobPath(lastJobId));
+ addRequired(ResourceStoreUtil.concatJobOutputPath(lastJobId));
+ for (ExecutablePO task : executablePO.getTasks()) {
+ addRequired(ResourceStoreUtil.concatJobPath(task.getUuid()));
+ addRequired(ResourceStoreUtil.concatJobOutputPath(task.getUuid()));
+ }
}
} catch (PersistentException e) {
throw new RuntimeException("PersistentException", e);
http://git-wip-us.apache.org/repos/asf/kylin/blob/baaa33d2/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
index ab845fb..5db7d83 100644
--- a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
@@ -51,6 +51,9 @@ public class DiagnosisInfoCLI extends AbstractInfoExtractor {
@SuppressWarnings("static-access")
private static final Option OPTION_INCLUDE_CLIENT = OptionBuilder.withArgName("includeClient").hasArg().isRequired(false).withDescription("Specify whether to include client info to extract. Default true.").create("includeClient");
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_INCLUDE_JOB = OptionBuilder.withArgName("includeJobs").hasArg().isRequired(false).withDescription("Specify whether to include job output to extract. Default true.").create("includeJobs");
+
public DiagnosisInfoCLI() {
super();
@@ -60,6 +63,7 @@ public class DiagnosisInfoCLI extends AbstractInfoExtractor {
options.addOption(OPTION_INCLUDE_CONF);
options.addOption(OPTION_INCLUDE_HBASE);
options.addOption(OPTION_INCLUDE_CLIENT);
+ options.addOption(OPTION_INCLUDE_JOB);
}
public static void main(String args[]) {
@@ -86,10 +90,11 @@ public class DiagnosisInfoCLI extends AbstractInfoExtractor {
boolean includeConf = optionsHelper.hasOption(OPTION_INCLUDE_CONF) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_CONF)) : true;
boolean includeHBase = optionsHelper.hasOption(OPTION_INCLUDE_HBASE) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_HBASE)) : true;
boolean includeClient = optionsHelper.hasOption(OPTION_INCLUDE_CLIENT) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_CLIENT)) : true;
+ boolean includeJob = optionsHelper.hasOption(OPTION_INCLUDE_JOB) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_JOB)) : true;
for (String project : getProjects(projectInput)) {
// export cube metadata
- String[] cubeMetaArgs = { "-destDir", new File(exportDir, "metadata").getAbsolutePath(), "-project", project, "-compress", "false", "-submodule", "true" };
+ String[] cubeMetaArgs = { "-destDir", new File(exportDir, "metadata").getAbsolutePath(), "-project", project, "-compress", "false", "-includeJobs", Boolean.toString(includeJob), "-submodule", "true" };
CubeMetaExtractor cubeMetaExtractor = new CubeMetaExtractor();
logger.info("CubeMetaExtractor args: " + Arrays.toString(cubeMetaArgs));
cubeMetaExtractor.execute(cubeMetaArgs);