You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/08/14 07:33:42 UTC
[1/2] incubator-kylin git commit: Don’t block cube build if couldn’t parse cube size from MR output
Repository: incubator-kylin
Updated Branches:
refs/heads/0.7-staging 5534ccccd -> ff4c0771e
Don’t block cube build if couldn’t parse cube size from MR output
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4c3381b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4c3381b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4c3381b0
Branch: refs/heads/0.7-staging
Commit: 4c3381b040138640df2e082db357218f1e8772d6
Parents: 5534ccc
Author: shaofengshi <sh...@apache.org>
Authored: Fri Aug 14 11:49:11 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Aug 14 11:49:11 2015 +0800
----------------------------------------------------------------------
.../job/cube/UpdateCubeInfoAfterBuildStep.java | 27 +++++++++++++-------
1 file changed, 18 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4c3381b0/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java b/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
index cc2b4d4..6f8febe 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
@@ -18,8 +18,6 @@
package org.apache.kylin.job.cube;
-import java.io.IOException;
-
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -31,7 +29,7 @@ import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
import org.apache.kylin.job.execution.Output;
-import com.google.common.base.Preconditions;
+import java.io.IOException;
/**
* Created by qianzhou on 1/4/15.
@@ -101,19 +99,30 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
Output baseCuboidOutput = executableManager.getOutput(getBaseCuboidStepId());
String sourceRecordsCount = baseCuboidOutput.getExtra().get(ExecutableConstants.SOURCE_RECORDS_COUNT);
- Preconditions.checkState(StringUtils.isNotEmpty(sourceRecordsCount), "Can't get cube source record count.");
- long sourceCount = Long.parseLong(sourceRecordsCount);
+ long sourceCount = 0l;
+ if (StringUtils.isNotEmpty(sourceRecordsCount)) {
+ sourceCount = Long.parseLong(sourceRecordsCount);
+ } else {
+ logger.warn("Can not get cube source record count.");
+ }
+ long sourceSize = 0l;
String sourceRecordsSize = baseCuboidOutput.getExtra().get(ExecutableConstants.SOURCE_RECORDS_SIZE);
- Preconditions.checkState(StringUtils.isNotEmpty(sourceRecordsSize), "Can't get cube source record size.");
- long sourceSize = Long.parseLong(sourceRecordsSize);
+ if (StringUtils.isNotEmpty(sourceRecordsSize)) {
+ sourceSize = Long.parseLong(sourceRecordsSize);
+ } else {
+ logger.warn("Can not get cube source record size.");
+ }
long size = 0;
boolean segmentReady = true;
if (!StringUtils.isBlank(getConvertToHfileStepId())) {
String cubeSizeString = executableManager.getOutput(getConvertToHfileStepId()).getExtra().get(ExecutableConstants.HDFS_BYTES_WRITTEN);
- Preconditions.checkState(StringUtils.isNotEmpty(cubeSizeString), "Can't get cube segment size.");
- size = Long.parseLong(cubeSizeString) / 1024;
+ if (StringUtils.isNotEmpty(cubeSizeString)) {
+ size = Long.parseLong(cubeSizeString) / 1024;
+ } else {
+ logger.warn("Can't get cube segment size.");
+ }
} else {
// for the increment & merge case, the increment segment is only built to be merged, won't serve query by itself
segmentReady = false;
[2/2] incubator-kylin git commit: Don’t block cube build if couldn’t parse cube size from MR output
Posted by sh...@apache.org.
Don’t block cube build if couldn’t parse cube size from MR output
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ff4c0771
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ff4c0771
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ff4c0771
Branch: refs/heads/0.7-staging
Commit: ff4c0771e1a3af3b2989588276aadaab6c9682c3
Parents: 4c3381b
Author: shaofengshi <sh...@apache.org>
Authored: Fri Aug 14 13:33:19 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Aug 14 13:33:19 2015 +0800
----------------------------------------------------------------------
.../kylin/job/cube/UpdateCubeInfoAfterMergeStep.java | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ff4c0771/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterMergeStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterMergeStep.java b/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterMergeStep.java
index 8386f8c..284e9c0 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterMergeStep.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterMergeStep.java
@@ -34,7 +34,6 @@ import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
-import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
/**
@@ -62,9 +61,14 @@ public class UpdateCubeInfoAfterMergeStep extends AbstractExecutable {
if (mergedSegment == null) {
return new ExecuteResult(ExecuteResult.State.FAILED, "there is no segment with id:" + getSegmentId());
}
+
+ long cubeSize = 0l;
String cubeSizeString = executableManager.getOutput(getConvertToHfileStepId()).getExtra().get(ExecutableConstants.HDFS_BYTES_WRITTEN);
- Preconditions.checkState(StringUtils.isNotEmpty(cubeSizeString), "Can't get cube segment size.");
- long cubeSize = Long.parseLong(cubeSizeString) / 1024;
+ if (StringUtils.isNotEmpty(cubeSizeString)) {
+ cubeSize = Long.parseLong(cubeSizeString) / 1024;
+ } else {
+ logger.warn("Can not get cube segment size.");
+ }
// collect source statistics
List<String> mergingSegmentIds = getMergingSegmentIds();