You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by qh...@apache.org on 2015/07/22 11:43:03 UTC
[1/2] incubator-kylin git commit: minor,
fix merge cube job does not trigger gc step
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8 fabe6bc7d -> dd5f2b2c6
minor, fix merge cube job does not trigger gc step
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/dd5f2b2c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/dd5f2b2c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/dd5f2b2c
Branch: refs/heads/0.8
Commit: dd5f2b2c622f20c0eb32d881b7e142d271fa1e42
Parents: dc6b350
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Jul 22 17:38:55 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Jul 22 17:43:21 2015 +0800
----------------------------------------------------------------------
.../kylin/engine/mr/BatchMergeJobBuilder2.java | 18 ++++++++----------
1 file changed, 8 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dd5f2b2c/job/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java b/job/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
index 8a5cb02..e0fc438 100644
--- a/job/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
+++ b/job/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
@@ -18,26 +18,24 @@
package org.apache.kylin.engine.mr;
-import java.util.List;
-
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.engine.mr.IMROutput2.IMRBatchCubingOutputSide2;
import org.apache.kylin.engine.mr.steps.MergeCuboidFromStorageJob;
import org.apache.kylin.engine.mr.steps.MergeStatisticsStep;
import org.apache.kylin.job.common.MapReduceExecutable;
import org.apache.kylin.job.constant.ExecutableConstants;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
+import java.util.List;
public class BatchMergeJobBuilder2 extends JobBuilderSupport {
- private final IMRBatchCubingOutputSide2 outputSide;
+ private final IMROutput2.IMRBatchMergeOutputSide2 outputSide;
public BatchMergeJobBuilder2(CubeSegment mergeSegment, String submitter) {
super(mergeSegment, submitter);
- this.outputSide = MRUtil.getBatchCubingOutputSide2(seg);
+ this.outputSide = MRUtil.getBatchMergeOutputSide2(seg);
}
public CubingJob build() {
@@ -56,16 +54,16 @@ public class BatchMergeJobBuilder2 extends JobBuilderSupport {
// Phase 1: Merge Dictionary
result.addTask(createMergeDictionaryStep(mergingSegmentIds));
result.addTask(createMergeStatisticsStep(seg, mergingSegmentIds, getStatisticsPath(jobId)));
- outputSide.addStepPhase2_BuildDictionary(result);
+ outputSide.addStepPhase1_MergeDictionary(result);
// Phase 2: Merge Cube
String formattedTables = StringUtil.join(mergingHTables, ",");
result.addTask(createMergeCuboidDataFromStorageStep(formattedTables, jobId));
- outputSide.addStepPhase3_BuildCube(result);
+ outputSide.addStepPhase2_BuildCube(result);
// Phase 3: Update Metadata & Cleanup
result.addTask(createUpdateCubeInfoAfterMergeStep(mergingSegmentIds, jobId));
- outputSide.addStepPhase4_Cleanup(result);
+ outputSide.addStepPhase3_Cleanup(result);
return result;
}
[2/2] incubator-kylin git commit: KYLIN-902 1. move margin into
StreamingConfig 2. set oneOff default value to true
Posted by qh...@apache.org.
KYLIN-902
1. move margin into StreamingConfig
2. set oneOff default value to true
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/dc6b350d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/dc6b350d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/dc6b350d
Branch: refs/heads/0.8
Commit: dc6b350d7d34e7447943ce57b28616883a96b7e2
Parents: fabe6bc
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Jul 22 15:55:16 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Jul 22 17:43:21 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/job/streaming/BootstrapConfig.java | 12 ++----------
.../apache/kylin/job/streaming/StreamingBootstrap.java | 4 ++--
.../org/apache/kylin/job/streaming/StreamingCLI.java | 3 ---
.../org/apache/kylin/job/BuildCubeWithStreamTest.java | 1 -
.../org/apache/kylin/streaming/StreamingConfig.java | 11 +++++++++++
5 files changed, 15 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dc6b350d/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java b/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
index e665cf4..adc8bbb 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
@@ -7,22 +7,14 @@ public class BootstrapConfig {
private String streaming;
private int partitionId = -1;
- private boolean oneOff = false;
+ //one off default value set to true
+ private boolean oneOff = true;
private long start = 0L;
private long end = 0L;
- private long margin = 0L;
private boolean fillGap;
- public long getMargin() {
- return margin;
- }
-
- public void setMargin(long margin) {
- this.margin = margin;
- }
-
public boolean isOneOff() {
return oneOff;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dc6b350d/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
index b15bee6..6b9e7dd 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
@@ -129,13 +129,13 @@ public class StreamingBootstrap {
final List<Pair<Long, Long>> gaps = StreamingMonitor.findGaps(streamingConfig.getCubeName());
logger.info("all gaps:" + StringUtils.join(gaps, ","));
for (Pair<Long, Long> gap : gaps) {
- startOneOffCubeStreaming(streamingConfig, gap.getFirst(), gap.getSecond(), bootstrapConfig.getMargin());
+ startOneOffCubeStreaming(streamingConfig, gap.getFirst(), gap.getSecond(), streamingConfig.getMargin());
}
} else {
if (bootstrapConfig.isOneOff()) {
Preconditions.checkArgument(bootstrapConfig.getStart() != 0);
Preconditions.checkArgument(bootstrapConfig.getEnd() != 0);
- startOneOffCubeStreaming(streamingConfig, bootstrapConfig.getStart(), bootstrapConfig.getEnd(), bootstrapConfig.getMargin());
+ startOneOffCubeStreaming(streamingConfig, bootstrapConfig.getStart(), bootstrapConfig.getEnd(), streamingConfig.getMargin());
} else {
startCubeStreaming(streamingConfig);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dc6b350d/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
index c11c7b5..78b1e93 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
@@ -76,9 +76,6 @@ public class StreamingCLI {
case "-partition":
bootstrapConfig.setPartitionId(Integer.parseInt(args[++i]));
break;
- case "-margin":
- bootstrapConfig.setMargin(Long.parseLong(args[++i]));
- break;
case "-fillGap":
bootstrapConfig.setFillGap(Boolean.parseBoolean(args[++i]));
break;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dc6b350d/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
index 7a99661..b1aad82 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
@@ -100,7 +100,6 @@ public class BuildCubeWithStreamTest {
BootstrapConfig bootstrapConfig = new BootstrapConfig();
bootstrapConfig.setStart(start);
bootstrapConfig.setEnd(start + batchInterval);
- bootstrapConfig.setMargin(0);
bootstrapConfig.setOneOff(true);
bootstrapConfig.setPartitionId(0);
bootstrapConfig.setStreaming(streamingName);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dc6b350d/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java b/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java
index f4d1e33..5cf61c2 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java
@@ -79,6 +79,9 @@ public class StreamingConfig extends RootPersistentEntity {
@JsonProperty("parserName")
private String parserName;
+ @JsonProperty("margin")
+ private long margin;
+
public List<KafkaClusterConfig> getKafkaClusterConfigs() {
return kafkaClusterConfigs;
@@ -149,6 +152,14 @@ public class StreamingConfig extends RootPersistentEntity {
this.name = name;
}
+ public long getMargin() {
+ return margin;
+ }
+
+ public void setMargin(long margin) {
+ this.margin = margin;
+ }
+
@Override
public StreamingConfig clone() {
try {