You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by qh...@apache.org on 2015/07/22 11:43:03 UTC

[1/2] incubator-kylin git commit: minor, fix merge cube job does not trigger gc step

Repository: incubator-kylin
Updated Branches:
  refs/heads/0.8 fabe6bc7d -> dd5f2b2c6


minor, fix merge cube job does not trigger gc step


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/dd5f2b2c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/dd5f2b2c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/dd5f2b2c

Branch: refs/heads/0.8
Commit: dd5f2b2c622f20c0eb32d881b7e142d271fa1e42
Parents: dc6b350
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Jul 22 17:38:55 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Jul 22 17:43:21 2015 +0800

----------------------------------------------------------------------
 .../kylin/engine/mr/BatchMergeJobBuilder2.java    | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dd5f2b2c/job/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java b/job/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
index 8a5cb02..e0fc438 100644
--- a/job/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
+++ b/job/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
@@ -18,26 +18,24 @@
 
 package org.apache.kylin.engine.mr;
 
-import java.util.List;
-
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
 import org.apache.kylin.common.util.StringUtil;
 import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.engine.mr.IMROutput2.IMRBatchCubingOutputSide2;
 import org.apache.kylin.engine.mr.steps.MergeCuboidFromStorageJob;
 import org.apache.kylin.engine.mr.steps.MergeStatisticsStep;
 import org.apache.kylin.job.common.MapReduceExecutable;
 import org.apache.kylin.job.constant.ExecutableConstants;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
+import java.util.List;
 
 public class BatchMergeJobBuilder2 extends JobBuilderSupport {
 
-    private final IMRBatchCubingOutputSide2 outputSide;
+    private final IMROutput2.IMRBatchMergeOutputSide2 outputSide;
     
     public BatchMergeJobBuilder2(CubeSegment mergeSegment, String submitter) {
         super(mergeSegment, submitter);
-        this.outputSide = MRUtil.getBatchCubingOutputSide2(seg);
+        this.outputSide = MRUtil.getBatchMergeOutputSide2(seg);
     }
 
     public CubingJob build() {
@@ -56,16 +54,16 @@ public class BatchMergeJobBuilder2 extends JobBuilderSupport {
         // Phase 1: Merge Dictionary
         result.addTask(createMergeDictionaryStep(mergingSegmentIds));
         result.addTask(createMergeStatisticsStep(seg, mergingSegmentIds, getStatisticsPath(jobId)));
-        outputSide.addStepPhase2_BuildDictionary(result);
+        outputSide.addStepPhase1_MergeDictionary(result);
 
         // Phase 2: Merge Cube
         String formattedTables = StringUtil.join(mergingHTables, ",");
         result.addTask(createMergeCuboidDataFromStorageStep(formattedTables, jobId));
-        outputSide.addStepPhase3_BuildCube(result);
+        outputSide.addStepPhase2_BuildCube(result);
 
         // Phase 3: Update Metadata & Cleanup
         result.addTask(createUpdateCubeInfoAfterMergeStep(mergingSegmentIds, jobId));
-        outputSide.addStepPhase4_Cleanup(result);
+        outputSide.addStepPhase3_Cleanup(result);
 
         return result;
     }


[2/2] incubator-kylin git commit: KYLIN-902 1. move margin into StreamingConfig 2. set oneOff default value to true

Posted by qh...@apache.org.
KYLIN-902
1. move margin into StreamingConfig
2. set oneOff default value to true


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/dc6b350d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/dc6b350d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/dc6b350d

Branch: refs/heads/0.8
Commit: dc6b350d7d34e7447943ce57b28616883a96b7e2
Parents: fabe6bc
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Jul 22 15:55:16 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Jul 22 17:43:21 2015 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/job/streaming/BootstrapConfig.java | 12 ++----------
 .../apache/kylin/job/streaming/StreamingBootstrap.java  |  4 ++--
 .../org/apache/kylin/job/streaming/StreamingCLI.java    |  3 ---
 .../org/apache/kylin/job/BuildCubeWithStreamTest.java   |  1 -
 .../org/apache/kylin/streaming/StreamingConfig.java     | 11 +++++++++++
 5 files changed, 15 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dc6b350d/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java b/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
index e665cf4..adc8bbb 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
@@ -7,22 +7,14 @@ public class BootstrapConfig {
     private String streaming;
     private int partitionId = -1;
 
-    private boolean oneOff = false;
+    //one off default value set to true
+    private boolean oneOff = true;
     private long start = 0L;
     private long end = 0L;
-    private long margin = 0L;
 
     private boolean fillGap;
 
 
-    public long getMargin() {
-        return margin;
-    }
-
-    public void setMargin(long margin) {
-        this.margin = margin;
-    }
-
     public boolean isOneOff() {
         return oneOff;
     }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dc6b350d/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
index b15bee6..6b9e7dd 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
@@ -129,13 +129,13 @@ public class StreamingBootstrap {
                 final List<Pair<Long, Long>> gaps = StreamingMonitor.findGaps(streamingConfig.getCubeName());
                 logger.info("all gaps:" + StringUtils.join(gaps, ","));
                 for (Pair<Long, Long> gap : gaps) {
-                    startOneOffCubeStreaming(streamingConfig, gap.getFirst(), gap.getSecond(), bootstrapConfig.getMargin());
+                    startOneOffCubeStreaming(streamingConfig, gap.getFirst(), gap.getSecond(), streamingConfig.getMargin());
                 }
             } else {
                 if (bootstrapConfig.isOneOff()) {
                     Preconditions.checkArgument(bootstrapConfig.getStart() != 0);
                     Preconditions.checkArgument(bootstrapConfig.getEnd() != 0);
-                    startOneOffCubeStreaming(streamingConfig, bootstrapConfig.getStart(), bootstrapConfig.getEnd(), bootstrapConfig.getMargin());
+                    startOneOffCubeStreaming(streamingConfig, bootstrapConfig.getStart(), bootstrapConfig.getEnd(), streamingConfig.getMargin());
                 } else {
                     startCubeStreaming(streamingConfig);
                 }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dc6b350d/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
index c11c7b5..78b1e93 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
@@ -76,9 +76,6 @@ public class StreamingCLI {
                 case "-partition":
                     bootstrapConfig.setPartitionId(Integer.parseInt(args[++i]));
                     break;
-                case "-margin":
-                    bootstrapConfig.setMargin(Long.parseLong(args[++i]));
-                    break;
                 case "-fillGap":
                     bootstrapConfig.setFillGap(Boolean.parseBoolean(args[++i]));
                     break;

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dc6b350d/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
index 7a99661..b1aad82 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
@@ -100,7 +100,6 @@ public class BuildCubeWithStreamTest {
             BootstrapConfig bootstrapConfig = new BootstrapConfig();
             bootstrapConfig.setStart(start);
             bootstrapConfig.setEnd(start + batchInterval);
-            bootstrapConfig.setMargin(0);
             bootstrapConfig.setOneOff(true);
             bootstrapConfig.setPartitionId(0);
             bootstrapConfig.setStreaming(streamingName);

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dc6b350d/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java b/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java
index f4d1e33..5cf61c2 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java
@@ -79,6 +79,9 @@ public class StreamingConfig extends RootPersistentEntity {
     @JsonProperty("parserName")
     private String parserName;
 
+    @JsonProperty("margin")
+    private long margin;
+
 
     public List<KafkaClusterConfig> getKafkaClusterConfigs() {
         return kafkaClusterConfigs;
@@ -149,6 +152,14 @@ public class StreamingConfig extends RootPersistentEntity {
         this.name = name;
     }
 
+    public long getMargin() {
+        return margin;
+    }
+
+    public void setMargin(long margin) {
+        this.margin = margin;
+    }
+
     @Override
     public StreamingConfig clone() {
         try {