You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by qh...@apache.org on 2015/07/27 11:21:08 UTC
[06/52] [abbrv] incubator-kylin git commit: KYLIN-902 1. move margin
into StreamingConfig 2. set oneOff default value to true
KYLIN-902
1. move margin into StreamingConfig
2. set oneOff default value to true
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/555f7f6d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/555f7f6d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/555f7f6d
Branch: refs/heads/0.8
Commit: 555f7f6da84db379b9ce80ce3c1560c600326842
Parents: 0063e2b
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Jul 22 15:55:16 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Jul 22 17:43:21 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/job/streaming/BootstrapConfig.java | 12 ++----------
.../apache/kylin/job/streaming/StreamingBootstrap.java | 4 ++--
.../org/apache/kylin/job/streaming/StreamingCLI.java | 3 ---
.../org/apache/kylin/job/BuildCubeWithStreamTest.java | 1 -
.../org/apache/kylin/streaming/StreamingConfig.java | 11 +++++++++++
5 files changed, 15 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/555f7f6d/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java b/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
index e665cf4..adc8bbb 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
@@ -7,22 +7,14 @@ public class BootstrapConfig {
private String streaming;
private int partitionId = -1;
- private boolean oneOff = false;
+ //one off default value set to true
+ private boolean oneOff = true;
private long start = 0L;
private long end = 0L;
- private long margin = 0L;
private boolean fillGap;
- public long getMargin() {
- return margin;
- }
-
- public void setMargin(long margin) {
- this.margin = margin;
- }
-
public boolean isOneOff() {
return oneOff;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/555f7f6d/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
index b15bee6..6b9e7dd 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
@@ -129,13 +129,13 @@ public class StreamingBootstrap {
final List<Pair<Long, Long>> gaps = StreamingMonitor.findGaps(streamingConfig.getCubeName());
logger.info("all gaps:" + StringUtils.join(gaps, ","));
for (Pair<Long, Long> gap : gaps) {
- startOneOffCubeStreaming(streamingConfig, gap.getFirst(), gap.getSecond(), bootstrapConfig.getMargin());
+ startOneOffCubeStreaming(streamingConfig, gap.getFirst(), gap.getSecond(), streamingConfig.getMargin());
}
} else {
if (bootstrapConfig.isOneOff()) {
Preconditions.checkArgument(bootstrapConfig.getStart() != 0);
Preconditions.checkArgument(bootstrapConfig.getEnd() != 0);
- startOneOffCubeStreaming(streamingConfig, bootstrapConfig.getStart(), bootstrapConfig.getEnd(), bootstrapConfig.getMargin());
+ startOneOffCubeStreaming(streamingConfig, bootstrapConfig.getStart(), bootstrapConfig.getEnd(), streamingConfig.getMargin());
} else {
startCubeStreaming(streamingConfig);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/555f7f6d/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
index c11c7b5..78b1e93 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
@@ -76,9 +76,6 @@ public class StreamingCLI {
case "-partition":
bootstrapConfig.setPartitionId(Integer.parseInt(args[++i]));
break;
- case "-margin":
- bootstrapConfig.setMargin(Long.parseLong(args[++i]));
- break;
case "-fillGap":
bootstrapConfig.setFillGap(Boolean.parseBoolean(args[++i]));
break;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/555f7f6d/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
index 7a99661..b1aad82 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
@@ -100,7 +100,6 @@ public class BuildCubeWithStreamTest {
BootstrapConfig bootstrapConfig = new BootstrapConfig();
bootstrapConfig.setStart(start);
bootstrapConfig.setEnd(start + batchInterval);
- bootstrapConfig.setMargin(0);
bootstrapConfig.setOneOff(true);
bootstrapConfig.setPartitionId(0);
bootstrapConfig.setStreaming(streamingName);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/555f7f6d/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java b/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java
index f4d1e33..5cf61c2 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java
@@ -79,6 +79,9 @@ public class StreamingConfig extends RootPersistentEntity {
@JsonProperty("parserName")
private String parserName;
+ @JsonProperty("margin")
+ private long margin;
+
public List<KafkaClusterConfig> getKafkaClusterConfigs() {
return kafkaClusterConfigs;
@@ -149,6 +152,14 @@ public class StreamingConfig extends RootPersistentEntity {
this.name = name;
}
+ public long getMargin() {
+ return margin;
+ }
+
+ public void setMargin(long margin) {
+ this.margin = margin;
+ }
+
@Override
public StreamingConfig clone() {
try {