You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/12/07 04:29:57 UTC
[40/50] [abbrv] kylin git commit: KYLIN-2245 further minor refactor
KYLIN-2245 further minor refactor
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9968c225
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9968c225
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9968c225
Branch: refs/heads/master-hbase1.x
Commit: 9968c2257c2a8605637a93830d1d95db94185dce
Parents: fc1e11a
Author: Li Yang <li...@apache.org>
Authored: Tue Dec 6 11:10:26 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Tue Dec 6 11:10:26 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeInstance.java | 2 +-
.../java/org/apache/kylin/cube/CubeSegment.java | 3 ++-
.../apache/kylin/metadata/model/ISegment.java | 3 +++
.../apache/kylin/metadata/model/Segments.java | 15 +++------------
.../test_case_data/sandbox/kylin_hive_conf.xml | 20 +-------------------
5 files changed, 10 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9968c225/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index a3665f5..1d60575 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -368,7 +368,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
public Segments calculateToBeSegments(CubeSegment newSegment) {
- return segments.calculateToBeSegments(newSegment, getModel().getPartitionDesc().isPartitioned());
+ return segments.calculateToBeSegments(newSegment);
}
public CubeSegment getLastSegment() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/9968c225/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 5339d81..e155f86 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -361,7 +361,8 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen
return Segments.sourceOffsetContains(this, seg);
}
- public void validate() {
+ @Override
+ public void validate() throws IllegalStateException {
if (cubeInstance.getDescriptor().getModel().getPartitionDesc().isPartitioned()) {
if (!isSourceOffsetsOn() && dateRangeStart >= dateRangeEnd)
throw new IllegalStateException("Invalid segment, dateRangeStart(" + dateRangeStart + ") must be smaller than dateRangeEnd(" + dateRangeEnd + ") in segment " + this);
http://git-wip-us.apache.org/repos/asf/kylin/blob/9968c225/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
index f006613..d46ea96 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
@@ -37,4 +37,7 @@ public interface ISegment {
public SegmentStatusEnum getStatus();
public long getLastBuildTime();
+
+ public void validate() throws IllegalStateException;
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9968c225/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
index 5198dc7..9371f76 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
@@ -229,7 +229,7 @@ public class Segments<T extends ISegment> extends ArrayList<T> {
* - Favors new segments over the old
* - Favors big segments over the small
*/
- public Segments calculateToBeSegments(ISegment newSegment, boolean isPartitioned) {
+ public Segments calculateToBeSegments(ISegment newSegment) {
Segments tobe = (Segments) this.clone();
if (newSegment != null && !tobe.contains(newSegment)) {
@@ -242,12 +242,12 @@ public class Segments<T extends ISegment> extends ArrayList<T> {
Collections.sort(tobe);
ISegment firstSeg = tobe.getFirst();
- validate(firstSeg, isPartitioned);
+ firstSeg.validate();
for (int i = 0, j = 1; j < tobe.size();) {
ISegment is = (ISegment) tobe.get(i);
ISegment js = (ISegment) tobe.get(j);
- validate(js, isPartitioned);
+ js.validate();
// check i is either ready or new
if (!isNew(is) && !isReady(is)) {
@@ -306,15 +306,6 @@ public class Segments<T extends ISegment> extends ArrayList<T> {
return tobe;
}
- private void validate(ISegment seg, boolean isPartitioned) {
- if (isPartitioned) {
- if (!seg.isSourceOffsetsOn() && seg.getDateRangeStart() >= seg.getDateRangeEnd())
- throw new IllegalStateException("Invalid segment, dateRangeStart(" + seg.getDateRangeStart() + ") must be smaller than dateRangeEnd(" + seg.getDateRangeEnd() + ") in segment " + seg);
- if (seg.isSourceOffsetsOn() && seg.getSourceOffsetStart() >= seg.getSourceOffsetEnd())
- throw new IllegalStateException("Invalid segment, sourceOffsetStart(" + seg.getSourceOffsetStart() + ") must be smaller than sourceOffsetEnd(" + seg.getSourceOffsetEnd() + ") in segment " + seg);
- }
- }
-
private boolean isReady(ISegment seg) {
return seg.getStatus() == SegmentStatusEnum.READY;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9968c225/examples/test_case_data/sandbox/kylin_hive_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin_hive_conf.xml b/examples/test_case_data/sandbox/kylin_hive_conf.xml
index c85731d..9a60bdd 100644
--- a/examples/test_case_data/sandbox/kylin_hive_conf.xml
+++ b/examples/test_case_data/sandbox/kylin_hive_conf.xml
@@ -29,29 +29,11 @@
</property>
<property>
- <name>hive.auto.convert.join.noconditionaltask</name>
- <value>true</value>
- <description>enable map-side join</description>
- </property>
-
- <property>
- <name>hive.auto.convert.join.noconditionaltask.size</name>
- <value>300000000</value>
- <description>enable map-side join</description>
- </property>
-
- <property>
<name>mapreduce.map.output.compress.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description></description>
</property>
- <property>
- <name>hive.execution.engine</name>
- <value>mr</value>
- <description></description>
- </property>
-
<!--
<property>
<name>mapreduce.output.fileoutputformat.compress.codec</name>
@@ -64,4 +46,4 @@
<description>Size for the merged file</description>
</property>
-->
-</configuration>
\ No newline at end of file
+</configuration>