You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/01/04 10:34:01 UTC
[1/2] kylin git commit: KYLIN-1281 Add "partition_date_end",
and move "partition_date_start" into cube descriptor
Repository: kylin
Updated Branches:
refs/heads/2.x-staging f554ebe71 -> 26c84f737
KYLIN-1281 Add "partition_date_end", and move "partition_date_start" into cube descriptor
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ce55fd8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ce55fd8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ce55fd8
Branch: refs/heads/2.x-staging
Commit: 3ce55fd86bf36ebfef220000a69eabf450b0a9bc
Parents: f554ebe
Author: shaofengshi <sh...@apache.org>
Authored: Mon Jan 4 17:33:00 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jan 4 17:33:00 2016 +0800
----------------------------------------------------------------------
.../engine/spark/BuildCubeWithSparkTest.java | 2 +-
.../kylin/job/BuildCubeWithEngineTest.java | 4 ++--
.../java/org/apache/kylin/cube/CubeManager.java | 19 +++++++++---------
.../org/apache/kylin/cube/model/CubeDesc.java | 21 ++++++++++++++++++++
.../cube/upgrade/CubeDescSignatureUpdate.java | 1 +
.../kylin/metadata/model/PartitionDesc.java | 3 +++
.../cube_desc/kylin_sales_cube_desc.json | 3 ++-
.../test_kylin_cube_with_slr_desc.json | 3 ++-
...test_kylin_cube_with_slr_left_join_desc.json | 3 ++-
.../test_kylin_cube_without_slr_desc.json | 3 ++-
...t_kylin_cube_without_slr_left_join_desc.json | 3 ++-
.../test_streaming_table_cube_desc.json | 3 ++-
12 files changed, 49 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java b/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
index 755b1ac..f5b9741 100644
--- a/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
+++ b/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
@@ -139,7 +139,7 @@ public class BuildCubeWithSparkTest {
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+ long dateStart = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart();
long dateEnd = f.parse("2050-11-12").getTime();
// this cube's start date is 0, end date is 20501112000000
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index 75a6c54..53db99a 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -254,7 +254,7 @@ public class BuildCubeWithEngineTest {
final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
clearSegment(cubeName);
- long date1 = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+ long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart();
long date2 = f.parse("2012-06-01").getTime();
long date3 = f.parse("2022-01-01").getTime();
long date4 = f.parse("2023-01-01").getTime();
@@ -280,7 +280,7 @@ public class BuildCubeWithEngineTest {
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long date1 = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+ long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart();
long date2 = f.parse("2013-01-01").getTime();
long date3 = f.parse("2013-07-01").getTime();
long date4 = f.parse("2022-01-01").getTime();
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 8e4906b..5bdbe8f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -19,14 +19,8 @@
package org.apache.kylin.cube;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Random;
-import java.util.UUID;
+import java.text.SimpleDateFormat;
+import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import javax.annotation.Nullable;
@@ -391,7 +385,7 @@ public class CubeManager implements IRealizationProvider {
long appendStart = calculateStartDateForAppendSegment(cube);
CubeSegment appendSegment = newSegment(cube, appendStart, endDate);
- long startDate = cube.getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+ long startDate = cube.getDescriptor().getPartitionDateStart();
CubeSegment mergeSegment = newSegment(cube, startDate, endDate);
validateNewSegments(cube, mergeSegment);
@@ -410,6 +404,11 @@ public class CubeManager implements IRealizationProvider {
long startDate = 0;
if (cube.getDescriptor().getModel().getPartitionDesc().isPartitioned()) {
startDate = calculateStartDateForAppendSegment(cube);
+ if (endDate > cube.getDescriptor().getPartitionDateEnd()) {
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+ throw new IllegalArgumentException("The selected date couldn't be later than cube's end date '" + f.format(new Date(cube.getDescriptor().getPartitionDateEnd())) + "'.");
+ }
} else {
endDate = Long.MAX_VALUE;
}
@@ -513,7 +512,7 @@ public class CubeManager implements IRealizationProvider {
private long calculateStartDateForAppendSegment(CubeInstance cube) {
List<CubeSegment> existing = cube.getSegments();
if (existing.isEmpty()) {
- return cube.getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+ return cube.getDescriptor().getPartitionDateStart();
} else {
return existing.get(existing.size() - 1).getDateRangeEnd();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 97fd1aa..737d4bd 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -122,6 +122,11 @@ public class CubeDesc extends RootPersistentEntity {
private List<String> notifyList;
@JsonProperty("status_need_notify")
private List<String> statusNeedNotify = Collections.emptyList();
+
+ @JsonProperty("partition_date_start")
+ private long partitionDateStart = 0L;
+ @JsonProperty("partition_date_end")
+ private long partitionDateEnd = Long.MAX_VALUE;
@JsonProperty("auto_merge_time_ranges")
private long[] autoMergeTimeRanges;
@JsonProperty("retention_range")
@@ -753,6 +758,22 @@ public class CubeDesc extends RootPersistentEntity {
this.engineType = engineType;
}
+ public long getPartitionDateStart() {
+ return partitionDateStart;
+ }
+
+ public void setPartitionDateStart(long partitionDateStart) {
+ this.partitionDateStart = partitionDateStart;
+ }
+
+ public long getPartitionDateEnd() {
+ return partitionDateEnd;
+ }
+
+ public void setPartitionDateEnd(long partitionDateEnd) {
+ this.partitionDateEnd = partitionDateEnd;
+ }
+
public List<TblColRef> getAllColumnsNeedDictionary() {
List<TblColRef> result = Lists.newArrayList();
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/core-cube/src/main/java/org/apache/kylin/cube/upgrade/CubeDescSignatureUpdate.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/CubeDescSignatureUpdate.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/CubeDescSignatureUpdate.java
index aa7feb8..ce45c00 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/CubeDescSignatureUpdate.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/CubeDescSignatureUpdate.java
@@ -115,6 +115,7 @@ public class CubeDescSignatureUpdate {
private void updateCubeDesc(CubeDesc cubeDesc) {
try {
+ cubeDesc.setPartitionDateStart(cubeDesc.getModel().getPartitionDesc().getPartitionDateStart());
String calculatedSign = cubeDesc.calculateSignature();
if (!cubeDesc.getSignature().equals(calculatedSign)) {
cubeDesc.setSignature(calculatedSign);
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index 0de905c..5f3468c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -43,6 +43,7 @@ public class PartitionDesc {
@JsonProperty("partition_date_column")
private String partitionDateColumn;
+
@JsonProperty("partition_date_start")
private long partitionDateStart = 0L;
@JsonProperty("partition_date_format")
@@ -92,10 +93,12 @@ public class PartitionDesc {
this.partitionDateColumn = partitionDateColumn;
}
+ @Deprecated
public long getPartitionDateStart() {
return partitionDateStart;
}
+ @Deprecated
public void setPartitionDateStart(long partitionDateStart) {
this.partitionDateStart = partitionDateStart;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json b/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
index 93195a7..732af79 100644
--- a/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
+++ b/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
@@ -163,5 +163,6 @@
"retention_range" : 0,
"engine_type" : 2,
"storage_type" : 2,
- "override_kylin_properties" : { }
+ "override_kylin_properties" : { },
+ "partition_date_start" : 1325376000000
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
index 048c147..98b584b 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
@@ -165,5 +165,6 @@
"auto_merge_time_ranges" : null,
"retention_range" : 0,
"engine_type" : 2,
- "storage_type" : 2
+ "storage_type" : 2,
+ "partition_date_start" : 0
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
index 524f924..cc95d28 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
@@ -165,5 +165,6 @@
"auto_merge_time_ranges" : null,
"retention_range" : 0,
"engine_type" : 0,
- "storage_type" : 0
+ "storage_type" : 0,
+ "partition_date_start" : 0
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
index 36c8d95..99fed60 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
@@ -220,5 +220,6 @@
"auto_merge_time_ranges" : null,
"retention_range" : 0,
"engine_type" : 2,
- "storage_type" : 0
+ "storage_type" : 0,
+ "partition_date_start" : 0
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index 16d7432..79e5799 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -220,5 +220,6 @@
"auto_merge_time_ranges" : null,
"retention_range" : 0,
"engine_type" : 2,
- "storage_type" : 2
+ "storage_type" : 2,
+ "partition_date_start": 0
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json b/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
index 8a1dedf..df88aca 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
@@ -109,5 +109,6 @@
"auto_merge_time_ranges" : null,
"retention_range" : 0,
"engine_type" : 2,
- "storage_type" : 2
+ "storage_type" : 2,
+ "partition_date_start": 0
}
\ No newline at end of file
[2/2] kylin git commit: KYLIN-1281 move “partition_date_start” to CubeDesc Conflicts: examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json examples/test_case_
Posted by sh...@apache.org.
KYLIN-1281 move “partition_date_start” to CubeDesc
Conflicts:
examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/26c84f73
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/26c84f73
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/26c84f73
Branch: refs/heads/2.x-staging
Commit: 26c84f7375326690e0ce91e3f74a882443cdf28c
Parents: 3ce55fd
Author: shaofengshi <sh...@apache.org>
Authored: Mon Jan 4 15:58:11 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jan 4 17:33:32 2016 +0800
----------------------------------------------------------------------
----------------------------------------------------------------------