You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/01/04 10:34:01 UTC

[1/2] kylin git commit: KYLIN-1281 Add "partition_date_end", and move "partition_date_start" into cube descriptor

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging f554ebe71 -> 26c84f737


KYLIN-1281 Add "partition_date_end", and move "partition_date_start" into cube descriptor

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ce55fd8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ce55fd8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ce55fd8

Branch: refs/heads/2.x-staging
Commit: 3ce55fd86bf36ebfef220000a69eabf450b0a9bc
Parents: f554ebe
Author: shaofengshi <sh...@apache.org>
Authored: Mon Jan 4 17:33:00 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jan 4 17:33:00 2016 +0800

----------------------------------------------------------------------
 .../engine/spark/BuildCubeWithSparkTest.java    |  2 +-
 .../kylin/job/BuildCubeWithEngineTest.java      |  4 ++--
 .../java/org/apache/kylin/cube/CubeManager.java | 19 +++++++++---------
 .../org/apache/kylin/cube/model/CubeDesc.java   | 21 ++++++++++++++++++++
 .../cube/upgrade/CubeDescSignatureUpdate.java   |  1 +
 .../kylin/metadata/model/PartitionDesc.java     |  3 +++
 .../cube_desc/kylin_sales_cube_desc.json        |  3 ++-
 .../test_kylin_cube_with_slr_desc.json          |  3 ++-
 ...test_kylin_cube_with_slr_left_join_desc.json |  3 ++-
 .../test_kylin_cube_without_slr_desc.json       |  3 ++-
 ...t_kylin_cube_without_slr_left_join_desc.json |  3 ++-
 .../test_streaming_table_cube_desc.json         |  3 ++-
 12 files changed, 49 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java b/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
index 755b1ac..f5b9741 100644
--- a/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
+++ b/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
@@ -139,7 +139,7 @@ public class BuildCubeWithSparkTest {
 
         SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
         f.setTimeZone(TimeZone.getTimeZone("GMT"));
-        long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+        long dateStart = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart();
         long dateEnd = f.parse("2050-11-12").getTime();
 
         // this cube's start date is 0, end date is 20501112000000

http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index 75a6c54..53db99a 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -254,7 +254,7 @@ public class BuildCubeWithEngineTest {
         final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
         clearSegment(cubeName);
 
-        long date1 = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+        long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart();
         long date2 = f.parse("2012-06-01").getTime();
         long date3 = f.parse("2022-01-01").getTime();
         long date4 = f.parse("2023-01-01").getTime();
@@ -280,7 +280,7 @@ public class BuildCubeWithEngineTest {
 
         SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
         f.setTimeZone(TimeZone.getTimeZone("GMT"));
-        long date1 = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+        long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart();
         long date2 = f.parse("2013-01-01").getTime();
         long date3 = f.parse("2013-07-01").getTime();
         long date4 = f.parse("2022-01-01").getTime();

http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 8e4906b..5bdbe8f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -19,14 +19,8 @@
 package org.apache.kylin.cube;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Random;
-import java.util.UUID;
+import java.text.SimpleDateFormat;
+import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 
 import javax.annotation.Nullable;
@@ -391,7 +385,7 @@ public class CubeManager implements IRealizationProvider {
         long appendStart = calculateStartDateForAppendSegment(cube);
         CubeSegment appendSegment = newSegment(cube, appendStart, endDate);
 
-        long startDate = cube.getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+        long startDate = cube.getDescriptor().getPartitionDateStart();
         CubeSegment mergeSegment = newSegment(cube, startDate, endDate);
 
         validateNewSegments(cube, mergeSegment);
@@ -410,6 +404,11 @@ public class CubeManager implements IRealizationProvider {
         long startDate = 0;
         if (cube.getDescriptor().getModel().getPartitionDesc().isPartitioned()) {
             startDate = calculateStartDateForAppendSegment(cube);
+            if (endDate > cube.getDescriptor().getPartitionDateEnd()) {
+                SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+                f.setTimeZone(TimeZone.getTimeZone("GMT"));
+                throw new IllegalArgumentException("The selected date couldn't be later than cube's end date '" + f.format(new Date(cube.getDescriptor().getPartitionDateEnd())) + "'.");
+            }
         } else {
             endDate = Long.MAX_VALUE;
         }
@@ -513,7 +512,7 @@ public class CubeManager implements IRealizationProvider {
     private long calculateStartDateForAppendSegment(CubeInstance cube) {
         List<CubeSegment> existing = cube.getSegments();
         if (existing.isEmpty()) {
-            return cube.getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+            return cube.getDescriptor().getPartitionDateStart();
         } else {
             return existing.get(existing.size() - 1).getDateRangeEnd();
         }

http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 97fd1aa..737d4bd 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -122,6 +122,11 @@ public class CubeDesc extends RootPersistentEntity {
     private List<String> notifyList;
     @JsonProperty("status_need_notify")
     private List<String> statusNeedNotify = Collections.emptyList();
+
+    @JsonProperty("partition_date_start")
+    private long partitionDateStart = 0L;
+    @JsonProperty("partition_date_end")
+    private long partitionDateEnd = Long.MAX_VALUE;
     @JsonProperty("auto_merge_time_ranges")
     private long[] autoMergeTimeRanges;
     @JsonProperty("retention_range")
@@ -753,6 +758,22 @@ public class CubeDesc extends RootPersistentEntity {
         this.engineType = engineType;
     }
 
+    public long getPartitionDateStart() {
+        return partitionDateStart;
+    }
+
+    public void setPartitionDateStart(long partitionDateStart) {
+        this.partitionDateStart = partitionDateStart;
+    }
+
+    public long getPartitionDateEnd() {
+        return partitionDateEnd;
+    }
+
+    public void setPartitionDateEnd(long partitionDateEnd) {
+        this.partitionDateEnd = partitionDateEnd;
+    }
+    
     public List<TblColRef> getAllColumnsNeedDictionary() {
         List<TblColRef> result = Lists.newArrayList();
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/core-cube/src/main/java/org/apache/kylin/cube/upgrade/CubeDescSignatureUpdate.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/CubeDescSignatureUpdate.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/CubeDescSignatureUpdate.java
index aa7feb8..ce45c00 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/CubeDescSignatureUpdate.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/CubeDescSignatureUpdate.java
@@ -115,6 +115,7 @@ public class CubeDescSignatureUpdate {
 
     private void updateCubeDesc(CubeDesc cubeDesc) {
         try {
+            cubeDesc.setPartitionDateStart(cubeDesc.getModel().getPartitionDesc().getPartitionDateStart());
             String calculatedSign = cubeDesc.calculateSignature();
             if (!cubeDesc.getSignature().equals(calculatedSign)) {
                 cubeDesc.setSignature(calculatedSign);

http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index 0de905c..5f3468c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -43,6 +43,7 @@ public class PartitionDesc {
 
     @JsonProperty("partition_date_column")
     private String partitionDateColumn;
+    
     @JsonProperty("partition_date_start")
     private long partitionDateStart = 0L;
     @JsonProperty("partition_date_format")
@@ -92,10 +93,12 @@ public class PartitionDesc {
         this.partitionDateColumn = partitionDateColumn;
     }
 
+    @Deprecated
     public long getPartitionDateStart() {
         return partitionDateStart;
     }
 
+    @Deprecated
     public void setPartitionDateStart(long partitionDateStart) {
         this.partitionDateStart = partitionDateStart;
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json b/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
index 93195a7..732af79 100644
--- a/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
+++ b/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
@@ -163,5 +163,6 @@
   "retention_range" : 0,
   "engine_type" : 2,
   "storage_type" : 2,
-  "override_kylin_properties" : { }
+  "override_kylin_properties" : { },
+  "partition_date_start" : 1325376000000
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
index 048c147..98b584b 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
@@ -165,5 +165,6 @@
   "auto_merge_time_ranges" : null,
   "retention_range" : 0,
   "engine_type" : 2,
-  "storage_type" : 2
+  "storage_type" : 2,
+  "partition_date_start" : 0
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
index 524f924..cc95d28 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
@@ -165,5 +165,6 @@
   "auto_merge_time_ranges" : null,
   "retention_range" : 0,
   "engine_type" : 0,
-  "storage_type" : 0
+  "storage_type" : 0,
+  "partition_date_start" : 0
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
index 36c8d95..99fed60 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
@@ -220,5 +220,6 @@
   "auto_merge_time_ranges" : null,
   "retention_range" : 0,
   "engine_type" : 2,
-  "storage_type" : 0
+  "storage_type" : 0,
+  "partition_date_start" : 0
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index 16d7432..79e5799 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -220,5 +220,6 @@
   "auto_merge_time_ranges" : null,
   "retention_range" : 0,
   "engine_type" : 2,
-  "storage_type" : 2
+  "storage_type" : 2,
+  "partition_date_start": 0
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce55fd8/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json b/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
index 8a1dedf..df88aca 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
@@ -109,5 +109,6 @@
   "auto_merge_time_ranges" : null,
   "retention_range" : 0,
   "engine_type" : 2,
-  "storage_type" : 2
+  "storage_type" : 2,
+  "partition_date_start": 0
 }
\ No newline at end of file


[2/2] kylin git commit: KYLIN-1281 move “partition_date_start” to CubeDesc Conflicts: examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json examples/test_case_

Posted by sh...@apache.org.
KYLIN-1281 move “partition_date_start” to CubeDesc
Conflicts:
	examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
	examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
	examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
	examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
	examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
	examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/26c84f73
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/26c84f73
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/26c84f73

Branch: refs/heads/2.x-staging
Commit: 26c84f7375326690e0ce91e3f74a882443cdf28c
Parents: 3ce55fd
Author: shaofengshi <sh...@apache.org>
Authored: Mon Jan 4 15:58:11 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jan 4 17:33:32 2016 +0800

----------------------------------------------------------------------

----------------------------------------------------------------------