You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/09/07 13:21:02 UTC
[27/28] incubator-kylin git commit: KYLIN-910,
add auto_merge_time_ranges to CubeDesc
KYLIN-910,add auto_merge_time_ranges to CubeDesc
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/c083e874
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/c083e874
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/c083e874
Branch: refs/heads/2.x-staging
Commit: c083e874079c51a44a580946a7161083d0937d34
Parents: 3ced3a4
Author: jiazhong <ji...@ebay.com>
Authored: Mon Sep 7 18:22:44 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Mon Sep 7 18:23:03 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeInstance.java | 25 ++-----------------
.../java/org/apache/kylin/cube/CubeManager.java | 2 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 20 +++++++++------
.../org/apache/kylin/cube/CubeManagerTest.java | 4 +--
.../kylin/rest/controller/CubeController.java | 1 -
.../rest/controller/CubeDescController.java | 1 -
.../apache/kylin/rest/request/CubeRequest.java | 8 ------
.../apache/kylin/rest/service/CubeService.java | 6 ++---
webapp/app/js/controllers/cubeAdvanceSetting.js | 7 ++++++
webapp/app/js/model/cubeDescModel.js | 4 +--
.../cubeDesigner/advanced_settings.html | 26 ++++++++++++++++++--
11 files changed, 53 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c083e874/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 81aa4d6..e95a5c6 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -66,8 +66,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
cubeInstance.setStatus(RealizationStatusEnum.DISABLED);
cubeInstance.updateRandomUuid();
cubeInstance.setProjectName(projectName);
- cubeInstance.setRetentionRange(cubeDesc.getRetentionRange());
-
+
// MR_V2 is the default engine since 0.8
cubeInstance.setEngineType(IEngineAware.ID_MR_V2);
cubeInstance.setStorageType(IStorageAware.ID_HBASE);
@@ -97,10 +96,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
@JsonProperty("create_time_utc")
private long createTimeUTC;
- @JsonProperty("auto_merge_time_ranges")
- private long[] autoMergeTimeRanges;
- @JsonProperty("retention_range")
- private long retentionRange = 0;
@JsonProperty("engine_type")
private int engineType = IEngineAware.ID_MR_V1;
@JsonProperty("storage_type")
@@ -434,14 +429,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
return Lists.newArrayList(getDescriptor().listDimensionColumnsIncludingDerived());
}
- public long[] getAutoMergeTimeRanges() {
- return autoMergeTimeRanges;
- }
-
- public void setAutoMergeTimeRanges(long[] autoMergeTimeRanges) {
- this.autoMergeTimeRanges = autoMergeTimeRanges;
- }
-
public boolean needAutoMerge() {
if (!this.getDescriptor().getModel().getPartitionDesc().isPartitioned())
return false;
@@ -449,15 +436,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
if (this.getDescriptor().hasHolisticCountDistinctMeasures())
return false;
- return autoMergeTimeRanges != null && autoMergeTimeRanges.length > 0;
- }
-
- public long getRetentionRange() {
- return retentionRange;
- }
-
- public void setRetentionRange(long retentionRange) {
- this.retentionRange = retentionRange;
+ return this.getDescriptor().getAutoMergeTimeRanges() != null && this.getDescriptor().getAutoMergeTimeRanges().length > 0;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c083e874/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index c6f066c..c6e6b88 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -609,7 +609,7 @@ public class CubeManager implements IRealizationProvider {
return null;
}
- long[] timeRanges = cube.getAutoMergeTimeRanges();
+ long[] timeRanges = cube.getDescriptor().getAutoMergeTimeRanges();
Arrays.sort(timeRanges);
CubeSegment newSeg = null;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c083e874/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index ae49eb0..c3ffd24 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -54,7 +54,6 @@ import org.apache.kylin.metadata.model.TblColRef;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.collect.Lists;
@@ -115,11 +114,10 @@ public class CubeDesc extends RootPersistentEntity {
private List<String> notifyList;
@JsonProperty("status_need_notify")
private List<String> statusNeedNotify = Collections.emptyList();
-
- /*
- * this field will not be serialized ,can be deserialized to front javascript
- */
- private long retentionRange;
+ @JsonProperty("auto_merge_time_ranges")
+ private long[] autoMergeTimeRanges;
+ @JsonProperty("retention_range")
+ private long retentionRange = 0;
private Map<String, Map<String, TblColRef>> columnMap = new HashMap<String, Map<String, TblColRef>>();
private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<TblColRef>();
@@ -718,16 +716,22 @@ public class CubeDesc extends RootPersistentEntity {
return false;
}
- @JsonProperty
public long getRetentionRange() {
return retentionRange;
}
- @JsonIgnore
public void setRetentionRange(long retentionRange) {
this.retentionRange = retentionRange;
}
+ public long[] getAutoMergeTimeRanges() {
+ return autoMergeTimeRanges;
+ }
+
+ public void setAutoMergeTimeRanges(long[] autoMergeTimeRanges) {
+ this.autoMergeTimeRanges = autoMergeTimeRanges;
+ }
+
/**
* Add error info and thrown exception out
*
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c083e874/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
index a8476e4..708bf32 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
@@ -95,7 +95,7 @@ public class CubeManagerTest extends LocalFileMetadataTestCase {
CubeManager mgr = CubeManager.getInstance(getTestConfig());
CubeInstance cube = mgr.getCube("test_kylin_cube_with_slr_empty");
- cube.setAutoMergeTimeRanges(new long[] { 2000, 6000 });
+ cube.getDescriptor().setAutoMergeTimeRanges(new long[]{2000, 6000});
mgr.updateCube(new CubeUpdate(cube));
assertTrue(cube.needAutoMerge());
@@ -138,7 +138,7 @@ public class CubeManagerTest extends LocalFileMetadataTestCase {
CubeManager mgr = CubeManager.getInstance(getTestConfig());
CubeInstance cube = mgr.getCube("test_kylin_cube_with_slr_empty");
- cube.setAutoMergeTimeRanges(new long[] { 2000, 6000 });
+ cube.getDescriptor().setAutoMergeTimeRanges(new long[] { 2000, 6000 });
mgr.updateCube(new CubeUpdate(cube));
assertTrue(cube.needAutoMerge());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c083e874/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index b8ec976..d61793d 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -340,7 +340,6 @@ public class CubeController extends BasicController {
try {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeRequest.getCubeName());
- cube.setRetentionRange(desc.getRetentionRange());
String projectName = (null == cubeRequest.getProject()) ? ProjectInstance.DEFAULT_PROJECT_NAME : cubeRequest.getProject();
desc = cubeService.updateCubeAndDesc(cube, desc, projectName);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c083e874/server/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
index 395365e..0a39634 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
@@ -58,7 +58,6 @@ public class CubeDescController {
}
CubeDesc cSchema = cubeInstance.getDescriptor();
if (cSchema != null) {
- cSchema.setRetentionRange(cubeInstance.getRetentionRange());
return new CubeDesc[] { cSchema };
} else {
return null;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c083e874/server/src/main/java/org/apache/kylin/rest/request/CubeRequest.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/request/CubeRequest.java b/server/src/main/java/org/apache/kylin/rest/request/CubeRequest.java
index 74712a7..1371a4f 100644
--- a/server/src/main/java/org/apache/kylin/rest/request/CubeRequest.java
+++ b/server/src/main/java/org/apache/kylin/rest/request/CubeRequest.java
@@ -26,7 +26,6 @@ public class CubeRequest {
private boolean successful;
private String message;
private String project;
- private long retentionRange;
public String getUuid() {
return uuid;
@@ -105,11 +104,4 @@ public class CubeRequest {
this.project = project;
}
- public long getRetentionRange() {
- return retentionRange;
- }
-
- public void setRetentionRange(long retentionRange) {
- this.retentionRange = retentionRange;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c083e874/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index ce51623..aac32e8 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -260,7 +260,6 @@ public class CubeService extends BasicService {
}
CubeDesc updatedCubeDesc = getCubeDescManager().updateCubeDesc(desc);
- cube = getCubeManager().updateCube(new CubeUpdate(cube));
int cuboidCount = CuboidCLI.simulateCuboidGeneration(updatedCubeDesc);
logger.info("Updated cube " + cube.getName() + " has " + cuboidCount + " cuboids");
@@ -591,7 +590,8 @@ public class CubeService extends BasicService {
private void keepCubeRetention(String cubeName) {
CubeInstance cube = getCubeManager().getCube(cubeName);
- if (cube.getRetentionRange() > 0) {
+ CubeDesc desc = cube.getDescriptor();
+ if (desc.getRetentionRange() > 0) {
synchronized (CubeService.class) {
cube = getCubeManager().getCube(cubeName);
List<CubeSegment> readySegs = cube.getSegment(SegmentStatusEnum.READY);
@@ -599,7 +599,7 @@ public class CubeService extends BasicService {
int position = readySegs.size() - 1;
while (position >= 0) {
currentRange += (readySegs.get(position).getDateRangeEnd() - readySegs.get(position).getDateRangeStart());
- if (currentRange >= cube.getRetentionRange()) {
+ if (currentRange >= desc.getRetentionRange()) {
break;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c083e874/webapp/app/js/controllers/cubeAdvanceSetting.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 93c13e3..5ced1bd 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -40,4 +40,11 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
}
}
+
+ $scope.refreshAutoMergeTimeRanges = function(list,index,item){
+ if (item) {
+ list[index] = item;
+ }
+ }
+
});
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c083e874/webapp/app/js/model/cubeDescModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeDescModel.js b/webapp/app/js/model/cubeDescModel.js
index 43efa93..86c8444 100644
--- a/webapp/app/js/model/cubeDescModel.js
+++ b/webapp/app/js/model/cubeDescModel.js
@@ -45,11 +45,11 @@ KylinApp.service('CubeDescModel',function(){
"aggregation_groups": []
},
"notify_list": [],
- "capacity": "",
"hbase_mapping": {
"column_family": []
},
- "retentionRange":"0"
+ "retention_range":"0",
+ "auto_merge_time_ranges":[604800000,2419200000]
};
return cubeMeta;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c083e874/webapp/app/partials/cubeDesigner/advanced_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html b/webapp/app/partials/cubeDesigner/advanced_settings.html
index b56983c..037a003 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -29,12 +29,34 @@
<!--Cube Size-->
<div class="form-group">
<div class="row">
+ <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Auto Merge Time Ranges(days)</b></label>
+ <div class="col-xs-12 col-sm-6">
+ <!--retention range is store in CubeInstance, will convert to cubeMetaFrame for front end-->
+ <div class="row" ng-if="state.mode=='edit'">
+ <div class="col-xs-3" ng-repeat="timeRange in cubeMetaFrame.auto_merge_time_ranges">
+ <input type="text" retention-format ng-change="refreshAutoMergeTimeRanges(cubeMetaFrame.auto_merge_time_ranges,$index,timeRange)" class="form-control" placeholder=".col-xs-3" ng-model="timeRange">
+ </div>
+ </div>
+ <span ng-if="state.mode=='view'">{{cubeMetaFrame.auto_merge_time_ranges[0] | millisecondsToDay}}</span>
+ <span ng-if="state.mode=='view'">,</span>
+ <span ng-if="state.mode=='view'">{{cubeMetaFrame.auto_merge_time_ranges[1] | millisecondsToDay}}</span>
+ </div>
+ </div>
+ </div>
+ </td>
+ </tr>
+
+ <tr>
+ <td>
+ <!--Cube Size-->
+ <div class="form-group">
+ <div class="row">
<label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Retention Range(days)</b></label>
<div class="col-xs-12 col-sm-6">
<!--retention range is store in CubeInstance, will convert to cubeMetaFrame for front end-->
- <input type="text" retention-format class="form-control ng-scope ng-pristine ng-valid" placeholder="how many days cube retention" ng-model="cubeMetaFrame.retentionRange" ng-if="state.mode=='edit'">
+ <input type="text" retention-format class="form-control ng-scope ng-pristine ng-valid" placeholder="how many days cube retention" ng-model="cubeMetaFrame.retention_range" ng-if="state.mode=='edit'">
<small class="help-block text-red" ng-show="state.mode=='edit'">(by default it's '0',which will keep all historic cube segments ,or will keep latest [Retention Range] days cube segments )</small>
- <span ng-if="state.mode=='view'">{{cube.retention_range | millisecondsToDay}}</span>
+ <span ng-if="state.mode=='view'">{{cubeMetaFrame.retention_range | millisecondsToDay}}</span>
</div>
</div>
</div>