You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2017/10/24 08:49:42 UTC
kylin git commit: KYLIN-1892 Support setting a volatile range for
auto merge
Repository: kylin
Updated Branches:
refs/heads/master febbf2623 -> 7e1680eda
KYLIN-1892 Support setting a volatile range for auto merge
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7e1680ed
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7e1680ed
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7e1680ed
Branch: refs/heads/master
Commit: 7e1680eda08de6851e9cea301c39cbeca761d10d
Parents: febbf26
Author: yanghao3 <ya...@xiaomi.com>
Authored: Tue Oct 24 12:26:56 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Oct 24 16:46:21 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeInstance.java | 2 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 11 +++
.../org/apache/kylin/cube/CubeManagerTest.java | 84 ++++++++++++++++++++
.../apache/kylin/metadata/model/Segments.java | 25 +++++-
webapp/app/js/model/cubeDescModel.js | 1 +
.../partials/cubeDesigner/refresh_settings.html | 25 ++++++
6 files changed, 146 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7e1680ed/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index f35cfa2..cc56727 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -479,7 +479,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
public SegmentRange autoMergeCubeSegments() throws IOException {
- return segments.autoMergeCubeSegments(needAutoMerge(), getName(), getDescriptor().getAutoMergeTimeRanges());
+ return segments.autoMergeCubeSegments(needAutoMerge(), getName(), getDescriptor().getAutoMergeTimeRanges(), getDescriptor().getVolatileRange());
}
public Segments calculateToBeSegments(CubeSegment newSegment) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7e1680ed/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index c9917ea..01dd302 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -162,6 +162,8 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
private long partitionDateEnd = 3153600000000L;
@JsonProperty("auto_merge_time_ranges")
private long[] autoMergeTimeRanges;
+ @JsonProperty("volatile_range")
+ private long volatileRange = 0;
@JsonProperty("retention_range")
private long retentionRange = 0;
@JsonProperty("engine_type")
@@ -1101,6 +1103,14 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
return result;
}
+ public long getVolatileRange() {
+ return volatileRange;
+ }
+
+ public void setVolatileRange(long volatileRange) {
+ this.volatileRange = volatileRange;
+ }
+
public long getRetentionRange() {
return retentionRange;
}
@@ -1338,6 +1348,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
newCubeDesc.setAutoMergeTimeRanges(cubeDesc.getAutoMergeTimeRanges());
newCubeDesc.setPartitionDateStart(cubeDesc.getPartitionDateStart());
newCubeDesc.setPartitionDateEnd(cubeDesc.getPartitionDateEnd());
+ newCubeDesc.setVolatileRange(cubeDesc.getVolatileRange());
newCubeDesc.setRetentionRange(cubeDesc.getRetentionRange());
newCubeDesc.setEngineType(cubeDesc.getEngineType());
newCubeDesc.setStorageType(cubeDesc.getStorageType());
http://git-wip-us.apache.org/repos/asf/kylin/blob/7e1680ed/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
index 3f68bab..bb91650 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
@@ -305,6 +305,90 @@ public class CubeManagerTest extends LocalFileMetadataTestCase {
}
@Test
+ public void testAutoMergeWithVolatileRange() throws Exception {
+ CubeManager mgr = CubeManager.getInstance(getTestConfig());
+ CubeInstance cube = mgr.getCube("test_kylin_cube_with_slr_empty");
+
+ cube.getDescriptor().setAutoMergeTimeRanges(new long[] { 2000, 6000 });
+
+ mgr.updateCube(new CubeUpdate(cube));
+
+ assertTrue(cube.needAutoMerge());
+
+ // no segment at first
+ assertEquals(0, cube.getSegments().size());
+
+ // append first
+ CubeSegment seg1 = mgr.appendSegment(cube, new TSRange(0L, 1000L));
+ seg1.setStatus(SegmentStatusEnum.READY);
+
+ CubeSegment seg3 = mgr.appendSegment(cube, new TSRange(2000L, 4000L));
+ seg3.setStatus(SegmentStatusEnum.READY);
+
+ assertEquals(2, cube.getSegments().size());
+
+ SegmentRange mergedSeg = cube.autoMergeCubeSegments();
+
+ assertTrue(mergedSeg == null);
+
+ assertEquals(2, cube.getSegments().size());
+
+ // append a new seg
+
+ CubeSegment seg4 = mgr.appendSegment(cube, new TSRange(4000L, 8000L));
+ seg4.setStatus(SegmentStatusEnum.READY);
+
+ assertEquals(3, cube.getSegments().size());
+
+ cube.getDescriptor().setVolatileRange(10000);
+
+ mergedSeg = cube.autoMergeCubeSegments();
+
+ assertTrue(mergedSeg == null);
+
+ //will merge after change the volatile_range
+
+ cube.getDescriptor().setVolatileRange(0);
+
+ mergedSeg = cube.autoMergeCubeSegments();
+
+ assertTrue(mergedSeg != null);
+
+ assertTrue((Long) mergedSeg.start.v == 2000 && (Long) mergedSeg.end.v == 8000);
+
+ // fill the gap
+
+ CubeSegment seg2 = mgr.appendSegment(cube, new TSRange(1000L, 2000L));
+ seg2.setStatus(SegmentStatusEnum.READY);
+
+ assertEquals(4, cube.getSegments().size());
+
+ cube.getDescriptor().setVolatileRange(10000);
+
+ mergedSeg = cube.autoMergeCubeSegments();
+
+ assertTrue(mergedSeg == null);
+
+ //will merge after change the volatile_range
+ cube.getDescriptor().setVolatileRange(0);
+
+ mergedSeg = cube.autoMergeCubeSegments();
+
+ assertTrue(mergedSeg != null);
+
+ assertTrue((Long) mergedSeg.start.v == 0 && (Long) mergedSeg.end.v == 8000);
+
+ cube.getDescriptor().setVolatileRange(1000);
+
+ mergedSeg = cube.autoMergeCubeSegments();
+
+ assertTrue(mergedSeg != null);
+
+ assertTrue((Long) mergedSeg.start.v == 0 && (Long) mergedSeg.end.v == 2000);
+
+ }
+
+ @Test
public void testGetCubeNameWithNamespace() {
System.setProperty("kylin.storage.hbase.table-name-prefix", "HELLO_");
try {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7e1680ed/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
index 7af1ebc..126d5f9 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
@@ -169,7 +169,28 @@ public class Segments<T extends ISegment> extends ArrayList<T> implements Serial
return result;
}
- public SegmentRange autoMergeCubeSegments(boolean needAutoMerge, String cubeName, long[] timeRanges) throws IOException {
+ public void removeLatestSegmentByVolatileRange(Segments<T> segs, long volatileRange) {
+ if(volatileRange <= 0) {
+ return;
+ }
+ long latestSegEndTs = Long.MIN_VALUE;
+ for(T seg: segs) {
+ latestSegEndTs = Math.max(latestSegEndTs, seg.getTSRange().end.v);
+ }
+ Segments volatileSegs = new Segments();
+ for(T seg: segs) {
+ if(seg.getTSRange().end.v + volatileRange >= latestSegEndTs) {
+ logger.warn("segment in volatile range: seg:" + seg.toString() +
+ "rangeStart:" + seg.getTSRange().start.v + ", rangeEnd" + seg.getTSRange().end.v);
+ volatileSegs.add(seg);
+ }
+ }
+
+ segs.removeAll(volatileSegs);
+
+ }
+
+ public SegmentRange autoMergeCubeSegments(boolean needAutoMerge, String cubeName, long[] timeRanges, long volatileRange) throws IOException {
if (!needAutoMerge) {
logger.debug("Cube " + cubeName + " doesn't need auto merge");
return null;
@@ -195,6 +216,8 @@ public class Segments<T extends ISegment> extends ArrayList<T> implements Serial
}
}
+ removeLatestSegmentByVolatileRange(readySegs, volatileRange);
+
// exclude those already under merging segments
readySegs.removeAll(mergingSegs);
http://git-wip-us.apache.org/repos/asf/kylin/blob/7e1680ed/webapp/app/js/model/cubeDescModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeDescModel.js b/webapp/app/js/model/cubeDescModel.js
index b9b95da..cf44a25 100644
--- a/webapp/app/js/model/cubeDescModel.js
+++ b/webapp/app/js/model/cubeDescModel.js
@@ -52,6 +52,7 @@ KylinApp.service('CubeDescModel', function (kylinConfig) {
"hbase_mapping": {
"column_family": []
},
+ "volatile_range": "0",
"retention_range": "0",
"status_need_notify":['ERROR', 'DISCARDED', 'SUCCEED'],
"auto_merge_time_ranges": [604800000, 2419200000],
http://git-wip-us.apache.org/repos/asf/kylin/blob/7e1680ed/webapp/app/partials/cubeDesigner/refresh_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/refresh_settings.html b/webapp/app/partials/cubeDesigner/refresh_settings.html
index fc5f657..2d2509d 100755
--- a/webapp/app/partials/cubeDesigner/refresh_settings.html
+++ b/webapp/app/partials/cubeDesigner/refresh_settings.html
@@ -100,6 +100,27 @@
<div class="form-group">
<div class="row middle-popover">
<label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default">
+ <b>Volatile Range</b> <i class="fa fa-info-circle" title="Volatile Range" kylinpopover template="volatileRangeTip.html" placement="right"></i>
+ </label>
+
+ <div class="col-xs-12 col-sm-6">
+ <!--volatile range is store in CubeInstance, will convert to cubeMetaFrame for front end-->
+ <input type="text" retention-format class="form-control ng-scope ng-pristine ng-valid"
+ placeholder="how many days cube volatile" ng-model="cubeMetaFrame.volatile_range"
+ ng-if="state.mode=='edit'">
+ <span ng-if="state.mode=='view'">{{cubeMetaFrame.volatile_range | millisecondsToDay}}</span>
+ </div>
+ </div>
+ </div>
+ </td>
+ </tr>
+
+ <tr>
+ <td>
+ <!--Cube Size-->
+ <div class="form-group">
+ <div class="row middle-popover">
+ <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default">
<b>Retention Threshold</b> <i class="fa fa-info-circle" title="Retention Threshold" kylinpopover template="retentionTip.html" placement="right"></i>
</label>
@@ -161,6 +182,10 @@
<p>The thresholds will be checked ascendingly to see if any consectuive segments' time range has exceeded it. For example the [7 days, 30 days] will result in daily incremental segments being merged every 7 days, and the 7-days segments will get merged every 30 days.</p>
</script>
+<script type="text/ng-template" id="volatileRangeTip.html">
+ <p>By default it's '0', which will auto merge all possile cube segments , or 'Auto Merge' will not merge latest [Volatile Range] days cube segments.</p>
+</script>
+
<script type="text/ng-template" id="retentionTip.html">
<p>By default it's '0',which will keep all historic cube segments ,or will keep latest [Retention Threshold] days cube segments.</p>
</script>