You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/09/10 08:51:43 UTC
[kylin] 04/04: KYLIN-3539 Hybrid segment overlap check
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit bffaa8a26a2d93b3aa62326484bc31a8717a7c0b
Author: chao long <wa...@qq.com>
AuthorDate: Mon Sep 10 16:00:46 2018 +0800
KYLIN-3539 Hybrid segment overlap check
---
.../org/apache/kylin/rest/job/HybridCubeCLI.java | 30 +++++++++-----
.../org/apache/kylin/tool/HybridCubeCLITest.java | 48 ++++++++++++++++++++++
2 files changed, 68 insertions(+), 10 deletions(-)
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java b/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java
index 303e000..443eba6 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
+import java.util.Collections;
import java.util.List;
import org.apache.commons.cli.Option;
@@ -38,6 +39,8 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.DataModelManager;
+import org.apache.kylin.metadata.model.SegmentRange;
+import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
import org.apache.kylin.metadata.realization.RealizationType;
@@ -46,6 +49,8 @@ import org.apache.kylin.storage.hybrid.HybridManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Lists;
+
/**
* 1. Create new HybridCube
* bin/kylin.sh org.apache.kylin.tool.HybridCubeCLI -action create -name hybrid_name -project project_name -model model_name -cubes cube1,cube2
@@ -202,23 +207,28 @@ public class HybridCubeCLI extends AbstractApplication {
}
private void checkSegmentOffset(List<RealizationEntry> realizationEntries) {
- long lastOffset = -1;
+ List<SegmentRange> segmentRanges = Lists.newArrayList();
+
for (RealizationEntry entry : realizationEntries) {
if (entry.getType() != RealizationType.CUBE) {
throw new IllegalArgumentException("Wrong realization type: " + entry.getType() + ", only cube supported. ");
}
CubeInstance cubeInstance = cubeManager.getCube(entry.getRealization());
- CubeSegment segment = cubeInstance.getLastSegment();
- if (segment == null)
- continue;
- if (lastOffset == -1) {
- lastOffset = (Long) segment.getSegRange().end.v;
- } else {
- if (lastOffset > (Long) segment.getSegRange().start.v) {
- throw new IllegalArgumentException("Segments has overlap, could not hybrid. Last Segment End: " + lastOffset + ", Next Segment Start: " + segment.getSegRange().start.v);
+ Segments<CubeSegment> segments = cubeInstance.getSegments();
+
+ for (CubeSegment segment : segments) {
+ segmentRanges.add(segment.getSegRange());
+ }
+ }
+
+ if (segmentRanges.size() >= 2) {
+ Collections.sort(segmentRanges);
+
+ for (int i = 0; i < segmentRanges.size() - 1; i++) {
+ if (segmentRanges.get(i).overlaps(segmentRanges.get(i + 1))) {
+ throw new IllegalArgumentException("Segments has overlap, could not hybrid. First Segment Range: [" + segmentRanges.get(i).start.v + "," + segmentRanges.get(i).end.v + "], Second Segment Range: [" + segmentRanges.get(i + 1).start.v + "," + segmentRanges.get(i + 1).end.v + "]");
}
- lastOffset = (Long) segment.getSegRange().end.v;
}
}
}
diff --git a/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java b/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java
index f4ed595..99b8319 100644
--- a/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java
+++ b/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java
@@ -22,6 +22,9 @@ import java.io.IOException;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.storage.hybrid.HybridInstance;
@@ -30,12 +33,17 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.FixMethodOrder;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import org.junit.runners.MethodSorters;
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class HybridCubeCLITest extends LocalFileMetadataTestCase {
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
@Before
public void setUp() throws Exception {
this.createTestMetadata();
@@ -89,4 +97,44 @@ public class HybridCubeCLITest extends LocalFileMetadataTestCase {
Assert.assertEquals(0, ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).findProjects(RealizationType.HYBRID, "ssb_hybrid").size());
}
+ @Test
+ public void testSegmentOverlap() throws IOException {
+ thrown.expect(RuntimeException.class);
+ thrown.expectMessage("Segments has overlap");
+
+ HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv());
+ Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid"));
+ HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" });
+
+ CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+ CubeInstance cube1 = cubeManager.getCube("ssb_cube1");
+ CubeInstance cube2 = cubeManager.getCube("ssb_cube2");
+
+ // 2012-01-01,2012-01-03
+ cubeManager.appendSegment(cube1, new SegmentRange.TSRange(1325376000000L, 1325548800000L));
+ // 2012-01-02,2012-01-04
+ cubeManager.appendSegment(cube2, new SegmentRange.TSRange(1325462400000L, 1325635200000L));
+
+ HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "update" });
+ }
+
+ @Test
+ public void testSegmentNotOverlap() throws IOException {
+ HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv());
+ Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid"));
+ HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" });
+
+ CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+ CubeInstance cube1 = cubeManager.getCube("ssb_cube1");
+ CubeInstance cube2 = cubeManager.getCube("ssb_cube2");
+
+ // 2012-01-01,2012-01-03
+ cubeManager.appendSegment(cube1, new SegmentRange.TSRange(1325376000000L, 1325548800000L));
+ // 2012-01-03,2012-01-04
+ cubeManager.appendSegment(cube2, new SegmentRange.TSRange(1325548800000L, 1325635200000L));
+
+ // not throw exception
+ HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "update" });
+ }
+
}