You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2020/10/22 02:36:31 UTC
[kylin] branch kylin-on-parquet-v2 updated: KYLIN-4792 Verify
several attribute values of segment in the build/merge test
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin-on-parquet-v2 by this push:
new be783f0 KYLIN-4792 Verify several attribute values of segment in the build/merge test
be783f0 is described below
commit be783f07342c420ddab3a3450a97802c6ee2d243
Author: yaqian.zhang <59...@qq.com>
AuthorDate: Thu Oct 22 09:18:17 2020 +0800
KYLIN-4792 Verify several attribute values of segment in the build/merge test
---
.../java/org/apache/kylin/cube/CubeSegment.java | 4 +++
.../engine/spark/LocalWithSparkSessionTest.java | 32 ++++++++++++++++++++++
.../kylin/engine/spark2/NBuildAndQueryTest.java | 24 ++++++++++++++++
3 files changed, 60 insertions(+)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index ef2395e..d0224fb 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -582,6 +582,10 @@ public class CubeSegment implements IBuildable, ISegment, Serializable {
this.cuboidShardNums = newCuboidShards;
}
+ public Map<Long, Short> getCuboidShardNums() {
+ return this.cuboidShardNums;
+ }
+
public int getTotalShards(long cuboidId) {
if (totalShards > 0) {
return totalShards;
diff --git a/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/LocalWithSparkSessionTest.java b/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/LocalWithSparkSessionTest.java
index 9e2328c..863af8f 100644
--- a/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/LocalWithSparkSessionTest.java
+++ b/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/LocalWithSparkSessionTest.java
@@ -221,6 +221,15 @@ public class LocalWithSparkSessionTest extends LocalFileMetadataTestCase impleme
cleanupSegments(cubeName);
ExecutableState state = buildCuboid(cubeName, null);
Assert.assertEquals(ExecutableState.SUCCEED, state);
+ if (cubeName.equals("ci_left_join_cube")) {
+ CubeManager cubeMgr = CubeManager.getInstance(config);
+ CubeSegment segment = cubeMgr.reloadCube(cubeName).getSegments().get(0);
+ Assert.assertEquals(10000, segment.getInputRecords());
+ Assert.assertEquals(2103495, segment.getInputRecordsSize());
+ Assert.assertTrue(segment.getSizeKB() > 0 );
+ Assert.assertEquals(17, segment.getCuboidShardNums().size());
+ Assert.assertEquals(leftJoinCubeCuboidShardNums(), segment.getCuboidShardNums());
+ }
}
protected void restoreAllSystemProp() {
@@ -328,4 +337,27 @@ public class LocalWithSparkSessionTest extends LocalFileMetadataTestCase impleme
} catch (IOException e) {
}
}
+
+ public Map<Long, Short> leftJoinCubeCuboidShardNums() {
+ Map<Long, Short> cuboidShardNums = Maps.newConcurrentMap();
+ cuboidShardNums.put((long)2097151, (short)1);
+ cuboidShardNums.put((long)14336, (short)1);
+ cuboidShardNums.put((long)112640, (short)1);
+ cuboidShardNums.put((long)79872, (short)1);
+ cuboidShardNums.put((long)114688, (short)1);
+ cuboidShardNums.put((long)98304, (short)1);
+ cuboidShardNums.put((long)65536, (short)1);
+ cuboidShardNums.put((long)245760, (short)1);
+ cuboidShardNums.put((long)276480, (short)1);
+ cuboidShardNums.put((long)262144, (short)1);
+ cuboidShardNums.put((long)342016, (short)1);
+ cuboidShardNums.put((long)376832, (short)1);
+ cuboidShardNums.put((long)360448, (short)1);
+ cuboidShardNums.put((long)327680, (short)1);
+ cuboidShardNums.put((long)507904, (short)1);
+ cuboidShardNums.put((long)1310735, (short)1);
+ cuboidShardNums.put((long)788464, (short)1);
+
+ return cuboidShardNums;
+ }
}
diff --git a/kylin-spark-project/kylin-spark-test/src/test/java/org/apache/kylin/engine/spark2/NBuildAndQueryTest.java b/kylin-spark-project/kylin-spark-test/src/test/java/org/apache/kylin/engine/spark2/NBuildAndQueryTest.java
index df72e8a..ddc0e2f 100644
--- a/kylin-spark-project/kylin-spark-test/src/test/java/org/apache/kylin/engine/spark2/NBuildAndQueryTest.java
+++ b/kylin-spark-project/kylin-spark-test/src/test/java/org/apache/kylin/engine/spark2/NBuildAndQueryTest.java
@@ -221,9 +221,26 @@ public class NBuildAndQueryTest extends LocalWithSparkSessionTest {
ExecutableState state;
state = buildCuboid(cubeName, new SegmentRange.TSRange(dateToLong("2010-01-01"), dateToLong("2012-01-01")));
Assert.assertEquals(ExecutableState.SUCCEED, state);
+ if (cubeName.equals("ci_left_join_cube")) {
+ CubeSegment segment1 = cubeMgr.reloadCube(cubeName).getSegments().get(0);
+
+ Assert.assertEquals(0, segment1.getInputRecords());
+ Assert.assertEquals(2103495, segment1.getInputRecordsSize());
+ Assert.assertEquals(0, segment1.getSizeKB());
+ Assert.assertEquals(17, segment1.getCuboidShardNums().size());
+ }
state = buildCuboid(cubeName, new SegmentRange.TSRange(dateToLong("2012-01-01"), dateToLong("2015-01-01")));
Assert.assertEquals(ExecutableState.SUCCEED, state);
+ if (cubeName.equals("ci_left_join_cube")) {
+ CubeSegment segment2 = cubeMgr.reloadCube(cubeName).getSegments().get(1);
+ Assert.assertEquals(10000, segment2.getInputRecords());
+ Assert.assertEquals(2103495, segment2.getInputRecordsSize());
+ Assert.assertTrue(segment2.getSizeKB() > 0);
+ Assert.assertEquals(17, segment2.getCuboidShardNums().size());
+ Assert.assertEquals(leftJoinCubeCuboidShardNums(), segment2.getCuboidShardNums());
+ }
+
// Round 2: Merge two segments
state = mergeSegments(cubeName, dateToLong("2010-01-01"), dateToLong("2015-01-01"), true);
@@ -231,6 +248,13 @@ public class NBuildAndQueryTest extends LocalWithSparkSessionTest {
// validate cube segment info
CubeSegment firstSegment = cubeMgr.reloadCube(cubeName).getSegments().get(0);
+ if (cubeName.equals("ci_left_join_cube")) {
+ Assert.assertEquals(10000, firstSegment.getInputRecords());
+ Assert.assertEquals(4206990, firstSegment.getInputRecordsSize());
+ Assert.assertTrue(firstSegment.getSizeKB() > 0);
+ Assert.assertEquals(17, firstSegment.getCuboidShardNums().size());
+ Assert.assertEquals(leftJoinCubeCuboidShardNums(), firstSegment.getCuboidShardNums());
+ }
Assert.assertEquals(new SegmentRange.TSRange(dateToLong("2010-01-01"), dateToLong("2015-01-01")),
firstSegment.getSegRange());