You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/01/07 15:46:59 UTC
[38/51] [partial] incubator-kylin git commit: migrate repo from
github.com to apache git
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java b/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java
new file mode 100644
index 0000000..f6f81c1
--- /dev/null
+++ b/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java
@@ -0,0 +1,710 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.cube;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.List;
+import java.util.TimeZone;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.kylinolap.common.persistence.ResourceStore;
+import com.kylinolap.common.util.JsonUtil;
+import com.kylinolap.common.util.LocalFileMetadataTestCase;
+import com.kylinolap.cube.exception.CubeIntegrityException;
+import com.kylinolap.cube.project.ProjectInstance;
+import com.kylinolap.cube.project.ProjectManager;
+import com.kylinolap.metadata.MetadataManager;
+import com.kylinolap.metadata.model.cube.CubeDesc;
+
+/**
+ * @author ysong1
+ *
+ */
+public class SegmentManagementTest extends LocalFileMetadataTestCase {
+
+ CubeManager cubeMgr = null;
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ MetadataManager.removeInstance(this.getTestConfig());
+ CubeManager.removeInstance(this.getTestConfig());
+ ProjectManager.removeInstance(this.getTestConfig());
+ cubeMgr = CubeManager.getInstance(this.getTestConfig());
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ private void createNewCube(CubeDesc cubeDesc) throws IOException {
+ ResourceStore store = getStore();
+ // clean legacy in case last run failed
+ store.deleteResource("/cube/a_whole_new_cube.json");
+
+ CubeInstance createdCube = cubeMgr.createCube("a_whole_new_cube", ProjectInstance.DEFAULT_PROJECT_NAME, cubeDesc, "username");
+ assertTrue(createdCube == cubeMgr.getCube("a_whole_new_cube"));
+
+ System.out.println(JsonUtil.writeValueAsIndentString(createdCube));
+ }
+
+ @Test
+ public void testInitialAndAppend() throws ParseException, IOException, CubeIntegrityException {
+ // create a new cube
+ MetadataManager metaMgr = getMetadataManager();
+ CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
+ createNewCube(desc);
+
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ long dateEnd = f.parse("2013-11-12").getTime();
+
+ CubeInstance cubeInstance = cubeMgr.getCube("a_whole_new_cube");
+ assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
+ assertEquals(0, cubeInstance.getSegments().size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getAllocatedEndDate());
+
+ // initial build
+ System.out.println("Initial Build");
+ CubeSegment initialSegment = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, desc.getCubePartitionDesc().getPartitionDateStart(), dateEnd).get(0);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
+ assertEquals(CubeSegmentStatusEnum.NEW, cubeInstance.getBuildingSegments().get(0).getStatus());
+ assertEquals(1, cubeInstance.getSegments().size());
+ assertEquals(1, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getRebuildingSegments().size());
+ assertTrue("".equals(initialSegment.getStorageLocationIdentifier()) == false);
+ assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
+
+ // initial build success
+ System.out.println("Initial Build Success");
+ cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, initialSegment.getName(), "job_1", System.currentTimeMillis(), 111L, 222L, 333L);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getRebuildingSegments().size());
+ assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
+
+ // incremental build
+ System.out.println("Incremental Build");
+ long dateEnd2 = f.parse("2013-12-12").getTime();
+ CubeSegment incrementalSegment = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, dateEnd, dateEnd2).get(0);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(2, cubeInstance.getSegments().size());
+ assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.NEW).size());
+ assertEquals(1, cubeInstance.getBuildingSegments().size());
+ assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(dateEnd2, cubeInstance.getAllocatedEndDate());
+ assertEquals(dateEnd, cubeInstance.getBuildingSegments().get(0).getDateRangeStart());
+ assertEquals(dateEnd2, cubeInstance.getBuildingSegments().get(0).getDateRangeEnd());
+
+ // incremental build success
+ System.out.println("Incremental Build Success");
+ cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, incrementalSegment.getName(), "job_2", System.currentTimeMillis(), 111L, 222L, 333L);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(2, cubeInstance.getSegments().size());
+ assertEquals(2, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(dateEnd2, cubeInstance.getAllocatedEndDate());
+ }
+
+ @Test
+ public void testRebuildSegment() throws IOException, CubeIntegrityException {
+ CubeInstance cubeInstance = cubeMgr.getCube("test_kylin_cube_with_slr_ready");
+
+ // rebuild segment
+ System.out.println("Rebuild Segment");
+ CubeSegment rebuildSegment = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, 1364688000000L, 1386806400000L).get(0);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(1, cubeInstance.getBuildingSegments().size());
+ assertEquals(CubeSegmentStatusEnum.NEW, cubeInstance.getBuildingSegments().get(0).getStatus());
+ assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
+ assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.NEW).size());
+ assertEquals(1, cubeInstance.getRebuildingSegments().size());
+ assertEquals(1364688000000L, cubeInstance.getAllocatedStartDate());
+ assertEquals(1386806400000L, cubeInstance.getAllocatedEndDate());
+
+ // rebuild success
+ System.out.println("Rebuild Success");
+ cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, rebuildSegment.getName(), "job_3", System.currentTimeMillis(), 111, 222, 333);
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(1, cubeInstance.getSegments().size());
+ assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getRebuildingSegments().size());
+ assertEquals(1364688000000L, cubeInstance.getAllocatedStartDate());
+ assertEquals(1386806400000L, cubeInstance.getAllocatedEndDate());
+ assertEquals("job_3", cubeInstance.getSegments().get(0).getLastBuildJobID());
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ }
+
+ @Test(expected = CubeIntegrityException.class)
+ public void testInvalidRebuild() throws IOException, CubeIntegrityException {
+ CubeInstance cubeInstance = cubeMgr.getCube("test_kylin_cube_with_slr_ready");
+
+ // rebuild segment
+ System.out.println("Rebuild Segment");
+ cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, 1364688000000L + 1000L, 1386806400000L).get(0);
+ }
+
+ @Test
+ public void testMergeSegments() throws IOException, CubeIntegrityException {
+ CubeInstance cubeInstance = cubeMgr.getCube("test_kylin_cube_with_slr_ready_2_segments");
+
+ // merge segments
+ System.out.println("Merge Segment");
+ CubeSegment mergedSegment = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.MERGE, 1384240200000L, 1386835200000L).get(0);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(CubeSegmentStatusEnum.NEW, cubeInstance.getBuildingSegments().get(0).getStatus());
+ assertEquals(2, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
+ assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.NEW).size());
+ assertEquals(2, cubeInstance.getMergingSegments().size());
+ assertEquals(1384240200000L, cubeInstance.getAllocatedStartDate());
+ assertEquals(1386835200000L, cubeInstance.getAllocatedEndDate());
+
+ // build success
+ System.out.println("Build Success");
+ cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.MERGE, mergedSegment.getName(), "job_4", System.currentTimeMillis(), 123, 20000L, 1216024L);
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getMergingSegments().size());
+ assertEquals(1384240200000L, cubeInstance.getAllocatedStartDate());
+ assertEquals(1386835200000L, cubeInstance.getAllocatedEndDate());
+ assertEquals("job_4", cubeInstance.getSegments().get(0).getLastBuildJobID());
+ assertEquals(20000L, cubeInstance.getSegments().get(0).getSourceRecords());
+ assertEquals(1216024L, cubeInstance.getSegments().get(0).getSourceRecordsSize());
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ }
+
+ @Test
+ public void testNonPartitionedCube() throws ParseException, IOException, CubeIntegrityException {
+ // create a new cube
+ MetadataManager metaMgr = getMetadataManager();
+ CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_without_slr_desc");
+ createNewCube(desc);
+
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ long dateEnd = f.parse("2013-11-12").getTime();
+
+ CubeInstance cubeInstance = cubeMgr.getCube("a_whole_new_cube");
+ assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
+ assertEquals(0, cubeInstance.getSegments().size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getAllocatedEndDate());
+
+ // initial build
+ System.out.println("Initial Build");
+ CubeSegment initialSegment = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, desc.getCubePartitionDesc().getPartitionDateStart(), dateEnd).get(0);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
+ assertEquals(CubeSegmentStatusEnum.NEW, cubeInstance.getBuildingSegments().get(0).getStatus());
+ assertEquals(1, cubeInstance.getSegments().size());
+ assertEquals(0, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
+ assertEquals(1, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getRebuildingSegments().size());
+ assertTrue("".equals(initialSegment.getStorageLocationIdentifier()) == false);
+ assertEquals("FULL_BUILD", initialSegment.getName());
+ assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(0, cubeInstance.getAllocatedEndDate());
+
+ // initial build success
+ System.out.println("Initial Build Success");
+ cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, initialSegment.getName(), "job_5", System.currentTimeMillis(), 111L, 222L, 333L);
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getRebuildingSegments().size());
+ assertEquals(0, cubeInstance.getAllocatedStartDate());
+ assertEquals(0, cubeInstance.getAllocatedEndDate());
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+
+ // rebuild segment
+ System.out.println("Rebuild Segment");
+ CubeSegment rebuildSegment = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, 1364688000000L, 1386806400000L).get(0);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(CubeSegmentStatusEnum.NEW, cubeInstance.getBuildingSegments().get(0).getStatus());
+ assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
+ assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.NEW).size());
+ assertEquals(1, cubeInstance.getBuildingSegments().size());
+ assertEquals(1, cubeInstance.getRebuildingSegments().size());
+ assertEquals(0, cubeInstance.getAllocatedStartDate());
+ assertEquals(0, cubeInstance.getAllocatedEndDate());
+
+ // rebuild success
+ System.out.println("Rebuild Success");
+ cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, rebuildSegment.getName(), "job_6", System.currentTimeMillis(), 111, 222, 333);
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(1, cubeInstance.getSegments().size());
+ assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getRebuildingSegments().size());
+ assertEquals(0, cubeInstance.getAllocatedStartDate());
+ assertEquals(0, cubeInstance.getAllocatedEndDate());
+ assertEquals("job_6", cubeInstance.getSegments().get(0).getLastBuildJobID());
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ }
+
+ @Test(expected = CubeIntegrityException.class)
+ public void testInvalidAppend() throws ParseException, IOException, CubeIntegrityException {
+ // create a new cube
+ MetadataManager metaMgr = getMetadataManager();
+ CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
+ createNewCube(desc);
+
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ long dateEnd = f.parse("2013-11-12").getTime();
+
+ CubeInstance cubeInstance = cubeMgr.getCube("a_whole_new_cube");
+ assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
+ assertEquals(0, cubeInstance.getSegments().size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getAllocatedEndDate());
+
+ // initial build
+ System.out.println("Initial Build");
+ CubeSegment initialSegment = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, desc.getCubePartitionDesc().getPartitionDateStart(), dateEnd).get(0);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
+ assertEquals(CubeSegmentStatusEnum.NEW, cubeInstance.getBuildingSegments().get(0).getStatus());
+ assertEquals(1, cubeInstance.getSegments().size());
+ assertEquals(1, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getRebuildingSegments().size());
+ assertTrue("".equals(initialSegment.getStorageLocationIdentifier()) == false);
+ assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
+
+ // initial build success
+ System.out.println("Initial Build Success");
+ cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, initialSegment.getName(), "job_1", System.currentTimeMillis(), 111L, 222L, 333L);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getRebuildingSegments().size());
+ assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
+
+ // incremental build
+ System.out.println("Invalid Incremental Build");
+ long dateEnd2 = f.parse("2013-12-12").getTime();
+ cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, dateEnd + 1000, dateEnd2);
+
+ }
+
+ @Test
+ public void testInitialAndUpsert() throws ParseException, IOException, CubeIntegrityException {
+ // create a new cube
+ MetadataManager metaMgr = getMetadataManager();
+ CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_without_slr_left_join_desc");
+ createNewCube(desc);
+
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ long dateEnd = f.parse("2013-11-12").getTime();
+
+ CubeInstance cubeInstance = cubeMgr.getCube("a_whole_new_cube");
+ assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
+ assertEquals(0, cubeInstance.getSegments().size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getAllocatedEndDate());
+
+ // initial build
+ System.out.println("Initial Build");
+ CubeSegment initialSegment = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, desc.getCubePartitionDesc().getPartitionDateStart(), dateEnd).get(0);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
+ for (CubeSegment cubeSegment : cubeInstance.getBuildingSegments()) {
+ assertEquals(CubeSegmentStatusEnum.NEW, cubeSegment.getStatus());
+ }
+ assertEquals(1, cubeInstance.getSegments().size());
+ assertEquals(1, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getRebuildingSegments().size());
+ assertTrue("".equals(initialSegment.getStorageLocationIdentifier()) == false);
+ assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
+
+ // initial build success
+ System.out.println("Initial Build Success");
+ cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, initialSegment.getName(), "job_1", System.currentTimeMillis(), 111L, 222L, 333L);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
+
+ // upsert build
+ System.out.println("Upsert Build");
+ long start = f.parse("2013-11-01").getTime();
+ long dateEnd2 = f.parse("2013-12-12").getTime();
+ List<CubeSegment> upsertSegments = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, start, dateEnd2);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(2, upsertSegments.size());
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(3, cubeInstance.getSegments().size());
+ assertEquals(2, cubeInstance.getSegments(CubeSegmentStatusEnum.NEW).size());
+ assertEquals(2, cubeInstance.getBuildingSegments().size());
+ assertEquals(1, cubeInstance.getRebuildingSegments().size());
+ assertEquals(0, cubeInstance.getMergingSegments().size());
+ assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(dateEnd2, cubeInstance.getAllocatedEndDate());
+ assertEquals(0L, cubeInstance.getBuildingSegments().get(0).getDateRangeStart());
+ assertEquals(dateEnd2, cubeInstance.getBuildingSegments().get(1).getDateRangeEnd());
+
+ // upsert build success
+ System.out.println("Upsert Build Success");
+ cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, upsertSegments.get(0).getName(), "job_2", System.currentTimeMillis(), 111L, 222L, 333L);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, upsertSegments.get(1).getName(), "job_3", System.currentTimeMillis(), 111L, 222L, 333L);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(2, cubeInstance.getSegments().size());
+ assertEquals(2, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(dateEnd2, cubeInstance.getAllocatedEndDate());
+
+ // upsert build again
+ System.out.println("Upsert Build");
+ long start2 = f.parse("2013-12-01").getTime();
+ long dateEnd3 = f.parse("2013-12-31").getTime();
+ List<CubeSegment> upsertSegments2 = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, start2, dateEnd3);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(2, upsertSegments2.size());
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(4, cubeInstance.getSegments().size());
+ assertEquals(2, cubeInstance.getSegments(CubeSegmentStatusEnum.NEW).size());
+ assertEquals(2, cubeInstance.getBuildingSegments().size());
+ assertEquals(1, cubeInstance.getRebuildingSegments().size());
+ assertEquals(0, cubeInstance.getMergingSegments().size());
+ assertEquals(dateEnd3, cubeInstance.getAllocatedEndDate());
+ // building segment 1 from 2013-11-01 to 2013-12-01
+ assertEquals(f.parse("2013-11-01").getTime(), cubeInstance.getBuildingSegments().get(0).getDateRangeStart());
+ assertEquals(f.parse("2013-12-01").getTime(), cubeInstance.getBuildingSegments().get(0).getDateRangeEnd());
+ // building segment 2 from 2013-12-01 to 2013-12-31
+ assertEquals(f.parse("2013-12-01").getTime(), cubeInstance.getBuildingSegments().get(1).getDateRangeStart());
+ assertEquals(f.parse("2013-12-31").getTime(), cubeInstance.getBuildingSegments().get(1).getDateRangeEnd());
+
+ // upsert build success
+ System.out.println("Upsert Build Success");
+ cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, upsertSegments2.get(1).getName(), "job_5", System.currentTimeMillis(), 111L, 222L, 333L);
+ cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, upsertSegments2.get(0).getName(), "job_4", System.currentTimeMillis(), 111L, 222L, 333L);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(3, cubeInstance.getSegments().size());
+ assertEquals(3, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getMergingSegments().size());
+ assertEquals(cubeInstance.getDescriptor().getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(dateEnd3, cubeInstance.getAllocatedEndDate());
+ // segment 1 from 1970-01-01 to 2013-11-01
+ assertEquals(cubeInstance.getDescriptor().getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getSegments().get(0).getDateRangeStart());
+ assertEquals(f.parse("2013-11-01").getTime(), cubeInstance.getSegments().get(0).getDateRangeEnd());
+ // segment 2 from 2013-11-01 to 2013-12-01
+ assertEquals(f.parse("2013-11-01").getTime(), cubeInstance.getSegments().get(1).getDateRangeStart());
+ assertEquals(f.parse("2013-12-01").getTime(), cubeInstance.getSegments().get(1).getDateRangeEnd());
+ // segment 3 from 2013-12-01 to 2013-12-31
+ assertEquals(f.parse("2013-12-01").getTime(), cubeInstance.getSegments().get(2).getDateRangeStart());
+ assertEquals(f.parse("2013-12-31").getTime(), cubeInstance.getSegments().get(2).getDateRangeEnd());
+
+ // upsert build again
+ System.out.println("Upsert Build");
+ List<CubeSegment> upsertSegments3 = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, f.parse("2013-10-01").getTime(), f.parse("2014-02-01").getTime());
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(2, upsertSegments3.size());
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(5, cubeInstance.getSegments().size());
+ assertEquals(2, cubeInstance.getSegments(CubeSegmentStatusEnum.NEW).size());
+ assertEquals(2, cubeInstance.getBuildingSegments().size());
+ assertEquals(3, cubeInstance.getRebuildingSegments().size());
+ assertEquals(0, cubeInstance.getMergingSegments().size());
+ assertEquals(f.parse("2014-02-01").getTime(), cubeInstance.getAllocatedEndDate());
+ // building segment 1 from 2013-11-01 to 2013-10-01
+ assertEquals(cubeInstance.getDescriptor().getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getBuildingSegments().get(0).getDateRangeStart());
+ assertEquals(f.parse("2013-10-01").getTime(), cubeInstance.getBuildingSegments().get(0).getDateRangeEnd());
+ // building segment 2 from 2013-10-01 to 2014-02-01
+ assertEquals(f.parse("2013-10-01").getTime(), cubeInstance.getBuildingSegments().get(1).getDateRangeStart());
+ assertEquals(f.parse("2014-02-01").getTime(), cubeInstance.getBuildingSegments().get(1).getDateRangeEnd());
+
+ // upsert build success
+ System.out.println("Upsert Build Success");
+ cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, upsertSegments3.get(1).getName(), "job_7", System.currentTimeMillis(), 111L, 222L, 333L);
+ cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, upsertSegments3.get(0).getName(), "job_6", System.currentTimeMillis(), 111L, 222L, 333L);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(2, cubeInstance.getSegments().size());
+ assertEquals(2, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getMergingSegments().size());
+ assertEquals(cubeInstance.getDescriptor().getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(f.parse("2014-02-01").getTime(), cubeInstance.getAllocatedEndDate());
+ // segment 1 from 1970-01-01 to 2013-10-01
+ assertEquals(cubeInstance.getDescriptor().getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getSegments().get(0).getDateRangeStart());
+ assertEquals(f.parse("2013-10-01").getTime(), cubeInstance.getSegments().get(0).getDateRangeEnd());
+ // segment 2 from 2013-10-01 to 2014-02-01
+ assertEquals(f.parse("2013-10-01").getTime(), cubeInstance.getSegments().get(1).getDateRangeStart());
+ assertEquals(f.parse("2014-02-01").getTime(), cubeInstance.getSegments().get(1).getDateRangeEnd());
+ }
+
+ @Test
+ public void testInitialAndUpsert2() throws ParseException, IOException, CubeIntegrityException {
+ // create a new cube
+ MetadataManager metaMgr = getMetadataManager();
+ CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_without_slr_left_join_desc");
+ createNewCube(desc);
+
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ long dateEnd = f.parse("2013-01-01").getTime();
+
+ CubeInstance cubeInstance = cubeMgr.getCube("a_whole_new_cube");
+ assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
+ assertEquals(0, cubeInstance.getSegments().size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getAllocatedEndDate());
+
+ // initial build
+ System.out.println("Initial Build");
+ CubeSegment initialSegment = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, desc.getCubePartitionDesc().getPartitionDateStart(), dateEnd).get(0);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
+ for (CubeSegment cubeSegment : cubeInstance.getBuildingSegments()) {
+ assertEquals(CubeSegmentStatusEnum.NEW, cubeSegment.getStatus());
+ }
+ assertEquals(1, cubeInstance.getSegments().size());
+ assertEquals(1, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getRebuildingSegments().size());
+ assertTrue("".equals(initialSegment.getStorageLocationIdentifier()) == false);
+ assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
+
+ // initial build success
+ System.out.println("Initial Build Success");
+ cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, initialSegment.getName(), "job_1", System.currentTimeMillis(), 111L, 222L, 333L);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
+
+ // upsert build
+ System.out.println("Upsert Build");
+ long start = f.parse("2013-01-01").getTime();
+ long dateEnd2 = f.parse("2014-01-01").getTime();
+ List<CubeSegment> upsertSegments = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, start, dateEnd2);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(1, upsertSegments.size());
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(2, cubeInstance.getSegments().size());
+ assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.NEW).size());
+ assertEquals(1, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getRebuildingSegments().size());
+ assertEquals(0, cubeInstance.getMergingSegments().size());
+ assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(dateEnd2, cubeInstance.getAllocatedEndDate());
+ assertEquals(start, cubeInstance.getBuildingSegments().get(0).getDateRangeStart());
+ assertEquals(dateEnd2, cubeInstance.getBuildingSegments().get(0).getDateRangeEnd());
+
+ // upsert build success
+ System.out.println("Upsert Build Success");
+ cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, upsertSegments.get(0).getName(), "job_2", System.currentTimeMillis(), 111L, 222L, 333L);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(2, cubeInstance.getSegments().size());
+ assertEquals(2, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(dateEnd2, cubeInstance.getAllocatedEndDate());
+
+ // upsert build again
+ System.out.println("Upsert Build");
+ List<CubeSegment> upsertSegments3 = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, f.parse("2013-10-01").getTime(), f.parse("2014-02-01").getTime());
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(2, upsertSegments3.size());
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(4, cubeInstance.getSegments().size());
+ assertEquals(2, cubeInstance.getSegments(CubeSegmentStatusEnum.NEW).size());
+ assertEquals(2, cubeInstance.getBuildingSegments().size());
+ assertEquals(1, cubeInstance.getRebuildingSegments().size());
+ assertEquals(0, cubeInstance.getMergingSegments().size());
+ assertEquals(f.parse("2014-02-01").getTime(), cubeInstance.getAllocatedEndDate());
+ // building segment 1 from 2013-11-01 to 2013-10-01
+ assertEquals(f.parse("2013-01-01").getTime(), cubeInstance.getBuildingSegments().get(0).getDateRangeStart());
+ assertEquals(f.parse("2013-10-01").getTime(), cubeInstance.getBuildingSegments().get(0).getDateRangeEnd());
+ // building segment 2 from 2013-10-01 to 2014-02-01
+ assertEquals(f.parse("2013-10-01").getTime(), cubeInstance.getBuildingSegments().get(1).getDateRangeStart());
+ assertEquals(f.parse("2014-02-01").getTime(), cubeInstance.getBuildingSegments().get(1).getDateRangeEnd());
+
+ // upsert build success
+ System.out.println("Upsert Build Success");
+ cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, upsertSegments3.get(1).getName(), "job_7", System.currentTimeMillis(), 111L, 222L, 333L);
+ cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, upsertSegments3.get(0).getName(), "job_6", System.currentTimeMillis(), 111L, 222L, 333L);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(3, cubeInstance.getSegments().size());
+ assertEquals(3, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getMergingSegments().size());
+ assertEquals(cubeInstance.getDescriptor().getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(f.parse("2014-02-01").getTime(), cubeInstance.getAllocatedEndDate());
+ // segment 1 from 1970-01-01 to 2013-10-01
+ assertEquals(cubeInstance.getDescriptor().getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getSegments().get(0).getDateRangeStart());
+ assertEquals(f.parse("2013-01-01").getTime(), cubeInstance.getSegments().get(0).getDateRangeEnd());
+ // segment 2 from 2013-10-01 to 2014-02-01
+ assertEquals(f.parse("2013-01-01").getTime(), cubeInstance.getSegments().get(1).getDateRangeStart());
+ assertEquals(f.parse("2013-10-01").getTime(), cubeInstance.getSegments().get(1).getDateRangeEnd());
+ // segment 3 from 2013-10-01 to 2014-02-01
+ assertEquals(f.parse("2013-10-01").getTime(), cubeInstance.getSegments().get(2).getDateRangeStart());
+ assertEquals(f.parse("2014-02-01").getTime(), cubeInstance.getSegments().get(2).getDateRangeEnd());
+ }
+
+ @Test(expected = CubeIntegrityException.class)
+ public void testInvalidUpsert() throws IOException, CubeIntegrityException, ParseException {
+ // create a new cube
+ MetadataManager metaMgr = getMetadataManager();
+ CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_without_slr_left_join_desc");
+ createNewCube(desc);
+
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ long dateEnd = f.parse("2013-11-12").getTime();
+
+ CubeInstance cubeInstance = cubeMgr.getCube("a_whole_new_cube");
+ assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
+ assertEquals(0, cubeInstance.getSegments().size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getAllocatedEndDate());
+
+ // initial build
+ System.out.println("Initial Build");
+ CubeSegment initialSegment = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, desc.getCubePartitionDesc().getPartitionDateStart(), dateEnd).get(0);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
+ for (CubeSegment cubeSegment : cubeInstance.getBuildingSegments()) {
+ assertEquals(CubeSegmentStatusEnum.NEW, cubeSegment.getStatus());
+ }
+ assertEquals(1, cubeInstance.getSegments().size());
+ assertEquals(1, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getRebuildingSegments().size());
+ assertTrue("".equals(initialSegment.getStorageLocationIdentifier()) == false);
+ assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
+
+ // initial build success
+ System.out.println("Initial Build Success");
+ cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, initialSegment.getName(), "job_1", System.currentTimeMillis(), 111L, 222L, 333L);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
+
+ // upsert build
+ // time gap in new segment
+ System.out.println("Upsert Build");
+ long start = f.parse("2013-11-13").getTime();
+ long dateEnd2 = f.parse("2013-12-12").getTime();
+ cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, start, dateEnd2);
+ }
+
+ @Test(expected = CubeIntegrityException.class)
+ public void testInvalidUpsert2() throws IOException, CubeIntegrityException, ParseException {
+ // create a new cube
+ MetadataManager metaMgr = getMetadataManager();
+ CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_without_slr_left_join_desc");
+ createNewCube(desc);
+
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ long dateEnd = f.parse("2013-11-12").getTime();
+
+ CubeInstance cubeInstance = cubeMgr.getCube("a_whole_new_cube");
+ assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
+ assertEquals(0, cubeInstance.getSegments().size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getAllocatedEndDate());
+
+ // initial build
+ System.out.println("Initial Build");
+ CubeSegment initialSegment = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, desc.getCubePartitionDesc().getPartitionDateStart(), dateEnd).get(0);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
+ for (CubeSegment cubeSegment : cubeInstance.getBuildingSegments()) {
+ assertEquals(CubeSegmentStatusEnum.NEW, cubeSegment.getStatus());
+ }
+ assertEquals(1, cubeInstance.getSegments().size());
+ assertEquals(1, cubeInstance.getBuildingSegments().size());
+ assertEquals(0, cubeInstance.getRebuildingSegments().size());
+ assertTrue("".equals(initialSegment.getStorageLocationIdentifier()) == false);
+ assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
+
+ // initial build success
+ System.out.println("Initial Build Success");
+ cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, initialSegment.getName(), "job_1", System.currentTimeMillis(), 111L, 222L, 333L);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
+ assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
+ assertEquals(0, cubeInstance.getBuildingSegments().size());
+ assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
+
+ // upsert build
+ // time gap in new segment
+ System.out.println("Upsert Build");
+ long start = f.parse("2013-11-01").getTime();
+ long dateEnd2 = f.parse("2013-11-02").getTime();
+
+ cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, start, dateEnd2);
+ System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
+ }
+
+ private MetadataManager getMetadataManager() {
+ return MetadataManager.getInstance(getTestConfig());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/cube/src/test/java/com/kylinolap/cube/common/BytesSplitterTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/com/kylinolap/cube/common/BytesSplitterTest.java b/cube/src/test/java/com/kylinolap/cube/common/BytesSplitterTest.java
new file mode 100644
index 0000000..6d378db
--- /dev/null
+++ b/cube/src/test/java/com/kylinolap/cube/common/BytesSplitterTest.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.cube.common;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+/**
+ * @author George Song (ysong1)
+ *
+ */
+public class BytesSplitterTest {
+
+ @Test
+ public void test() {
+ BytesSplitter bytesSplitter = new BytesSplitter(10, 15);
+ byte[] input = "2013-02-17Collectibles".getBytes();
+ bytesSplitter.split(input, input.length, (byte) 127);
+
+ assertEquals(2, bytesSplitter.getBufferSize());
+ assertEquals("2013-02-17", new String(bytesSplitter.getSplitBuffers()[0].value, 0, bytesSplitter.getSplitBuffers()[0].length));
+ assertEquals("Collectibles", new String(bytesSplitter.getSplitBuffers()[1].value, 0, bytesSplitter.getSplitBuffers()[1].length));
+ }
+
+ @Test
+ public void testNullValue() {
+ BytesSplitter bytesSplitter = new BytesSplitter(10, 15);
+ byte[] input = "2013-02-17Collectibles".getBytes();
+ bytesSplitter.split(input, input.length, (byte) 127);
+
+ assertEquals(3, bytesSplitter.getBufferSize());
+ assertEquals("2013-02-17", new String(bytesSplitter.getSplitBuffers()[0].value, 0, bytesSplitter.getSplitBuffers()[0].length));
+ assertEquals("", new String(bytesSplitter.getSplitBuffers()[1].value, 0, bytesSplitter.getSplitBuffers()[1].length));
+ assertEquals("Collectibles", new String(bytesSplitter.getSplitBuffers()[2].value, 0, bytesSplitter.getSplitBuffers()[2].length));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/cube/src/test/java/com/kylinolap/cube/common/RowKeySplitterTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/com/kylinolap/cube/common/RowKeySplitterTest.java b/cube/src/test/java/com/kylinolap/cube/common/RowKeySplitterTest.java
new file mode 100644
index 0000000..5d5d377
--- /dev/null
+++ b/cube/src/test/java/com/kylinolap/cube/common/RowKeySplitterTest.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.cube.common;
+
+import static org.junit.Assert.*;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.kylinolap.common.util.LocalFileMetadataTestCase;
+import com.kylinolap.cube.CubeInstance;
+import com.kylinolap.cube.CubeManager;
+import com.kylinolap.metadata.MetadataManager;
+
+/**
+ * @author George Song (ysong1)
+ *
+ */
+public class RowKeySplitterTest extends LocalFileMetadataTestCase {
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ MetadataManager.removeInstance(this.getTestConfig());
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testWithSlr() throws Exception {
+ CubeInstance cube = CubeManager.getInstance(this.getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");
+
+ RowKeySplitter rowKeySplitter = new RowKeySplitter(cube.getFirstSegment(), 10, 20);
+ // base cuboid rowkey
+ byte[] input = { 0, 0, 0, 0, 0, 0, 1, -1, 49, 48, 48, 48, 48, 48, 48, 48, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 11, 54, -105, 55, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
+ rowKeySplitter.split(input, input.length);
+
+ assertEquals(10, rowKeySplitter.getBufferSize());
+ }
+
+ @Test
+ public void testWithoutSlr() throws Exception {
+ CubeInstance cube = CubeManager.getInstance(this.getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");
+
+ RowKeySplitter rowKeySplitter = new RowKeySplitter(cube.getFirstSegment(), 10, 20);
+ // base cuboid rowkey
+ byte[] input = { 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 };
+ rowKeySplitter.split(input, input.length);
+
+ assertEquals(9, rowKeySplitter.getBufferSize());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/cube/src/test/java/com/kylinolap/cube/cuboid/CombinationTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/com/kylinolap/cube/cuboid/CombinationTest.java b/cube/src/test/java/com/kylinolap/cube/cuboid/CombinationTest.java
new file mode 100644
index 0000000..1cd48bd
--- /dev/null
+++ b/cube/src/test/java/com/kylinolap/cube/cuboid/CombinationTest.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.cube.cuboid;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+/**
+ * @author xjiang
+ *
+ */
+public class CombinationTest {
+
+ public int findSmallerSibling(long valueBits, long valueMask) {
+ if ((valueBits | valueMask) != valueMask) {
+ throw new IllegalArgumentException("Dismatch " + Long.toBinaryString(valueBits) + " from " + Long.toBinaryString(valueMask));
+ }
+
+ int n = Long.bitCount(valueMask);
+ int k = Long.bitCount(valueBits);
+ long[] bitMasks = new long[n];
+ long leftBits = valueMask;
+ for (int i = 0; i < n; i++) {
+ long lowestBit = Long.lowestOneBit(leftBits);
+ bitMasks[i] = lowestBit;
+ leftBits &= ~lowestBit;
+ }
+ return combination(valueBits, bitMasks, 0, 0L, k);
+ }
+
+ private int combination(long valueBits, long[] bitMasks, int offset, long prefix, int k) {
+ if (k == 0) {
+ if (prefix < valueBits) {
+ System.out.println(Long.toBinaryString(prefix));
+ return 1;
+ } else {
+ return 0;
+ }
+ } else {
+ int count = 0;
+ for (int i = offset; i < bitMasks.length; i++) {
+ long newPrefix = prefix | bitMasks[i];
+ if (newPrefix < valueBits) {
+ count += combination(valueBits, bitMasks, i + 1, newPrefix, k - 1);
+ }
+ }
+ return count;
+ }
+ }
+
+ private long calculateCombination(int n, int k) {
+ if (n < k) {
+ throw new IllegalArgumentException("N < K");
+ }
+ long res = 1;
+ for (int i = n - k + 1; i <= n; i++) {
+ res *= i;
+ }
+ for (int i = 1; i <= k; i++) {
+ res /= i;
+ }
+ return res;
+ }
+
+ @Test
+ public void testComb3() {
+ long valueBits = 1 << 4 | 1 << 6 | 1 << 8;
+ System.out.println("value = " + Long.toBinaryString(valueBits) + ", count = " + Long.bitCount(valueBits));
+ long valueMask = (long) Math.pow(2, 10) - 1;
+ System.out.println("mask = " + Long.toBinaryString(valueMask) + ", count = " + Long.bitCount(valueMask));
+ System.out.println("************");
+ int count = findSmallerSibling(valueBits, valueMask);
+ System.out.println("smaller sibling count = " + count);
+ int cnk = (int) calculateCombination(Long.bitCount(valueMask), Long.bitCount(valueBits));
+ assertTrue(cnk > count);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidSchedulerTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidSchedulerTest.java b/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidSchedulerTest.java
new file mode 100644
index 0000000..c3e4540
--- /dev/null
+++ b/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidSchedulerTest.java
@@ -0,0 +1,309 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.cube.cuboid;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.kylinolap.common.util.LocalFileMetadataTestCase;
+import com.kylinolap.metadata.MetadataManager;
+import com.kylinolap.metadata.model.cube.CubeDesc;
+
+/**
+ * @author George Song (ysong1)
+ *
+ */
+public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ MetadataManager.removeInstance(this.getTestConfig());
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ static long toLong(String bin) {
+ return Long.parseLong(bin, 2);
+ }
+
+ static String toString(long l) {
+ return Long.toBinaryString(l);
+ }
+
+ static String toString(Collection<Long> cuboids) {
+ StringBuilder buf = new StringBuilder();
+ buf.append("[");
+ for (Long l : cuboids) {
+ if (buf.length() > 1)
+ buf.append(",");
+ buf.append(l).append("(").append(Long.toBinaryString(l)).append(")");
+ }
+ buf.append("]");
+ return buf.toString();
+ }
+
+ private CubeDesc getTestKylinCubeWithoutSeller() {
+ return getMetadataManager().getCubeDesc("test_kylin_cube_without_slr_desc");
+ }
+
+ private CubeDesc getTestKylinCubeWithSeller() {
+ return getMetadataManager().getCubeDesc("test_kylin_cube_with_slr_desc");
+ }
+
+ private CubeDesc getTestKylinCubeWithoutSellerLeftJoin() {
+ return getMetadataManager().getCubeDesc("test_kylin_cube_without_slr_left_join_desc");
+ }
+
+ @Test
+ public void testFindSmallerSibling1() {
+ CubeDesc cube = getTestKylinCubeWithoutSeller();
+ CuboidScheduler scheduler = new CuboidScheduler(cube);
+
+ Collection<Long> siblings;
+
+ siblings = scheduler.findSmallerSibling(255);
+ assertEquals("[]", siblings.toString());
+
+ siblings = scheduler.findSmallerSibling(133);
+ assertEquals("[131]", siblings.toString());
+
+ siblings = scheduler.findSmallerSibling(127);
+ assertEquals("[]", siblings.toString());
+
+ siblings = scheduler.findSmallerSibling(134);
+ assertEquals("[131, 133]", sortToString(siblings));
+
+ siblings = scheduler.findSmallerSibling(130);
+ assertEquals("[129]", siblings.toString());
+
+ siblings = scheduler.findSmallerSibling(5);
+ assertEquals("[]", siblings.toString());
+
+ siblings = scheduler.findSmallerSibling(135);
+ assertEquals("[]", siblings.toString());
+ }
+
+ private void testSpanningAndGetParent(CuboidScheduler scheduler, CubeDesc cube, long[] cuboidIds) {
+ for (long cuboidId : cuboidIds) {
+ Collection<Long> spannings = scheduler.getSpanningCuboid(cuboidId);
+ System.out.println("Spanning result for " + cuboidId + "(" + Long.toBinaryString(cuboidId) + "): " + toString(spannings));
+
+ for (long child : spannings) {
+ assertTrue(Cuboid.isValid(cube, child));
+ }
+ }
+ }
+
+ @Test
+ public void testFindSmallerSibling2() {
+ CubeDesc cube = getTestKylinCubeWithSeller();
+ CuboidScheduler scheduler = new CuboidScheduler(cube);
+
+ Collection<Long> siblings;
+
+ siblings = scheduler.findSmallerSibling(511);
+ assertEquals("[]", siblings.toString());
+
+ siblings = scheduler.findSmallerSibling(toLong("110111111"));
+ assertEquals("[383]", siblings.toString());
+
+ siblings = scheduler.findSmallerSibling(toLong("101110111"));
+ assertEquals("[319]", siblings.toString());
+
+ siblings = scheduler.findSmallerSibling(toLong("111111000"));
+ assertEquals("[]", siblings.toString());
+
+ siblings = scheduler.findSmallerSibling(toLong("111111000"));
+ assertEquals("[]", siblings.toString());
+
+ siblings = scheduler.findSmallerSibling(toLong("110000000"));
+ assertEquals("[288, 320]", sortToString(siblings));
+ }
+
+ @Test
+ public void testGetSpanningCuboid2() {
+ CubeDesc cube = getTestKylinCubeWithSeller();
+ CuboidScheduler scheduler = new CuboidScheduler(cube);
+
+ // generate 8d
+ System.out.println("Spanning for 8D Cuboids");
+ testSpanningAndGetParent(scheduler, cube, new long[] { 511 });
+ // generate 7d
+ System.out.println("Spanning for 7D Cuboids");
+ testSpanningAndGetParent(scheduler, cube, new long[] { 504, 447, 503, 383 });
+ // generate 6d
+ System.out.println("Spanning for 6D Cuboids");
+ testSpanningAndGetParent(scheduler, cube, new long[] { 440, 496, 376, 439, 487, 319, 375 });
+ // generate 5d
+ System.out.println("Spanning for 5D Cuboids");
+ testSpanningAndGetParent(scheduler, cube, new long[] { 432, 480, 312, 368, 423, 455, 311, 359 });
+ // generate 4d
+ System.out.println("Spanning for 4D Cuboids");
+ testSpanningAndGetParent(scheduler, cube, new long[] { 416, 448, 304, 352, 391, 295, 327 });
+ // generate 3d
+ System.out.println("Spanning for 3D Cuboids");
+ testSpanningAndGetParent(scheduler, cube, new long[] { 384, 288, 320, 263 });
+ // generate 2d
+ // generate 1d
+ // generate 0d
+ }
+
+ @Test
+ public void testGetSpanningCuboid1() {
+ CubeDesc cube = getTestKylinCubeWithoutSeller();
+ CuboidScheduler scheduler = new CuboidScheduler(cube);
+
+ long quiz = toLong("01100111");
+ testSpanningAndGetParent(scheduler, cube, new long[] { quiz });
+
+ // generate 7d
+ System.out.println("Spanning for 7D Cuboids");
+ testSpanningAndGetParent(scheduler, cube, new long[] { 255 });
+ // generate 6d
+ System.out.println("Spanning for 6D Cuboids");
+ testSpanningAndGetParent(scheduler, cube, new long[] { 135, 251, 253, 254 });
+ // generate 5d
+ System.out.println("Spanning for 5D Cuboids");
+ testSpanningAndGetParent(scheduler, cube, new long[] { 131, 133, 134, 249, 250, 252 });
+ // generate 4d
+ System.out.println("Spanning for 4D Cuboids");
+ testSpanningAndGetParent(scheduler, cube, new long[] { 129, 130, 132, 248 });
+ // generate 3d
+ System.out.println("Spanning for 3D Cuboids");
+ testSpanningAndGetParent(scheduler, cube, new long[] { 184, 240 });
+ // generate 2d
+ System.out.println("Spanning for 2D Cuboids");
+ testSpanningAndGetParent(scheduler, cube, new long[] { 176, 224 });
+ // generate 1d
+ System.out.println("Spanning for 1D Cuboids");
+ testSpanningAndGetParent(scheduler, cube, new long[] { 160, 192 });
+ // generate 0d
+ }
+
+ @Test
+ public void testGetSpanningCuboid() {
+ CubeDesc cube = getTestKylinCubeWithoutSeller();
+ CuboidScheduler scheduler = new CuboidScheduler(cube);
+
+ Collection<Long> spnanningCuboids = scheduler.getSpanningCuboid(248);
+
+ assertEquals("[]", spnanningCuboids.toString());
+ }
+
+ @Test
+ public void testGetCardinality() {
+ CubeDesc cube = getTestKylinCubeWithSeller();
+ CuboidScheduler scheduler = new CuboidScheduler(cube);
+
+ assertEquals(0, scheduler.getCardinality(0));
+ assertEquals(7, scheduler.getCardinality(127));
+ assertEquals(1, scheduler.getCardinality(1));
+ assertEquals(1, scheduler.getCardinality(8));
+ assertEquals(6, scheduler.getCardinality(126));
+ }
+
+ @Test
+ public void testCuboidGeneration1() {
+ CubeDesc cube = getTestKylinCubeWithoutSeller();
+ CuboidCLI.simulateCuboidGeneration(cube);
+ }
+
+ @Test
+ public void testCuboidGeneration2() {
+ CubeDesc cube = getTestKylinCubeWithSeller();
+ CuboidCLI.simulateCuboidGeneration(cube);
+ }
+
+ @Test
+ public void testCuboidGeneration3() {
+ CubeDesc cube = getTestKylinCubeWithoutSellerLeftJoin();
+ CuboidCLI.simulateCuboidGeneration(cube);
+ }
+
+ @Test
+ @Ignore
+ public void testCuboidGeneration4() {
+ CubeDesc cube = getMetadataManager().getCubeDesc("geox_trans_mtrc_sd_cube_desc");
+ CuboidCLI.simulateCuboidGeneration(cube);
+ }
+
+ @Test
+ @Ignore
+ public void testCuboidGeneration5() {
+ CubeDesc cube = getMetadataManager().getCubeDesc("clsfd_ga_dayweek");
+ CuboidCLI.simulateCuboidGeneration(cube);
+ }
+
+ @Test
+ @Ignore
+ public void testCuboidGeneration6() {
+ CubeDesc cube = getMetadataManager().getCubeDesc("clsfd_ga_day");
+ CuboidCLI.simulateCuboidGeneration(cube);
+ }
+
+ @Test
+ @Ignore
+ public void testCuboidGeneration7() {
+ CubeDesc cube = getMetadataManager().getCubeDesc("clsfd_ga_week");
+ CuboidCLI.simulateCuboidGeneration(cube);
+ }
+
+ @Test
+ public void testCuboidCounts1() {
+ CubeDesc cube = getTestKylinCubeWithoutSeller();
+ int[] counts = CuboidCLI.calculateAllLevelCount(cube);
+ printCount(counts);
+ assertArrayEquals(new int[] { 1, 4, 6, 6, 4, 4, 2, 0 }, counts);
+ }
+
+ @Test
+ public void testCuboidCounts2() {
+ CubeDesc cube = getTestKylinCubeWithSeller();
+ CuboidCLI.calculateAllLevelCount(cube);
+ int[] counts = CuboidCLI.calculateAllLevelCount(cube);
+ printCount(counts);
+ assertArrayEquals(new int[] { 1, 4, 7, 8, 7, 4 }, counts);
+ }
+
+ private String sortToString(Collection<Long> longs) {
+ ArrayList<Long> copy = new ArrayList<Long>(longs);
+ Collections.sort(copy);
+ return copy.toString();
+ }
+
+ private MetadataManager getMetadataManager() {
+ return MetadataManager.getInstance(getTestConfig());
+ }
+
+ private void printCount(int[] counts) {
+ int sum = 0;
+ for (int c : counts)
+ sum += c;
+ System.out.println(sum + " = " + Arrays.toString(counts));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidTest.java b/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidTest.java
new file mode 100644
index 0000000..6682c9d
--- /dev/null
+++ b/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidTest.java
@@ -0,0 +1,181 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.cube.cuboid;
+
+import static org.junit.Assert.*;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.kylinolap.common.util.LocalFileMetadataTestCase;
+import com.kylinolap.metadata.MetadataManager;
+import com.kylinolap.metadata.model.cube.CubeDesc;
+
+/**
+ * @author yangli9
+ */
+public class CuboidTest extends LocalFileMetadataTestCase {
+
+ private long toLong(String bin) {
+ return Long.parseLong(bin, 2);
+ }
+
+ private MetadataManager getMetadataManager() {
+ return MetadataManager.getInstance(getTestConfig());
+ }
+
+ private CubeDesc getTestKylinCubeII() {
+ return getMetadataManager().getCubeDesc("test_kylin_cube_ii");
+ }
+
+ private CubeDesc getTestKylinCubeWithoutSeller() {
+ return getMetadataManager().getCubeDesc("test_kylin_cube_without_slr_desc");
+ }
+
+ private CubeDesc getTestKylinCubeWithSeller() {
+ return getMetadataManager().getCubeDesc("test_kylin_cube_with_slr_desc");
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ MetadataManager.removeInstance(this.getTestConfig());
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testIsValid() {
+
+ CubeDesc cube = getTestKylinCubeWithSeller();
+
+ // base
+ assertEquals(false, Cuboid.isValid(cube, 0));
+ assertEquals(true, Cuboid.isValid(cube, toLong("111111111")));
+
+ // mandatory column
+ assertEquals(false, Cuboid.isValid(cube, toLong("011111110")));
+ assertEquals(false, Cuboid.isValid(cube, toLong("100000000")));
+
+ // zero tail
+ assertEquals(true, Cuboid.isValid(cube, toLong("111111000")));
+
+ // aggregation group & zero tail
+ assertEquals(true, Cuboid.isValid(cube, toLong("110000111")));
+ assertEquals(true, Cuboid.isValid(cube, toLong("110111000")));
+ assertEquals(true, Cuboid.isValid(cube, toLong("111110111")));
+ assertEquals(false, Cuboid.isValid(cube, toLong("111110001")));
+ assertEquals(false, Cuboid.isValid(cube, toLong("111110100")));
+ assertEquals(false, Cuboid.isValid(cube, toLong("110000100")));
+ }
+
+ @Test
+ public void testCuboid1() {
+ CubeDesc cube = getTestKylinCubeWithoutSeller();
+ Cuboid cuboid;
+
+ cuboid = Cuboid.findById(cube, 0);
+ assertEquals(toLong("10000001"), cuboid.getId());
+
+ cuboid = Cuboid.findById(cube, 1);
+ assertEquals(toLong("10000001"), cuboid.getId());
+
+ cuboid = Cuboid.findById(cube, toLong("00000010"));
+ assertEquals(toLong("10000010"), cuboid.getId());
+
+ cuboid = Cuboid.findById(cube, toLong("00100000"));
+ assertEquals(toLong("10100000"), cuboid.getId());
+
+ cuboid = Cuboid.findById(cube, toLong("01001000"));
+ assertEquals(toLong("11111000"), cuboid.getId());
+
+ cuboid = Cuboid.findById(cube, toLong("01000111"));
+ assertEquals(toLong("11111111"), cuboid.getId());
+
+ cuboid = Cuboid.findById(cube, toLong("11111111"));
+ assertEquals(toLong("11111111"), cuboid.getId());
+ }
+
+ @Test
+ public void testIsValid2() {
+ CubeDesc cube = getTestKylinCubeWithoutSeller();
+ try {
+ assertEquals(false, Cuboid.isValid(cube, toLong("111111111")));
+ fail();
+ } catch (IllegalArgumentException ex) {
+ // expected
+ }
+
+ // base
+ assertEquals(false, Cuboid.isValid(cube, 0));
+ assertEquals(true, Cuboid.isValid(cube, toLong("11111111")));
+
+ // aggregation group & zero tail
+ assertEquals(true, Cuboid.isValid(cube, toLong("10000111")));
+ assertEquals(false, Cuboid.isValid(cube, toLong("10001111")));
+ assertEquals(false, Cuboid.isValid(cube, toLong("11001111")));
+ assertEquals(true, Cuboid.isValid(cube, toLong("10000001")));
+ assertEquals(true, Cuboid.isValid(cube, toLong("10000101")));
+
+ // hierarchy
+ assertEquals(true, Cuboid.isValid(cube, toLong("10100000")));
+ assertEquals(true, Cuboid.isValid(cube, toLong("10110000")));
+ assertEquals(true, Cuboid.isValid(cube, toLong("10111000")));
+ assertEquals(false, Cuboid.isValid(cube, toLong("10001000")));
+ assertEquals(false, Cuboid.isValid(cube, toLong("10011000")));
+ }
+
+ @Test
+ public void testCuboid2() {
+ CubeDesc cube = getTestKylinCubeWithSeller();
+ Cuboid cuboid;
+
+ cuboid = Cuboid.findById(cube, 0);
+ assertEquals(toLong("100100000"), cuboid.getId());
+
+ cuboid = Cuboid.findById(cube, 1);
+ assertEquals(toLong("100000111"), cuboid.getId());
+
+ cuboid = Cuboid.findById(cube, toLong("010"));
+ assertEquals(toLong("100000111"), cuboid.getId());
+
+ cuboid = Cuboid.findById(cube, toLong("0100000"));
+ assertEquals(toLong("100100000"), cuboid.getId());
+
+ cuboid = Cuboid.findById(cube, toLong("1001000"));
+ assertEquals(toLong("101111000"), cuboid.getId());
+
+ cuboid = Cuboid.findById(cube, toLong("1000111"));
+ assertEquals(toLong("101000111"), cuboid.getId());
+
+ cuboid = Cuboid.findById(cube, toLong("111111111"));
+ assertEquals(toLong("111111111"), cuboid.getId());
+ }
+
+ @Test
+ public void testII() {
+ CubeDesc cube = getTestKylinCubeII();
+ assertEquals(toLong("111111111"), Cuboid.getBaseCuboidId(cube));
+ assertEquals(true, Cuboid.isValid(cube, toLong("111111111")));
+ assertEquals(false, Cuboid.isValid(cube, toLong("111111011")));
+ assertEquals(false, Cuboid.isValid(cube, toLong("101011011")));
+ assertEquals(false, Cuboid.isValid(cube, toLong("000000000")));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/cube/src/test/java/com/kylinolap/cube/dataGen/DataGenTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/com/kylinolap/cube/dataGen/DataGenTest.java b/cube/src/test/java/com/kylinolap/cube/dataGen/DataGenTest.java
new file mode 100644
index 0000000..8cd0532
--- /dev/null
+++ b/cube/src/test/java/com/kylinolap/cube/dataGen/DataGenTest.java
@@ -0,0 +1,32 @@
+package com.kylinolap.cube.dataGen;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.kylinolap.common.util.LocalFileMetadataTestCase;
+import com.kylinolap.metadata.MetadataManager;
+
+/**
+ * Created by honma on 6/19/14.
+ */
+public class DataGenTest extends LocalFileMetadataTestCase {
+
+ @Before
+ public void before() throws Exception {
+ this.createTestMetadata();
+ MetadataManager.removeInstance(this.getTestConfig());
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testBasics() throws Exception {
+ FactTableGenerator.generate("test_kylin_cube_with_slr_ready", "10000", "0.6", null, "left");// default
+ // settings
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/cube/src/test/java/com/kylinolap/cube/invertedindex/InvertedIndexLocalTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/com/kylinolap/cube/invertedindex/InvertedIndexLocalTest.java b/cube/src/test/java/com/kylinolap/cube/invertedindex/InvertedIndexLocalTest.java
new file mode 100644
index 0000000..2c42c5b
--- /dev/null
+++ b/cube/src/test/java/com/kylinolap/cube/invertedindex/InvertedIndexLocalTest.java
@@ -0,0 +1,198 @@
+package com.kylinolap.cube.invertedindex;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Pair;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+import com.kylinolap.common.util.BytesUtil;
+import com.kylinolap.common.util.LocalFileMetadataTestCase;
+import com.kylinolap.cube.CubeInstance;
+import com.kylinolap.cube.CubeManager;
+import com.kylinolap.dict.Dictionary;
+
+public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
+
+ CubeInstance cube;
+ TableRecordInfo info;
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ this.cube = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_ii");
+ this.info = new TableRecordInfo(cube.getFirstSegment());
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testBitMapContainer() {
+ // create container
+ BitMapContainer container = new BitMapContainer(info, 0);
+ Dictionary<String> dict = info.dict(0);
+ for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
+ container.append(v);
+ }
+ container.append(Dictionary.NULL_ID[dict.getSizeOfId()]);
+ container.closeForChange();
+
+ // copy by serialization
+ List<ImmutableBytesWritable> bytes = container.toBytes();
+ BitMapContainer container2 = new BitMapContainer(info, 0);
+ container2.fromBytes(bytes);
+
+ // check the copy
+ int i = 0;
+ for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
+ int value = container2.getValueIntAt(i++);
+ assertEquals(v, value);
+ }
+ assertEquals(Dictionary.NULL_ID[dict.getSizeOfId()], container2.getValueIntAt(i++));
+ assertEquals(container, container2);
+ }
+
+ @Test
+ public void testCompressedValueContainer() {
+ // create container
+ CompressedValueContainer container = new CompressedValueContainer(info, 0, 500);
+ Dictionary<String> dict = info.dict(0);
+
+ byte[] buf = new byte[dict.getSizeOfId()];
+ ImmutableBytesWritable bytes = new ImmutableBytesWritable(buf);
+
+ for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
+ BytesUtil.writeUnsigned(v, buf, 0, dict.getSizeOfId());
+ container.append(bytes);
+ }
+ BytesUtil.writeUnsigned(Dictionary.NULL_ID[dict.getSizeOfId()], buf, 0, dict.getSizeOfId());
+ container.append(bytes);
+ container.closeForChange();
+
+ // copy by serialization
+ ImmutableBytesWritable copy = container.toBytes();
+ CompressedValueContainer container2 = new CompressedValueContainer(info, 0, 500);
+ container2.fromBytes(copy);
+
+ // check the copy
+ int i = 0;
+ for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
+ container2.getValueAt(i++, bytes);
+ int value = BytesUtil.readUnsigned(bytes.get(), bytes.getOffset(), bytes.getLength());
+ assertEquals(v, value);
+ }
+ container2.getValueAt(i++, bytes);
+ int value = BytesUtil.readUnsigned(bytes.get(), bytes.getOffset(), bytes.getLength());
+ assertEquals(Dictionary.NULL_ID[dict.getSizeOfId()], value);
+ assertEquals(container, container2);
+ }
+
+ @Test
+ public void testCodec() throws IOException {
+ List<TableRecord> records = loadRecordsSorted();
+ System.out.println(records.size() + " records");
+ List<Slice> slices = buildTimeSlices(records);
+ System.out.println(slices.size() + " slices");
+
+ IIKeyValueCodec codec = new IIKeyValueCodec(info);
+ List<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> kvs = encodeKVs(codec, slices);
+ System.out.println(kvs.size() + " KV pairs");
+
+ List<Slice> slicesCopy = decodeKVs(codec, kvs);
+ assertEquals(slices, slicesCopy);
+
+ List<TableRecord> recordsCopy = iterateRecords(slicesCopy);
+ assertEquals(new HashSet<TableRecord>(records), new HashSet<TableRecord>(recordsCopy));
+ dump(recordsCopy);
+ }
+
+ private List<TableRecord> loadRecordsSorted() throws IOException {
+ File file = new File(LOCALMETA_TEST_DATA, "data/TEST_KYLIN_FACT.csv");
+ FileInputStream in = new FileInputStream(file);
+ List<String> lines = IOUtils.readLines(in, "UTF-8");
+ in.close();
+
+ List<TableRecord> records = Lists.newArrayList();
+ for (String line : lines) {
+ String[] fields = line.split(",");
+ TableRecord rec = new TableRecord(info);
+ for (int col = 0; col < fields.length; col++) {
+ rec.setValueString(col, fields[col]);
+ }
+ records.add(rec);
+ }
+
+ Collections.sort(records, new Comparator<TableRecord>() {
+ @Override
+ public int compare(TableRecord a, TableRecord b) {
+ return (int) (a.getTimestamp() - b.getTimestamp());
+ }
+ });
+
+ return records;
+ }
+
+ private List<Slice> buildTimeSlices(List<TableRecord> records) throws IOException {
+ ShardingSliceBuilder builder = new ShardingSliceBuilder(info);
+ List<Slice> slices = Lists.newArrayList();
+ for (TableRecord rec : records) {
+ Slice slice = builder.append(rec);
+ if (slice != null)
+ slices.add(slice);
+ }
+ List<Slice> finals = builder.close();
+ slices.addAll(finals);
+
+ Collections.sort(slices);
+ return slices;
+ }
+
+ private List<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> encodeKVs(IIKeyValueCodec codec, List<Slice> slices) {
+
+ List<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> kvs = Lists.newArrayList();
+ for (Slice slice : slices) {
+ kvs.addAll(codec.encodeKeyValue(slice));
+ }
+ return kvs;
+ }
+
+ private List<Slice> decodeKVs(IIKeyValueCodec codec, List<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> kvs) {
+ List<Slice> slices = Lists.newArrayList();
+ for (Slice slice : codec.decodeKeyValue(kvs)) {
+ slices.add(slice);
+ }
+ return slices;
+ }
+
+ private List<TableRecord> iterateRecords(List<Slice> slices) {
+ List<TableRecord> records = Lists.newArrayList();
+ for (Slice slice : slices) {
+ for (TableRecordBytes rec : slice) {
+ records.add((TableRecord) rec.clone());
+ }
+ }
+ return records;
+ }
+
+ private void dump(Iterable<TableRecord> records) {
+ for (TableRecord rec : records) {
+ System.out.println(rec.toString());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/cube/src/test/java/com/kylinolap/cube/kv/RowKeyDecoderTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/com/kylinolap/cube/kv/RowKeyDecoderTest.java b/cube/src/test/java/com/kylinolap/cube/kv/RowKeyDecoderTest.java
new file mode 100644
index 0000000..b78b2fb
--- /dev/null
+++ b/cube/src/test/java/com/kylinolap/cube/kv/RowKeyDecoderTest.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.cube.kv;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.kylinolap.common.util.LocalFileMetadataTestCase;
+import com.kylinolap.cube.CubeInstance;
+import com.kylinolap.cube.CubeManager;
+import com.kylinolap.cube.cuboid.Cuboid;
+import com.kylinolap.metadata.MetadataManager;
+import com.kylinolap.metadata.model.cube.CubeDesc;
+
+/**
+ * @author George Song (ysong1)
+ *
+ */
+public class RowKeyDecoderTest extends LocalFileMetadataTestCase {
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ MetadataManager.removeInstance(this.getTestConfig());
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testDecodeWithoutSlr() throws Exception {
+ CubeInstance cube = CubeManager.getInstance(this.getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");
+
+ RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment());
+
+ byte[] key = { 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 };
+
+ rowKeyDecoder.decode(key);
+ List<String> names = rowKeyDecoder.getNames(null);
+ List<String> values = rowKeyDecoder.getValues();
+
+ assertEquals("[CAL_DT, LEAF_CATEG_ID, META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME, LSTG_FORMAT_NAME, LSTG_SITE_ID, SLR_SEGMENT_CD]", names.toString());
+ assertEquals("[2012-12-15, 11848, Health & Beauty, Fragrances, Women, FP-GTC, 0, 15]", values.toString());
+
+ }
+
+ @Test
+ public void testDecodeWithSlr() throws Exception {
+ CubeInstance cube = CubeManager.getInstance(this.getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");
+
+ RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment());
+
+ byte[] key = { 0, 0, 0, 0, 0, 0, 1, -1, 49, 48, 48, 48, 48, 48, 48, 48, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 11, 54, -105, 55, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
+
+ rowKeyDecoder.decode(key);
+ List<String> names = rowKeyDecoder.getNames(null);
+ List<String> values = rowKeyDecoder.getValues();
+
+ assertEquals("[SELLER_ID, CAL_DT, LEAF_CATEG_ID, META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME, LSTG_FORMAT_NAME, LSTG_SITE_ID, SLR_SEGMENT_CD]", names.toString());
+ assertEquals("[10000000, 2012-01-02, 20213, Collectibles, Postcards, US StateCities & Towns, ABIN, 0, -99]", values.toString());
+
+ }
+
+ @Test
+ public void testEncodeAndDecodeWithUtf8() throws IOException {
+ CubeInstance cube = CubeManager.getInstance(this.getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");
+ CubeDesc cubeDesc = cube.getDescriptor();
+
+ byte[][] data = new byte[8][];
+ data[0] = Bytes.toBytes("2012-12-15");
+ data[1] = Bytes.toBytes("11848");
+ data[2] = Bytes.toBytes("Health & Beauty");
+ data[3] = Bytes.toBytes("Fragrances");
+ data[4] = Bytes.toBytes("Women");
+ data[5] = Bytes.toBytes("刊登格式测试");// UTF-8
+ data[6] = Bytes.toBytes("0");
+ data[7] = Bytes.toBytes("15");
+
+ long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
+ Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+ AbstractRowKeyEncoder rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cube.getFirstSegment(), baseCuboid);
+
+ byte[] encodedKey = rowKeyEncoder.encode(data);
+ assertEquals(30, encodedKey.length);
+
+ RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment());
+ rowKeyDecoder.decode(encodedKey);
+ List<String> names = rowKeyDecoder.getNames(null);
+ List<String> values = rowKeyDecoder.getValues();
+ assertEquals("[CAL_DT, LEAF_CATEG_ID, META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME, LSTG_FORMAT_NAME, LSTG_SITE_ID, SLR_SEGMENT_CD]", names.toString());
+ assertEquals("[2012-12-15, 11848, Health & Beauty, Fragrances, Women, 刊登格式, 0, 15]", values.toString());
+ }
+}