You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/01/14 15:16:13 UTC
[38/51] [partial] incubator-kylin git commit: cleanup for migration
from github.com
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java b/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java
deleted file mode 100644
index f6f81c1..0000000
--- a/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java
+++ /dev/null
@@ -1,710 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.IOException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.List;
-import java.util.TimeZone;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.kylinolap.common.persistence.ResourceStore;
-import com.kylinolap.common.util.JsonUtil;
-import com.kylinolap.common.util.LocalFileMetadataTestCase;
-import com.kylinolap.cube.exception.CubeIntegrityException;
-import com.kylinolap.cube.project.ProjectInstance;
-import com.kylinolap.cube.project.ProjectManager;
-import com.kylinolap.metadata.MetadataManager;
-import com.kylinolap.metadata.model.cube.CubeDesc;
-
-/**
- * @author ysong1
- *
- */
-public class SegmentManagementTest extends LocalFileMetadataTestCase {
-
- CubeManager cubeMgr = null;
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- MetadataManager.removeInstance(this.getTestConfig());
- CubeManager.removeInstance(this.getTestConfig());
- ProjectManager.removeInstance(this.getTestConfig());
- cubeMgr = CubeManager.getInstance(this.getTestConfig());
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- private void createNewCube(CubeDesc cubeDesc) throws IOException {
- ResourceStore store = getStore();
- // clean legacy in case last run failed
- store.deleteResource("/cube/a_whole_new_cube.json");
-
- CubeInstance createdCube = cubeMgr.createCube("a_whole_new_cube", ProjectInstance.DEFAULT_PROJECT_NAME, cubeDesc, "username");
- assertTrue(createdCube == cubeMgr.getCube("a_whole_new_cube"));
-
- System.out.println(JsonUtil.writeValueAsIndentString(createdCube));
- }
-
- @Test
- public void testInitialAndAppend() throws ParseException, IOException, CubeIntegrityException {
- // create a new cube
- MetadataManager metaMgr = getMetadataManager();
- CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
- createNewCube(desc);
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- long dateEnd = f.parse("2013-11-12").getTime();
-
- CubeInstance cubeInstance = cubeMgr.getCube("a_whole_new_cube");
- assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
- assertEquals(0, cubeInstance.getSegments().size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getAllocatedEndDate());
-
- // initial build
- System.out.println("Initial Build");
- CubeSegment initialSegment = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, desc.getCubePartitionDesc().getPartitionDateStart(), dateEnd).get(0);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
- assertEquals(CubeSegmentStatusEnum.NEW, cubeInstance.getBuildingSegments().get(0).getStatus());
- assertEquals(1, cubeInstance.getSegments().size());
- assertEquals(1, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getRebuildingSegments().size());
- assertTrue("".equals(initialSegment.getStorageLocationIdentifier()) == false);
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
- assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
-
- // initial build success
- System.out.println("Initial Build Success");
- cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, initialSegment.getName(), "job_1", System.currentTimeMillis(), 111L, 222L, 333L);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getRebuildingSegments().size());
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
- assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
-
- // incremental build
- System.out.println("Incremental Build");
- long dateEnd2 = f.parse("2013-12-12").getTime();
- CubeSegment incrementalSegment = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, dateEnd, dateEnd2).get(0);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(2, cubeInstance.getSegments().size());
- assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.NEW).size());
- assertEquals(1, cubeInstance.getBuildingSegments().size());
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
- assertEquals(dateEnd2, cubeInstance.getAllocatedEndDate());
- assertEquals(dateEnd, cubeInstance.getBuildingSegments().get(0).getDateRangeStart());
- assertEquals(dateEnd2, cubeInstance.getBuildingSegments().get(0).getDateRangeEnd());
-
- // incremental build success
- System.out.println("Incremental Build Success");
- cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, incrementalSegment.getName(), "job_2", System.currentTimeMillis(), 111L, 222L, 333L);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(2, cubeInstance.getSegments().size());
- assertEquals(2, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
- assertEquals(dateEnd2, cubeInstance.getAllocatedEndDate());
- }
-
- @Test
- public void testRebuildSegment() throws IOException, CubeIntegrityException {
- CubeInstance cubeInstance = cubeMgr.getCube("test_kylin_cube_with_slr_ready");
-
- // rebuild segment
- System.out.println("Rebuild Segment");
- CubeSegment rebuildSegment = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, 1364688000000L, 1386806400000L).get(0);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(1, cubeInstance.getBuildingSegments().size());
- assertEquals(CubeSegmentStatusEnum.NEW, cubeInstance.getBuildingSegments().get(0).getStatus());
- assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
- assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.NEW).size());
- assertEquals(1, cubeInstance.getRebuildingSegments().size());
- assertEquals(1364688000000L, cubeInstance.getAllocatedStartDate());
- assertEquals(1386806400000L, cubeInstance.getAllocatedEndDate());
-
- // rebuild success
- System.out.println("Rebuild Success");
- cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, rebuildSegment.getName(), "job_3", System.currentTimeMillis(), 111, 222, 333);
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(1, cubeInstance.getSegments().size());
- assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getRebuildingSegments().size());
- assertEquals(1364688000000L, cubeInstance.getAllocatedStartDate());
- assertEquals(1386806400000L, cubeInstance.getAllocatedEndDate());
- assertEquals("job_3", cubeInstance.getSegments().get(0).getLastBuildJobID());
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- }
-
- @Test(expected = CubeIntegrityException.class)
- public void testInvalidRebuild() throws IOException, CubeIntegrityException {
- CubeInstance cubeInstance = cubeMgr.getCube("test_kylin_cube_with_slr_ready");
-
- // rebuild segment
- System.out.println("Rebuild Segment");
- cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, 1364688000000L + 1000L, 1386806400000L).get(0);
- }
-
- @Test
- public void testMergeSegments() throws IOException, CubeIntegrityException {
- CubeInstance cubeInstance = cubeMgr.getCube("test_kylin_cube_with_slr_ready_2_segments");
-
- // merge segments
- System.out.println("Merge Segment");
- CubeSegment mergedSegment = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.MERGE, 1384240200000L, 1386835200000L).get(0);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(CubeSegmentStatusEnum.NEW, cubeInstance.getBuildingSegments().get(0).getStatus());
- assertEquals(2, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
- assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.NEW).size());
- assertEquals(2, cubeInstance.getMergingSegments().size());
- assertEquals(1384240200000L, cubeInstance.getAllocatedStartDate());
- assertEquals(1386835200000L, cubeInstance.getAllocatedEndDate());
-
- // build success
- System.out.println("Build Success");
- cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.MERGE, mergedSegment.getName(), "job_4", System.currentTimeMillis(), 123, 20000L, 1216024L);
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getMergingSegments().size());
- assertEquals(1384240200000L, cubeInstance.getAllocatedStartDate());
- assertEquals(1386835200000L, cubeInstance.getAllocatedEndDate());
- assertEquals("job_4", cubeInstance.getSegments().get(0).getLastBuildJobID());
- assertEquals(20000L, cubeInstance.getSegments().get(0).getSourceRecords());
- assertEquals(1216024L, cubeInstance.getSegments().get(0).getSourceRecordsSize());
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- }
-
- @Test
- public void testNonPartitionedCube() throws ParseException, IOException, CubeIntegrityException {
- // create a new cube
- MetadataManager metaMgr = getMetadataManager();
- CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_without_slr_desc");
- createNewCube(desc);
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- long dateEnd = f.parse("2013-11-12").getTime();
-
- CubeInstance cubeInstance = cubeMgr.getCube("a_whole_new_cube");
- assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
- assertEquals(0, cubeInstance.getSegments().size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getAllocatedEndDate());
-
- // initial build
- System.out.println("Initial Build");
- CubeSegment initialSegment = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, desc.getCubePartitionDesc().getPartitionDateStart(), dateEnd).get(0);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
- assertEquals(CubeSegmentStatusEnum.NEW, cubeInstance.getBuildingSegments().get(0).getStatus());
- assertEquals(1, cubeInstance.getSegments().size());
- assertEquals(0, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
- assertEquals(1, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getRebuildingSegments().size());
- assertTrue("".equals(initialSegment.getStorageLocationIdentifier()) == false);
- assertEquals("FULL_BUILD", initialSegment.getName());
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
- assertEquals(0, cubeInstance.getAllocatedEndDate());
-
- // initial build success
- System.out.println("Initial Build Success");
- cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, initialSegment.getName(), "job_5", System.currentTimeMillis(), 111L, 222L, 333L);
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getRebuildingSegments().size());
- assertEquals(0, cubeInstance.getAllocatedStartDate());
- assertEquals(0, cubeInstance.getAllocatedEndDate());
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
-
- // rebuild segment
- System.out.println("Rebuild Segment");
- CubeSegment rebuildSegment = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, 1364688000000L, 1386806400000L).get(0);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(CubeSegmentStatusEnum.NEW, cubeInstance.getBuildingSegments().get(0).getStatus());
- assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
- assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.NEW).size());
- assertEquals(1, cubeInstance.getBuildingSegments().size());
- assertEquals(1, cubeInstance.getRebuildingSegments().size());
- assertEquals(0, cubeInstance.getAllocatedStartDate());
- assertEquals(0, cubeInstance.getAllocatedEndDate());
-
- // rebuild success
- System.out.println("Rebuild Success");
- cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, rebuildSegment.getName(), "job_6", System.currentTimeMillis(), 111, 222, 333);
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(1, cubeInstance.getSegments().size());
- assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getRebuildingSegments().size());
- assertEquals(0, cubeInstance.getAllocatedStartDate());
- assertEquals(0, cubeInstance.getAllocatedEndDate());
- assertEquals("job_6", cubeInstance.getSegments().get(0).getLastBuildJobID());
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- }
-
- @Test(expected = CubeIntegrityException.class)
- public void testInvalidAppend() throws ParseException, IOException, CubeIntegrityException {
- // create a new cube
- MetadataManager metaMgr = getMetadataManager();
- CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
- createNewCube(desc);
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- long dateEnd = f.parse("2013-11-12").getTime();
-
- CubeInstance cubeInstance = cubeMgr.getCube("a_whole_new_cube");
- assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
- assertEquals(0, cubeInstance.getSegments().size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getAllocatedEndDate());
-
- // initial build
- System.out.println("Initial Build");
- CubeSegment initialSegment = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, desc.getCubePartitionDesc().getPartitionDateStart(), dateEnd).get(0);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
- assertEquals(CubeSegmentStatusEnum.NEW, cubeInstance.getBuildingSegments().get(0).getStatus());
- assertEquals(1, cubeInstance.getSegments().size());
- assertEquals(1, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getRebuildingSegments().size());
- assertTrue("".equals(initialSegment.getStorageLocationIdentifier()) == false);
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
- assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
-
- // initial build success
- System.out.println("Initial Build Success");
- cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, initialSegment.getName(), "job_1", System.currentTimeMillis(), 111L, 222L, 333L);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getRebuildingSegments().size());
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
- assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
-
- // incremental build
- System.out.println("Invalid Incremental Build");
- long dateEnd2 = f.parse("2013-12-12").getTime();
- cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, dateEnd + 1000, dateEnd2);
-
- }
-
- @Test
- public void testInitialAndUpsert() throws ParseException, IOException, CubeIntegrityException {
- // create a new cube
- MetadataManager metaMgr = getMetadataManager();
- CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_without_slr_left_join_desc");
- createNewCube(desc);
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- long dateEnd = f.parse("2013-11-12").getTime();
-
- CubeInstance cubeInstance = cubeMgr.getCube("a_whole_new_cube");
- assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
- assertEquals(0, cubeInstance.getSegments().size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getAllocatedEndDate());
-
- // initial build
- System.out.println("Initial Build");
- CubeSegment initialSegment = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, desc.getCubePartitionDesc().getPartitionDateStart(), dateEnd).get(0);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
- for (CubeSegment cubeSegment : cubeInstance.getBuildingSegments()) {
- assertEquals(CubeSegmentStatusEnum.NEW, cubeSegment.getStatus());
- }
- assertEquals(1, cubeInstance.getSegments().size());
- assertEquals(1, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getRebuildingSegments().size());
- assertTrue("".equals(initialSegment.getStorageLocationIdentifier()) == false);
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
- assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
-
- // initial build success
- System.out.println("Initial Build Success");
- cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, initialSegment.getName(), "job_1", System.currentTimeMillis(), 111L, 222L, 333L);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
- assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
-
- // upsert build
- System.out.println("Upsert Build");
- long start = f.parse("2013-11-01").getTime();
- long dateEnd2 = f.parse("2013-12-12").getTime();
- List<CubeSegment> upsertSegments = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, start, dateEnd2);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(2, upsertSegments.size());
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(3, cubeInstance.getSegments().size());
- assertEquals(2, cubeInstance.getSegments(CubeSegmentStatusEnum.NEW).size());
- assertEquals(2, cubeInstance.getBuildingSegments().size());
- assertEquals(1, cubeInstance.getRebuildingSegments().size());
- assertEquals(0, cubeInstance.getMergingSegments().size());
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
- assertEquals(dateEnd2, cubeInstance.getAllocatedEndDate());
- assertEquals(0L, cubeInstance.getBuildingSegments().get(0).getDateRangeStart());
- assertEquals(dateEnd2, cubeInstance.getBuildingSegments().get(1).getDateRangeEnd());
-
- // upsert build success
- System.out.println("Upsert Build Success");
- cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, upsertSegments.get(0).getName(), "job_2", System.currentTimeMillis(), 111L, 222L, 333L);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, upsertSegments.get(1).getName(), "job_3", System.currentTimeMillis(), 111L, 222L, 333L);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(2, cubeInstance.getSegments().size());
- assertEquals(2, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
- assertEquals(dateEnd2, cubeInstance.getAllocatedEndDate());
-
- // upsert build again
- System.out.println("Upsert Build");
- long start2 = f.parse("2013-12-01").getTime();
- long dateEnd3 = f.parse("2013-12-31").getTime();
- List<CubeSegment> upsertSegments2 = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, start2, dateEnd3);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(2, upsertSegments2.size());
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(4, cubeInstance.getSegments().size());
- assertEquals(2, cubeInstance.getSegments(CubeSegmentStatusEnum.NEW).size());
- assertEquals(2, cubeInstance.getBuildingSegments().size());
- assertEquals(1, cubeInstance.getRebuildingSegments().size());
- assertEquals(0, cubeInstance.getMergingSegments().size());
- assertEquals(dateEnd3, cubeInstance.getAllocatedEndDate());
- // building segment 1 from 2013-11-01 to 2013-12-01
- assertEquals(f.parse("2013-11-01").getTime(), cubeInstance.getBuildingSegments().get(0).getDateRangeStart());
- assertEquals(f.parse("2013-12-01").getTime(), cubeInstance.getBuildingSegments().get(0).getDateRangeEnd());
- // building segment 2 from 2013-12-01 to 2013-12-31
- assertEquals(f.parse("2013-12-01").getTime(), cubeInstance.getBuildingSegments().get(1).getDateRangeStart());
- assertEquals(f.parse("2013-12-31").getTime(), cubeInstance.getBuildingSegments().get(1).getDateRangeEnd());
-
- // upsert build success
- System.out.println("Upsert Build Success");
- cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, upsertSegments2.get(1).getName(), "job_5", System.currentTimeMillis(), 111L, 222L, 333L);
- cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, upsertSegments2.get(0).getName(), "job_4", System.currentTimeMillis(), 111L, 222L, 333L);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(3, cubeInstance.getSegments().size());
- assertEquals(3, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getMergingSegments().size());
- assertEquals(cubeInstance.getDescriptor().getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
- assertEquals(dateEnd3, cubeInstance.getAllocatedEndDate());
- // segment 1 from 1970-01-01 to 2013-11-01
- assertEquals(cubeInstance.getDescriptor().getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getSegments().get(0).getDateRangeStart());
- assertEquals(f.parse("2013-11-01").getTime(), cubeInstance.getSegments().get(0).getDateRangeEnd());
- // segment 2 from 2013-11-01 to 2013-12-01
- assertEquals(f.parse("2013-11-01").getTime(), cubeInstance.getSegments().get(1).getDateRangeStart());
- assertEquals(f.parse("2013-12-01").getTime(), cubeInstance.getSegments().get(1).getDateRangeEnd());
- // segment 3 from 2013-12-01 to 2013-12-31
- assertEquals(f.parse("2013-12-01").getTime(), cubeInstance.getSegments().get(2).getDateRangeStart());
- assertEquals(f.parse("2013-12-31").getTime(), cubeInstance.getSegments().get(2).getDateRangeEnd());
-
- // upsert build again
- System.out.println("Upsert Build");
- List<CubeSegment> upsertSegments3 = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, f.parse("2013-10-01").getTime(), f.parse("2014-02-01").getTime());
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(2, upsertSegments3.size());
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(5, cubeInstance.getSegments().size());
- assertEquals(2, cubeInstance.getSegments(CubeSegmentStatusEnum.NEW).size());
- assertEquals(2, cubeInstance.getBuildingSegments().size());
- assertEquals(3, cubeInstance.getRebuildingSegments().size());
- assertEquals(0, cubeInstance.getMergingSegments().size());
- assertEquals(f.parse("2014-02-01").getTime(), cubeInstance.getAllocatedEndDate());
- // building segment 1 from 2013-11-01 to 2013-10-01
- assertEquals(cubeInstance.getDescriptor().getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getBuildingSegments().get(0).getDateRangeStart());
- assertEquals(f.parse("2013-10-01").getTime(), cubeInstance.getBuildingSegments().get(0).getDateRangeEnd());
- // building segment 2 from 2013-10-01 to 2014-02-01
- assertEquals(f.parse("2013-10-01").getTime(), cubeInstance.getBuildingSegments().get(1).getDateRangeStart());
- assertEquals(f.parse("2014-02-01").getTime(), cubeInstance.getBuildingSegments().get(1).getDateRangeEnd());
-
- // upsert build success
- System.out.println("Upsert Build Success");
- cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, upsertSegments3.get(1).getName(), "job_7", System.currentTimeMillis(), 111L, 222L, 333L);
- cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, upsertSegments3.get(0).getName(), "job_6", System.currentTimeMillis(), 111L, 222L, 333L);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(2, cubeInstance.getSegments().size());
- assertEquals(2, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getMergingSegments().size());
- assertEquals(cubeInstance.getDescriptor().getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
- assertEquals(f.parse("2014-02-01").getTime(), cubeInstance.getAllocatedEndDate());
- // segment 1 from 1970-01-01 to 2013-10-01
- assertEquals(cubeInstance.getDescriptor().getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getSegments().get(0).getDateRangeStart());
- assertEquals(f.parse("2013-10-01").getTime(), cubeInstance.getSegments().get(0).getDateRangeEnd());
- // segment 2 from 2013-10-01 to 2014-02-01
- assertEquals(f.parse("2013-10-01").getTime(), cubeInstance.getSegments().get(1).getDateRangeStart());
- assertEquals(f.parse("2014-02-01").getTime(), cubeInstance.getSegments().get(1).getDateRangeEnd());
- }
-
- @Test
- public void testInitialAndUpsert2() throws ParseException, IOException, CubeIntegrityException {
- // create a new cube
- MetadataManager metaMgr = getMetadataManager();
- CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_without_slr_left_join_desc");
- createNewCube(desc);
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- long dateEnd = f.parse("2013-01-01").getTime();
-
- CubeInstance cubeInstance = cubeMgr.getCube("a_whole_new_cube");
- assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
- assertEquals(0, cubeInstance.getSegments().size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getAllocatedEndDate());
-
- // initial build
- System.out.println("Initial Build");
- CubeSegment initialSegment = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, desc.getCubePartitionDesc().getPartitionDateStart(), dateEnd).get(0);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
- for (CubeSegment cubeSegment : cubeInstance.getBuildingSegments()) {
- assertEquals(CubeSegmentStatusEnum.NEW, cubeSegment.getStatus());
- }
- assertEquals(1, cubeInstance.getSegments().size());
- assertEquals(1, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getRebuildingSegments().size());
- assertTrue("".equals(initialSegment.getStorageLocationIdentifier()) == false);
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
- assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
-
- // initial build success
- System.out.println("Initial Build Success");
- cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, initialSegment.getName(), "job_1", System.currentTimeMillis(), 111L, 222L, 333L);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
- assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
-
- // upsert build
- System.out.println("Upsert Build");
- long start = f.parse("2013-01-01").getTime();
- long dateEnd2 = f.parse("2014-01-01").getTime();
- List<CubeSegment> upsertSegments = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, start, dateEnd2);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(1, upsertSegments.size());
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(2, cubeInstance.getSegments().size());
- assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.NEW).size());
- assertEquals(1, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getRebuildingSegments().size());
- assertEquals(0, cubeInstance.getMergingSegments().size());
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
- assertEquals(dateEnd2, cubeInstance.getAllocatedEndDate());
- assertEquals(start, cubeInstance.getBuildingSegments().get(0).getDateRangeStart());
- assertEquals(dateEnd2, cubeInstance.getBuildingSegments().get(0).getDateRangeEnd());
-
- // upsert build success
- System.out.println("Upsert Build Success");
- cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, upsertSegments.get(0).getName(), "job_2", System.currentTimeMillis(), 111L, 222L, 333L);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
-
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(2, cubeInstance.getSegments().size());
- assertEquals(2, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
- assertEquals(dateEnd2, cubeInstance.getAllocatedEndDate());
-
- // upsert build again
- System.out.println("Upsert Build");
- List<CubeSegment> upsertSegments3 = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, f.parse("2013-10-01").getTime(), f.parse("2014-02-01").getTime());
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(2, upsertSegments3.size());
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(4, cubeInstance.getSegments().size());
- assertEquals(2, cubeInstance.getSegments(CubeSegmentStatusEnum.NEW).size());
- assertEquals(2, cubeInstance.getBuildingSegments().size());
- assertEquals(1, cubeInstance.getRebuildingSegments().size());
- assertEquals(0, cubeInstance.getMergingSegments().size());
- assertEquals(f.parse("2014-02-01").getTime(), cubeInstance.getAllocatedEndDate());
- // building segment 1 from 2013-11-01 to 2013-10-01
- assertEquals(f.parse("2013-01-01").getTime(), cubeInstance.getBuildingSegments().get(0).getDateRangeStart());
- assertEquals(f.parse("2013-10-01").getTime(), cubeInstance.getBuildingSegments().get(0).getDateRangeEnd());
- // building segment 2 from 2013-10-01 to 2014-02-01
- assertEquals(f.parse("2013-10-01").getTime(), cubeInstance.getBuildingSegments().get(1).getDateRangeStart());
- assertEquals(f.parse("2014-02-01").getTime(), cubeInstance.getBuildingSegments().get(1).getDateRangeEnd());
-
- // upsert build success
- System.out.println("Upsert Build Success");
- cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, upsertSegments3.get(1).getName(), "job_7", System.currentTimeMillis(), 111L, 222L, 333L);
- cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, upsertSegments3.get(0).getName(), "job_6", System.currentTimeMillis(), 111L, 222L, 333L);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(3, cubeInstance.getSegments().size());
- assertEquals(3, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getMergingSegments().size());
- assertEquals(cubeInstance.getDescriptor().getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
- assertEquals(f.parse("2014-02-01").getTime(), cubeInstance.getAllocatedEndDate());
- // segment 1 from 1970-01-01 to 2013-10-01
- assertEquals(cubeInstance.getDescriptor().getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getSegments().get(0).getDateRangeStart());
- assertEquals(f.parse("2013-01-01").getTime(), cubeInstance.getSegments().get(0).getDateRangeEnd());
- // segment 2 from 2013-10-01 to 2014-02-01
- assertEquals(f.parse("2013-01-01").getTime(), cubeInstance.getSegments().get(1).getDateRangeStart());
- assertEquals(f.parse("2013-10-01").getTime(), cubeInstance.getSegments().get(1).getDateRangeEnd());
- // segment 3 from 2013-10-01 to 2014-02-01
- assertEquals(f.parse("2013-10-01").getTime(), cubeInstance.getSegments().get(2).getDateRangeStart());
- assertEquals(f.parse("2014-02-01").getTime(), cubeInstance.getSegments().get(2).getDateRangeEnd());
- }
-
- @Test(expected = CubeIntegrityException.class)
- public void testInvalidUpsert() throws IOException, CubeIntegrityException, ParseException {
- // create a new cube
- MetadataManager metaMgr = getMetadataManager();
- CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_without_slr_left_join_desc");
- createNewCube(desc);
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- long dateEnd = f.parse("2013-11-12").getTime();
-
- CubeInstance cubeInstance = cubeMgr.getCube("a_whole_new_cube");
- assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
- assertEquals(0, cubeInstance.getSegments().size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getAllocatedEndDate());
-
- // initial build
- System.out.println("Initial Build");
- CubeSegment initialSegment = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, desc.getCubePartitionDesc().getPartitionDateStart(), dateEnd).get(0);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
- for (CubeSegment cubeSegment : cubeInstance.getBuildingSegments()) {
- assertEquals(CubeSegmentStatusEnum.NEW, cubeSegment.getStatus());
- }
- assertEquals(1, cubeInstance.getSegments().size());
- assertEquals(1, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getRebuildingSegments().size());
- assertTrue("".equals(initialSegment.getStorageLocationIdentifier()) == false);
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
- assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
-
- // initial build success
- System.out.println("Initial Build Success");
- cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, initialSegment.getName(), "job_1", System.currentTimeMillis(), 111L, 222L, 333L);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
- assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
-
- // upsert build
- // time gap in new segment
- System.out.println("Upsert Build");
- long start = f.parse("2013-11-13").getTime();
- long dateEnd2 = f.parse("2013-12-12").getTime();
- cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, start, dateEnd2);
- }
-
- @Test(expected = CubeIntegrityException.class)
- public void testInvalidUpsert2() throws IOException, CubeIntegrityException, ParseException {
- // create a new cube
- MetadataManager metaMgr = getMetadataManager();
- CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_without_slr_left_join_desc");
- createNewCube(desc);
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- long dateEnd = f.parse("2013-11-12").getTime();
-
- CubeInstance cubeInstance = cubeMgr.getCube("a_whole_new_cube");
- assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
- assertEquals(0, cubeInstance.getSegments().size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getAllocatedEndDate());
-
- // initial build
- System.out.println("Initial Build");
- CubeSegment initialSegment = cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, desc.getCubePartitionDesc().getPartitionDateStart(), dateEnd).get(0);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(CubeStatusEnum.DISABLED, cubeInstance.getStatus());
- for (CubeSegment cubeSegment : cubeInstance.getBuildingSegments()) {
- assertEquals(CubeSegmentStatusEnum.NEW, cubeSegment.getStatus());
- }
- assertEquals(1, cubeInstance.getSegments().size());
- assertEquals(1, cubeInstance.getBuildingSegments().size());
- assertEquals(0, cubeInstance.getRebuildingSegments().size());
- assertTrue("".equals(initialSegment.getStorageLocationIdentifier()) == false);
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
- assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
-
- // initial build success
- System.out.println("Initial Build Success");
- cubeMgr.updateSegmentOnJobSucceed(cubeInstance, CubeBuildTypeEnum.BUILD, initialSegment.getName(), "job_1", System.currentTimeMillis(), 111L, 222L, 333L);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- assertEquals(CubeStatusEnum.READY, cubeInstance.getStatus());
- assertEquals(1, cubeInstance.getSegments(CubeSegmentStatusEnum.READY).size());
- assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
- assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
-
- // upsert build
- // time gap in new segment
- System.out.println("Upsert Build");
- long start = f.parse("2013-11-01").getTime();
- long dateEnd2 = f.parse("2013-11-02").getTime();
-
- cubeMgr.allocateSegments(cubeInstance, CubeBuildTypeEnum.BUILD, start, dateEnd2);
- System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
- }
-
- private MetadataManager getMetadataManager() {
- return MetadataManager.getInstance(getTestConfig());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/test/java/com/kylinolap/cube/common/BytesSplitterTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/com/kylinolap/cube/common/BytesSplitterTest.java b/cube/src/test/java/com/kylinolap/cube/common/BytesSplitterTest.java
deleted file mode 100644
index 6d378db..0000000
--- a/cube/src/test/java/com/kylinolap/cube/common/BytesSplitterTest.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.cube.common;
-
-import static org.junit.Assert.*;
-
-import org.junit.Test;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class BytesSplitterTest {
-
- @Test
- public void test() {
- BytesSplitter bytesSplitter = new BytesSplitter(10, 15);
- byte[] input = "2013-02-17Collectibles".getBytes();
- bytesSplitter.split(input, input.length, (byte) 127);
-
- assertEquals(2, bytesSplitter.getBufferSize());
- assertEquals("2013-02-17", new String(bytesSplitter.getSplitBuffers()[0].value, 0, bytesSplitter.getSplitBuffers()[0].length));
- assertEquals("Collectibles", new String(bytesSplitter.getSplitBuffers()[1].value, 0, bytesSplitter.getSplitBuffers()[1].length));
- }
-
- @Test
- public void testNullValue() {
- BytesSplitter bytesSplitter = new BytesSplitter(10, 15);
- byte[] input = "2013-02-17Collectibles".getBytes();
- bytesSplitter.split(input, input.length, (byte) 127);
-
- assertEquals(3, bytesSplitter.getBufferSize());
- assertEquals("2013-02-17", new String(bytesSplitter.getSplitBuffers()[0].value, 0, bytesSplitter.getSplitBuffers()[0].length));
- assertEquals("", new String(bytesSplitter.getSplitBuffers()[1].value, 0, bytesSplitter.getSplitBuffers()[1].length));
- assertEquals("Collectibles", new String(bytesSplitter.getSplitBuffers()[2].value, 0, bytesSplitter.getSplitBuffers()[2].length));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/test/java/com/kylinolap/cube/common/RowKeySplitterTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/com/kylinolap/cube/common/RowKeySplitterTest.java b/cube/src/test/java/com/kylinolap/cube/common/RowKeySplitterTest.java
deleted file mode 100644
index 5d5d377..0000000
--- a/cube/src/test/java/com/kylinolap/cube/common/RowKeySplitterTest.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.cube.common;
-
-import static org.junit.Assert.*;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.kylinolap.common.util.LocalFileMetadataTestCase;
-import com.kylinolap.cube.CubeInstance;
-import com.kylinolap.cube.CubeManager;
-import com.kylinolap.metadata.MetadataManager;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class RowKeySplitterTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- MetadataManager.removeInstance(this.getTestConfig());
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testWithSlr() throws Exception {
- CubeInstance cube = CubeManager.getInstance(this.getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");
-
- RowKeySplitter rowKeySplitter = new RowKeySplitter(cube.getFirstSegment(), 10, 20);
- // base cuboid rowkey
- byte[] input = { 0, 0, 0, 0, 0, 0, 1, -1, 49, 48, 48, 48, 48, 48, 48, 48, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 11, 54, -105, 55, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
- rowKeySplitter.split(input, input.length);
-
- assertEquals(10, rowKeySplitter.getBufferSize());
- }
-
- @Test
- public void testWithoutSlr() throws Exception {
- CubeInstance cube = CubeManager.getInstance(this.getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");
-
- RowKeySplitter rowKeySplitter = new RowKeySplitter(cube.getFirstSegment(), 10, 20);
- // base cuboid rowkey
- byte[] input = { 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 };
- rowKeySplitter.split(input, input.length);
-
- assertEquals(9, rowKeySplitter.getBufferSize());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/test/java/com/kylinolap/cube/cuboid/CombinationTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/com/kylinolap/cube/cuboid/CombinationTest.java b/cube/src/test/java/com/kylinolap/cube/cuboid/CombinationTest.java
deleted file mode 100644
index 1cd48bd..0000000
--- a/cube/src/test/java/com/kylinolap/cube/cuboid/CombinationTest.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.cuboid;
-
-import static org.junit.Assert.*;
-
-import org.junit.Test;
-
-/**
- * @author xjiang
- *
- */
-public class CombinationTest {
-
- public int findSmallerSibling(long valueBits, long valueMask) {
- if ((valueBits | valueMask) != valueMask) {
- throw new IllegalArgumentException("Dismatch " + Long.toBinaryString(valueBits) + " from " + Long.toBinaryString(valueMask));
- }
-
- int n = Long.bitCount(valueMask);
- int k = Long.bitCount(valueBits);
- long[] bitMasks = new long[n];
- long leftBits = valueMask;
- for (int i = 0; i < n; i++) {
- long lowestBit = Long.lowestOneBit(leftBits);
- bitMasks[i] = lowestBit;
- leftBits &= ~lowestBit;
- }
- return combination(valueBits, bitMasks, 0, 0L, k);
- }
-
- private int combination(long valueBits, long[] bitMasks, int offset, long prefix, int k) {
- if (k == 0) {
- if (prefix < valueBits) {
- System.out.println(Long.toBinaryString(prefix));
- return 1;
- } else {
- return 0;
- }
- } else {
- int count = 0;
- for (int i = offset; i < bitMasks.length; i++) {
- long newPrefix = prefix | bitMasks[i];
- if (newPrefix < valueBits) {
- count += combination(valueBits, bitMasks, i + 1, newPrefix, k - 1);
- }
- }
- return count;
- }
- }
-
- private long calculateCombination(int n, int k) {
- if (n < k) {
- throw new IllegalArgumentException("N < K");
- }
- long res = 1;
- for (int i = n - k + 1; i <= n; i++) {
- res *= i;
- }
- for (int i = 1; i <= k; i++) {
- res /= i;
- }
- return res;
- }
-
- @Test
- public void testComb3() {
- long valueBits = 1 << 4 | 1 << 6 | 1 << 8;
- System.out.println("value = " + Long.toBinaryString(valueBits) + ", count = " + Long.bitCount(valueBits));
- long valueMask = (long) Math.pow(2, 10) - 1;
- System.out.println("mask = " + Long.toBinaryString(valueMask) + ", count = " + Long.bitCount(valueMask));
- System.out.println("************");
- int count = findSmallerSibling(valueBits, valueMask);
- System.out.println("smaller sibling count = " + count);
- int cnk = (int) calculateCombination(Long.bitCount(valueMask), Long.bitCount(valueBits));
- assertTrue(cnk > count);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidSchedulerTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidSchedulerTest.java b/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidSchedulerTest.java
deleted file mode 100644
index c3e4540..0000000
--- a/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidSchedulerTest.java
+++ /dev/null
@@ -1,309 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.cube.cuboid;
-
-import static org.junit.Assert.*;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import com.kylinolap.common.util.LocalFileMetadataTestCase;
-import com.kylinolap.metadata.MetadataManager;
-import com.kylinolap.metadata.model.cube.CubeDesc;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- MetadataManager.removeInstance(this.getTestConfig());
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- static long toLong(String bin) {
- return Long.parseLong(bin, 2);
- }
-
- static String toString(long l) {
- return Long.toBinaryString(l);
- }
-
- static String toString(Collection<Long> cuboids) {
- StringBuilder buf = new StringBuilder();
- buf.append("[");
- for (Long l : cuboids) {
- if (buf.length() > 1)
- buf.append(",");
- buf.append(l).append("(").append(Long.toBinaryString(l)).append(")");
- }
- buf.append("]");
- return buf.toString();
- }
-
- private CubeDesc getTestKylinCubeWithoutSeller() {
- return getMetadataManager().getCubeDesc("test_kylin_cube_without_slr_desc");
- }
-
- private CubeDesc getTestKylinCubeWithSeller() {
- return getMetadataManager().getCubeDesc("test_kylin_cube_with_slr_desc");
- }
-
- private CubeDesc getTestKylinCubeWithoutSellerLeftJoin() {
- return getMetadataManager().getCubeDesc("test_kylin_cube_without_slr_left_join_desc");
- }
-
- @Test
- public void testFindSmallerSibling1() {
- CubeDesc cube = getTestKylinCubeWithoutSeller();
- CuboidScheduler scheduler = new CuboidScheduler(cube);
-
- Collection<Long> siblings;
-
- siblings = scheduler.findSmallerSibling(255);
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(133);
- assertEquals("[131]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(127);
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(134);
- assertEquals("[131, 133]", sortToString(siblings));
-
- siblings = scheduler.findSmallerSibling(130);
- assertEquals("[129]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(5);
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(135);
- assertEquals("[]", siblings.toString());
- }
-
- private void testSpanningAndGetParent(CuboidScheduler scheduler, CubeDesc cube, long[] cuboidIds) {
- for (long cuboidId : cuboidIds) {
- Collection<Long> spannings = scheduler.getSpanningCuboid(cuboidId);
- System.out.println("Spanning result for " + cuboidId + "(" + Long.toBinaryString(cuboidId) + "): " + toString(spannings));
-
- for (long child : spannings) {
- assertTrue(Cuboid.isValid(cube, child));
- }
- }
- }
-
- @Test
- public void testFindSmallerSibling2() {
- CubeDesc cube = getTestKylinCubeWithSeller();
- CuboidScheduler scheduler = new CuboidScheduler(cube);
-
- Collection<Long> siblings;
-
- siblings = scheduler.findSmallerSibling(511);
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(toLong("110111111"));
- assertEquals("[383]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(toLong("101110111"));
- assertEquals("[319]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(toLong("111111000"));
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(toLong("111111000"));
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(toLong("110000000"));
- assertEquals("[288, 320]", sortToString(siblings));
- }
-
- @Test
- public void testGetSpanningCuboid2() {
- CubeDesc cube = getTestKylinCubeWithSeller();
- CuboidScheduler scheduler = new CuboidScheduler(cube);
-
- // generate 8d
- System.out.println("Spanning for 8D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 511 });
- // generate 7d
- System.out.println("Spanning for 7D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 504, 447, 503, 383 });
- // generate 6d
- System.out.println("Spanning for 6D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 440, 496, 376, 439, 487, 319, 375 });
- // generate 5d
- System.out.println("Spanning for 5D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 432, 480, 312, 368, 423, 455, 311, 359 });
- // generate 4d
- System.out.println("Spanning for 4D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 416, 448, 304, 352, 391, 295, 327 });
- // generate 3d
- System.out.println("Spanning for 3D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 384, 288, 320, 263 });
- // generate 2d
- // generate 1d
- // generate 0d
- }
-
- @Test
- public void testGetSpanningCuboid1() {
- CubeDesc cube = getTestKylinCubeWithoutSeller();
- CuboidScheduler scheduler = new CuboidScheduler(cube);
-
- long quiz = toLong("01100111");
- testSpanningAndGetParent(scheduler, cube, new long[] { quiz });
-
- // generate 7d
- System.out.println("Spanning for 7D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 255 });
- // generate 6d
- System.out.println("Spanning for 6D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 135, 251, 253, 254 });
- // generate 5d
- System.out.println("Spanning for 5D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 131, 133, 134, 249, 250, 252 });
- // generate 4d
- System.out.println("Spanning for 4D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 129, 130, 132, 248 });
- // generate 3d
- System.out.println("Spanning for 3D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 184, 240 });
- // generate 2d
- System.out.println("Spanning for 2D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 176, 224 });
- // generate 1d
- System.out.println("Spanning for 1D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 160, 192 });
- // generate 0d
- }
-
- @Test
- public void testGetSpanningCuboid() {
- CubeDesc cube = getTestKylinCubeWithoutSeller();
- CuboidScheduler scheduler = new CuboidScheduler(cube);
-
- Collection<Long> spnanningCuboids = scheduler.getSpanningCuboid(248);
-
- assertEquals("[]", spnanningCuboids.toString());
- }
-
- @Test
- public void testGetCardinality() {
- CubeDesc cube = getTestKylinCubeWithSeller();
- CuboidScheduler scheduler = new CuboidScheduler(cube);
-
- assertEquals(0, scheduler.getCardinality(0));
- assertEquals(7, scheduler.getCardinality(127));
- assertEquals(1, scheduler.getCardinality(1));
- assertEquals(1, scheduler.getCardinality(8));
- assertEquals(6, scheduler.getCardinality(126));
- }
-
- @Test
- public void testCuboidGeneration1() {
- CubeDesc cube = getTestKylinCubeWithoutSeller();
- CuboidCLI.simulateCuboidGeneration(cube);
- }
-
- @Test
- public void testCuboidGeneration2() {
- CubeDesc cube = getTestKylinCubeWithSeller();
- CuboidCLI.simulateCuboidGeneration(cube);
- }
-
- @Test
- public void testCuboidGeneration3() {
- CubeDesc cube = getTestKylinCubeWithoutSellerLeftJoin();
- CuboidCLI.simulateCuboidGeneration(cube);
- }
-
- @Test
- @Ignore
- public void testCuboidGeneration4() {
- CubeDesc cube = getMetadataManager().getCubeDesc("geox_trans_mtrc_sd_cube_desc");
- CuboidCLI.simulateCuboidGeneration(cube);
- }
-
- @Test
- @Ignore
- public void testCuboidGeneration5() {
- CubeDesc cube = getMetadataManager().getCubeDesc("clsfd_ga_dayweek");
- CuboidCLI.simulateCuboidGeneration(cube);
- }
-
- @Test
- @Ignore
- public void testCuboidGeneration6() {
- CubeDesc cube = getMetadataManager().getCubeDesc("clsfd_ga_day");
- CuboidCLI.simulateCuboidGeneration(cube);
- }
-
- @Test
- @Ignore
- public void testCuboidGeneration7() {
- CubeDesc cube = getMetadataManager().getCubeDesc("clsfd_ga_week");
- CuboidCLI.simulateCuboidGeneration(cube);
- }
-
- @Test
- public void testCuboidCounts1() {
- CubeDesc cube = getTestKylinCubeWithoutSeller();
- int[] counts = CuboidCLI.calculateAllLevelCount(cube);
- printCount(counts);
- assertArrayEquals(new int[] { 1, 4, 6, 6, 4, 4, 2, 0 }, counts);
- }
-
- @Test
- public void testCuboidCounts2() {
- CubeDesc cube = getTestKylinCubeWithSeller();
- CuboidCLI.calculateAllLevelCount(cube);
- int[] counts = CuboidCLI.calculateAllLevelCount(cube);
- printCount(counts);
- assertArrayEquals(new int[] { 1, 4, 7, 8, 7, 4 }, counts);
- }
-
- private String sortToString(Collection<Long> longs) {
- ArrayList<Long> copy = new ArrayList<Long>(longs);
- Collections.sort(copy);
- return copy.toString();
- }
-
- private MetadataManager getMetadataManager() {
- return MetadataManager.getInstance(getTestConfig());
- }
-
- private void printCount(int[] counts) {
- int sum = 0;
- for (int c : counts)
- sum += c;
- System.out.println(sum + " = " + Arrays.toString(counts));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidTest.java b/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidTest.java
deleted file mode 100644
index 6682c9d..0000000
--- a/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidTest.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.cube.cuboid;
-
-import static org.junit.Assert.*;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.kylinolap.common.util.LocalFileMetadataTestCase;
-import com.kylinolap.metadata.MetadataManager;
-import com.kylinolap.metadata.model.cube.CubeDesc;
-
-/**
- * @author yangli9
- */
-public class CuboidTest extends LocalFileMetadataTestCase {
-
- private long toLong(String bin) {
- return Long.parseLong(bin, 2);
- }
-
- private MetadataManager getMetadataManager() {
- return MetadataManager.getInstance(getTestConfig());
- }
-
- private CubeDesc getTestKylinCubeII() {
- return getMetadataManager().getCubeDesc("test_kylin_cube_ii");
- }
-
- private CubeDesc getTestKylinCubeWithoutSeller() {
- return getMetadataManager().getCubeDesc("test_kylin_cube_without_slr_desc");
- }
-
- private CubeDesc getTestKylinCubeWithSeller() {
- return getMetadataManager().getCubeDesc("test_kylin_cube_with_slr_desc");
- }
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- MetadataManager.removeInstance(this.getTestConfig());
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testIsValid() {
-
- CubeDesc cube = getTestKylinCubeWithSeller();
-
- // base
- assertEquals(false, Cuboid.isValid(cube, 0));
- assertEquals(true, Cuboid.isValid(cube, toLong("111111111")));
-
- // mandatory column
- assertEquals(false, Cuboid.isValid(cube, toLong("011111110")));
- assertEquals(false, Cuboid.isValid(cube, toLong("100000000")));
-
- // zero tail
- assertEquals(true, Cuboid.isValid(cube, toLong("111111000")));
-
- // aggregation group & zero tail
- assertEquals(true, Cuboid.isValid(cube, toLong("110000111")));
- assertEquals(true, Cuboid.isValid(cube, toLong("110111000")));
- assertEquals(true, Cuboid.isValid(cube, toLong("111110111")));
- assertEquals(false, Cuboid.isValid(cube, toLong("111110001")));
- assertEquals(false, Cuboid.isValid(cube, toLong("111110100")));
- assertEquals(false, Cuboid.isValid(cube, toLong("110000100")));
- }
-
- @Test
- public void testCuboid1() {
- CubeDesc cube = getTestKylinCubeWithoutSeller();
- Cuboid cuboid;
-
- cuboid = Cuboid.findById(cube, 0);
- assertEquals(toLong("10000001"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, 1);
- assertEquals(toLong("10000001"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, toLong("00000010"));
- assertEquals(toLong("10000010"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, toLong("00100000"));
- assertEquals(toLong("10100000"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, toLong("01001000"));
- assertEquals(toLong("11111000"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, toLong("01000111"));
- assertEquals(toLong("11111111"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, toLong("11111111"));
- assertEquals(toLong("11111111"), cuboid.getId());
- }
-
- @Test
- public void testIsValid2() {
- CubeDesc cube = getTestKylinCubeWithoutSeller();
- try {
- assertEquals(false, Cuboid.isValid(cube, toLong("111111111")));
- fail();
- } catch (IllegalArgumentException ex) {
- // expected
- }
-
- // base
- assertEquals(false, Cuboid.isValid(cube, 0));
- assertEquals(true, Cuboid.isValid(cube, toLong("11111111")));
-
- // aggregation group & zero tail
- assertEquals(true, Cuboid.isValid(cube, toLong("10000111")));
- assertEquals(false, Cuboid.isValid(cube, toLong("10001111")));
- assertEquals(false, Cuboid.isValid(cube, toLong("11001111")));
- assertEquals(true, Cuboid.isValid(cube, toLong("10000001")));
- assertEquals(true, Cuboid.isValid(cube, toLong("10000101")));
-
- // hierarchy
- assertEquals(true, Cuboid.isValid(cube, toLong("10100000")));
- assertEquals(true, Cuboid.isValid(cube, toLong("10110000")));
- assertEquals(true, Cuboid.isValid(cube, toLong("10111000")));
- assertEquals(false, Cuboid.isValid(cube, toLong("10001000")));
- assertEquals(false, Cuboid.isValid(cube, toLong("10011000")));
- }
-
- @Test
- public void testCuboid2() {
- CubeDesc cube = getTestKylinCubeWithSeller();
- Cuboid cuboid;
-
- cuboid = Cuboid.findById(cube, 0);
- assertEquals(toLong("100100000"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, 1);
- assertEquals(toLong("100000111"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, toLong("010"));
- assertEquals(toLong("100000111"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, toLong("0100000"));
- assertEquals(toLong("100100000"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, toLong("1001000"));
- assertEquals(toLong("101111000"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, toLong("1000111"));
- assertEquals(toLong("101000111"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, toLong("111111111"));
- assertEquals(toLong("111111111"), cuboid.getId());
- }
-
- @Test
- public void testII() {
- CubeDesc cube = getTestKylinCubeII();
- assertEquals(toLong("111111111"), Cuboid.getBaseCuboidId(cube));
- assertEquals(true, Cuboid.isValid(cube, toLong("111111111")));
- assertEquals(false, Cuboid.isValid(cube, toLong("111111011")));
- assertEquals(false, Cuboid.isValid(cube, toLong("101011011")));
- assertEquals(false, Cuboid.isValid(cube, toLong("000000000")));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/test/java/com/kylinolap/cube/dataGen/DataGenTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/com/kylinolap/cube/dataGen/DataGenTest.java b/cube/src/test/java/com/kylinolap/cube/dataGen/DataGenTest.java
deleted file mode 100644
index 8cd0532..0000000
--- a/cube/src/test/java/com/kylinolap/cube/dataGen/DataGenTest.java
+++ /dev/null
@@ -1,32 +0,0 @@
-package com.kylinolap.cube.dataGen;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.kylinolap.common.util.LocalFileMetadataTestCase;
-import com.kylinolap.metadata.MetadataManager;
-
-/**
- * Created by honma on 6/19/14.
- */
-public class DataGenTest extends LocalFileMetadataTestCase {
-
- @Before
- public void before() throws Exception {
- this.createTestMetadata();
- MetadataManager.removeInstance(this.getTestConfig());
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testBasics() throws Exception {
- FactTableGenerator.generate("test_kylin_cube_with_slr_ready", "10000", "0.6", null, "left");// default
- // settings
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/test/java/com/kylinolap/cube/invertedindex/InvertedIndexLocalTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/com/kylinolap/cube/invertedindex/InvertedIndexLocalTest.java b/cube/src/test/java/com/kylinolap/cube/invertedindex/InvertedIndexLocalTest.java
deleted file mode 100644
index 2c42c5b..0000000
--- a/cube/src/test/java/com/kylinolap/cube/invertedindex/InvertedIndexLocalTest.java
+++ /dev/null
@@ -1,198 +0,0 @@
-package com.kylinolap.cube.invertedindex;
-
-import static org.junit.Assert.*;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.List;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Pair;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-import com.kylinolap.common.util.BytesUtil;
-import com.kylinolap.common.util.LocalFileMetadataTestCase;
-import com.kylinolap.cube.CubeInstance;
-import com.kylinolap.cube.CubeManager;
-import com.kylinolap.dict.Dictionary;
-
-public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
-
- CubeInstance cube;
- TableRecordInfo info;
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- this.cube = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_ii");
- this.info = new TableRecordInfo(cube.getFirstSegment());
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testBitMapContainer() {
- // create container
- BitMapContainer container = new BitMapContainer(info, 0);
- Dictionary<String> dict = info.dict(0);
- for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
- container.append(v);
- }
- container.append(Dictionary.NULL_ID[dict.getSizeOfId()]);
- container.closeForChange();
-
- // copy by serialization
- List<ImmutableBytesWritable> bytes = container.toBytes();
- BitMapContainer container2 = new BitMapContainer(info, 0);
- container2.fromBytes(bytes);
-
- // check the copy
- int i = 0;
- for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
- int value = container2.getValueIntAt(i++);
- assertEquals(v, value);
- }
- assertEquals(Dictionary.NULL_ID[dict.getSizeOfId()], container2.getValueIntAt(i++));
- assertEquals(container, container2);
- }
-
- @Test
- public void testCompressedValueContainer() {
- // create container
- CompressedValueContainer container = new CompressedValueContainer(info, 0, 500);
- Dictionary<String> dict = info.dict(0);
-
- byte[] buf = new byte[dict.getSizeOfId()];
- ImmutableBytesWritable bytes = new ImmutableBytesWritable(buf);
-
- for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
- BytesUtil.writeUnsigned(v, buf, 0, dict.getSizeOfId());
- container.append(bytes);
- }
- BytesUtil.writeUnsigned(Dictionary.NULL_ID[dict.getSizeOfId()], buf, 0, dict.getSizeOfId());
- container.append(bytes);
- container.closeForChange();
-
- // copy by serialization
- ImmutableBytesWritable copy = container.toBytes();
- CompressedValueContainer container2 = new CompressedValueContainer(info, 0, 500);
- container2.fromBytes(copy);
-
- // check the copy
- int i = 0;
- for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
- container2.getValueAt(i++, bytes);
- int value = BytesUtil.readUnsigned(bytes.get(), bytes.getOffset(), bytes.getLength());
- assertEquals(v, value);
- }
- container2.getValueAt(i++, bytes);
- int value = BytesUtil.readUnsigned(bytes.get(), bytes.getOffset(), bytes.getLength());
- assertEquals(Dictionary.NULL_ID[dict.getSizeOfId()], value);
- assertEquals(container, container2);
- }
-
- @Test
- public void testCodec() throws IOException {
- List<TableRecord> records = loadRecordsSorted();
- System.out.println(records.size() + " records");
- List<Slice> slices = buildTimeSlices(records);
- System.out.println(slices.size() + " slices");
-
- IIKeyValueCodec codec = new IIKeyValueCodec(info);
- List<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> kvs = encodeKVs(codec, slices);
- System.out.println(kvs.size() + " KV pairs");
-
- List<Slice> slicesCopy = decodeKVs(codec, kvs);
- assertEquals(slices, slicesCopy);
-
- List<TableRecord> recordsCopy = iterateRecords(slicesCopy);
- assertEquals(new HashSet<TableRecord>(records), new HashSet<TableRecord>(recordsCopy));
- dump(recordsCopy);
- }
-
- private List<TableRecord> loadRecordsSorted() throws IOException {
- File file = new File(LOCALMETA_TEST_DATA, "data/TEST_KYLIN_FACT.csv");
- FileInputStream in = new FileInputStream(file);
- List<String> lines = IOUtils.readLines(in, "UTF-8");
- in.close();
-
- List<TableRecord> records = Lists.newArrayList();
- for (String line : lines) {
- String[] fields = line.split(",");
- TableRecord rec = new TableRecord(info);
- for (int col = 0; col < fields.length; col++) {
- rec.setValueString(col, fields[col]);
- }
- records.add(rec);
- }
-
- Collections.sort(records, new Comparator<TableRecord>() {
- @Override
- public int compare(TableRecord a, TableRecord b) {
- return (int) (a.getTimestamp() - b.getTimestamp());
- }
- });
-
- return records;
- }
-
- private List<Slice> buildTimeSlices(List<TableRecord> records) throws IOException {
- ShardingSliceBuilder builder = new ShardingSliceBuilder(info);
- List<Slice> slices = Lists.newArrayList();
- for (TableRecord rec : records) {
- Slice slice = builder.append(rec);
- if (slice != null)
- slices.add(slice);
- }
- List<Slice> finals = builder.close();
- slices.addAll(finals);
-
- Collections.sort(slices);
- return slices;
- }
-
- private List<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> encodeKVs(IIKeyValueCodec codec, List<Slice> slices) {
-
- List<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> kvs = Lists.newArrayList();
- for (Slice slice : slices) {
- kvs.addAll(codec.encodeKeyValue(slice));
- }
- return kvs;
- }
-
- private List<Slice> decodeKVs(IIKeyValueCodec codec, List<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> kvs) {
- List<Slice> slices = Lists.newArrayList();
- for (Slice slice : codec.decodeKeyValue(kvs)) {
- slices.add(slice);
- }
- return slices;
- }
-
- private List<TableRecord> iterateRecords(List<Slice> slices) {
- List<TableRecord> records = Lists.newArrayList();
- for (Slice slice : slices) {
- for (TableRecordBytes rec : slice) {
- records.add((TableRecord) rec.clone());
- }
- }
- return records;
- }
-
- private void dump(Iterable<TableRecord> records) {
- for (TableRecord rec : records) {
- System.out.println(rec.toString());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/test/java/com/kylinolap/cube/kv/RowKeyDecoderTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/com/kylinolap/cube/kv/RowKeyDecoderTest.java b/cube/src/test/java/com/kylinolap/cube/kv/RowKeyDecoderTest.java
deleted file mode 100644
index b78b2fb..0000000
--- a/cube/src/test/java/com/kylinolap/cube/kv/RowKeyDecoderTest.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.cube.kv;
-
-import static org.junit.Assert.*;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.kylinolap.common.util.LocalFileMetadataTestCase;
-import com.kylinolap.cube.CubeInstance;
-import com.kylinolap.cube.CubeManager;
-import com.kylinolap.cube.cuboid.Cuboid;
-import com.kylinolap.metadata.MetadataManager;
-import com.kylinolap.metadata.model.cube.CubeDesc;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class RowKeyDecoderTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- MetadataManager.removeInstance(this.getTestConfig());
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testDecodeWithoutSlr() throws Exception {
- CubeInstance cube = CubeManager.getInstance(this.getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");
-
- RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment());
-
- byte[] key = { 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 };
-
- rowKeyDecoder.decode(key);
- List<String> names = rowKeyDecoder.getNames(null);
- List<String> values = rowKeyDecoder.getValues();
-
- assertEquals("[CAL_DT, LEAF_CATEG_ID, META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME, LSTG_FORMAT_NAME, LSTG_SITE_ID, SLR_SEGMENT_CD]", names.toString());
- assertEquals("[2012-12-15, 11848, Health & Beauty, Fragrances, Women, FP-GTC, 0, 15]", values.toString());
-
- }
-
- @Test
- public void testDecodeWithSlr() throws Exception {
- CubeInstance cube = CubeManager.getInstance(this.getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");
-
- RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment());
-
- byte[] key = { 0, 0, 0, 0, 0, 0, 1, -1, 49, 48, 48, 48, 48, 48, 48, 48, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 11, 54, -105, 55, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
-
- rowKeyDecoder.decode(key);
- List<String> names = rowKeyDecoder.getNames(null);
- List<String> values = rowKeyDecoder.getValues();
-
- assertEquals("[SELLER_ID, CAL_DT, LEAF_CATEG_ID, META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME, LSTG_FORMAT_NAME, LSTG_SITE_ID, SLR_SEGMENT_CD]", names.toString());
- assertEquals("[10000000, 2012-01-02, 20213, Collectibles, Postcards, US StateCities & Towns, ABIN, 0, -99]", values.toString());
-
- }
-
- @Test
- public void testEncodeAndDecodeWithUtf8() throws IOException {
- CubeInstance cube = CubeManager.getInstance(this.getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");
- CubeDesc cubeDesc = cube.getDescriptor();
-
- byte[][] data = new byte[8][];
- data[0] = Bytes.toBytes("2012-12-15");
- data[1] = Bytes.toBytes("11848");
- data[2] = Bytes.toBytes("Health & Beauty");
- data[3] = Bytes.toBytes("Fragrances");
- data[4] = Bytes.toBytes("Women");
- data[5] = Bytes.toBytes("刊登格式测试");// UTF-8
- data[6] = Bytes.toBytes("0");
- data[7] = Bytes.toBytes("15");
-
- long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
- Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
- AbstractRowKeyEncoder rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cube.getFirstSegment(), baseCuboid);
-
- byte[] encodedKey = rowKeyEncoder.encode(data);
- assertEquals(30, encodedKey.length);
-
- RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment());
- rowKeyDecoder.decode(encodedKey);
- List<String> names = rowKeyDecoder.getNames(null);
- List<String> values = rowKeyDecoder.getValues();
- assertEquals("[CAL_DT, LEAF_CATEG_ID, META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME, LSTG_FORMAT_NAME, LSTG_SITE_ID, SLR_SEGMENT_CD]", names.toString());
- assertEquals("[2012-12-15, 11848, Health & Beauty, Fragrances, Women, 刊登格式, 0, 15]", values.toString());
- }
-}