You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/11/16 03:23:24 UTC
[4/5] incubator-kylin git commit: KYLIN-1136 Distinguish fast build
mode and complete build mode
KYLIN-1136 Distinguish fast build mode and complete build mode
bug fix
fix
temp
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/d256a7f3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/d256a7f3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/d256a7f3
Branch: refs/heads/2.x-staging
Commit: d256a7f30f1174aa0efa74ab58ca17a329e6c7f7
Parents: fce575b
Author: honma <ho...@ebay.com>
Authored: Tue Nov 10 16:34:39 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon Nov 16 10:28:24 2015 +0800
----------------------------------------------------------------------
.../kylin/job/BuildCubeWithEngineTest.java | 114 ++++++++++++-------
.../org/apache/kylin/cube/CubeManagerTest.java | 5 +
...test_kylin_cube_with_slr_left_join_desc.json | 4 +-
...t_kylin_cube_without_slr_left_join_desc.json | 6 +-
4 files changed, 86 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d256a7f3/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index af3dc43..d2a101d 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -62,6 +62,7 @@ public class BuildCubeWithEngineTest {
private CubeManager cubeManager;
private DefaultScheduler scheduler;
protected ExecutableManager jobService;
+ private static boolean fastBuildMode = true;
private static final Log logger = LogFactory.getLog(BuildCubeWithEngineTest.class);
@@ -84,6 +85,15 @@ public class BuildCubeWithEngineTest {
public static void beforeClass() throws Exception {
logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+
+ String fastModeStr = System.getProperty("fastBuildMode");
+ if (fastModeStr != null && fastModeStr.equalsIgnoreCase("false")) {
+ fastBuildMode = false;
+ logger.info("Will not use fast build mode");
+ } else {
+ logger.info("Will use fast build mode");
+ }
+
System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
if (System.getProperty("hdp.version") == null) {
throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
@@ -93,6 +103,7 @@ public class BuildCubeWithEngineTest {
@Before
public void before() throws Exception {
+
HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
DeployUtil.initCliWorkDir();
@@ -128,12 +139,12 @@ public class BuildCubeWithEngineTest {
}
private void testInner() throws Exception {
- String[] testCase = new String[] { "testInnerJoinCube", "testInnerJoinCube2" };
+ String[] testCase = new String[] { "testInnerJoinCubeWithoutSlr", "testInnerJoinCubeWithSlr" };
runTestAndAssertSucceed(testCase);
}
private void testLeft() throws Exception {
- String[] testCase = new String[] { "testLeftJoinCube", "testLeftJoinCube2" };
+ String[] testCase = new String[] { "testLeftJoinCubeWithSlr", "testLeftJoinCubeWithoutSlr" };
runTestAndAssertSucceed(testCase);
}
@@ -180,6 +191,9 @@ public class BuildCubeWithEngineTest {
final Method method = BuildCubeWithEngineTest.class.getDeclaredMethod(methodName);
method.setAccessible(true);
return (List<String>) method.invoke(BuildCubeWithEngineTest.this);
+ } catch (Exception e) {
+ logger.error(e.getMessage());
+ throw e;
} finally {
countDownLatch.countDown();
}
@@ -188,63 +202,71 @@ public class BuildCubeWithEngineTest {
@SuppressWarnings("unused")
// called by reflection
- private List<String> testInnerJoinCube2() throws Exception {
+ private List<String> testInnerJoinCubeWithSlr() throws Exception {
clearSegment("test_kylin_cube_with_slr_empty");
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
long date1 = 0;
- long date2 = f.parse("2013-01-01").getTime();
+ long date2 = f.parse("2015-01-01").getTime();
long date3 = f.parse("2022-01-01").getTime();
List<String> result = Lists.newArrayList();
- result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2));
- result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3));
+
+ if (fastBuildMode) {
+ result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date3));
+ } else {
+ result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2));
+ result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3));//empty segment
+ }
return result;
}
@SuppressWarnings("unused")
// called by reflection
- private List<String> testInnerJoinCube() throws Exception {
- clearSegment("test_kylin_cube_without_slr_empty");
+ private List<String> testInnerJoinCubeWithoutSlr() throws Exception {
+ clearSegment("test_kylin_cube_without_slr_empty");
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- // this cube's start date is 0, end date is 20501112000000
long date1 = 0;
- long date2 = f.parse("2050-01-11").getTime();
-
- // this cube doesn't support incremental build, always do full build
-
+ long date2 = f.parse("2013-01-01").getTime();
+ long date3 = f.parse("2013-07-01").getTime();
+ long date4 = f.parse("2022-01-01").getTime();
List<String> result = Lists.newArrayList();
- result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2));
+
+ if (fastBuildMode) {
+ result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date4));
+ } else {
+ result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2));
+ result.add(buildSegment("test_kylin_cube_without_slr_empty", date2, date3));
+ result.add(buildSegment("test_kylin_cube_without_slr_empty", date3, date4));
+ result.add(mergeSegment("test_kylin_cube_without_slr_empty", date1, date3));//don't merge all segments
+ }
return result;
+
}
-
+
@SuppressWarnings("unused")
// called by reflection
- private List<String> testLeftJoinCube2() throws Exception {
+ private List<String> testLeftJoinCubeWithoutSlr() throws Exception {
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
List<String> result = Lists.newArrayList();
final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
- // this cube's start date is 0, end date is 20120601000000
- long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
-
- long dateEnd = f.parse("2012-06-01").getTime();
-
clearSegment(cubeName);
- result.add(buildSegment(cubeName, dateStart, dateEnd));
- // then submit an append job, start date is 20120601000000, end
- // date is 20220101000000
- dateStart = dateEnd;
- dateEnd = f.parse("2022-01-01").getTime();
- result.add(buildSegment(cubeName, dateStart, dateEnd));
-
- // build an empty segment which doesn't have data
- dateStart = dateEnd;
- dateEnd = f.parse("2023-01-01").getTime();
- result.add(buildSegment(cubeName, dateStart, dateEnd));
+ long date1 = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+ long date2 = f.parse("2012-06-01").getTime();
+ long date3 = f.parse("2022-01-01").getTime();
+ long date4 = f.parse("2023-01-01").getTime();
+
+ if (fastBuildMode) {
+ result.add(buildSegment("test_kylin_cube_without_slr_left_join_empty", date1, date4));
+ } else {
+ result.add(buildSegment("test_kylin_cube_without_slr_left_join_empty", date1, date2));
+ result.add(buildSegment("test_kylin_cube_without_slr_left_join_empty", date2, date3));
+ result.add(buildSegment("test_kylin_cube_without_slr_left_join_empty", date3, date4));//empty segment
+ result.add(mergeSegment("test_kylin_cube_without_slr_left_join_empty", date1, date3));//don't merge all segments
+ }
return result;
@@ -252,18 +274,26 @@ public class BuildCubeWithEngineTest {
@SuppressWarnings("unused")
// called by reflection
- private List<String> testLeftJoinCube() throws Exception {
+ private List<String> testLeftJoinCubeWithSlr() throws Exception {
String cubeName = "test_kylin_cube_with_slr_left_join_empty";
clearSegment(cubeName);
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- long dateEnd = f.parse("2050-11-12").getTime();
+ long date1 = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+ long date2 = f.parse("2013-01-01").getTime();
+ long date3 = f.parse("2013-07-01").getTime();
+ long date4 = f.parse("2022-01-01").getTime();
- // this cube's start date is 0, end date is 20501112000000
List<String> result = Lists.newArrayList();
- result.add(buildSegment(cubeName, dateStart, dateEnd));
+ if (fastBuildMode) {
+ result.add(buildSegment(cubeName, date1, date4));
+ } else {
+ result.add(buildSegment(cubeName, date1, date2));
+ result.add(buildSegment(cubeName, date2, date3));
+ result.add(buildSegment(cubeName, date3, date4));
+ result.add(mergeSegment(cubeName, date1, date3));//don't merge all segments
+ }
return result;
}
@@ -276,6 +306,14 @@ public class BuildCubeWithEngineTest {
cubeManager.updateCube(cubeBuilder);
}
+ private String mergeSegment(String cubeName, long startDate, long endDate) throws Exception {
+ CubeSegment segment = cubeManager.mergeSegments(cubeManager.getCube(cubeName), startDate, endDate, true);
+ DefaultChainedExecutable job = EngineFactory.createBatchMergeJob(segment, "TEST");
+ jobService.addJob(job);
+ waitForJob(job.getId());
+ return job.getId();
+ }
+
private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate);
DefaultChainedExecutable job = EngineFactory.createBatchCubingJob(segment, "TEST");
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d256a7f3/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
index 708bf32..fcfa67d 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
@@ -22,10 +22,13 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.TimeZone;
+import com.google.common.collect.Lists;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.JsonUtil;
@@ -55,6 +58,8 @@ public class CubeManagerTest extends LocalFileMetadataTestCase {
@Test
public void testBasics() throws Exception {
+
+
CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_without_slr_ready");
CubeDesc desc = cube.getDescriptor();
System.out.println(JsonUtil.writeValueAsIndentString(desc));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d256a7f3/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
index 8e22615..32536e1 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
@@ -237,6 +237,6 @@
]
},
"notify_list": null,
- "engine_type": 2,
- "storage_type": 2
+ "engine_type": 0,
+ "storage_type": 0
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d256a7f3/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index 08a132e..dfa62f7 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -15,7 +15,7 @@
},
{
"id": 1,
- "name": "CATEGORY",
+ "name": "CATEGORY"
"table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
"column": null,
"derived": [
@@ -289,6 +289,6 @@
]
},
"notify_list": null,
- "engine_type": 0,
- "storage_type": 0
+ "engine_type": 2,
+ "storage_type": 2
}
\ No newline at end of file