You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/11/10 09:29:45 UTC
[3/3] incubator-kylin git commit: KYLIN-1136 Distinguish fast build
mode and complete build mode
KYLIN-1136 Distinguish fast build mode and complete build mode
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a9e33cde
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a9e33cde
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a9e33cde
Branch: refs/heads/KYLIN-1126
Commit: a9e33cdea5d7fc3cb02e7a1b00456e8237557b81
Parents: 1dd1afd
Author: honma <ho...@ebay.com>
Authored: Tue Nov 10 16:34:39 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Nov 10 16:34:39 2015 +0800
----------------------------------------------------------------------
.../kylin/job/BuildCubeWithEngineTest.java | 110 ++++++++++++-------
1 file changed, 73 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a9e33cde/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index af3dc43..8864170 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -62,6 +62,7 @@ public class BuildCubeWithEngineTest {
private CubeManager cubeManager;
private DefaultScheduler scheduler;
protected ExecutableManager jobService;
+ private static boolean fastBuildMode = true;
private static final Log logger = LogFactory.getLog(BuildCubeWithEngineTest.class);
@@ -84,6 +85,15 @@ public class BuildCubeWithEngineTest {
public static void beforeClass() throws Exception {
logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+
+ String fastModeStr = System.getProperty("fastBuildMode");
+ if (fastModeStr != null && fastModeStr.equalsIgnoreCase("false")) {
+ fastBuildMode = false;
+ logger.info("Will not use fast build mode");
+ } else {
+ logger.info("Will use fast build mode");
+ }
+
System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
if (System.getProperty("hdp.version") == null) {
throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
@@ -93,6 +103,7 @@ public class BuildCubeWithEngineTest {
@Before
public void before() throws Exception {
+
HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
DeployUtil.initCliWorkDir();
@@ -128,12 +139,12 @@ public class BuildCubeWithEngineTest {
}
private void testInner() throws Exception {
- String[] testCase = new String[] { "testInnerJoinCube", "testInnerJoinCube2" };
+ String[] testCase = new String[] { "testInnerJoinCubeWithoutSlr", "testInnerJoinCubeWithSlr" };
runTestAndAssertSucceed(testCase);
}
private void testLeft() throws Exception {
- String[] testCase = new String[] { "testLeftJoinCube", "testLeftJoinCube2" };
+ String[] testCase = new String[] { "testLeftJoinCubeWithSlr", "testLeftJoinCubeWithoutSlr" };
runTestAndAssertSucceed(testCase);
}
@@ -188,63 +199,74 @@ public class BuildCubeWithEngineTest {
@SuppressWarnings("unused")
// called by reflection
- private List<String> testInnerJoinCube2() throws Exception {
+ private List<String> testInnerJoinCubeWithSlr() throws Exception {
clearSegment("test_kylin_cube_with_slr_empty");
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
long date1 = 0;
long date2 = f.parse("2013-01-01").getTime();
- long date3 = f.parse("2022-01-01").getTime();
+ long date3 = f.parse("2013-07-01").getTime();
+ long date4 = f.parse("2022-01-01").getTime();
List<String> result = Lists.newArrayList();
- result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2));
- result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3));
+
+ if (fastBuildMode) {
+ result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date4));
+ } else {
+ result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2));
+ result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3));
+ result.add(buildSegment("test_kylin_cube_with_slr_empty", date3, date4));
+ result.add(mergeSegment("test_kylin_cube_with_slr_empty", date1, date3));//don't merge all segments
+ }
return result;
}
@SuppressWarnings("unused")
// called by reflection
- private List<String> testInnerJoinCube() throws Exception {
- clearSegment("test_kylin_cube_without_slr_empty");
+ private List<String> testInnerJoinCubeWithoutSlr() throws Exception {
+ clearSegment("test_kylin_cube_without_slr_empty");
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- // this cube's start date is 0, end date is 20501112000000
long date1 = 0;
- long date2 = f.parse("2050-01-11").getTime();
-
- // this cube doesn't support incremental build, always do full build
-
+ long date2 = f.parse("2013-01-01").getTime();
+ long date3 = f.parse("2013-07-01").getTime();
+ long date4 = f.parse("2022-01-01").getTime();
List<String> result = Lists.newArrayList();
- result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2));
+
+ if (fastBuildMode) {
+ result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date4));
+ } else {
+ result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2));
+ result.add(buildSegment("test_kylin_cube_without_slr_empty", date2, date3));
+ result.add(buildSegment("test_kylin_cube_without_slr_empty", date3, date4));
+ result.add(mergeSegment("test_kylin_cube_without_slr_empty", date1, date3));//don't merge all segments
+ }
return result;
+
}
-
+
@SuppressWarnings("unused")
// called by reflection
- private List<String> testLeftJoinCube2() throws Exception {
+ private List<String> testLeftJoinCubeWithoutSlr() throws Exception {
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
List<String> result = Lists.newArrayList();
final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
- // this cube's start date is 0, end date is 20120601000000
- long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
-
- long dateEnd = f.parse("2012-06-01").getTime();
-
clearSegment(cubeName);
- result.add(buildSegment(cubeName, dateStart, dateEnd));
-
- // then submit an append job, start date is 20120601000000, end
- // date is 20220101000000
- dateStart = dateEnd;
- dateEnd = f.parse("2022-01-01").getTime();
- result.add(buildSegment(cubeName, dateStart, dateEnd));
- // build an empty segment which doesn't have data
- dateStart = dateEnd;
- dateEnd = f.parse("2023-01-01").getTime();
- result.add(buildSegment(cubeName, dateStart, dateEnd));
+ long date1 = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+ long date2 = f.parse("2012-06-01").getTime();
+ long date3 = f.parse("2022-01-01").getTime();
+ long date4 = f.parse("2023-01-01").getTime();
+
+ if (fastBuildMode) {
+ result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date4));
+ } else {
+ result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2));
+ result.add(buildSegment("test_kylin_cube_without_slr_empty", date2, date3));
+ result.add(buildSegment("test_kylin_cube_without_slr_empty", date3, date4));
+ result.add(mergeSegment("test_kylin_cube_without_slr_empty", date1, date3));//don't merge all segments
+ }
return result;
@@ -252,18 +274,24 @@ public class BuildCubeWithEngineTest {
@SuppressWarnings("unused")
// called by reflection
- private List<String> testLeftJoinCube() throws Exception {
+ private List<String> testLeftJoinCubeWithSlr() throws Exception {
String cubeName = "test_kylin_cube_with_slr_left_join_empty";
clearSegment(cubeName);
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- long dateEnd = f.parse("2050-11-12").getTime();
+ long dateEnd = f.parse("2020-11-12").getTime();
+ long dateFuture = f.parse("2021-11-12").getTime();
- // this cube's start date is 0, end date is 20501112000000
List<String> result = Lists.newArrayList();
- result.add(buildSegment(cubeName, dateStart, dateEnd));
+ if (fastBuildMode) {
+ result.add(buildSegment(cubeName, dateStart, dateFuture));
+ } else {
+ result.add(buildSegment(cubeName, dateStart, dateEnd));
+ // build an empty segment which doesn't have data
+ result.add(buildSegment(cubeName, dateEnd, dateFuture));
+ }
return result;
}
@@ -276,6 +304,14 @@ public class BuildCubeWithEngineTest {
cubeManager.updateCube(cubeBuilder);
}
+ private String mergeSegment(String cubeName, long startDate, long endDate) throws Exception {
+ CubeSegment segment = cubeManager.mergeSegments(cubeManager.getCube(cubeName), startDate, endDate, true);
+ DefaultChainedExecutable job = EngineFactory.createBatchMergeJob(segment, "TEST");
+ jobService.addJob(job);
+ waitForJob(job.getId());
+ return job.getId();
+ }
+
private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate);
DefaultChainedExecutable job = EngineFactory.createBatchCubingJob(segment, "TEST");