You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/10/27 02:48:30 UTC
kylin git commit: Add step result check for BuildCubeWithEngine
Repository: kylin
Updated Branches:
refs/heads/master 3047b53d6 -> 71963cbeb
Add step result check for BuildCubeWithEngine
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/71963cbe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/71963cbe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/71963cbe
Branch: refs/heads/master
Commit: 71963cbeb332a4a035d83dc18a2b090b87c15d42
Parents: 3047b53
Author: shaofengshi <sh...@apache.org>
Authored: Thu Oct 27 10:48:10 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Oct 27 10:48:10 2016 +0800
----------------------------------------------------------------------
.../kylin/provision/BuildCubeWithEngine.java | 119 ++++++++++---------
1 file changed, 61 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/71963cbe/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index f6c8801..3ae9bac 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -149,8 +149,7 @@ public class BuildCubeWithEngine {
}
-
- public void after(){
+ public void after() {
DefaultScheduler.destroyInstance();
}
@@ -167,11 +166,11 @@ public class BuildCubeWithEngine {
KylinConfig.getInstanceFromEnv().setHBaseHFileSizeGB(0.0f);
}
- protected void waitForJob(String jobId) {
+ protected ExecutableState waitForJob(String jobId) {
while (true) {
AbstractExecutable job = jobService.getJob(jobId);
if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
- break;
+ return job.getStatus();
} else {
try {
Thread.sleep(5000);
@@ -200,17 +199,17 @@ public class BuildCubeWithEngine {
private void runTestAndAssertSucceed(String[] testCase) throws Exception {
ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
- List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
+ List<Future<Boolean>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
for (int i = 0; i < testCase.length; i++) {
tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
}
countDownLatch.await();
try {
for (int i = 0; i < tasks.size(); ++i) {
- Future<List<String>> task = tasks.get(i);
- final List<String> jobIds = task.get();
- for (String jobId : jobIds) {
- assertJobSucceed(jobId);
+ Future<Boolean> task = tasks.get(i);
+ final Boolean result = task.get();
+ if (result == false) {
+ throw new RuntimeException("The test '" + testCase[i] + "' is failed.");
}
}
} catch (Exception ex) {
@@ -219,13 +218,7 @@ public class BuildCubeWithEngine {
}
}
- private void assertJobSucceed(String jobId) {
- if (jobService.getOutput(jobId).getState() != ExecutableState.SUCCEED) {
- throw new RuntimeException("The job '" + jobId + "' is failed.");
- }
- }
-
- private class TestCallable implements Callable<List<String>> {
+ private class TestCallable implements Callable<Boolean> {
private final String methodName;
private final CountDownLatch countDownLatch;
@@ -237,11 +230,11 @@ public class BuildCubeWithEngine {
@SuppressWarnings("unchecked")
@Override
- public List<String> call() throws Exception {
+ public Boolean call() throws Exception {
try {
final Method method = BuildCubeWithEngine.class.getDeclaredMethod(methodName);
method.setAccessible(true);
- return (List<String>) method.invoke(BuildCubeWithEngine.this);
+ return (Boolean) method.invoke(BuildCubeWithEngine.this);
} catch (Exception e) {
logger.error(e.getMessage());
throw e;
@@ -251,9 +244,13 @@ public class BuildCubeWithEngine {
}
}
+ private void assertJobSuccess() {
+
+ }
+
@SuppressWarnings("unused")
// called by reflection
- private List<String> testInnerJoinCubeWithSlr() throws Exception {
+ private boolean testInnerJoinCubeWithSlr() throws Exception {
final String cubeName = "test_kylin_cube_with_slr_empty";
clearSegment(cubeName);
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
@@ -264,17 +261,18 @@ public class BuildCubeWithEngine {
List<String> result = Lists.newArrayList();
if (fastBuildMode) {
- result.add(buildSegment(cubeName, date1, date3));
+ return buildSegment(cubeName, date1, date3);
} else {
- result.add(buildSegment(cubeName, date1, date2));
- result.add(buildSegment(cubeName, date2, date3));//empty segment
+ if (buildSegment(cubeName, date1, date2) == true) {
+ return buildSegment(cubeName, date2, date3);//empty segment
+ }
}
- return result;
+ return false;
}
@SuppressWarnings("unused")
// called by reflection
- private List<String> testInnerJoinCubeWithoutSlr() throws Exception {
+ private Boolean testInnerJoinCubeWithoutSlr() throws Exception {
final String cubeName = "test_kylin_cube_without_slr_empty";
clearSegment(cubeName);
@@ -287,20 +285,23 @@ public class BuildCubeWithEngine {
List<String> result = Lists.newArrayList();
if (fastBuildMode) {
- result.add(buildSegment(cubeName, date1, date4));
+ return buildSegment(cubeName, date1, date4);
} else {
- result.add(buildSegment(cubeName, date1, date2));
- result.add(buildSegment(cubeName, date2, date3));
- result.add(buildSegment(cubeName, date3, date4));
- result.add(mergeSegment(cubeName, date1, date3));//don't merge all segments
+ if (buildSegment(cubeName, date1, date2) == true) {
+ if (buildSegment(cubeName, date2, date3) == true) {
+ if (buildSegment(cubeName, date3, date4) == true) {
+ return mergeSegment(cubeName, date1, date3);//don't merge all segments
+ }
+ }
+ }
}
- return result;
+ return false;
}
@SuppressWarnings("unused")
// called by reflection
- private List<String> testLeftJoinCubeWithoutSlr() throws Exception {
+ private boolean testLeftJoinCubeWithoutSlr() throws Exception {
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
List<String> result = Lists.newArrayList();
@@ -313,21 +314,24 @@ public class BuildCubeWithEngine {
long date4 = f.parse("2023-01-01").getTime();
if (fastBuildMode) {
- result.add(buildSegment(cubeName, date1, date4));
+ return buildSegment(cubeName, date1, date4);
} else {
- result.add(buildSegment(cubeName, date1, date2));
- result.add(buildSegment(cubeName, date2, date3));
- result.add(buildSegment(cubeName, date3, date4));//empty segment
- result.add(mergeSegment(cubeName, date1, date3));//don't merge all segments
+ if (buildSegment(cubeName, date1, date2) == true) {
+ if (buildSegment(cubeName, date2, date3) == true) {
+ if (buildSegment(cubeName, date3, date4) == true) { //empty segment
+ return mergeSegment(cubeName, date1, date3);//don't merge all segments
+ }
+ }
+ }
}
- return result;
+ return false;
}
@SuppressWarnings("unused")
// called by reflection
- private List<String> testLeftJoinCubeWithView() throws Exception {
+ private boolean testLeftJoinCubeWithView() throws Exception {
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
List<String> result = Lists.newArrayList();
@@ -337,15 +341,13 @@ public class BuildCubeWithEngine {
long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart();
long date4 = f.parse("2023-01-01").getTime();
- result.add(buildSegment(cubeName, date1, date4));
-
- return result;
+ return buildSegment(cubeName, date1, date4);
}
@SuppressWarnings("unused")
// called by reflection
- private List<String> testInnerJoinCubeWithView() throws Exception {
+ private boolean testInnerJoinCubeWithView() throws Exception {
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
List<String> result = Lists.newArrayList();
@@ -355,15 +357,13 @@ public class BuildCubeWithEngine {
long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart();
long date4 = f.parse("2023-01-01").getTime();
- result.add(buildSegment(cubeName, date1, date4));
-
- return result;
+ return buildSegment(cubeName, date1, date4);
}
@SuppressWarnings("unused")
// called by reflection
- private List<String> testLeftJoinCubeWithSlr() throws Exception {
+ private boolean testLeftJoinCubeWithSlr() throws Exception {
String cubeName = "test_kylin_cube_with_slr_left_join_empty";
clearSegment(cubeName);
@@ -376,14 +376,17 @@ public class BuildCubeWithEngine {
List<String> result = Lists.newArrayList();
if (fastBuildMode) {
- result.add(buildSegment(cubeName, date1, date4));
+ return buildSegment(cubeName, date1, date4);
} else {
- result.add(buildSegment(cubeName, date1, date2));
- result.add(buildSegment(cubeName, date2, date3));
- result.add(buildSegment(cubeName, date3, date4));
- result.add(mergeSegment(cubeName, date1, date3));//don't merge all segments
+ if (buildSegment(cubeName, date1, date2) == true) {
+ if (buildSegment(cubeName, date2, date3) == true) {
+ if (buildSegment(cubeName, date3, date4) == true) {
+ return mergeSegment(cubeName, date1, date3);//don't merge all segments
+ }
+ }
+ }
}
- return result;
+ return false;
}
@@ -395,24 +398,24 @@ public class BuildCubeWithEngine {
cubeManager.updateCube(cubeBuilder);
}
- private String mergeSegment(String cubeName, long startDate, long endDate) throws Exception {
+ private Boolean mergeSegment(String cubeName, long startDate, long endDate) throws Exception {
CubeSegment segment = cubeManager.mergeSegments(cubeManager.getCube(cubeName), startDate, endDate, 0, 0, true);
DefaultChainedExecutable job = EngineFactory.createBatchMergeJob(segment, "TEST");
jobService.addJob(job);
- waitForJob(job.getId());
- return job.getId();
+ ExecutableState state = waitForJob(job.getId());
+ return Boolean.valueOf(ExecutableState.SUCCEED == state);
}
- private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
+ private Boolean buildSegment(String cubeName, long startDate, long endDate) throws Exception {
CubeSegment segment = cubeManager.appendSegment(cubeManager.getCube(cubeName), 0, endDate);
DefaultChainedExecutable job = EngineFactory.createBatchCubingJob(segment, "TEST");
jobService.addJob(job);
- waitForJob(job.getId());
+ ExecutableState state = waitForJob(job.getId());
// if (segment.getCubeDesc().getEngineType() == IEngineAware.ID_MR_V1
// || segment.getCubeDesc().getStorageType() == IStorageAware.ID_SHARDED_HBASE) {
// checkHFilesInHBase(segment);
// }
- return job.getId();
+ return Boolean.valueOf(ExecutableState.SUCCEED == state);
}
@SuppressWarnings("unused")