You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/03/14 01:01:00 UTC
[25/50] [abbrv] incubator-kylin git commit: refactor test
refactor test
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7d119e9b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7d119e9b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7d119e9b
Branch: refs/heads/streaming-localdict
Commit: 7d119e9b68cd73b184adf6a60c4536a964e8f21a
Parents: 0939359
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Mar 11 18:17:41 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Mar 11 18:17:41 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/job/BuildIIWithEngineTest.java | 4 +-
.../apache/kylin/job/BuildIIWithStreamTest.java | 61 +++++++++++++++++---
2 files changed, 56 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d119e9b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
index 2a6f16b..80c075e 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -65,7 +65,7 @@ public class BuildIIWithEngineTest {
protected ExecutableManager jobService;
// protected static final String[] TEST_II_INSTANCES = new String[]{ "test_kylin_ii_inner_join", "test_kylin_ii_left_join"};
- protected static final String[] TEST_II_INSTANCES = new String[]{"test_kylin_ii_left_join"};
+ protected static final String[] TEST_II_INSTANCES = new String[]{};
private static final Log logger = LogFactory.getLog(BuildIIWithEngineTest.class);
@@ -106,7 +106,6 @@ public class BuildIIWithEngineTest {
if (!scheduler.hasStarted()) {
throw new RuntimeException("scheduler has not been started");
}
- iiManager = IIManager.getInstance(kylinConfig);
jobEngineConfig = new JobEngineConfig(kylinConfig);
for (String jobId : jobService.getAllJobIds()) {
if(jobService.getJob(jobId) instanceof IIJob){
@@ -114,6 +113,7 @@ public class BuildIIWithEngineTest {
}
}
+ iiManager = IIManager.getInstance(kylinConfig);
for (String iiInstance : TEST_II_INSTANCES) {
IIInstance ii = iiManager.getII(iiInstance);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d119e9b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index c892b5f..af3c8e4 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -53,11 +53,14 @@ import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.job.common.ShellExecutable;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
import org.apache.kylin.job.hadoop.hive.IIJoinedFlatTableDesc;
import org.apache.kylin.job.hadoop.invertedindex.IICreateHTableJob;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.streaming.Stream;
import org.apache.kylin.streaming.invertedindex.IIStreamBuilder;
+import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -77,7 +80,7 @@ public class BuildIIWithStreamTest {
private static final Log logger = LogFactory.getLog(BuildIIWithStreamTest.class);
- private static final String II_NAME = "test_kylin_ii_inner_join";
+ private static final String[] II_NAME = new String[]{"test_kylin_ii_inner_join", "test_kylin_ii_left_join"};
private IIManager iiManager;
private KylinConfig kylinConfig;
@@ -94,6 +97,46 @@ public class BuildIIWithStreamTest {
kylinConfig = KylinConfig.getInstanceFromEnv();
iiManager = IIManager.getInstance(kylinConfig);
+ iiManager = IIManager.getInstance(kylinConfig);
+ for (String iiInstance : II_NAME) {
+
+ IIInstance ii = iiManager.getII(iiInstance);
+ if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
+ ii.setStatus(RealizationStatusEnum.DISABLED);
+ iiManager.updateII(ii);
+ }
+ }
+ }
+
+ @After
+ public void after() throws Exception {
+ for (String iiInstance : II_NAME) {
+ IIInstance ii = iiManager.getII(iiInstance);
+ if (ii.getStatus() != RealizationStatusEnum.READY) {
+ ii.setStatus(RealizationStatusEnum.READY);
+ iiManager.updateII(ii);
+ }
+ }
+ backup();
+ }
+
+ private int cleanupOldStorage() throws Exception {
+ String[] args = {"--delete", "true"};
+
+ int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+ return exitCode;
+ }
+
+ private void backup() throws Exception {
+ int exitCode = cleanupOldStorage();
+ if (exitCode == 0) {
+ exportHBaseData();
+ }
+ }
+
+ private void exportHBaseData() throws IOException {
+ ExportHBaseData export = new ExportHBaseData();
+ export.exportTables();
}
private String createIntermediateTable(IIDesc desc, KylinConfig kylinConfig) throws IOException {
@@ -149,11 +192,9 @@ public class BuildIIWithStreamTest {
return segment;
}
- @Test
- public void test() throws Exception {
- final IIDesc desc = iiManager.getII(II_NAME).getDescriptor();
+ private void buildII(String iiName) throws Exception {
+ final IIDesc desc = iiManager.getII(iiName).getDescriptor();
final String tableName = createIntermediateTable(desc, kylinConfig);
-// final String tableName = "kylin_intermediate_ii_test_kylin_ii_inner_join_desc_f24b8e1f_1c1f_4835_8d78_8f21ce79a536";
final Configuration conf = new Configuration();
HCatInputFormat.setInput(conf, "default", tableName);
final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
@@ -168,8 +209,8 @@ public class BuildIIWithStreamTest {
}
}
LinkedBlockingDeque<Stream> queue = new LinkedBlockingDeque<Stream>();
- final IISegment segment = createSegment(II_NAME);
- String[] args = new String[]{"-iiname", II_NAME, "-htablename", segment.getStorageLocationIdentifier()};
+ final IISegment segment = createSegment(iiName);
+ String[] args = new String[]{"-iiname", iiName, "-htablename", segment.getStorageLocationIdentifier()};
ToolRunner.run(new IICreateHTableJob(), args);
@@ -183,7 +224,13 @@ public class BuildIIWithStreamTest {
thread.join();
logger.info("stream build finished, htable name:" + segment.getStorageLocationIdentifier());
+ }
+ @Test
+ public void test() throws Exception {
+ for (String iiName : II_NAME) {
+ buildII(iiName);
+ }
}
private Stream parse(String[] row) {