You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/03/14 01:00:58 UTC
[23/50] [abbrv] incubator-kylin git commit: refactor
refactor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e07c7020
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e07c7020
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e07c7020
Branch: refs/heads/streaming-localdict
Commit: e07c702061df5656735578a2f400ce3770e384cf
Parents: 63695e4
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Mar 11 16:50:40 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Mar 11 16:50:40 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/job/BuildIIWithEngineTest.java | 3 +-
.../apache/kylin/job/BuildIIWithStreamTest.java | 45 ++++++++++++++++++--
pom.xml | 1 +
3 files changed, 44 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e07c7020/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
index 03fe34a..2a6f16b 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -64,7 +64,8 @@ public class BuildIIWithEngineTest {
private DefaultScheduler scheduler;
protected ExecutableManager jobService;
- protected static final String[] TEST_II_INSTANCES = new String[]{ "test_kylin_ii_inner_join", "test_kylin_ii_left_join"};
+// protected static final String[] TEST_II_INSTANCES = new String[]{ "test_kylin_ii_inner_join", "test_kylin_ii_left_join"};
+ protected static final String[] TEST_II_INSTANCES = new String[]{"test_kylin_ii_left_join"};
private static final Log logger = LogFactory.getLog(BuildIIWithEngineTest.class);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e07c7020/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index c7c07e3..c892b5f 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -38,6 +38,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ToolRunner;
import org.apache.hive.hcatalog.data.schema.HCatSchema;
import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
import org.apache.kylin.common.KylinConfig;
@@ -45,12 +46,15 @@ import org.apache.kylin.common.util.AbstractKylinTestCase;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.apache.kylin.dict.lookup.HiveTableReader;
+import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.IISegment;
import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.job.common.ShellExecutable;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.hadoop.hive.IIJoinedFlatTableDesc;
+import org.apache.kylin.job.hadoop.invertedindex.IICreateHTableJob;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.streaming.Stream;
import org.apache.kylin.streaming.invertedindex.IIStreamBuilder;
@@ -60,7 +64,9 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
+import java.text.SimpleDateFormat;
import java.util.List;
+import java.util.TimeZone;
import java.util.UUID;
import java.util.concurrent.LinkedBlockingDeque;
@@ -119,11 +125,35 @@ public class BuildIIWithStreamTest {
return intermediateTableDesc.getTableName(uuid);
}
+ private void clearSegment(String iiName) throws Exception {
+ IIInstance ii = iiManager.getII(iiName);
+ ii.getSegments().clear();
+ iiManager.updateII(ii);
+ }
+
+ private IISegment createSegment(String iiName) throws Exception {
+ clearSegment(iiName);
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ long date1 = 0;
+ long date2 = f.parse("2015-01-01").getTime();
+ return buildSegment(iiName, date1, date2);
+ }
+
+ private IISegment buildSegment(String iiName, long startDate, long endDate) throws Exception {
+ IIInstance iiInstance = iiManager.getII(iiName);
+ IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
+ iiInstance.getSegments().add(segment);
+ iiManager.updateII(iiInstance);
+ return segment;
+ }
+
@Test
- public void test() throws IOException, InterruptedException {
+ public void test() throws Exception {
final IIDesc desc = iiManager.getII(II_NAME).getDescriptor();
-// final String tableName = createIntermediateTable(desc, kylinConfig);
- final String tableName = "kylin_intermediate_ii_test_kylin_ii_inner_join_desc_f24b8e1f_1c1f_4835_8d78_8f21ce79a536";
+ final String tableName = createIntermediateTable(desc, kylinConfig);
+// final String tableName = "kylin_intermediate_ii_test_kylin_ii_inner_join_desc_f24b8e1f_1c1f_4835_8d78_8f21ce79a536";
final Configuration conf = new Configuration();
HCatInputFormat.setInput(conf, "default", tableName);
final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
@@ -138,7 +168,12 @@ public class BuildIIWithStreamTest {
}
}
LinkedBlockingDeque<Stream> queue = new LinkedBlockingDeque<Stream>();
- final IIStreamBuilder streamBuilder = new IIStreamBuilder(queue, "test_htable", desc, 0);
+ final IISegment segment = createSegment(II_NAME);
+ String[] args = new String[]{"-iiname", II_NAME, "-htablename", segment.getStorageLocationIdentifier()};
+ ToolRunner.run(new IICreateHTableJob(), args);
+
+
+ final IIStreamBuilder streamBuilder = new IIStreamBuilder(queue, segment.getStorageLocationIdentifier(), desc, 0);
final Thread thread = new Thread(streamBuilder);
thread.start();
while (reader.next()) {
@@ -147,6 +182,8 @@ public class BuildIIWithStreamTest {
queue.put(new Stream(-1, null));
thread.join();
+ logger.info("stream build finished, htable name:" + segment.getStorageLocationIdentifier());
+
}
private Stream parse(String[] row) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e07c7020/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e99f77d..94f1ca3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -615,6 +615,7 @@
<excludes>
<exclude>**/BuildCubeWithEngineTest.java</exclude>
<exclude>**/BuildIIWithEngineTest.java</exclude>
+ <exclude>**/BuildIIWithStreamTest.java</exclude>
<exclude>**/SampleCubeSetupTest.java</exclude>
<exclude>**/KylinQueryTest.java</exclude>
<exclude>**/Kafka*Test.java</exclude>