You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/03/14 01:01:05 UTC
[30/50] [abbrv] incubator-kylin git commit: fix ut
fix ut
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a1b0177a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a1b0177a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a1b0177a
Branch: refs/heads/streaming-localdict
Commit: a1b0177a668a1fde1754387931407df10fb0fef0
Parents: 8be38e7
Author: qianhao.zhou <qi...@ebay.com>
Authored: Thu Mar 12 15:02:02 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Thu Mar 12 15:02:02 2015 +0800
----------------------------------------------------------------------
.../kylin/invertedindex/index/TableRecordInfo.java | 12 +++++++++---
.../kylin/storage/hbase/HBaseClientKVIterator.java | 4 +---
.../kylin/storage/hbase/InvertedIndexHBaseTest.java | 6 +++++-
.../kylin/streaming/invertedindex/IIStreamBuilder.java | 6 ++++++
4 files changed, 21 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1b0177a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
index 5535fcc..0cfb05e 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
@@ -92,10 +92,16 @@ public class TableRecordInfo {
isMetric[i] = desc.isMetricsCol(i);
dataTypes[i] = tblColRef.getDatatype();
if (isMetric[i]) {
- lengths[i] = measureCodecMap.get(tblColRef).getLength();
+ final FixedLenMeasureCodec<?> fixedLenMeasureCodec = measureCodecMap.get(tblColRef);
+ if (fixedLenMeasureCodec != null) {
+ lengths[i] = fixedLenMeasureCodec.getLength();
+ }
} else {
- lengths[i] = dictionaryMap.get(tblColRef).getSizeOfId();
- dictMaxIds[i] = dictionaryMap.get(tblColRef).getMaxId();
+ final Dictionary<?> dictionary = dictionaryMap.get(tblColRef);
+ if (dictionary != null) {
+ lengths[i] = dictionary.getSizeOfId();
+ dictMaxIds[i] = dictionaryMap.get(tblColRef).getMaxId();
+ }
}
}
// offsets
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1b0177a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
index 339612d..29b2bcf 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
@@ -40,7 +40,6 @@ import org.apache.kylin.invertedindex.model.KeyValuePair;
public class HBaseClientKVIterator implements Iterable<KeyValuePair>, Closeable {
byte[] family;
- byte[] qualifier;
HTableInterface table;
ResultScanner scanner;
@@ -48,10 +47,9 @@ public class HBaseClientKVIterator implements Iterable<KeyValuePair>, Closeable
public HBaseClientKVIterator(HConnection hconn, String tableName, byte[] family, byte[] qualifier) throws IOException {
this.family = family;
- this.qualifier = qualifier;
this.table = hconn.getTable(tableName);
- this.scanner = table.getScanner(family, qualifier);
+ this.scanner = table.getScanner(family);
this.iterator = scanner.iterator();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1b0177a/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
index f38ed49..3ac832a 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
@@ -18,11 +18,15 @@
package org.apache.kylin.storage.hbase;
+import java.util.Collections;
import java.util.List;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.*;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
+import org.apache.kylin.metadata.model.TblColRef;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
@@ -65,7 +69,7 @@ public class InvertedIndexHBaseTest extends HBaseMetadataTestCase {
Configuration hconf = HadoopUtil.newHBaseConfiguration(hbaseUrl);
hconn = HConnectionManager.createConnection(hconf);
- this.info = new TableRecordInfo(seg);
+ this.info = new TableRecordInfo(seg.getIIDesc(), Collections.<TblColRef, Dictionary<?>>emptyMap(), Collections.<TblColRef, FixedLenMeasureCodec<?>>emptyMap());
}
@After
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1b0177a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
index a4c7167..5ed48cf 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
@@ -35,6 +35,7 @@
package org.apache.kylin.streaming.invertedindex;
import com.google.common.base.Function;
+import com.google.common.base.Stopwatch;
import com.google.common.collect.*;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HConnectionManager;
@@ -67,6 +68,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.TimeUnit;
/**
* Created by qianzhou on 3/3/15.
@@ -93,6 +95,8 @@ public class IIStreamBuilder extends StreamBuilder {
@Override
protected void build(List<Stream> streamsToBuild) throws IOException {
+ logger.info("stream build start, size:" + streamsToBuild.size());
+ Stopwatch stopwatch = new Stopwatch().start();
List<List<String>> table = Lists.transform(streamsToBuild, new Function<Stream, List<String>>() {
@Nullable
@Override
@@ -114,6 +118,8 @@ public class IIStreamBuilder extends StreamBuilder {
final Slice slice = buildSlice(table, sliceBuilder, tableRecordInfo);
loadToHBase(hTable, slice, new IIKeyValueCodec(tableRecordInfo.getDigest()), desc.listAllColumns(), dictionaryMap);
submitOffset();
+ stopwatch.stop();
+ logger.info("stream build finished, size:" + streamsToBuild.size() + " elapsed time:" + stopwatch.elapsedTime(TimeUnit.MILLISECONDS) + TimeUnit.MILLISECONDS);
}
private Map<TblColRef, Dictionary<?>> buildDictionary(List<List<String>> table, IIDesc desc) {