You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/03/14 01:01:08 UTC
[33/50] [abbrv] incubator-kylin git commit: refactor
refactor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/bb673625
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/bb673625
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/bb673625
Branch: refs/heads/streaming-localdict
Commit: bb6736258fd9146b9f7fb04745e1ad1990738193
Parents: ab458df
Author: qianhao.zhou <qi...@ebay.com>
Authored: Thu Mar 12 18:25:11 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Thu Mar 12 18:25:11 2015 +0800
----------------------------------------------------------------------
.../index/CompressedValueContainer.java | 8 ++++++--
.../invertedindex/model/IIKeyValueCodec.java | 18 +++++++++++++++++-
.../storage/hbase/HBaseClientKVIterator.java | 2 +-
3 files changed, 24 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bb673625/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
index 7a65489..7bd2266 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
@@ -42,8 +42,12 @@ public class CompressedValueContainer implements ColumnValueContainer {
byte[] compressed;
public CompressedValueContainer(TableRecordInfoDigest digest, int col, int cap) {
- this.valueLen = digest.length(col);
- this.nValues = digest.getMaxID(col) + 1;
+ this(digest.length(col), digest.getMaxID(col) + 1, cap);
+ }
+
+ public CompressedValueContainer(int valueLen, int nValues, int cap) {
+ this.valueLen = valueLen;
+ this.nValues = nValues;
this.cap = cap;
this.size = 0;
this.uncompressed = null;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bb673625/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index 2ebd53d..6955229 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -22,12 +22,14 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.invertedindex.index.ColumnValueContainer;
import org.apache.kylin.invertedindex.index.CompressedValueContainer;
import org.apache.kylin.invertedindex.index.Slice;
import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
+import java.io.*;
import java.util.*;
/**
@@ -137,6 +139,18 @@ public class IIKeyValueCodec implements KeyValueCodec {
this.iterator = kvs.iterator();
}
+ private Dictionary<?> deserialize(ImmutableBytesWritable dictBytes) {
+ try {
+ final DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(dictBytes.get(), dictBytes.getOffset(), dictBytes.getLength()));
+ final String type = dataInputStream.readUTF();
+ final Dictionary dictionary = ClassUtil.forName(type, Dictionary.class).newInstance();
+ dictionary.readFields(dataInputStream);
+ return dictionary;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
private void goToNext() {
if (slice != null) { // was not fetched
return;
@@ -148,7 +162,9 @@ public class IIKeyValueCodec implements KeyValueCodec {
ImmutableBytesWritable k = kv.getKey();
ImmutableBytesWritable v = kv.getValue();
decodeKey(k);
-// localDictionaries.put(curCol, kv.getDictionary());
+ final Dictionary<?> dictionary = deserialize(kv.getDictionary());
+ addContainer(curCol, new CompressedValueContainer(dictionary.getSizeOfId(), (dictionary.getMaxId() - dictionary.getMinId() + 1), (dictionary.getMaxId() - dictionary.getMinId() + 1)));
+ localDictionaries.put(curCol, dictionary);
if (curShard != lastShard
|| curSliceTimestamp != lastSliceTimestamp) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bb673625/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
index 05d2421..6c1085e 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
@@ -67,7 +67,7 @@ public class HBaseClientKVIterator implements Iterable<IIRow>, Closeable {
ImmutableBytesWritable key = new ImmutableBytesWritable();
ImmutableBytesWritable value = new ImmutableBytesWritable();
- IIRow pair = new IIRow(key, value);
+ IIRow pair = new IIRow(key, value, null);
@Override
public boolean hasNext() {