You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/03/14 01:01:07 UTC
[32/50] [abbrv] incubator-kylin git commit: refactor
refactor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ab458df4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ab458df4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ab458df4
Branch: refs/heads/streaming-localdict
Commit: ab458df401c23ea86dd5537e35c6d93669961246
Parents: 7b48b51
Author: qianhao.zhou <qi...@ebay.com>
Authored: Thu Mar 12 17:47:02 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Thu Mar 12 17:47:02 2015 +0800
----------------------------------------------------------------------
.../invertedindex/index/BitMapContainer.java | 382 +++++++++----------
.../kylin/invertedindex/index/SliceBuilder.java | 3 -
.../invertedindex/index/TableRecordInfo.java | 22 +-
.../invertedindex/model/IIKeyValueCodec.java | 164 +++-----
.../apache/kylin/invertedindex/model/IIRow.java | 67 ++++
.../invertedindex/model/KeyValueCodec.java | 49 +++
.../kylin/invertedindex/model/KeyValuePair.java | 78 ----
.../apache/kylin/invertedindex/tools/IICLI.java | 16 +-
.../invertedindex/InvertedIndexLocalTest.java | 40 +-
.../invertedindex/InvertedIndexReducer.java | 6 +-
.../storage/hbase/HBaseClientKVIterator.java | 13 +-
.../endpoint/HbaseServerKVIterator.java | 110 +++---
.../hbase/coprocessor/endpoint/IIEndpoint.java | 24 +-
.../invertedindex/IIStreamBuilder.java | 57 +--
14 files changed, 489 insertions(+), 542 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java
index 066a3c8..ad3b0c1 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java
@@ -34,194 +34,194 @@ import org.apache.kylin.dict.Dictionary;
/**
* @author yangli9
*/
-public class BitMapContainer implements ColumnValueContainer {
-
- int valueLen;
- int nValues;
- int size;
- ConciseSet[] sets;
- boolean closedForChange;
-
- transient byte[] temp;
-
- public BitMapContainer(TableRecordInfoDigest digest, int col) {
- this.valueLen = digest.length(col);
- this.size = 0;
- this.nValues = digest.getMaxID(col) + 1;
- this.sets = null;
- this.closedForChange = false;
-
- this.temp = new byte[valueLen];
- }
-
- @Override
- public void append(ImmutableBytesWritable valueBytes) {
- int value = BytesUtil.readUnsigned(valueBytes.get(), valueBytes.getOffset(), valueLen);
- append(value);
- }
-
- public void append(int value) {
- checkUpdateMode();
- if (value == Dictionary.NULL_ID[valueLen]) {
- value = nValues; // set[nValues] holds NULL
- }
- sets[value].add(size);
- size++;
- }
-
- @Override
- public void getValueAt(int i, ImmutableBytesWritable valueBytes) {
- int value = getValueIntAt(i);
- BytesUtil.writeUnsigned(value, temp, 0, valueLen);
- valueBytes.set(temp, 0, valueLen);
- }
-
- @Override
- public ConciseSet getBitMap(Integer startId, Integer endId) {
- if (startId == null && endId == null) {
- return sets[this.nValues];
- }
-
- int start = 0;
- int end = this.nValues - 1;
- if (startId != null) {
- start = startId;
- }
- if (endId != null) {
- end = endId;
- }
-
- ConciseSet ret = new ConciseSet();
- for (int i = start; i <= end; ++i) {
- ConciseSet temp = getBitMap(i);
- ret.addAll(temp);
- }
- return ret;
- }
-
- private ConciseSet getBitMap(int valueId) {
- if (valueId >= 0 && valueId <= getMaxValueId())
- return sets[valueId];
- else
- return sets[this.nValues];
- }
-
- @Override
- public int getMaxValueId() {
- return this.nValues - 1;
- }
-
- public int getValueIntAt(int i) {
- for (int v = 0; v < nValues; v++) {
- if (sets[v].contains(i)) {
- return v;
- }
- }
- // if v is not in [0..nValues-1], then it must be nValue (NULL)
- return Dictionary.NULL_ID[valueLen];
- }
-
- private void checkUpdateMode() {
- if (isClosedForChange()) {
- throw new IllegalStateException();
- }
- if (sets == null) {
- sets = new ConciseSet[nValues + 1];
- for (int i = 0; i <= nValues; i++) {
- sets[i] = new ConciseSet();
- }
- }
- }
-
- private boolean isClosedForChange() {
- return closedForChange;
- }
-
- @Override
- public void closeForChange() {
- closedForChange = true;
- }
-
- @Override
- public int getSize() {
- return size;
- }
-
- public List<ImmutableBytesWritable> toBytes() {
- if (isClosedForChange() == false)
- closeForChange();
-
- List<ImmutableBytesWritable> r = new ArrayList<ImmutableBytesWritable>(nValues + 1);
- for (int i = 0; i <= nValues; i++) {
- r.add(setToBytes(sets[i]));
- }
- return r;
- }
-
- public void fromBytes(List<ImmutableBytesWritable> bytes) {
- assert nValues + 1 == bytes.size();
- sets = new ConciseSet[nValues + 1];
- size = 0;
- for (int i = 0; i <= nValues; i++) {
- sets[i] = bytesToSet(bytes.get(i));
- size += sets[i].size();
- }
- closedForChange = true;
- }
-
- private ImmutableBytesWritable setToBytes(ConciseSet set) {
- byte[] array;
- if (set.isEmpty()) // ConciseSet.toByteBuffer() throws exception when
- // set is empty
- array = BytesUtil.EMPTY_BYTE_ARRAY;
- else
- array = set.toByteBuffer().array();
- return new ImmutableBytesWritable(array);
- }
-
- private ConciseSet bytesToSet(ImmutableBytesWritable bytes) {
- if (bytes.get() == null || bytes.getLength() == 0) {
- return new ConciseSet();
- } else {
- IntBuffer intBuffer = ByteBuffer.wrap(bytes.get(), bytes.getOffset(), bytes.getLength()).asIntBuffer();
- int[] words = new int[intBuffer.capacity()];
- intBuffer.get(words);
- return new ConciseSet(words, false);
- }
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + (closedForChange ? 1231 : 1237);
- result = prime * result + nValues;
- result = prime * result + Arrays.hashCode(sets);
- result = prime * result + size;
- result = prime * result + valueLen;
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- BitMapContainer other = (BitMapContainer) obj;
- if (closedForChange != other.closedForChange)
- return false;
- if (nValues != other.nValues)
- return false;
- if (!Arrays.equals(sets, other.sets))
- return false;
- if (size != other.size)
- return false;
- if (valueLen != other.valueLen)
- return false;
- return true;
- }
-
-}
+//public class BitMapContainer implements ColumnValueContainer {
+//
+// int valueLen;
+// int nValues;
+// int size;
+// ConciseSet[] sets;
+// boolean closedForChange;
+//
+// transient byte[] temp;
+//
+// public BitMapContainer(TableRecordInfoDigest digest, int col) {
+// this.valueLen = digest.length(col);
+// this.size = 0;
+// this.nValues = digest.getMaxID(col) + 1;
+// this.sets = null;
+// this.closedForChange = false;
+//
+// this.temp = new byte[valueLen];
+// }
+//
+// @Override
+// public void append(ImmutableBytesWritable valueBytes) {
+// int value = BytesUtil.readUnsigned(valueBytes.get(), valueBytes.getOffset(), valueLen);
+// append(value);
+// }
+//
+// public void append(int value) {
+// checkUpdateMode();
+// if (value == Dictionary.NULL_ID[valueLen]) {
+// value = nValues; // set[nValues] holds NULL
+// }
+// sets[value].add(size);
+// size++;
+// }
+//
+// @Override
+// public void getValueAt(int i, ImmutableBytesWritable valueBytes) {
+// int value = getValueIntAt(i);
+// BytesUtil.writeUnsigned(value, temp, 0, valueLen);
+// valueBytes.set(temp, 0, valueLen);
+// }
+//
+// @Override
+// public ConciseSet getBitMap(Integer startId, Integer endId) {
+// if (startId == null && endId == null) {
+// return sets[this.nValues];
+// }
+//
+// int start = 0;
+// int end = this.nValues - 1;
+// if (startId != null) {
+// start = startId;
+// }
+// if (endId != null) {
+// end = endId;
+// }
+//
+// ConciseSet ret = new ConciseSet();
+// for (int i = start; i <= end; ++i) {
+// ConciseSet temp = getBitMap(i);
+// ret.addAll(temp);
+// }
+// return ret;
+// }
+//
+// private ConciseSet getBitMap(int valueId) {
+// if (valueId >= 0 && valueId <= getMaxValueId())
+// return sets[valueId];
+// else
+// return sets[this.nValues];
+// }
+//
+// @Override
+// public int getMaxValueId() {
+// return this.nValues - 1;
+// }
+//
+// public int getValueIntAt(int i) {
+// for (int v = 0; v < nValues; v++) {
+// if (sets[v].contains(i)) {
+// return v;
+// }
+// }
+// // if v is not in [0..nValues-1], then it must be nValue (NULL)
+// return Dictionary.NULL_ID[valueLen];
+// }
+//
+// private void checkUpdateMode() {
+// if (isClosedForChange()) {
+// throw new IllegalStateException();
+// }
+// if (sets == null) {
+// sets = new ConciseSet[nValues + 1];
+// for (int i = 0; i <= nValues; i++) {
+// sets[i] = new ConciseSet();
+// }
+// }
+// }
+//
+// private boolean isClosedForChange() {
+// return closedForChange;
+// }
+//
+// @Override
+// public void closeForChange() {
+// closedForChange = true;
+// }
+//
+// @Override
+// public int getSize() {
+// return size;
+// }
+//
+// public List<ImmutableBytesWritable> toBytes() {
+// if (isClosedForChange() == false)
+// closeForChange();
+//
+// List<ImmutableBytesWritable> r = new ArrayList<ImmutableBytesWritable>(nValues + 1);
+// for (int i = 0; i <= nValues; i++) {
+// r.add(setToBytes(sets[i]));
+// }
+// return r;
+// }
+//
+// public void fromBytes(List<ImmutableBytesWritable> bytes) {
+// assert nValues + 1 == bytes.size();
+// sets = new ConciseSet[nValues + 1];
+// size = 0;
+// for (int i = 0; i <= nValues; i++) {
+// sets[i] = bytesToSet(bytes.get(i));
+// size += sets[i].size();
+// }
+// closedForChange = true;
+// }
+//
+// private ImmutableBytesWritable setToBytes(ConciseSet set) {
+// byte[] array;
+// if (set.isEmpty()) // ConciseSet.toByteBuffer() throws exception when
+// // set is empty
+// array = BytesUtil.EMPTY_BYTE_ARRAY;
+// else
+// array = set.toByteBuffer().array();
+// return new ImmutableBytesWritable(array);
+// }
+//
+// private ConciseSet bytesToSet(ImmutableBytesWritable bytes) {
+// if (bytes.get() == null || bytes.getLength() == 0) {
+// return new ConciseSet();
+// } else {
+// IntBuffer intBuffer = ByteBuffer.wrap(bytes.get(), bytes.getOffset(), bytes.getLength()).asIntBuffer();
+// int[] words = new int[intBuffer.capacity()];
+// intBuffer.get(words);
+// return new ConciseSet(words, false);
+// }
+// }
+//
+// @Override
+// public int hashCode() {
+// final int prime = 31;
+// int result = 1;
+// result = prime * result + (closedForChange ? 1231 : 1237);
+// result = prime * result + nValues;
+// result = prime * result + Arrays.hashCode(sets);
+// result = prime * result + size;
+// result = prime * result + valueLen;
+// return result;
+// }
+//
+// @Override
+// public boolean equals(Object obj) {
+// if (this == obj)
+// return true;
+// if (obj == null)
+// return false;
+// if (getClass() != obj.getClass())
+// return false;
+// BitMapContainer other = (BitMapContainer) obj;
+// if (closedForChange != other.closedForChange)
+// return false;
+// if (nValues != other.nValues)
+// return false;
+// if (!Arrays.equals(sets, other.sets))
+// return false;
+// if (size != other.size)
+// return false;
+// if (valueLen != other.valueLen)
+// return false;
+// return true;
+// }
+//
+//}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
index e217df0..72050a1 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
@@ -61,9 +61,6 @@ public class SliceBuilder {
// reset for next slice
nRecords = 0;
containers = new ColumnValueContainer[nColumns];
- for (int i : info.getDescriptor().getBitmapColumns()) {
- containers[i] = new BitMapContainer(info.getDigest(), i);
- }
for (int i : info.getDescriptor().getValueColumns()) {
containers[i] = new CompressedValueContainer(info.getDigest(), i,
nRecordsCap);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
index 0cfb05e..3e72f43 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
@@ -43,7 +43,7 @@ public class TableRecordInfo {
final IIDesc desc;
final TableRecordInfoDigest digest;
- final Map<TblColRef, Dictionary<?>> dictionaryMap;
+ final Map<Integer, Dictionary<?>> dictionaryMap;
public TableRecordInfo(IISegment iiSegment) {
@@ -59,10 +59,12 @@ public class TableRecordInfo {
measureCodecMap.put(tblColRef, FixedLenMeasureCodec.get(col.getType()));
} else {
String dictPath = iiSegment.getDictResPath(tblColRef);
- try {
- dictionaryMap.put(tblColRef, dictMgr.getDictionary(dictPath));
- } catch (IOException e) {
- throw new RuntimeException("dictionary " + dictPath + " does not exist ", e);
+ if (dictPath != null) {
+ try {
+ dictionaryMap.put(index, dictMgr.getDictionary(dictPath));
+ } catch (IOException e) {
+ throw new RuntimeException("dictionary " + dictPath + " does not exist ", e);
+ }
}
}
index++;
@@ -71,7 +73,7 @@ public class TableRecordInfo {
digest = createDigest(dictionaryMap, measureCodecMap);
}
- public TableRecordInfo(IIDesc desc, Map<TblColRef, Dictionary<?>> dictionaryMap, Map<TblColRef, FixedLenMeasureCodec<?>> measureCodecMap) {
+ public TableRecordInfo(IIDesc desc, Map<Integer, Dictionary<?>> dictionaryMap, Map<TblColRef, FixedLenMeasureCodec<?>> measureCodecMap) {
this.desc = desc;
this.dictionaryMap = dictionaryMap;
this.digest = createDigest(dictionaryMap, measureCodecMap);
@@ -81,7 +83,7 @@ public class TableRecordInfo {
return digest;
}
- private TableRecordInfoDigest createDigest(Map<TblColRef, Dictionary<?>> dictionaryMap, Map<TblColRef, FixedLenMeasureCodec<?>> measureCodecMap) {
+ private TableRecordInfoDigest createDigest(Map<Integer, Dictionary<?>> dictionaryMap, Map<TblColRef, FixedLenMeasureCodec<?>> measureCodecMap) {
int nColumns = getColumns().size();
boolean[] isMetric = new boolean[nColumns];
int[] lengths = new int[nColumns];
@@ -97,10 +99,10 @@ public class TableRecordInfo {
lengths[i] = fixedLenMeasureCodec.getLength();
}
} else {
- final Dictionary<?> dictionary = dictionaryMap.get(tblColRef);
+ final Dictionary<?> dictionary = dictionaryMap.get(i);
if (dictionary != null) {
lengths[i] = dictionary.getSizeOfId();
- dictMaxIds[i] = dictionaryMap.get(tblColRef).getMaxId();
+ dictMaxIds[i] = dictionary.getMaxId();
}
}
}
@@ -149,7 +151,7 @@ public class TableRecordInfo {
@SuppressWarnings("unchecked")
public Dictionary<String> dict(int col) {
// yes, all dictionaries are string based
- return (Dictionary<String>) dictionaryMap.get(desc.listAllColumns().get(col));
+ return (Dictionary<String>) dictionaryMap.get(col);
}
public int getTimestampColumn() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index 8ea4e70..2ebd53d 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -18,41 +18,37 @@
package org.apache.kylin.invertedindex.model;
-import java.util.*;
-
+import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Pair;
-
-import com.google.common.collect.Lists;
-
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.invertedindex.index.*;
+import org.apache.kylin.invertedindex.index.ColumnValueContainer;
+import org.apache.kylin.invertedindex.index.CompressedValueContainer;
+import org.apache.kylin.invertedindex.index.Slice;
+import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
+
+import java.util.*;
/**
* @author yangli9
*/
-public class IIKeyValueCodec {
+public class IIKeyValueCodec implements KeyValueCodec {
public static final int SHARD_LEN = 2;
public static final int TIMEPART_LEN = 8;
public static final int COLNO_LEN = 2;
- private TableRecordInfoDigest infoDigest;
-
- public IIKeyValueCodec(TableRecordInfoDigest digest) {
- this.infoDigest = digest;
+ public IIKeyValueCodec() {
}
- public Collection<KeyValuePair> encodeKeyValue(Slice slice) {
- ArrayList<KeyValuePair> result = Lists
+ @Override
+ public Collection<IIRow> encodeKeyValue(Slice slice) {
+ ArrayList<IIRow> result = Lists
.newArrayList();
ColumnValueContainer[] containers = slice.getColumnValueContainers();
for (int col = 0; col < containers.length; col++) {
- if (containers[col] instanceof BitMapContainer) {
- result.addAll(collectKeyValues(slice, col, (BitMapContainer) containers[col]));
- } else if (containers[col] instanceof CompressedValueContainer) {
+ if (containers[col] instanceof CompressedValueContainer) {
result.add(collectKeyValues(slice, col, (CompressedValueContainer) containers[col]));
} else {
throw new IllegalArgumentException("Unknown container class "
@@ -62,37 +58,21 @@ public class IIKeyValueCodec {
return result;
}
- private KeyValuePair collectKeyValues(Slice slice,
+ private IIRow collectKeyValues(Slice slice,
int col,
CompressedValueContainer container) {
- ImmutableBytesWritable key = encodeKey(slice.getShard(),
- slice.getTimestamp(), col, -1);
+ ImmutableBytesWritable key = encodeKey(slice.getShard(), slice.getTimestamp(), col);
ImmutableBytesWritable value = container.toBytes();
- return new KeyValuePair(col, key, value);
- }
-
- private List<KeyValuePair> collectKeyValues(Slice slice,
- int col,
- BitMapContainer container) {
- List<ImmutableBytesWritable> values = container.toBytes();
- ArrayList<KeyValuePair> list = Lists.newArrayListWithExpectedSize(values.size());
- for (int v = 0; v < values.size(); v++) {
- ImmutableBytesWritable key = encodeKey(slice.getShard(),
- slice.getTimestamp(), col, v);
- list.add(new KeyValuePair(col, key, values.get(v)));
- }
- return list;
+ return new IIRow(key, value, null);
}
- ImmutableBytesWritable encodeKey(short shard, long timestamp, int col,
- int colValue) {
+ ImmutableBytesWritable encodeKey(short shard, long timestamp, int col) {
byte[] bytes = new byte[20];
- int len = encodeKey(shard, timestamp, col, colValue, bytes, 0);
+ int len = encodeKey(shard, timestamp, col, bytes, 0);
return new ImmutableBytesWritable(bytes, 0, len);
}
- int encodeKey(short shard, long timestamp, int col, int colValue,
- byte[] buf, int offset) {
+ int encodeKey(short shard, long timestamp, int col, byte[] buf, int offset) {
int i = offset;
BytesUtil.writeUnsigned(shard, buf, i, SHARD_LEN);
@@ -103,55 +83,72 @@ public class IIKeyValueCodec {
BytesUtil.writeUnsigned(col, buf, i, COLNO_LEN);
i += COLNO_LEN;
- if (colValue >= 0) {
- int colLen = infoDigest.length(col);
- BytesUtil.writeUnsigned(colValue, buf, i, colLen);
- i += colLen;
- }
-
return i - offset;
}
- public Iterable<Slice> decodeKeyValue(
- Iterable<KeyValuePair> kvs) {
- return new Decoder(infoDigest, kvs);
+ @Override
+ public Iterable<Slice> decodeKeyValue(Iterable<IIRow> kvs) {
+ return new Decoder(kvs);
}
+ private static class SliceIterator implements Iterator<Slice> {
+
+ private final Iterator<IIRow> rowIterator;
+
+ public SliceIterator(Iterator<IIRow> rowIterator) {
+ this.rowIterator = rowIterator;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (!rowIterator.hasNext()) {
+ return false;
+ }
+ return false;
+ }
+
+ @Override
+ public Slice next() {
+ return null;
+ }
+
+ @Override
+ public void remove() {
+
+ }
+ }
+
private static class Decoder implements Iterable<Slice> {
TableRecordInfoDigest info;
- Iterator<KeyValuePair> iterator;
+ Iterator<IIRow> iterator;
- Slice next = null;
+ Slice slice = null;
short curShard = Short.MIN_VALUE;
long curSliceTimestamp = Long.MIN_VALUE;
int curCol = -1;
- int curColValue = -1;
short lastShard = Short.MIN_VALUE;
long lastSliceTimestamp = Long.MIN_VALUE;
int lastCol = -1;
ColumnValueContainer[] containers = null;
- List<ImmutableBytesWritable> bitMapValues = Lists.newArrayList();
Map<Integer, Dictionary<?>> localDictionaries = Maps.newHashMap();
- Decoder(TableRecordInfoDigest info,
- Iterable<KeyValuePair> kvs) {
- this.info = info;
+ Decoder(Iterable<IIRow> kvs) {
this.iterator = kvs.iterator();
}
private void goToNext() {
- if (next != null) { // was not fetched
+ if (slice != null) { // was not fetched
return;
}
// NOTE the input keys are ordered
- while (next == null && iterator.hasNext()) {
- KeyValuePair kv = iterator.next();
+ while (slice == null && iterator.hasNext()) {
+ IIRow kv = iterator.next();
ImmutableBytesWritable k = kv.getKey();
ImmutableBytesWritable v = kv.getValue();
decodeKey(k);
- localDictionaries.put(curCol, kv.getDictionary());
+// localDictionaries.put(curCol, kv.getDictionary());
if (curShard != lastShard
|| curSliceTimestamp != lastSliceTimestamp) {
@@ -159,7 +156,7 @@ public class IIKeyValueCodec {
}
consumeCurrent(v);
}
- if (next == null) {
+ if (slice == null) {
makeNext();
}
}
@@ -176,63 +173,26 @@ public class IIKeyValueCodec {
curCol = BytesUtil.readUnsigned(buf, i, COLNO_LEN);
i += COLNO_LEN;
- if (i - k.getOffset() < k.getLength()) {
- // bitmap
- int colLen = info.length(curCol);
- curColValue = BytesUtil.readUnsigned(buf, i, colLen);
- i += colLen;
- } else {
- // value list
- curColValue = -1;
- }
}
private void consumeCurrent(ImmutableBytesWritable v) {
- if (curCol != lastCol && bitMapValues.size() > 0) { // end of a
- // bitmap
- // container
- addBitMapContainer(lastCol);
- }
- if (curColValue < 0) {
- CompressedValueContainer c = new CompressedValueContainer(info,
- curCol, 0);
- c.fromBytes(v);
- addContainer(curCol, c);
- } else {
- assert curColValue == bitMapValues.size();
- // make a copy, the value object from caller is typically reused
- // through iteration
- bitMapValues.add(new ImmutableBytesWritable(v));
- }
-
lastShard = curShard;
lastSliceTimestamp = curSliceTimestamp;
lastCol = curCol;
}
private void makeNext() {
- if (bitMapValues.isEmpty() == false) {
- addBitMapContainer(lastCol);
- }
if (containers != null) {
- next = new Slice(info, lastShard, lastSliceTimestamp,
+ slice = new Slice(info, lastShard, lastSliceTimestamp,
containers);
- next.setLocalDictionaries(Maps.newHashMap(localDictionaries));
+ slice.setLocalDictionaries(Maps.newHashMap(localDictionaries));
}
lastSliceTimestamp = Long.MIN_VALUE;
lastCol = -1;
containers = null;
- bitMapValues.clear();
localDictionaries.clear();
}
- private void addBitMapContainer(int col) {
- BitMapContainer c = new BitMapContainer(info, col);
- c.fromBytes(bitMapValues);
- addContainer(col, c);
- bitMapValues.clear();
- }
-
private void addContainer(int col, ColumnValueContainer c) {
if (containers == null) {
containers = new ColumnValueContainer[info.getColumnCount()];
@@ -246,13 +206,13 @@ public class IIKeyValueCodec {
@Override
public boolean hasNext() {
goToNext();
- return next != null;
+ return slice != null;
}
@Override
public Slice next() {
- Slice result = next;
- next = null;
+ Slice result = slice;
+ slice = null;
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java
new file mode 100644
index 0000000..aba4fff
--- /dev/null
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java
@@ -0,0 +1,67 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.invertedindex.model;
+
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+
+/**
+ * Created by qianzhou on 3/10/15.
+ */
+public final class IIRow {
+
+ private final ImmutableBytesWritable key;
+ private final ImmutableBytesWritable value;
+ private final ImmutableBytesWritable dictionary;
+
+ public IIRow(ImmutableBytesWritable key, ImmutableBytesWritable value, ImmutableBytesWritable dictionary) {
+ this.key = key;
+ this.value = value;
+ this.dictionary = dictionary;
+ }
+ public IIRow() {
+ this(new ImmutableBytesWritable(), new ImmutableBytesWritable(), new ImmutableBytesWritable());
+ }
+
+ public ImmutableBytesWritable getKey() {
+ return key;
+ }
+
+ public ImmutableBytesWritable getValue() {
+ return value;
+ }
+ public ImmutableBytesWritable getDictionary() {
+ return dictionary;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java
new file mode 100644
index 0000000..bbd8d61
--- /dev/null
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java
@@ -0,0 +1,49 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.invertedindex.model;
+
+import org.apache.kylin.invertedindex.index.Slice;
+
+import java.util.Collection;
+
+/**
+ * Created by qianzhou on 3/12/15.
+ */
+public interface KeyValueCodec {
+
+ Collection<IIRow> encodeKeyValue(Slice slice);
+
+ Iterable<Slice> decodeKeyValue(Iterable<IIRow> kvs);
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java
deleted file mode 100644
index 48c45f6..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.invertedindex.model;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.dict.Dictionary;
-
-/**
- * Created by qianzhou on 3/10/15.
- */
-public final class KeyValuePair {
-
- private final ImmutableBytesWritable key;
- private final ImmutableBytesWritable value;
- private final int id;
- private Dictionary<?> dictionary;
-
- public KeyValuePair(ImmutableBytesWritable key, ImmutableBytesWritable value) {
- this(-1, key, value);
- }
- public KeyValuePair(int id, ImmutableBytesWritable key, ImmutableBytesWritable value) {
- this.id = id;
- this.key = key;
- this.value = value;
- }
-
- public void setDictionary(Dictionary<?> dictionary) {
- this.dictionary = dictionary;
- }
-
- public int getId() {
- return this.id;
- }
-
- public Dictionary<?> getDictionary() {
- return dictionary;
- }
-
- public ImmutableBytesWritable getKey() {
- return key;
- }
-
- public ImmutableBytesWritable getValue() {
- return value;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
index eb65ca8..aecc37d 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
@@ -33,7 +33,7 @@ import org.apache.kylin.invertedindex.index.Slice;
import org.apache.kylin.invertedindex.index.TableRecord;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.invertedindex.model.KeyValuePair;
+import org.apache.kylin.invertedindex.model.IIRow;
import java.io.IOException;
import java.util.Iterator;
@@ -54,7 +54,7 @@ public class IICLI {
System.out.println("Reading from " + path + " ...");
TableRecordInfo info = new TableRecordInfo(ii.getFirstSegment());
- IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
+ IIKeyValueCodec codec = new IIKeyValueCodec();
int count = 0;
for (Slice slice : codec.decodeKeyValue(readSequenceKVs(hconf, path))) {
for (RawTableRecord rec : slice) {
@@ -65,17 +65,17 @@ public class IICLI {
System.out.println("Total " + count + " records");
}
- public static Iterable<KeyValuePair> readSequenceKVs(
+ public static Iterable<IIRow> readSequenceKVs(
Configuration hconf, String path) throws IOException {
final Reader reader = new Reader(hconf,
SequenceFile.Reader.file(new Path(path)));
- return new Iterable<KeyValuePair>() {
+ return new Iterable<IIRow>() {
@Override
- public Iterator<KeyValuePair> iterator() {
- return new Iterator<KeyValuePair>() {
+ public Iterator<IIRow> iterator() {
+ return new Iterator<IIRow>() {
ImmutableBytesWritable k = new ImmutableBytesWritable();
ImmutableBytesWritable v = new ImmutableBytesWritable();
- KeyValuePair pair = new KeyValuePair(k, v);
+ IIRow pair = new IIRow(k, v, null);
@Override
public boolean hasNext() {
@@ -93,7 +93,7 @@ public class IICLI {
}
@Override
- public KeyValuePair next() {
+ public IIRow next() {
return pair;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
index f040820..b152cdd 100644
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
@@ -29,8 +29,7 @@ import org.apache.kylin.invertedindex.index.*;
import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.kylin.invertedindex.model.KeyValuePair;
+import org.apache.kylin.invertedindex.model.IIRow;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -63,33 +62,6 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
}
@Test
- public void testBitMapContainer() {
- // create container
- BitMapContainer container = new BitMapContainer(info.getDigest(), 0);
- Dictionary<String> dict = info.dict(0);
- for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
- container.append(v);
- }
- container.append(Dictionary.NULL_ID[dict.getSizeOfId()]);
- container.closeForChange();
-
- // copy by serialization
- List<ImmutableBytesWritable> bytes = container.toBytes();
- BitMapContainer container2 = new BitMapContainer(info.getDigest(), 0);
- container2.fromBytes(bytes);
-
- // check the copy
- int i = 0;
- for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
- int value = container2.getValueIntAt(i++);
- assertEquals(v, value);
- }
- assertEquals(Dictionary.NULL_ID[dict.getSizeOfId()],
- container2.getValueIntAt(i++));
- assertEquals(container, container2);
- }
-
- @Test
public void testCompressedValueContainer() {
// create container
CompressedValueContainer container = new CompressedValueContainer(
@@ -136,8 +108,8 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
List<Slice> slices = buildTimeSlices(records);
System.out.println(slices.size() + " slices");
- IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
- List<KeyValuePair> kvs = encodeKVs(codec, slices);
+ IIKeyValueCodec codec = new IIKeyValueCodec();
+ List<IIRow> kvs = encodeKVs(codec, slices);
System.out.println(kvs.size() + " KV pairs");
List<Slice> slicesCopy = decodeKVs(codec, kvs);
@@ -198,10 +170,10 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
return slices;
}
- private List<KeyValuePair> encodeKVs(
+ private List<IIRow> encodeKVs(
IIKeyValueCodec codec, List<Slice> slices) {
- List<KeyValuePair> kvs = Lists
+ List<IIRow> kvs = Lists
.newArrayList();
for (Slice slice : slices) {
kvs.addAll(codec.encodeKeyValue(slice));
@@ -210,7 +182,7 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
}
private List<Slice> decodeKVs(IIKeyValueCodec codec,
- List<KeyValuePair> kvs) {
+ List<IIRow> kvs) {
List<Slice> slices = Lists.newArrayList();
for (Slice slice : codec.decodeKeyValue(kvs)) {
slices.add(slice);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
index 40cca36..285b058 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
@@ -31,7 +31,7 @@ import org.apache.kylin.invertedindex.index.SliceBuilder;
import org.apache.kylin.invertedindex.index.TableRecord;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.invertedindex.model.KeyValuePair;
+import org.apache.kylin.invertedindex.model.IIRow;
import org.apache.kylin.job.constant.BatchConstants;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
@@ -60,7 +60,7 @@ public class InvertedIndexReducer extends KylinReducer<LongWritable, ImmutableBy
info = new TableRecordInfo(seg);
rec = info.createTableRecord();
builder = null;
- kv = new IIKeyValueCodec(info.getDigest());
+ kv = new IIKeyValueCodec();
}
@Override
@@ -92,7 +92,7 @@ public class InvertedIndexReducer extends KylinReducer<LongWritable, ImmutableBy
}
private void output(Slice slice, Context context) throws IOException, InterruptedException {
- for (KeyValuePair pair : kv.encodeKeyValue(slice)) {
+ for (IIRow pair : kv.encodeKeyValue(slice)) {
context.write(pair.getKey(), pair.getValue());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
index 29b2bcf..05d2421 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
@@ -29,15 +29,14 @@ import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Pair;
import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.KeyValuePair;
+import org.apache.kylin.invertedindex.model.IIRow;
/**
* @author yangli9
*
*/
-public class HBaseClientKVIterator implements Iterable<KeyValuePair>, Closeable {
+public class HBaseClientKVIterator implements Iterable<IIRow>, Closeable {
byte[] family;
@@ -60,15 +59,15 @@ public class HBaseClientKVIterator implements Iterable<KeyValuePair>, Closeable
}
@Override
- public Iterator<KeyValuePair> iterator() {
+ public Iterator<IIRow> iterator() {
return new MyIterator();
}
- private class MyIterator implements Iterator<KeyValuePair> {
+ private class MyIterator implements Iterator<IIRow> {
ImmutableBytesWritable key = new ImmutableBytesWritable();
ImmutableBytesWritable value = new ImmutableBytesWritable();
- KeyValuePair pair = new KeyValuePair(key, value);
+ IIRow pair = new IIRow(key, value);
@Override
public boolean hasNext() {
@@ -76,7 +75,7 @@ public class HBaseClientKVIterator implements Iterable<KeyValuePair>, Closeable
}
@Override
- public KeyValuePair next() {
+ public IIRow next() {
Result r = iterator.next();
Cell c = r.getColumnLatestCell(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES);
key.set(c.getRowArray(), c.getRowOffset(), c.getRowLength());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
index 47aa7d9..3dbc8ea 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
@@ -18,21 +18,17 @@
package org.apache.kylin.storage.hbase.coprocessor.endpoint;
+import com.google.common.collect.Lists;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.KeyValuePair;
+import org.apache.kylin.invertedindex.model.IIRow;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.ByteArrayInputStream;
import java.io.Closeable;
-import java.io.DataInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
@@ -41,13 +37,11 @@ import java.util.List;
/**
* Created by honma on 11/10/14.
*/
-public class HbaseServerKVIterator implements Iterable<KeyValuePair>, Closeable {
+public class HbaseServerKVIterator implements Iterable<IIRow>, Closeable {
private RegionScanner innerScanner;
private Logger logger = LoggerFactory.getLogger(HbaseServerKVIterator.class);
- List<Cell> results = new ArrayList<Cell>();
-
public HbaseServerKVIterator(RegionScanner innerScanner) {
this.innerScanner = innerScanner;
}
@@ -57,69 +51,55 @@ public class HbaseServerKVIterator implements Iterable<KeyValuePair>, Closeable
IOUtils.closeQuietly(this.innerScanner);
}
- @Override
- public Iterator<KeyValuePair> iterator() {
- return new Iterator<KeyValuePair>() {
+ private static class IIRowIterator implements Iterator<IIRow> {
- ImmutableBytesWritable key = new ImmutableBytesWritable();
- ImmutableBytesWritable value = new ImmutableBytesWritable();
- ImmutableBytesWritable dict = new ImmutableBytesWritable();
- KeyValuePair pair = new KeyValuePair(key, value);
+ private final RegionScanner regionScanner;
+ private final IIRow row = new IIRow();
+ private boolean hasMore = true;
+ List<Cell> results = Lists.newArrayList();
- private boolean hasMore = true;
+ IIRowIterator(RegionScanner innerScanner) {
+ this.regionScanner = innerScanner;
+ }
- @Override
- public boolean hasNext() {
- return hasMore;
- }
- @Override
- public KeyValuePair next() {
- if (hasNext()) {
- try {
- hasMore = innerScanner.nextRaw(results);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- if (results.size() < 1)
- throw new IllegalStateException("Hbase row contains less than 1 cell");
-
- Dictionary<?> dictionary = null;
- boolean hasDictData = false;
- for (Cell c : results) {
- if (BytesUtil.compareBytes(IIDesc.HBASE_QUALIFIER_BYTES, 0, c.getQualifierArray(), c.getQualifierOffset(), IIDesc.HBASE_QUALIFIER_BYTES.length) == 0) {
- key.set(c.getRowArray(), c.getRowOffset(), c.getRowLength());
- value.set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
- } else if (BytesUtil.compareBytes(IIDesc.HBASE_DICTIONARY_BYTES, 0, c.getQualifierArray(), c.getQualifierOffset(), IIDesc.HBASE_DICTIONARY_BYTES.length) == 0) {
- dict.set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
- hasDictData = true;
- }
- }
- if (hasDictData) {
- try {
- final DataInputStream in = new DataInputStream(new ByteArrayInputStream(dict.get(), dict.getOffset(), dict.getLength()));
- String type = in.readUTF();
- dictionary = (Dictionary<?>) ClassUtil.forName(type, Dictionary.class).newInstance();
- dictionary.readFields(in);
- } catch (Exception e) {
- logger.error("error create dictionary", e);
- }
- }
- pair.setDictionary(dictionary);
-
- results.clear();
- return pair;
- } else {
- return null;
+ @Override
+ public boolean hasNext() {
+ return hasMore;
+ }
+
+ @Override
+ public IIRow next() {
+ results.clear();
+ try {
+ hasMore = regionScanner.next(results);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ if (results.size() < 1) {
+ throw new IllegalStateException("Hbase row contains less than 1 cell");
+ }
+ for (Cell c : results) {
+ if (BytesUtil.compareBytes(IIDesc.HBASE_QUALIFIER_BYTES, 0, c.getQualifierArray(), c.getQualifierOffset(), IIDesc.HBASE_QUALIFIER_BYTES.length) == 0) {
+ row.getKey().set(c.getRowArray(), c.getRowOffset(), c.getRowLength());
+ row.getValue().set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
+ } else if (BytesUtil.compareBytes(IIDesc.HBASE_DICTIONARY_BYTES, 0, c.getQualifierArray(), c.getQualifierOffset(), IIDesc.HBASE_DICTIONARY_BYTES.length) == 0) {
+ row.getDictionary().set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
}
}
+ return row;
+ }
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+
+ @Override
+ public Iterator<IIRow> iterator() {
+ return new IIRowIterator(innerScanner);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
index fbe3e8b..a20104d 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
@@ -22,19 +22,7 @@ import com.google.protobuf.ByteString;
import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
import com.google.protobuf.Service;
-
-import org.apache.kylin.invertedindex.index.RawTableRecord;
-import org.apache.kylin.invertedindex.index.Slice;
-import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
-import org.apache.kylin.storage.filter.BitMapFilterEvaluator;
-import org.apache.kylin.storage.hbase.coprocessor.*;
-import org.apache.kylin.storage.hbase.coprocessor.endpoint.generated.IIProtos;
-
import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
@@ -45,7 +33,15 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.protobuf.ResponseConverter;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.invertedindex.index.RawTableRecord;
+import org.apache.kylin.invertedindex.index.Slice;
+import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
+import org.apache.kylin.invertedindex.model.IIDesc;
+import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
+import org.apache.kylin.metadata.measure.MeasureAggregator;
+import org.apache.kylin.storage.filter.BitMapFilterEvaluator;
+import org.apache.kylin.storage.hbase.coprocessor.*;
+import org.apache.kylin.storage.hbase.coprocessor.endpoint.generated.IIProtos;
import java.io.IOException;
import java.util.Iterator;
@@ -94,7 +90,7 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop
region.startRegionOperation();
synchronized (innerScanner) {
- IIKeyValueCodec codec = new IIKeyValueCodec(tableRecordInfoDigest);
+ IIKeyValueCodec codec = new IIKeyValueCodec();
//TODO pass projector to codec to skip loading columns
Iterable<Slice> slices = codec.decodeKeyValue(new HbaseServerKVIterator(innerScanner));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
index 5ed48cf..10d212a 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Pair;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.invertedindex.index.Slice;
@@ -51,7 +50,7 @@ import org.apache.kylin.invertedindex.index.TableRecord;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.invertedindex.model.KeyValuePair;
+import org.apache.kylin.invertedindex.model.IIRow;
import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TblColRef;
@@ -104,7 +103,7 @@ public class IIStreamBuilder extends StreamBuilder {
return parseStream(input, desc);
}
});
- final Map<TblColRef, Dictionary<?>> dictionaryMap = buildDictionary(table, desc);
+ final Map<Integer, Dictionary<?>> dictionaryMap = buildDictionary(table, desc);
Map<TblColRef, FixedLenMeasureCodec<?>> measureCodecMap = Maps.newHashMap();
int index = 0;
for (TblColRef tblColRef : desc.listAllColumns()) {
@@ -115,14 +114,14 @@ public class IIStreamBuilder extends StreamBuilder {
}
TableRecordInfo tableRecordInfo = new TableRecordInfo(desc, dictionaryMap, measureCodecMap);
SliceBuilder sliceBuilder = new SliceBuilder(tableRecordInfo, (short) partitionId);
- final Slice slice = buildSlice(table, sliceBuilder, tableRecordInfo);
- loadToHBase(hTable, slice, new IIKeyValueCodec(tableRecordInfo.getDigest()), desc.listAllColumns(), dictionaryMap);
+ final Slice slice = buildSlice(table, sliceBuilder, tableRecordInfo, dictionaryMap);
+ loadToHBase(hTable, slice, new IIKeyValueCodec());
submitOffset();
stopwatch.stop();
logger.info("stream build finished, size:" + streamsToBuild.size() + " elapsed time:" + stopwatch.elapsedTime(TimeUnit.MILLISECONDS) + TimeUnit.MILLISECONDS);
}
- private Map<TblColRef, Dictionary<?>> buildDictionary(List<List<String>> table, IIDesc desc) {
+ private Map<Integer, Dictionary<?>> buildDictionary(List<List<String>> table, IIDesc desc) {
SetMultimap<TblColRef, String> valueMap = HashMultimap.create();
Set<TblColRef> dimensionColumns = Sets.newHashSet();
for (int i = 0; i < desc.listAllColumns().size(); i++) {
@@ -136,9 +135,9 @@ public class IIStreamBuilder extends StreamBuilder {
valueMap.put(desc.listAllColumns().get(i), cell);
}
}
- Map<TblColRef, Dictionary<?>> result = Maps.newHashMap();
+ Map<Integer, Dictionary<?>> result = Maps.newHashMap();
for (TblColRef tblColRef : valueMap.keys()) {
- result.put(tblColRef, DictionaryGenerator.buildDictionaryFromValueList(Collections2.transform(valueMap.get(tblColRef), new Function<String, byte[]>() {
+ result.put(desc.findColumn(tblColRef), DictionaryGenerator.buildDictionaryFromValueList(Collections2.transform(valueMap.get(tblColRef), new Function<String, byte[]>() {
@Nullable
@Override
public byte[] apply(String input) {
@@ -153,7 +152,7 @@ public class IIStreamBuilder extends StreamBuilder {
return Lists.newArrayList(new String(stream.getRawData()).split(","));
}
- private Slice buildSlice(List<List<String>> table, SliceBuilder sliceBuilder, TableRecordInfo tableRecordInfo) {
+ private Slice buildSlice(List<List<String>> table, SliceBuilder sliceBuilder, TableRecordInfo tableRecordInfo, Map<Integer, Dictionary<?>> localDictionary) {
for (List<String> row : table) {
TableRecord tableRecord = tableRecordInfo.createTableRecord();
for (int i = 0; i < row.size(); i++) {
@@ -161,31 +160,35 @@ public class IIStreamBuilder extends StreamBuilder {
}
sliceBuilder.append(tableRecord);
}
- return sliceBuilder.close();
+ final Slice slice = sliceBuilder.close();
+ slice.setLocalDictionaries(localDictionary);
+ return slice;
}
- private void loadToHBase(HTableInterface hTable, Slice slice, IIKeyValueCodec codec, List<TblColRef> allColumns, Map<TblColRef, Dictionary<?>> localDictionaries) throws IOException {
+ private byte[] getDictBytes(Dictionary<?> dict) throws IOException {
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(baos);
+ out.writeUTF(dict.getClass().getName());
+ dict.write(out);
+ return baos.toByteArray();
+ }
+
+ private void loadToHBase(HTableInterface hTable, Slice slice, IIKeyValueCodec codec) throws IOException {
try {
List<Put> data = Lists.newArrayList();
- for (KeyValuePair pair : codec.encodeKeyValue(slice)) {
- final byte[] key = pair.getKey().get();
- final byte[] value = pair.getValue().get();
+ for (IIRow row : codec.encodeKeyValue(slice)) {
+ final byte[] key = row.getKey().get();
+ final byte[] value = row.getValue().get();
Put put = new Put(key);
put.add(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES, value);
-
- //dictionary
- final int id = pair.getId();
- if (id >= 0 && id < allColumns.size()) {
- final Dictionary<?> dictionary = localDictionaries.get(allColumns.get(id));
- if (dictionary != null) {
- final ByteArrayOutputStream baos = new ByteArrayOutputStream();
- DataOutputStream out = new DataOutputStream(baos);
- out.writeUTF(dictionary.getClass().getName());
- dictionary.write(out);
- put.add(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_DICTIONARY_BYTES, baos.toByteArray());
- }
- data.add(put);
+ final ImmutableBytesWritable dictionary = row.getDictionary();
+ final byte[] dictBytes = dictionary.get();
+ if (dictionary.getOffset() == 0 && dictionary.getLength() == dictBytes.length) {
+ put.add(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_DICTIONARY_BYTES, dictBytes);
+ } else {
+ throw new RuntimeException("dict offset should be 0, and dict length should be " + dictBytes.length + " but they are" + dictionary.getOffset() + " " + dictionary.getLength());
}
+ data.add(put);
}
hTable.put(data);
} finally {