You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/01/15 13:26:49 UTC
[33/50] [abbrv] incubator-kylin git commit: refactor bitmap evaluator
refactor bitmap evaluator
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/aabc3fdc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/aabc3fdc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/aabc3fdc
Branch: refs/heads/inverted-index
Commit: aabc3fdc587e7841578926b9146d446ffcc0f71d
Parents: b3ac674
Author: honma <ho...@ebay.com>
Authored: Wed Jan 14 16:59:30 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Jan 14 18:05:45 2015 +0800
----------------------------------------------------------------------
.../invertedindex/index/BitMapContainer.java | 34 ++++++++------------
.../index/ColumnValueContainer.java | 2 +-
.../index/CompressedValueContainer.java | 20 +++++++++---
.../storage/filter/BitMapFilterEvaluator.java | 21 ++++++------
.../endpoint/SliceBitMapProvider.java | 5 +--
.../filter/BitMapFilterEvaluatorTest.java | 28 ++++++++--------
6 files changed, 57 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/aabc3fdc/invertedindex/src/main/java/com/kylinolap/invertedindex/index/BitMapContainer.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/com/kylinolap/invertedindex/index/BitMapContainer.java b/invertedindex/src/main/java/com/kylinolap/invertedindex/index/BitMapContainer.java
index 0b86bc2..d019ad1 100644
--- a/invertedindex/src/main/java/com/kylinolap/invertedindex/index/BitMapContainer.java
+++ b/invertedindex/src/main/java/com/kylinolap/invertedindex/index/BitMapContainer.java
@@ -16,7 +16,6 @@
package com.kylinolap.invertedindex.index;
-import com.kylinolap.metadata.model.ParameterDesc;
import it.uniroma3.mat.extendedset.intset.ConciseSet;
import java.nio.ByteBuffer;
@@ -75,32 +74,27 @@ public class BitMapContainer implements ColumnValueContainer {
valueBytes.set(temp, 0, valueLen);
}
- /**
- * if endExclusiveId == startInclusiveId + 1, the performance should be
- * nearly as good as {@link #getBitMap(int startInclusiveId )}
- */
@Override
- public ConciseSet getBitMap(int startInclusiveId, int endExclusiveId) {
- if (startInclusiveId == endExclusiveId) {
- //entry for getting null value
+ public ConciseSet getBitMap(Integer startId, Integer endId) {
+ if (startId == null && endId == null) {
return sets[this.nValues];
}
- ConciseSet ret = null;
- for (int i = startInclusiveId; i < endExclusiveId; ++i) {
- ConciseSet temp = getBitMap(i);
- if (ret == null) {
- ret = temp;
- } else {
- ret.addAll(temp);
- }
+ int start = 0;
+ int end = this.nValues - 1;
+ if (startId != null) {
+ start = startId;
+ }
+ if (endId != null) {
+ end = endId;
}
- if (ret != null) {
- return ret;
- } else {
- return new ConciseSet();
+ ConciseSet ret = new ConciseSet();
+ for (int i = start; i <= end; ++i) {
+ ConciseSet temp = getBitMap(i);
+ ret.addAll(temp);
}
+ return ret;
}
private ConciseSet getBitMap(int valueId) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/aabc3fdc/invertedindex/src/main/java/com/kylinolap/invertedindex/index/ColumnValueContainer.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/com/kylinolap/invertedindex/index/ColumnValueContainer.java b/invertedindex/src/main/java/com/kylinolap/invertedindex/index/ColumnValueContainer.java
index 0cb9957..586db6a 100644
--- a/invertedindex/src/main/java/com/kylinolap/invertedindex/index/ColumnValueContainer.java
+++ b/invertedindex/src/main/java/com/kylinolap/invertedindex/index/ColumnValueContainer.java
@@ -33,7 +33,7 @@ public interface ColumnValueContainer {
// works only after closeForChange()
void getValueAt(int i, ImmutableBytesWritable valueBytes);
- ConciseSet getBitMap(int startIncludsiveId, int endExclusiveId);
+ ConciseSet getBitMap(Integer startId,Integer endId);
int getMaxValueId();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/aabc3fdc/invertedindex/src/main/java/com/kylinolap/invertedindex/index/CompressedValueContainer.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/com/kylinolap/invertedindex/index/CompressedValueContainer.java b/invertedindex/src/main/java/com/kylinolap/invertedindex/index/CompressedValueContainer.java
index 043105f..4f469a0 100644
--- a/invertedindex/src/main/java/com/kylinolap/invertedindex/index/CompressedValueContainer.java
+++ b/invertedindex/src/main/java/com/kylinolap/invertedindex/index/CompressedValueContainer.java
@@ -60,12 +60,12 @@ public class CompressedValueContainer implements ColumnValueContainer {
}
@Override
- public ConciseSet getBitMap(int startInclusiveId, int endExclusiveId) {
+ public ConciseSet getBitMap(Integer startId, Integer endId) {
ConciseSet ret = new ConciseSet();
+ int nullId = com.kylinolap.dict.Dictionary.NULL_ID[valueLen];
- if (startInclusiveId == endExclusiveId) {
+ if (startId == null && endId == null) {
//entry for getting null values
- int nullId = com.kylinolap.dict.Dictionary.NULL_ID[valueLen];
for (int i = 0; i < size; ++i) {
int valueID = BytesUtil.readUnsigned(uncompressed, i * valueLen, valueLen);
if (nullId == valueID) {
@@ -78,9 +78,19 @@ public class CompressedValueContainer implements ColumnValueContainer {
//normal values
for (int i = 0; i < size; ++i) {
int valueID = BytesUtil.readUnsigned(uncompressed, i * valueLen, valueLen);
- if (valueID >= startInclusiveId && valueID < endExclusiveId) {
- ret.add(i);
+ if (valueID == nullId) {
+ continue;
}
+
+ if (startId != null && valueID < startId) {
+ continue;
+ }
+
+ if (endId != null && valueID > endId) {
+ continue;
+ }
+
+ ret.add(i);
}
return ret;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/aabc3fdc/storage/src/main/java/com/kylinolap/storage/filter/BitMapFilterEvaluator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/filter/BitMapFilterEvaluator.java b/storage/src/main/java/com/kylinolap/storage/filter/BitMapFilterEvaluator.java
index b857432..d121aee 100644
--- a/storage/src/main/java/com/kylinolap/storage/filter/BitMapFilterEvaluator.java
+++ b/storage/src/main/java/com/kylinolap/storage/filter/BitMapFilterEvaluator.java
@@ -21,7 +21,7 @@ public class BitMapFilterEvaluator {
public static interface BitMapProvider {
/** return records whose specified column having specified value */
- ConciseSet getBitMap(TblColRef col, int startIncludsiveId, int endExclusiveId);
+ ConciseSet getBitMap(TblColRef col, Integer startId, Integer endId);
/** return the size of the group */
int getRecordCount();
@@ -82,31 +82,31 @@ public class BitMapFilterEvaluator {
private ConciseSet evalCompareLT(CompareTupleFilter filter) {
int id = Dictionary.stringToDictId(filter.getFirstValue());
- return collectRange(filter.getColumn(), 0, id - 1);
+ return collectRange(filter.getColumn(), null, id - 1);
}
private ConciseSet evalCompareLTE(CompareTupleFilter filter) {
int id = Dictionary.stringToDictId(filter.getFirstValue());
- return collectRange(filter.getColumn(), 0, id);
+ return collectRange(filter.getColumn(), null, id);
}
private ConciseSet evalCompareGT(CompareTupleFilter filter) {
int id = Dictionary.stringToDictId(filter.getFirstValue());
- return collectRange(filter.getColumn(), id + 1, provider.getMaxValueId(filter.getColumn()));
+ return collectRange(filter.getColumn(), id + 1, null);
}
private ConciseSet evalCompareGTE(CompareTupleFilter filter) {
int id = Dictionary.stringToDictId(filter.getFirstValue());
- return collectRange(filter.getColumn(), id, provider.getMaxValueId(filter.getColumn()));
+ return collectRange(filter.getColumn(), id, null);
}
- private ConciseSet collectRange(TblColRef column, int from, int to) {
- return provider.getBitMap(column,from,to+1);
+ private ConciseSet collectRange(TblColRef column, Integer startId, Integer endId) {
+ return provider.getBitMap(column, startId, endId);
}
private ConciseSet evalCompareEqual(CompareTupleFilter filter) {
int id = Dictionary.stringToDictId(filter.getFirstValue());
- ConciseSet bitMap = provider.getBitMap(filter.getColumn(), id);
+ ConciseSet bitMap = provider.getBitMap(filter.getColumn(), id, id);
if (bitMap == null)
return null;
return bitMap.clone(); // NOTE the clone() to void messing provider's cache
@@ -123,7 +123,7 @@ public class BitMapFilterEvaluator {
ConciseSet set = new ConciseSet();
for (String value : filter.getValues()) {
int id = Dictionary.stringToDictId(value);
- ConciseSet bitMap = provider.getBitMap(filter.getColumn(), id);
+ ConciseSet bitMap = provider.getBitMap(filter.getColumn(), id, id);
if (bitMap == null)
return null;
set.addAll(bitMap);
@@ -147,8 +147,7 @@ public class BitMapFilterEvaluator {
}
private ConciseSet evalCompareIsNull(CompareTupleFilter filter) {
- int nullId = Dictionary.stringToDictId(filter.getNullString());
- ConciseSet bitMap = provider.getBitMap(filter.getColumn(), nullId);
+ ConciseSet bitMap = provider.getBitMap(filter.getColumn(), null, null);
if (bitMap == null)
return null;
return bitMap.clone(); // NOTE the clone() to void messing provider's cache
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/aabc3fdc/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/endpoint/SliceBitMapProvider.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/endpoint/SliceBitMapProvider.java b/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/endpoint/SliceBitMapProvider.java
index 5535a4e..aa01088 100644
--- a/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/endpoint/SliceBitMapProvider.java
+++ b/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/endpoint/SliceBitMapProvider.java
@@ -21,9 +21,10 @@ public class SliceBitMapProvider implements BitMapFilterEvaluator.BitMapProvider
this.type = type;
}
+
@Override
- public ConciseSet getBitMap(TblColRef col, int startIncludsiveId, int endExclusiveId) {
- return slice.getColumnValueContainer(type.getColIndexByTblColRef(col)).getBitMap(startIncludsiveId,endExclusiveId);
+ public ConciseSet getBitMap(TblColRef col, Integer startId, Integer endId) {
+ return slice.getColumnValueContainer(type.getColIndexByTblColRef(col)).getBitMap(startId,endId);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/aabc3fdc/storage/src/test/java/com/kylinolap/storage/filter/BitMapFilterEvaluatorTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/com/kylinolap/storage/filter/BitMapFilterEvaluatorTest.java b/storage/src/test/java/com/kylinolap/storage/filter/BitMapFilterEvaluatorTest.java
index 3c14fae..fd12d46 100644
--- a/storage/src/test/java/com/kylinolap/storage/filter/BitMapFilterEvaluatorTest.java
+++ b/storage/src/test/java/com/kylinolap/storage/filter/BitMapFilterEvaluatorTest.java
@@ -43,33 +43,33 @@ public class BitMapFilterEvaluatorTest {
private static final int REC_COUNT = 10;
@Override
- public ConciseSet getBitMap(TblColRef col, int startInclusiveId, int endExclusiveId) {
+ public ConciseSet getBitMap(TblColRef col, Integer startId, Integer endId) {
if (!col.equals(colA))
return null;
// i-th record has value ID i, and last record has value null
- if (startInclusiveId == endExclusiveId) {
+ if (startId == null && endId == null) {
//entry for getting null value
ConciseSet s = new ConciseSet();
s.add(getRecordCount() - 1);
return s;
}
- ConciseSet ret = null;
- for (int i = startInclusiveId; i < endExclusiveId; ++i) {
- ConciseSet temp = getBitMap(col,i);
- if (ret == null) {
- ret = temp;
- } else {
- ret.addAll(temp);
- }
+ int start = 0;
+ int end = MAX_ID;
+ if (startId != null) {
+ start = startId;
+ }
+ if (endId != null) {
+ end = endId;
}
- if (ret != null) {
- return ret;
- } else {
- return new ConciseSet();
+ ConciseSet ret = new ConciseSet();
+ for (int i = start; i <= end; ++i) {
+ ConciseSet temp = getBitMap(col, i);
+ ret.addAll(temp);
}
+ return ret;
}
public ConciseSet getBitMap(TblColRef col, int valueId) {