You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/03/28 01:05:24 UTC
[47/50] incubator-kylin git commit: KYLIN-625, filter convert pass
KYLIN-625, filter convert pass
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/d7fc2312
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/d7fc2312
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/d7fc2312
Branch: refs/heads/streaming-localdict
Commit: d7fc2312c4c800cb2fec5264b6997feda6527521
Parents: 24acccc
Author: Li, Yang <ya...@ebay.com>
Authored: Fri Mar 27 18:26:39 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Fri Mar 27 18:26:39 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/ByteArray.java | 9 +-
.../metadata/filter/ColumnTupleFilter.java | 6 +-
.../metadata/filter/CompareTupleFilter.java | 2 +-
.../metadata/filter/ConstantTupleFilter.java | 4 +-
.../metadata/filter/LogicalTupleFilter.java | 2 +-
.../metadata/serializer/DataTypeSerializer.java | 3 +-
.../gridtable/GTDictionaryCodeSystem.java | 47 +++-
.../kylin/storage/gridtable/GTRecord.java | 29 ++-
.../storage/gridtable/GTSampleCodeSystem.java | 5 +-
.../kylin/storage/gridtable/GTScanRange.java | 11 +-
.../kylin/storage/gridtable/GTScanRequest.java | 8 +-
.../apache/kylin/storage/gridtable/GTUtil.java | 35 ++-
.../storage/gridtable/DictGridTableTest.java | 214 +++++++++++++++++++
.../storage/gridtable/GTInvertedIndexTest.java | 165 --------------
.../kylin/storage/gridtable/GridTableTest.java | 208 ------------------
.../storage/gridtable/SimpleGridTableTest.java | 208 ++++++++++++++++++
.../gridtable/SimpleInvertedIndexTest.java | 165 ++++++++++++++
17 files changed, 690 insertions(+), 431 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/ByteArray.java b/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
index d09b350..8c6ae91 100644
--- a/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
+++ b/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
@@ -30,7 +30,7 @@ public class ByteArray implements Comparable<ByteArray> {
public static ByteArray allocate(int length) {
return new ByteArray(new byte[length]);
}
-
+
public static ByteArray copyOf(byte[] array, int offset, int length) {
byte[] space = new byte[length];
System.arraycopy(array, offset, space, 0, length);
@@ -52,7 +52,7 @@ public class ByteArray implements Comparable<ByteArray> {
}
public ByteArray(byte[] data) {
- set(data, 0, data.length);
+ set(data, 0, data == null ? 0 : data.length);
}
public ByteArray(byte[] data, int offset, int length) {
@@ -148,7 +148,10 @@ public class ByteArray implements Comparable<ByteArray> {
@Override
public String toString() {
- return Bytes.toString(data, offset, length);
+ if (data == null)
+ return null;
+ else
+ return Bytes.toStringBinary(data, offset, length);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
index f689ccb..fde41b1 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
@@ -63,7 +63,7 @@ public class ColumnTupleFilter extends TupleFilter {
@Override
public String toString() {
- return "ColumnFilter [column=" + columnRef + "]";
+ return "" + columnRef;
}
@Override
@@ -79,7 +79,7 @@ public class ColumnTupleFilter extends TupleFilter {
@Override
public Collection<?> getValues() {
- this.values.set(0, (String) this.tupleValue);
+ this.values.set(0, this.tupleValue);
return this.values;
}
@@ -114,7 +114,7 @@ public class ColumnTupleFilter extends TupleFilter {
table = new TableDesc();
table.setName(tableName);
}
-
+
column.setId(BytesUtil.readUTFString(buffer));
column.setName(BytesUtil.readUTFString(buffer));
column.setDatatype(BytesUtil.readUTFString(buffer));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index 2b68469..57b50b7 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -126,7 +126,7 @@ public class CompareTupleFilter extends TupleFilter {
@Override
public String toString() {
- return "CompareFilter [" + column + " " + operator + " " + conditionValues + ", children=" + children + "]";
+ return column + " " + operator + " " + conditionValues;
}
// TODO requires generalize, currently only evaluates COLUMN {op} CONST
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
index f372b4a..cc3add2 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
@@ -34,7 +34,7 @@ import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
public class ConstantTupleFilter extends TupleFilter {
public static final ConstantTupleFilter FALSE = new ConstantTupleFilter();
- public static final ConstantTupleFilter TRUE = new ConstantTupleFilter("TRUE");
+ public static final ConstantTupleFilter TRUE = new ConstantTupleFilter((Object) null); // not sure of underlying code system, null is the only value that applies to all types
private Collection<Object> constantValues;
@@ -60,7 +60,7 @@ public class ConstantTupleFilter extends TupleFilter {
@Override
public String toString() {
- return "ConstantFilter [constant=" + constantValues + "]";
+ return "" + constantValues;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 1844392..4d38565 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -67,7 +67,7 @@ public class LogicalTupleFilter extends TupleFilter {
@Override
public String toString() {
- return "LogicalFilter [operator=" + operator + ", children=" + children + "]";
+ return operator + " " + children;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/metadata/src/main/java/org/apache/kylin/metadata/serializer/DataTypeSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/serializer/DataTypeSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/serializer/DataTypeSerializer.java
index 63d4ddd..aafb1c2 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/serializer/DataTypeSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/serializer/DataTypeSerializer.java
@@ -70,9 +70,10 @@ abstract public class DataTypeSerializer<T> implements BytesSerializer<T> {
/** peek into buffer and return the length of serialization */
abstract public int peekLength(ByteBuffer in);
- /** convert from String to obj */
+ /** convert from String to obj (string often come as byte[] in mapred) */
abstract public T valueOf(byte[] value);
+ /** convert from String to obj */
public T valueOf(String value) {
try {
return valueOf(value.getBytes("UTF-8"));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/storage/src/main/java/org/apache/kylin/storage/gridtable/GTDictionaryCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTDictionaryCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTDictionaryCodeSystem.java
index ada4ed7..c94c604 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTDictionaryCodeSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTDictionaryCodeSystem.java
@@ -2,10 +2,13 @@ package org.apache.kylin.storage.gridtable;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.metadata.filter.IFilterCodeSystem;
import org.apache.kylin.metadata.measure.MeasureAggregator;
import org.apache.kylin.metadata.serializer.DataTypeSerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer;
import java.util.Map;
@@ -15,13 +18,15 @@ import java.util.Map;
*/
@SuppressWarnings({ "rawtypes", "unchecked" })
public class GTDictionaryCodeSystem implements IGTCodeSystem {
+ private static final Logger logger = LoggerFactory.getLogger(CubeManager.class);
+
private GTInfo info;
- private Map<Integer, Dictionary> dictionaryMaps = null; // key: column index; value: dictionary for this column;
+ private Map<Integer, Dictionary> dictionaryMap = null; // key: column index; value: dictionary for this column;
private IFilterCodeSystem<ByteArray> filterCS;
private DataTypeSerializer[] serializers;
- public GTDictionaryCodeSystem(Map<Integer, Dictionary> dictionaryMaps) {
- this.dictionaryMaps = dictionaryMaps;
+ public GTDictionaryCodeSystem(Map<Integer, Dictionary> dictionaryMap) {
+ this.dictionaryMap = dictionaryMap;
}
@Override
@@ -30,8 +35,8 @@ public class GTDictionaryCodeSystem implements IGTCodeSystem {
serializers = new DataTypeSerializer[info.nColumns];
for (int i = 0; i < info.nColumns; i++) {
- if (dictionaryMaps.get(i) != null) {
- serializers[i] = new DictionarySerializer(dictionaryMaps.get(i));
+ if (dictionaryMap.get(i) != null) {
+ serializers[i] = new DictionarySerializer(dictionaryMap.get(i));
} else {
serializers[i] = DataTypeSerializer.create(info.colTypes[i]);
}
@@ -56,7 +61,10 @@ public class GTDictionaryCodeSystem implements IGTCodeSystem {
@Override
public void serialize(ByteArray code, ByteBuffer buffer) {
- BytesUtil.writeByteArray(code.array(), code.offset(), code.length(), buffer);
+ if (code == null)
+ BytesUtil.writeByteArray(null, 0, 0, buffer);
+ else
+ BytesUtil.writeByteArray(code.array(), code.offset(), code.length(), buffer);
}
@Override
@@ -78,16 +86,33 @@ public class GTDictionaryCodeSystem implements IGTCodeSystem {
@Override
public void encodeColumnValue(int col, Object value, ByteBuffer buf) {
- serializers[col].serialize(value, buf);
+ encodeColumnValue(col, value, 0, buf);
}
@Override
public void encodeColumnValue(int col, Object value, int roundingFlag, ByteBuffer buf) {
+ // this is a bit too complicated, but encoding only happens once at build time, so it is OK
DataTypeSerializer serializer = serializers[col];
- if (serializer instanceof DictionarySerializer) {
- ((DictionarySerializer) serializer).serializeWithRounding(value, roundingFlag, buf);
- } else {
- serializer.serialize(value, buf);
+ try {
+ if (serializer instanceof DictionarySerializer) {
+ ((DictionarySerializer) serializer).serializeWithRounding(value, roundingFlag, buf);
+ } else {
+ serializer.serialize(value, buf);
+ }
+ } catch (ClassCastException ex) {
+ // try convert string into a correct object type
+ try {
+ if (value instanceof String) {
+ Object converted = serializer.valueOf((String) value);
+ if ((converted instanceof String) == false) {
+ encodeColumnValue(col, converted, roundingFlag, buf);
+ return;
+ }
+ }
+ } catch (Throwable e) {
+ logger.error("Fail to encode value '" + value + "'", e);
+ }
+ throw ex;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
index 605a469..aeefc2b 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
@@ -21,7 +21,7 @@ public class GTRecord implements Comparable<GTRecord> {
this.cols[i] = new ByteArray();
this.maskForEqualHashComp = info.colAll;
}
-
+
public ByteArray get(int i) {
return cols[i];
}
@@ -51,16 +51,17 @@ public class GTRecord implements Comparable<GTRecord> {
/** decode and return the values of this record */
public Object[] getValues() {
- return getValues(new Object[info.nColumns]);
+ return getValues(info.colAll, new Object[info.nColumns]);
}
/** decode and return the values of this record */
- public Object[] getValues(Object[] result) {
- for (int i = 0; i < info.nColumns; i++) {
- if (cols[i].array() == null)
+ public Object[] getValues(BitSet selectedColumns, Object[] result) {
+ assert selectedColumns.cardinality() <= result.length;
+ for (int i = 0, c = selectedColumns.nextSetBit(0); c >= 0; i++, c = selectedColumns.nextSetBit(c + 1)) {
+ if (cols[c].array() == null)
result[i] = null;
else
- result[i] = info.codeSystem.decodeColumnValue(i, cols[i].asBuffer());
+ result[i] = info.codeSystem.decodeColumnValue(c, cols[c].asBuffer());
}
return result;
}
@@ -92,11 +93,11 @@ public class GTRecord implements Comparable<GTRecord> {
public BitSet maskForEqualHashComp() {
return maskForEqualHashComp;
}
-
+
public void maskForEqualHashComp(BitSet set) {
this.maskForEqualHashComp = set;
}
-
+
@Override
public boolean equals(Object obj) {
if (this == obj)
@@ -132,7 +133,7 @@ public class GTRecord implements Comparable<GTRecord> {
assert this.info == o.info;
assert this.maskForEqualHashComp == o.maskForEqualHashComp; // reference equal for performance
IFilterCodeSystem<ByteArray> cs = info.codeSystem.getFilterCodeSystem();
-
+
int comp = 0;
for (int i = maskForEqualHashComp.nextSetBit(0); i >= 0; i = maskForEqualHashComp.nextSetBit(i + 1)) {
comp = cs.compare(cols[i], o.cols[i]);
@@ -141,10 +142,16 @@ public class GTRecord implements Comparable<GTRecord> {
}
return comp;
}
-
+
@Override
public String toString() {
- return Arrays.toString(getValues());
+ return toString(info.colAll);
+ }
+
+ public String toString(BitSet selectedColumns) {
+ Object[] values = new Object[selectedColumns.cardinality()];
+ getValues(selectedColumns, values);
+ return Arrays.toString(values);
}
// ============================================================================
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
index 083d8c2..aea4e49 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
@@ -54,7 +54,10 @@ public class GTSampleCodeSystem implements IGTCodeSystem {
@Override
public void serialize(ByteArray code, ByteBuffer buffer) {
- BytesUtil.writeByteArray(code.array(), code.offset(), code.length(), buffer);
+ if (code == null)
+ BytesUtil.writeByteArray(null, 0, 0, buffer);
+ else
+ BytesUtil.writeByteArray(code.array(), code.offset(), code.length(), buffer);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRange.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRange.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRange.java
index 08513f7..b09a01d 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRange.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRange.java
@@ -14,12 +14,9 @@ public class GTScanRange {
}
public GTScanRange(GTRecord pkStart, GTRecord pkEnd, List<GTRecord> hbaseFuzzyKeys) {
- assert pkStart.info == pkEnd.info;
- assert pkStart.maskForEqualHashComp() == pkStart.info.primaryKey;
- assert pkEnd.maskForEqualHashComp() == pkEnd.info.primaryKey;
this.pkStart = pkStart;
this.pkEnd = pkEnd;
- this.hbaseFuzzyKeys = hbaseFuzzyKeys == null ? Collections.<GTRecord>emptyList() : hbaseFuzzyKeys;
+ this.hbaseFuzzyKeys = hbaseFuzzyKeys == null ? Collections.<GTRecord> emptyList() : hbaseFuzzyKeys;
}
@Override
@@ -58,4 +55,10 @@ public class GTScanRange {
return false;
return true;
}
+
+ @Override
+ public String toString() {
+ return (pkStart == null ? "null" : pkStart.toString(pkStart.info.primaryKey)) //
+ + "-" + (pkEnd == null ? "null" : pkEnd.toString(pkEnd.info.primaryKey));
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java
index c92cba4..b71032c 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java
@@ -1,5 +1,6 @@
package org.apache.kylin.storage.gridtable;
+import java.util.Arrays;
import java.util.BitSet;
import java.util.Set;
@@ -87,7 +88,7 @@ public class GTScanRequest {
// un-evaluatable filter must be removed
if (TupleFilter.isEvaluableRecursively(filterPushDown) == false) {
Set<TblColRef> unevaluableColumns = Sets.newHashSet();
- filterPushDown = GTUtil.convertFilterUnevaluatable(filterPushDown, unevaluableColumns);
+ filterPushDown = GTUtil.convertFilterUnevaluatable(filterPushDown, info, unevaluableColumns);
// columns in un-evaluatable filter must be returned without loss so upper layer can do final evaluation
if (hasAggregation()) {
@@ -138,4 +139,9 @@ public class GTScanRequest {
return aggrMetricsFuncs;
}
+ @Override
+ public String toString() {
+ return "GTScanRequest [range=" + range + ", columns=" + columns + ", filterPushDown=" + filterPushDown + ", aggrGroupBy=" + aggrGroupBy + ", aggrMetrics=" + aggrMetrics + ", aggrMetricsFuncs=" + Arrays.toString(aggrMetricsFuncs) + "]";
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
index 1fb0376..7d042eb 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
@@ -34,18 +34,18 @@ public class GTUtil {
return new TblColRef(desc);
}
- public static TupleFilter convertFilterUnevaluatable(TupleFilter rootFilter, //
- final Set<TblColRef> unevaluatableColumnCollector) {
- return convertFilter(rootFilter, null, null, false, unevaluatableColumnCollector);
+ public static TupleFilter convertFilterUnevaluatable(TupleFilter rootFilter, GTInfo info, //
+ Set<TblColRef> unevaluatableColumnCollector) {
+ return convertFilter(rootFilter, info, null, false, unevaluatableColumnCollector);
}
- public static TupleFilter convertFilterConstants(TupleFilter rootFilter, final GTInfo info) {
+ public static TupleFilter convertFilterConstants(TupleFilter rootFilter, GTInfo info) {
return convertFilter(rootFilter, info, null, true, null);
}
- public static TupleFilter convertFilterColumnsAndConstants(TupleFilter rootFilter, final GTInfo info, //
- final Map<TblColRef, Integer> colMapping, //
- final Set<TblColRef> unevaluatableColumnCollector) {
+ public static TupleFilter convertFilterColumnsAndConstants(TupleFilter rootFilter, GTInfo info, //
+ Map<TblColRef, Integer> colMapping, //
+ Set<TblColRef> unevaluatableColumnCollector) {
return convertFilter(rootFilter, info, colMapping, true, unevaluatableColumnCollector);
}
@@ -68,6 +68,12 @@ public class GTUtil {
return ConstantTupleFilter.TRUE;
}
+ // shortcut for unEvaluatable filter
+ if (filter.isEvaluable() == false) {
+ TupleFilter.collectColumns(filter, unevaluatableColumnCollector);
+ return ConstantTupleFilter.TRUE;
+ }
+
// map to column onto grid table
if (colMapping != null && filter instanceof ColumnTupleFilter) {
ColumnTupleFilter colFilter = (ColumnTupleFilter) filter;
@@ -75,18 +81,9 @@ public class GTUtil {
return new ColumnTupleFilter(info.colRef(gtColIdx));
}
- // below consider compare filter only
- if (filter instanceof CompareTupleFilter) {
-
- // shortcut for unEvaluatable compare filter
- if (TupleFilter.isEvaluableRecursively(filter) == false) {
- TupleFilter.collectColumns(filter, unevaluatableColumnCollector);
- return ConstantTupleFilter.TRUE;
- }
-
- if (encodeConstants) {
- return encodeConstants((CompareTupleFilter) filter);
- }
+ // encode constants
+ if (encodeConstants && filter instanceof CompareTupleFilter) {
+ return encodeConstants((CompareTupleFilter) filter);
}
return filter;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/storage/src/test/java/org/apache/kylin/storage/gridtable/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/gridtable/DictGridTableTest.java b/storage/src/test/java/org/apache/kylin/storage/gridtable/DictGridTableTest.java
new file mode 100644
index 0000000..46ec66c
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/gridtable/DictGridTableTest.java
@@ -0,0 +1,214 @@
+package org.apache.kylin.storage.gridtable;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.BitSet;
+import java.util.Map;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.dict.NumberDictionaryBuilder;
+import org.apache.kylin.dict.StringBytesConverter;
+import org.apache.kylin.dict.TrieDictionaryBuilder;
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.ExtractTupleFilter;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.DataType;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.gridtable.GTInfo.Builder;
+import org.apache.kylin.storage.gridtable.memstore.GTSimpleMemStore;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+
+public class DictGridTableTest {
+
+ @Test
+ public void test() throws IOException {
+ GridTable table = newTestTable();
+ verifyFirstRow(table);
+ verifyScanWithUnevaluatableFilter(table);
+ verifyScanWithEvaluatableFilter(table);
+ }
+
+ private void verifyFirstRow(GridTable table) throws IOException {
+ doScanAndVerify(table, new GTScanRequest(table.getInfo()), "[1421193600000, 30, Yang, 10, 10.5]");
+ }
+
+ private void verifyScanWithUnevaluatableFilter(GridTable table) throws IOException {
+ GTInfo info = table.getInfo();
+
+ CompareTupleFilter fcomp = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
+ ExtractTupleFilter funevaluatable = unevaluatable(info.colRef(1));
+ LogicalTupleFilter filter = and(fcomp, funevaluatable);
+
+ GTScanRequest req = new GTScanRequest(info, null, setOf(0), setOf(3), new String[] { "sum" }, filter);
+ // note the unEvaluatable column 1 in filter is added to group by
+ assertEquals("GTScanRequest [range=null-null, columns={0, 1, 3}, filterPushDown=AND [NULL.GT_MOCKUP_TABLE.1 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], [null]], aggrGroupBy={0, 1}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", req.toString());
+
+ doScanAndVerify(table, req, "[1421280000000, 20, null, 20, null]");
+ }
+
+ private void verifyScanWithEvaluatableFilter(GridTable table) throws IOException {
+ GTInfo info = table.getInfo();
+
+ CompareTupleFilter fcomp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
+ CompareTupleFilter fcomp2 = compare(info.colRef(1), FilterOperatorEnum.GT, enc(info, 1, "10"));
+ LogicalTupleFilter filter = and(fcomp1, fcomp2);
+
+ GTScanRequest req = new GTScanRequest(info, null, setOf(0), setOf(3), new String[] { "sum" }, filter);
+ // note the evaluatable column 1 in filter is added to returned columns but not in group by
+ assertEquals("GTScanRequest [range=null-null, columns={0, 1, 3}, filterPushDown=AND [NULL.GT_MOCKUP_TABLE.1 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.2 GT [\\x00]], aggrGroupBy={0}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", req.toString());
+
+ doScanAndVerify(table, req, "[1421280000000, 30, null, 30, null]", "[1421366400000, 20, null, 40, null]");
+ }
+
+ private void doScanAndVerify(GridTable table, GTScanRequest req, String... verifyRows) throws IOException {
+ System.out.println(req);
+ IGTScanner scanner = table.scan(req);
+ int i = 0;
+ for (GTRecord r : scanner) {
+ System.out.println(r);
+ if (verifyRows != null && i < verifyRows.length) {
+ assertEquals(verifyRows[i], r.toString());
+ }
+ i++;
+ }
+ scanner.close();
+ }
+
+ private Object enc(GTInfo info, int col, String value) {
+ ByteBuffer buf = ByteBuffer.allocate(info.maxRecordLength);
+ info.codeSystem.encodeColumnValue(col, value, buf);
+ return ByteArray.copyOf(buf.array(), buf.arrayOffset(), buf.position());
+ }
+
+ private ExtractTupleFilter unevaluatable(TblColRef col) {
+ ExtractTupleFilter r = new ExtractTupleFilter(FilterOperatorEnum.EXTRACT);
+ r.addChild(new ColumnTupleFilter(col));
+ return r;
+ }
+
+ private CompareTupleFilter compare(TblColRef col, FilterOperatorEnum op, Object value) {
+ CompareTupleFilter result = new CompareTupleFilter(op);
+ result.addChild(new ColumnTupleFilter(col));
+ result.addChild(new ConstantTupleFilter(value));
+ return result;
+ }
+
+ private LogicalTupleFilter and(TupleFilter... children) {
+ return logic(FilterOperatorEnum.AND, children);
+ }
+
+ private LogicalTupleFilter or(TupleFilter... children) {
+ return logic(FilterOperatorEnum.AND, children);
+ }
+
+ private LogicalTupleFilter not(TupleFilter child) {
+ return logic(FilterOperatorEnum.AND, child);
+ }
+
+ private LogicalTupleFilter logic(FilterOperatorEnum op, TupleFilter... children) {
+ LogicalTupleFilter result = new LogicalTupleFilter(op);
+ for (TupleFilter c : children) {
+ result.addChild(c);
+ }
+ return result;
+ }
+
+ static GridTable newTestTable() throws IOException {
+ GTInfo info = newInfo();
+ GTSimpleMemStore store = new GTSimpleMemStore(info);
+ GridTable table = new GridTable(info, store);
+
+ GTRecord r = new GTRecord(table.getInfo());
+ GTBuilder builder = table.rebuild();
+
+ builder.write(r.setValues("2015-01-14", "30", "Yang", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-14", "30", "Luke", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "30", "Xu", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "20", "Dong", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "20", "Jason", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "20", "Mahone", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "20", "Qianhao", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "30", "George", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-17", "10", "Kejia", new LongWritable(10), new BigDecimal("10.5")));
+ builder.close();
+
+ return table;
+ }
+
+ static GTInfo newInfo() {
+ Builder builder = GTInfo.builder();
+ builder.setCodeSystem(newDictCodeSystem());
+ builder.setColumns( //
+ DataType.getInstance("timestamp"), //
+ DataType.getInstance("integer"), //
+ DataType.getInstance("varchar"), //
+ DataType.getInstance("bigint"), //
+ DataType.getInstance("decimal") //
+ );
+ builder.setPrimaryKey(setOf(0));
+ builder.setColumnPreferIndex(setOf(0));
+ builder.enableColumnBlock(new BitSet[] { setOf(0, 1, 2), setOf(3, 4) });
+ builder.enableRowBlock(4);
+ GTInfo info = builder.build();
+ return info;
+ }
+
+ @SuppressWarnings("rawtypes")
+ private static GTDictionaryCodeSystem newDictCodeSystem() {
+ Map<Integer, Dictionary> dictionaryMap = Maps.newHashMap();
+ dictionaryMap.put(1, newDictionaryOfInteger());
+ dictionaryMap.put(2, newDictionaryOfString());
+ return new GTDictionaryCodeSystem(dictionaryMap);
+ }
+
+ @SuppressWarnings("rawtypes")
+ private static Dictionary newDictionaryOfString() {
+ TrieDictionaryBuilder<String> builder = new TrieDictionaryBuilder<>(new StringBytesConverter());
+ builder.addValue("Dong");
+ builder.addValue("George");
+ builder.addValue("Jason");
+ builder.addValue("Kejia");
+ builder.addValue("Luke");
+ builder.addValue("Mahone");
+ builder.addValue("Qianhao");
+ builder.addValue("Shaofeng");
+ builder.addValue("Xu");
+ builder.addValue("Yang");
+ return builder.build(0);
+ }
+
+ @SuppressWarnings("rawtypes")
+ private static Dictionary newDictionaryOfInteger() {
+ NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<>(new StringBytesConverter());
+ builder.addValue("10");
+ builder.addValue("20");
+ builder.addValue("30");
+ builder.addValue("40");
+ builder.addValue("50");
+ builder.addValue("60");
+ builder.addValue("70");
+ builder.addValue("80");
+ builder.addValue("90");
+ builder.addValue("100");
+ return builder.build(0);
+ }
+
+ private static BitSet setOf(int... values) {
+ BitSet set = new BitSet();
+ for (int i : values)
+ set.set(i);
+ return set;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/storage/src/test/java/org/apache/kylin/storage/gridtable/GTInvertedIndexTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/gridtable/GTInvertedIndexTest.java b/storage/src/test/java/org/apache/kylin/storage/gridtable/GTInvertedIndexTest.java
deleted file mode 100644
index 1460039..0000000
--- a/storage/src/test/java/org/apache/kylin/storage/gridtable/GTInvertedIndexTest.java
+++ /dev/null
@@ -1,165 +0,0 @@
-package org.apache.kylin.storage.gridtable;
-
-import static org.junit.Assert.*;
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.serializer.StringSerializer;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-public class GTInvertedIndexTest {
-
- GTInfo info;
- GTInvertedIndex index;
- ArrayList<CompareTupleFilter> basicFilters = Lists.newArrayList();
- ArrayList<ConciseSet> basicResults = Lists.newArrayList();
-
- public GTInvertedIndexTest() {
-
- info = GridTableTest.advancedInfo();
- TblColRef colA = info.colRef(0);
-
- // block i contains value "i", the last is NULL
- index = new GTInvertedIndex(info);
- GTRowBlock mockBlock = GTRowBlock.allocate(info);
- GTRowBlock.Writer writer = mockBlock.getWriter();
- GTRecord record = new GTRecord(info);
- for (int i = 0; i < 10; i++) {
- record.setValues(i < 9 ? "" + i : null, "", "", new LongWritable(0), new BigDecimal(0));
- for (int j = 0; j < info.getRowBlockSize(); j++) {
- writer.append(record);
- }
- writer.readyForFlush();
- index.add(mockBlock);
-
- writer.clearForNext();
- }
-
- basicFilters.add(compare(colA, FilterOperatorEnum.ISNULL));
- basicResults.add(set(9));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.ISNOTNULL));
- basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.EQ, 0));
- basicResults.add(set(0));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.NEQ, 0));
- basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.IN, 0, 5));
- basicResults.add(set(0, 5));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.NOTIN, 0, 5));
- basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.LT, 3));
- basicResults.add(set(0, 1, 2));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.LTE, 3));
- basicResults.add(set(0, 1, 2, 3));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.GT, 3));
- basicResults.add(set(4, 5, 6, 7, 8));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.GTE, 3));
- basicResults.add(set(3, 4, 5, 6, 7, 8));
- }
-
- @Test
- public void testBasics() {
- for (int i = 0; i < basicFilters.size(); i++) {
- assertEquals(basicResults.get(i), index.filter(basicFilters.get(i)));
- }
- }
-
- @Test
- public void testLogicalAnd() {
- for (int i = 0; i < basicFilters.size(); i++) {
- for (int j = 0; j < basicFilters.size(); j++) {
- LogicalTupleFilter f = logical(FilterOperatorEnum.AND, basicFilters.get(i), basicFilters.get(j));
- ConciseSet r = basicResults.get(i).clone();
- r.retainAll(basicResults.get(j));
- assertEquals(r, index.filter(f));
- }
- }
- }
-
- @Test
- public void testLogicalOr() {
- for (int i = 0; i < basicFilters.size(); i++) {
- for (int j = 0; j < basicFilters.size(); j++) {
- LogicalTupleFilter f = logical(FilterOperatorEnum.OR, basicFilters.get(i), basicFilters.get(j));
- ConciseSet r = basicResults.get(i).clone();
- r.addAll(basicResults.get(j));
- assertEquals(r, index.filter(f));
- }
- }
- }
-
- @Test
- public void testNotEvaluable() {
- ConciseSet all = set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
-
- CompareTupleFilter notEvaluable = compare(info.colRef(1), FilterOperatorEnum.EQ, 0);
- assertEquals(all, index.filter(notEvaluable));
-
- LogicalTupleFilter or = logical(FilterOperatorEnum.OR, basicFilters.get(0), notEvaluable);
- assertEquals(all, index.filter(or));
-
- LogicalTupleFilter and = logical(FilterOperatorEnum.AND, basicFilters.get(0), notEvaluable);
- assertEquals(basicResults.get(0), index.filter(and));
- }
-
- public static CompareTupleFilter compare(TblColRef col, TupleFilter.FilterOperatorEnum op, int... ids) {
- CompareTupleFilter filter = new CompareTupleFilter(op);
- filter.addChild(columnFilter(col));
- for (int i : ids) {
- filter.addChild(constFilter(i));
- }
- return filter;
- }
-
- public static LogicalTupleFilter logical(TupleFilter.FilterOperatorEnum op, TupleFilter... filters) {
- LogicalTupleFilter filter = new LogicalTupleFilter(op);
- for (TupleFilter f : filters)
- filter.addChild(f);
- return filter;
- }
-
- public static ColumnTupleFilter columnFilter(TblColRef col) {
- return new ColumnTupleFilter(col);
- }
-
- public static ConstantTupleFilter constFilter(int id) {
- byte[] space = new byte[10];
- ByteBuffer buf = ByteBuffer.wrap(space);
- StringSerializer stringSerializer = new StringSerializer();
- stringSerializer.serialize("" + id, buf);
- ByteArray data = new ByteArray(buf.array(), buf.arrayOffset(), buf.position());
- return new ConstantTupleFilter(data);
- }
-
- public static ConciseSet set(int... ints) {
- ConciseSet set = new ConciseSet();
- for (int i : ints)
- set.add(i);
- return set;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/storage/src/test/java/org/apache/kylin/storage/gridtable/GridTableTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/gridtable/GridTableTest.java b/storage/src/test/java/org/apache/kylin/storage/gridtable/GridTableTest.java
deleted file mode 100644
index 6561c6e..0000000
--- a/storage/src/test/java/org/apache/kylin/storage/gridtable/GridTableTest.java
+++ /dev/null
@@ -1,208 +0,0 @@
-package org.apache.kylin.storage.gridtable;
-
-import static org.junit.Assert.*;
-
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.util.BitSet;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.kylin.metadata.model.DataType;
-import org.apache.kylin.storage.gridtable.GTInfo.Builder;
-import org.apache.kylin.storage.gridtable.memstore.GTSimpleMemStore;
-import org.junit.Test;
-
-public class GridTableTest {
-
- @Test
- public void testBasics() throws IOException {
- GTInfo info = basicInfo();
- GTSimpleMemStore store = new GTSimpleMemStore(info);
- GridTable table = new GridTable(info, store);
-
- GTBuilder builder = rebuild(table);
- IGTScanner scanner = scan(table);
- assertEquals(builder.getWrittenRowBlockCount(), scanner.getScannedRowBlockCount());
- assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount());
- }
-
- @Test
- public void testAdvanced() throws IOException {
- GTInfo info = advancedInfo();
- GTSimpleMemStore store = new GTSimpleMemStore(info);
- GridTable table = new GridTable(info, store);
-
- GTBuilder builder = rebuild(table);
- IGTScanner scanner = scan(table);
- assertEquals(builder.getWrittenRowBlockCount(), scanner.getScannedRowBlockCount());
- assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount());
- }
-
- @Test
- public void testAggregate() throws IOException {
- GTInfo info = advancedInfo();
- GTSimpleMemStore store = new GTSimpleMemStore(info);
- GridTable table = new GridTable(info, store);
-
- GTBuilder builder = rebuild(table);
- IGTScanner scanner = scanAndAggregate(table);
- assertEquals(builder.getWrittenRowBlockCount(), scanner.getScannedRowBlockCount());
- assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount());
- }
-
- @Test
- public void testAppend() throws IOException {
- GTInfo info = advancedInfo();
- GTSimpleMemStore store = new GTSimpleMemStore(info);
- GridTable table = new GridTable(info, store);
-
- rebuildViaAppend(table);
- IGTScanner scanner = scan(table);
- assertEquals(3, scanner.getScannedRowBlockCount());
- assertEquals(10, scanner.getScannedRowCount());
- }
-
- private IGTScanner scan(GridTable table) throws IOException {
- GTScanRequest req = new GTScanRequest(table.getInfo());
- IGTScanner scanner = table.scan(req);
- for (GTRecord r : scanner) {
- Object[] v = r.getValues();
- assertTrue(((String) v[0]).startsWith("2015-"));
- assertTrue(((String) v[2]).equals("Food"));
- assertTrue(((LongWritable) v[3]).get() == 10);
- assertTrue(((BigDecimal) v[4]).doubleValue() == 10.5);
- System.out.println(r);
- }
- scanner.close();
- System.out.println("Scanned Row Block Count: " + scanner.getScannedRowBlockCount());
- System.out.println("Scanned Row Count: " + scanner.getScannedRowCount());
- return scanner;
- }
-
- private IGTScanner scanAndAggregate(GridTable table) throws IOException {
- GTScanRequest req = new GTScanRequest(table.getInfo(), null, setOf(0, 2), setOf(3, 4), new String[] { "count", "sum" }, null);
- IGTScanner scanner = table.scan(req);
- int i = 0;
- for (GTRecord r : scanner) {
- Object[] v = r.getValues();
- switch (i) {
- case 0:
- assertTrue(((LongWritable) v[3]).get() == 20);
- assertTrue(((BigDecimal) v[4]).doubleValue() == 21.0);
- break;
- case 1:
- assertTrue(((LongWritable) v[3]).get() == 30);
- assertTrue(((BigDecimal) v[4]).doubleValue() == 31.5);
- break;
- case 2:
- assertTrue(((LongWritable) v[3]).get() == 40);
- assertTrue(((BigDecimal) v[4]).doubleValue() == 42.0);
- break;
- case 3:
- assertTrue(((LongWritable) v[3]).get() == 10);
- assertTrue(((BigDecimal) v[4]).doubleValue() == 10.5);
- break;
- default:
- fail();
- }
- i++;
- System.out.println(r);
- }
- scanner.close();
- System.out.println("Scanned Row Block Count: " + scanner.getScannedRowBlockCount());
- System.out.println("Scanned Row Count: " + scanner.getScannedRowCount());
- return scanner;
- }
-
- static GTBuilder rebuild(GridTable table) throws IOException {
- GTRecord r = new GTRecord(table.getInfo());
- GTBuilder builder = table.rebuild();
-
- builder.write(r.setValues("2015-01-14", "Yang", "Food", new LongWritable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-14", "Luke", "Food", new LongWritable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-15", "Xu", "Food", new LongWritable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-15", "Dong", "Food", new LongWritable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-15", "Jason", "Food", new LongWritable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "Mahone", "Food", new LongWritable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "Shaofeng", "Food", new LongWritable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "Qianhao", "Food", new LongWritable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "George", "Food", new LongWritable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-17", "Kejia", "Food", new LongWritable(10), new BigDecimal("10.5")));
- builder.close();
-
- System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount());
- System.out.println("Written Row Count: " + builder.getWrittenRowCount());
- return builder;
- }
-
- static void rebuildViaAppend(GridTable table) throws IOException {
- GTRecord r = new GTRecord(table.getInfo());
- GTBuilder builder;
-
- builder = table.append();
- builder.write(r.setValues("2015-01-14", "Yang", "Food", new LongWritable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-14", "Luke", "Food", new LongWritable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-15", "Xu", "Food", new LongWritable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-15", "Dong", "Food", new LongWritable(10), new BigDecimal("10.5")));
- builder.close();
- System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount());
- System.out.println("Written Row Count: " + builder.getWrittenRowCount());
-
- builder = table.append();
- builder.write(r.setValues("2015-01-15", "Jason", "Food", new LongWritable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "Mahone", "Food", new LongWritable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "Shaofeng", "Food", new LongWritable(10), new BigDecimal("10.5")));
- builder.close();
- System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount());
- System.out.println("Written Row Count: " + builder.getWrittenRowCount());
-
- builder = table.append();
- builder.write(r.setValues("2015-01-16", "Qianhao", "Food", new LongWritable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "George", "Food", new LongWritable(10), new BigDecimal("10.5")));
- builder.close();
- System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount());
- System.out.println("Written Row Count: " + builder.getWrittenRowCount());
-
- builder = table.append();
- builder.write(r.setValues("2015-01-17", "Kejia", "Food", new LongWritable(10), new BigDecimal("10.5")));
- builder.close();
- System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount());
- System.out.println("Written Row Count: " + builder.getWrittenRowCount());
- }
-
- static GTInfo basicInfo() {
- Builder builder = infoBuilder();
- GTInfo info = builder.build();
- return info;
- }
-
- static GTInfo advancedInfo() {
- Builder builder = infoBuilder();
- builder.enableColumnBlock(new BitSet[] { setOf(0, 1, 2), setOf(3, 4) });
- builder.enableRowBlock(4);
- GTInfo info = builder.build();
- return info;
- }
-
- private static Builder infoBuilder() {
- Builder builder = GTInfo.builder();
- builder.setCodeSystem(new GTSampleCodeSystem());
- builder.setColumns( //
- DataType.getInstance("varchar"), //
- DataType.getInstance("varchar"), //
- DataType.getInstance("varchar"), //
- DataType.getInstance("bigint"), //
- DataType.getInstance("decimal") //
- );
- builder.setPrimaryKey(setOf(0));
- builder.setColumnPreferIndex(setOf(0));
- return builder;
- }
-
- private static BitSet setOf(int... values) {
- BitSet set = new BitSet();
- for (int i : values)
- set.set(i);
- return set;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleGridTableTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleGridTableTest.java b/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleGridTableTest.java
new file mode 100644
index 0000000..c5878b3
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleGridTableTest.java
@@ -0,0 +1,208 @@
+package org.apache.kylin.storage.gridtable;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.BitSet;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.kylin.metadata.model.DataType;
+import org.apache.kylin.storage.gridtable.GTInfo.Builder;
+import org.apache.kylin.storage.gridtable.memstore.GTSimpleMemStore;
+import org.junit.Test;
+
+public class SimpleGridTableTest {
+
+ @Test
+ public void testBasics() throws IOException {
+ GTInfo info = basicInfo();
+ GTSimpleMemStore store = new GTSimpleMemStore(info);
+ GridTable table = new GridTable(info, store);
+
+ GTBuilder builder = rebuild(table);
+ IGTScanner scanner = scan(table);
+ assertEquals(builder.getWrittenRowBlockCount(), scanner.getScannedRowBlockCount());
+ assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount());
+ }
+
+ @Test
+ public void testAdvanced() throws IOException {
+ GTInfo info = advancedInfo();
+ GTSimpleMemStore store = new GTSimpleMemStore(info);
+ GridTable table = new GridTable(info, store);
+
+ GTBuilder builder = rebuild(table);
+ IGTScanner scanner = scan(table);
+ assertEquals(builder.getWrittenRowBlockCount(), scanner.getScannedRowBlockCount());
+ assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount());
+ }
+
+ @Test
+ public void testAggregate() throws IOException {
+ GTInfo info = advancedInfo();
+ GTSimpleMemStore store = new GTSimpleMemStore(info);
+ GridTable table = new GridTable(info, store);
+
+ GTBuilder builder = rebuild(table);
+ IGTScanner scanner = scanAndAggregate(table);
+ assertEquals(builder.getWrittenRowBlockCount(), scanner.getScannedRowBlockCount());
+ assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount());
+ }
+
+ @Test
+ public void testAppend() throws IOException {
+ GTInfo info = advancedInfo();
+ GTSimpleMemStore store = new GTSimpleMemStore(info);
+ GridTable table = new GridTable(info, store);
+
+ rebuildViaAppend(table);
+ IGTScanner scanner = scan(table);
+ assertEquals(3, scanner.getScannedRowBlockCount());
+ assertEquals(10, scanner.getScannedRowCount());
+ }
+
+ private IGTScanner scan(GridTable table) throws IOException {
+ GTScanRequest req = new GTScanRequest(table.getInfo());
+ IGTScanner scanner = table.scan(req);
+ for (GTRecord r : scanner) {
+ Object[] v = r.getValues();
+ assertTrue(((String) v[0]).startsWith("2015-"));
+ assertTrue(((String) v[2]).equals("Food"));
+ assertTrue(((LongWritable) v[3]).get() == 10);
+ assertTrue(((BigDecimal) v[4]).doubleValue() == 10.5);
+ System.out.println(r);
+ }
+ scanner.close();
+ System.out.println("Scanned Row Block Count: " + scanner.getScannedRowBlockCount());
+ System.out.println("Scanned Row Count: " + scanner.getScannedRowCount());
+ return scanner;
+ }
+
+ private IGTScanner scanAndAggregate(GridTable table) throws IOException {
+ GTScanRequest req = new GTScanRequest(table.getInfo(), null, setOf(0, 2), setOf(3, 4), new String[] { "count", "sum" }, null);
+ IGTScanner scanner = table.scan(req);
+ int i = 0;
+ for (GTRecord r : scanner) {
+ Object[] v = r.getValues();
+ switch (i) {
+ case 0:
+ assertTrue(((LongWritable) v[3]).get() == 20);
+ assertTrue(((BigDecimal) v[4]).doubleValue() == 21.0);
+ break;
+ case 1:
+ assertTrue(((LongWritable) v[3]).get() == 30);
+ assertTrue(((BigDecimal) v[4]).doubleValue() == 31.5);
+ break;
+ case 2:
+ assertTrue(((LongWritable) v[3]).get() == 40);
+ assertTrue(((BigDecimal) v[4]).doubleValue() == 42.0);
+ break;
+ case 3:
+ assertTrue(((LongWritable) v[3]).get() == 10);
+ assertTrue(((BigDecimal) v[4]).doubleValue() == 10.5);
+ break;
+ default:
+ fail();
+ }
+ i++;
+ System.out.println(r);
+ }
+ scanner.close();
+ System.out.println("Scanned Row Block Count: " + scanner.getScannedRowBlockCount());
+ System.out.println("Scanned Row Count: " + scanner.getScannedRowCount());
+ return scanner;
+ }
+
+ static GTBuilder rebuild(GridTable table) throws IOException {
+ GTRecord r = new GTRecord(table.getInfo());
+ GTBuilder builder = table.rebuild();
+
+ builder.write(r.setValues("2015-01-14", "Yang", "Food", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-14", "Luke", "Food", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "Xu", "Food", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "Dong", "Food", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "Jason", "Food", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "Mahone", "Food", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "Shaofeng", "Food", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "Qianhao", "Food", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "George", "Food", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-17", "Kejia", "Food", new LongWritable(10), new BigDecimal("10.5")));
+ builder.close();
+
+ System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount());
+ System.out.println("Written Row Count: " + builder.getWrittenRowCount());
+ return builder;
+ }
+
+ static void rebuildViaAppend(GridTable table) throws IOException {
+ GTRecord r = new GTRecord(table.getInfo());
+ GTBuilder builder;
+
+ builder = table.append();
+ builder.write(r.setValues("2015-01-14", "Yang", "Food", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-14", "Luke", "Food", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "Xu", "Food", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "Dong", "Food", new LongWritable(10), new BigDecimal("10.5")));
+ builder.close();
+ System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount());
+ System.out.println("Written Row Count: " + builder.getWrittenRowCount());
+
+ builder = table.append();
+ builder.write(r.setValues("2015-01-15", "Jason", "Food", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "Mahone", "Food", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "Shaofeng", "Food", new LongWritable(10), new BigDecimal("10.5")));
+ builder.close();
+ System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount());
+ System.out.println("Written Row Count: " + builder.getWrittenRowCount());
+
+ builder = table.append();
+ builder.write(r.setValues("2015-01-16", "Qianhao", "Food", new LongWritable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "George", "Food", new LongWritable(10), new BigDecimal("10.5")));
+ builder.close();
+ System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount());
+ System.out.println("Written Row Count: " + builder.getWrittenRowCount());
+
+ builder = table.append();
+ builder.write(r.setValues("2015-01-17", "Kejia", "Food", new LongWritable(10), new BigDecimal("10.5")));
+ builder.close();
+ System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount());
+ System.out.println("Written Row Count: " + builder.getWrittenRowCount());
+ }
+
+ static GTInfo basicInfo() {
+ Builder builder = infoBuilder();
+ GTInfo info = builder.build();
+ return info;
+ }
+
+ static GTInfo advancedInfo() {
+ Builder builder = infoBuilder();
+ builder.enableColumnBlock(new BitSet[] { setOf(0, 1, 2), setOf(3, 4) });
+ builder.enableRowBlock(4);
+ GTInfo info = builder.build();
+ return info;
+ }
+
+ private static Builder infoBuilder() {
+ Builder builder = GTInfo.builder();
+ builder.setCodeSystem(new GTSampleCodeSystem());
+ builder.setColumns( //
+ DataType.getInstance("varchar"), //
+ DataType.getInstance("varchar"), //
+ DataType.getInstance("varchar"), //
+ DataType.getInstance("bigint"), //
+ DataType.getInstance("decimal") //
+ );
+ builder.setPrimaryKey(setOf(0));
+ builder.setColumnPreferIndex(setOf(0));
+ return builder;
+ }
+
+ private static BitSet setOf(int... values) {
+ BitSet set = new BitSet();
+ for (int i : values)
+ set.set(i);
+ return set;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleInvertedIndexTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleInvertedIndexTest.java b/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleInvertedIndexTest.java
new file mode 100644
index 0000000..f96b709
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleInvertedIndexTest.java
@@ -0,0 +1,165 @@
+package org.apache.kylin.storage.gridtable;
+
+import static org.junit.Assert.*;
+import it.uniroma3.mat.extendedset.intset.ConciseSet;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.serializer.StringSerializer;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+public class SimpleInvertedIndexTest {
+
+ GTInfo info;
+ GTInvertedIndex index;
+ ArrayList<CompareTupleFilter> basicFilters = Lists.newArrayList();
+ ArrayList<ConciseSet> basicResults = Lists.newArrayList();
+
+ public SimpleInvertedIndexTest() {
+
+ info = SimpleGridTableTest.advancedInfo();
+ TblColRef colA = info.colRef(0);
+
+ // block i contains value "i", the last is NULL
+ index = new GTInvertedIndex(info);
+ GTRowBlock mockBlock = GTRowBlock.allocate(info);
+ GTRowBlock.Writer writer = mockBlock.getWriter();
+ GTRecord record = new GTRecord(info);
+ for (int i = 0; i < 10; i++) {
+ record.setValues(i < 9 ? "" + i : null, "", "", new LongWritable(0), new BigDecimal(0));
+ for (int j = 0; j < info.getRowBlockSize(); j++) {
+ writer.append(record);
+ }
+ writer.readyForFlush();
+ index.add(mockBlock);
+
+ writer.clearForNext();
+ }
+
+ basicFilters.add(compare(colA, FilterOperatorEnum.ISNULL));
+ basicResults.add(set(9));
+
+ basicFilters.add(compare(colA, FilterOperatorEnum.ISNOTNULL));
+ basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
+
+ basicFilters.add(compare(colA, FilterOperatorEnum.EQ, 0));
+ basicResults.add(set(0));
+
+ basicFilters.add(compare(colA, FilterOperatorEnum.NEQ, 0));
+ basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
+
+ basicFilters.add(compare(colA, FilterOperatorEnum.IN, 0, 5));
+ basicResults.add(set(0, 5));
+
+ basicFilters.add(compare(colA, FilterOperatorEnum.NOTIN, 0, 5));
+ basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
+
+ basicFilters.add(compare(colA, FilterOperatorEnum.LT, 3));
+ basicResults.add(set(0, 1, 2));
+
+ basicFilters.add(compare(colA, FilterOperatorEnum.LTE, 3));
+ basicResults.add(set(0, 1, 2, 3));
+
+ basicFilters.add(compare(colA, FilterOperatorEnum.GT, 3));
+ basicResults.add(set(4, 5, 6, 7, 8));
+
+ basicFilters.add(compare(colA, FilterOperatorEnum.GTE, 3));
+ basicResults.add(set(3, 4, 5, 6, 7, 8));
+ }
+
+ @Test
+ public void testBasics() {
+ for (int i = 0; i < basicFilters.size(); i++) {
+ assertEquals(basicResults.get(i), index.filter(basicFilters.get(i)));
+ }
+ }
+
+ @Test
+ public void testLogicalAnd() {
+ for (int i = 0; i < basicFilters.size(); i++) {
+ for (int j = 0; j < basicFilters.size(); j++) {
+ LogicalTupleFilter f = logical(FilterOperatorEnum.AND, basicFilters.get(i), basicFilters.get(j));
+ ConciseSet r = basicResults.get(i).clone();
+ r.retainAll(basicResults.get(j));
+ assertEquals(r, index.filter(f));
+ }
+ }
+ }
+
+ @Test
+ public void testLogicalOr() {
+ for (int i = 0; i < basicFilters.size(); i++) {
+ for (int j = 0; j < basicFilters.size(); j++) {
+ LogicalTupleFilter f = logical(FilterOperatorEnum.OR, basicFilters.get(i), basicFilters.get(j));
+ ConciseSet r = basicResults.get(i).clone();
+ r.addAll(basicResults.get(j));
+ assertEquals(r, index.filter(f));
+ }
+ }
+ }
+
+ @Test
+ public void testNotEvaluable() {
+ ConciseSet all = set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+
+ CompareTupleFilter notEvaluable = compare(info.colRef(1), FilterOperatorEnum.EQ, 0);
+ assertEquals(all, index.filter(notEvaluable));
+
+ LogicalTupleFilter or = logical(FilterOperatorEnum.OR, basicFilters.get(0), notEvaluable);
+ assertEquals(all, index.filter(or));
+
+ LogicalTupleFilter and = logical(FilterOperatorEnum.AND, basicFilters.get(0), notEvaluable);
+ assertEquals(basicResults.get(0), index.filter(and));
+ }
+
+ public static CompareTupleFilter compare(TblColRef col, TupleFilter.FilterOperatorEnum op, int... ids) {
+ CompareTupleFilter filter = new CompareTupleFilter(op);
+ filter.addChild(columnFilter(col));
+ for (int i : ids) {
+ filter.addChild(constFilter(i));
+ }
+ return filter;
+ }
+
+ public static LogicalTupleFilter logical(TupleFilter.FilterOperatorEnum op, TupleFilter... filters) {
+ LogicalTupleFilter filter = new LogicalTupleFilter(op);
+ for (TupleFilter f : filters)
+ filter.addChild(f);
+ return filter;
+ }
+
+ public static ColumnTupleFilter columnFilter(TblColRef col) {
+ return new ColumnTupleFilter(col);
+ }
+
+ public static ConstantTupleFilter constFilter(int id) {
+ byte[] space = new byte[10];
+ ByteBuffer buf = ByteBuffer.wrap(space);
+ StringSerializer stringSerializer = new StringSerializer();
+ stringSerializer.serialize("" + id, buf);
+ ByteArray data = new ByteArray(buf.array(), buf.arrayOffset(), buf.position());
+ return new ConstantTupleFilter(data);
+ }
+
+ public static ConciseSet set(int... ints) {
+ ConciseSet set = new ConciseSet();
+ for (int i : ints)
+ set.add(i);
+ return set;
+ }
+
+
+}