You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/03/14 01:01:38 UTC
[25/50] incubator-kylin git commit: KYLIN-625,
extract ICodeSystem to separate query level filter and coprocessor
level filter
KYLIN-625, extract ICodeSystem to separate query level filter and coprocessor level filter
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/78e9ed04
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/78e9ed04
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/78e9ed04
Branch: refs/heads/streaming
Commit: 78e9ed04892b8fe6e96432c67720c6999f6b1c35
Parents: fd0fbd5
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Mar 9 18:06:56 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Mon Mar 9 18:06:56 2015 +0800
----------------------------------------------------------------------
.../kylin/metadata/filter/CaseTupleFilter.java | 16 ++---
.../metadata/filter/ColumnTupleFilter.java | 24 ++++---
.../metadata/filter/CompareTupleFilter.java | 71 +++++++-------------
.../metadata/filter/ConstantTupleFilter.java | 22 +++---
.../metadata/filter/DynamicTupleFilter.java | 8 +--
.../metadata/filter/ExtractTupleFilter.java | 18 ++---
.../kylin/metadata/filter/ICodeSystem.java | 23 +++++++
.../metadata/filter/LogicalTupleFilter.java | 26 +++----
.../kylin/metadata/filter/StringCodeSystem.java | 40 +++++++++++
.../kylin/metadata/filter/TupleFilter.java | 8 +--
.../metadata/filter/TupleFilterSerializer.java | 26 +++----
.../kylin/metadata/model/DatabaseDesc.java | 2 +-
.../apache/kylin/metadata/model/TableDesc.java | 3 +-
.../kylin/query/enumerator/CubeEnumerator.java | 16 ++---
.../kylin/query/relnode/OLAPFilterRel.java | 2 +-
.../apache/kylin/query/test/KylinQueryTest.java | 18 +++--
.../storage/filter/BitMapFilterEvaluator.java | 14 ++--
.../kylin/storage/hbase/CubeStorageEngine.java | 5 +-
.../storage/hbase/DerivedFilterTranslator.java | 3 +-
.../hbase/coprocessor/CoprocessorFilter.java | 42 +++++-------
.../hbase/coprocessor/DictCodeSystem.java | 45 +++++++++++++
.../filter/BitMapFilterEvaluatorTest.java | 1 -
.../kylin/storage/filter/FilterBaseTest.java | 4 +-
.../storage/filter/FilterEvaluateTest.java | 36 +++++-----
.../storage/filter/FilterSerializeTest.java | 48 ++++++-------
25 files changed, 304 insertions(+), 217 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
index 5acbe5c..d3ec3e3 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
@@ -33,7 +33,7 @@ public class CaseTupleFilter extends TupleFilter {
private List<TupleFilter> whenFilters;
private List<TupleFilter> thenFilters;
private TupleFilter elseFilter;
- private Collection<String> values;
+ private Collection<?> values;
private int filterIndex;
public CaseTupleFilter() {
@@ -62,16 +62,16 @@ public class CaseTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple) {
+ public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs) {
if (whenFilters.size() != thenFilters.size()) {
elseFilter = whenFilters.remove(whenFilters.size() - 1);
}
boolean matched = false;
for (int i = 0; i < whenFilters.size(); i++) {
TupleFilter whenFilter = whenFilters.get(i);
- if (whenFilter.evaluate(tuple)) {
+ if (whenFilter.evaluate(tuple, cs)) {
TupleFilter thenFilter = thenFilters.get(i);
- thenFilter.evaluate(tuple);
+ thenFilter.evaluate(tuple, cs);
values = thenFilter.getValues();
matched = true;
break;
@@ -79,7 +79,7 @@ public class CaseTupleFilter extends TupleFilter {
}
if (!matched) {
if (elseFilter != null) {
- elseFilter.evaluate(tuple);
+ elseFilter.evaluate(tuple, cs);
values = elseFilter.getValues();
} else {
values = Collections.emptyList();
@@ -95,17 +95,17 @@ public class CaseTupleFilter extends TupleFilter {
}
@Override
- public Collection<String> getValues() {
+ public Collection<?> getValues() {
return this.values;
}
@Override
- public byte[] serialize() {
+ public byte[] serialize(ICodeSystem cs) {
return new byte[0];
}
@Override
- public void deserialize(byte[] bytes) {
+ public void deserialize(byte[] bytes, ICodeSystem cs) {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
index 1cf1f3c..a294112 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
@@ -38,12 +38,12 @@ public class ColumnTupleFilter extends TupleFilter {
private TblColRef columnRef;
private Object tupleValue;
- private List<String> values;
+ private List<Object> values;
public ColumnTupleFilter(TblColRef column) {
super(Collections.<TupleFilter> emptyList(), FilterOperatorEnum.COLUMN);
this.columnRef = column;
- this.values = new ArrayList<String>(1);
+ this.values = new ArrayList<Object>(1);
this.values.add(null);
}
@@ -66,7 +66,7 @@ public class ColumnTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple) {
+ public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs) {
this.tupleValue = tuple.getValue(columnRef);
return true;
}
@@ -77,17 +77,20 @@ public class ColumnTupleFilter extends TupleFilter {
}
@Override
- public Collection<String> getValues() {
+ public Collection<?> getValues() {
this.values.set(0, (String) this.tupleValue);
return this.values;
}
@Override
- public byte[] serialize() {
+ public byte[] serialize(ICodeSystem cs) {
ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
String table = columnRef.getTable();
BytesUtil.writeUTFString(table, buffer);
+ String columnId = columnRef.getColumn().getId();
+ BytesUtil.writeUTFString(columnId, buffer);
+
String columnName = columnRef.getName();
BytesUtil.writeUTFString(columnName, buffer);
@@ -100,18 +103,21 @@ public class ColumnTupleFilter extends TupleFilter {
}
@Override
- public void deserialize(byte[] bytes) {
+ public void deserialize(byte[] bytes, ICodeSystem cs) {
+ TableDesc table = null;
ColumnDesc column = new ColumnDesc();
ByteBuffer buffer = ByteBuffer.wrap(bytes);
+
String tableName = BytesUtil.readUTFString(buffer);
if (tableName != null) {
- TableDesc table = new TableDesc();
+ table = new TableDesc();
table.setName(tableName);
- column.setTable(table);
}
-
+
+ column.setId(BytesUtil.readUTFString(buffer));
column.setName(BytesUtil.readUTFString(buffer));
column.setDatatype(BytesUtil.readUTFString(buffer));
+ column.init(table);
this.columnRef = new TblColRef(column);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index 16114c5..48b31ea 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -34,15 +34,14 @@ import org.apache.kylin.metadata.model.TblColRef;
public class CompareTupleFilter extends TupleFilter {
private TblColRef column;
- private Collection<String> conditionValues;
- private String firstCondValue;
- private Map<String, String> dynamicVariables;
- private String nullString;
+ private Collection<Object> conditionValues;
+ private Object firstCondValue;
+ private Map<String, Object> dynamicVariables;
public CompareTupleFilter(FilterOperatorEnum op) {
super(new ArrayList<TupleFilter>(2), op);
- this.conditionValues = new HashSet<String>();
- this.dynamicVariables = new HashMap<String, String>();
+ this.conditionValues = new HashSet<Object>();
+ this.dynamicVariables = new HashMap<String, Object>();
boolean opGood = (op == FilterOperatorEnum.EQ || op == FilterOperatorEnum.NEQ //
|| op == FilterOperatorEnum.LT || op == FilterOperatorEnum.LTE //
|| op == FilterOperatorEnum.GT || op == FilterOperatorEnum.GTE //
@@ -55,9 +54,9 @@ public class CompareTupleFilter extends TupleFilter {
private CompareTupleFilter(CompareTupleFilter another) {
super(new ArrayList<TupleFilter>(another.children), another.operator);
this.column = another.column;
- this.conditionValues = new HashSet<String>();
+ this.conditionValues = new HashSet<Object>();
this.conditionValues.addAll(another.conditionValues);
- this.dynamicVariables = new HashMap<String, String>();
+ this.dynamicVariables = new HashMap<String, Object>();
this.dynamicVariables.putAll(another.dynamicVariables);
}
@@ -81,10 +80,6 @@ public class CompareTupleFilter extends TupleFilter {
DynamicTupleFilter dynamicFilter = (DynamicTupleFilter) child;
this.dynamicVariables.put(dynamicFilter.getVariableName(), null);
}
- //TODO
- // else if (child instanceof ExtractTupleFilter) {
- // } else if (child instanceof CaseTupleFilter) {
- // }
}
private boolean needSwapOperator() {
@@ -92,11 +87,11 @@ public class CompareTupleFilter extends TupleFilter {
}
@Override
- public Collection<String> getValues() {
+ public Collection<?> getValues() {
return conditionValues;
}
- public String getFirstValue() {
+ public Object getFirstValue() {
return firstCondValue;
}
@@ -104,24 +99,16 @@ public class CompareTupleFilter extends TupleFilter {
return column;
}
- public Map<String, String> getVariables() {
+ public Map<String, Object> getVariables() {
return dynamicVariables;
}
- public void bindVariable(String variable, String value) {
+ public void bindVariable(String variable, Object value) {
this.dynamicVariables.put(variable, value);
this.conditionValues.add(value);
this.firstCondValue = this.conditionValues.iterator().next();
}
- public String getNullString() {
- return nullString;
- }
-
- public void setNullString(String nullString) {
- this.nullString = nullString;
- }
-
@Override
public TupleFilter copy() {
return new CompareTupleFilter(this);
@@ -141,34 +128,30 @@ public class CompareTupleFilter extends TupleFilter {
// TODO requires generalize, currently only evaluates COLUMN {op} CONST
@Override
- public boolean evaluate(IEvaluatableTuple tuple) {
+ public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs) {
// extract tuple value
- String tupleValue = null;
+ Object tupleValue = null;
for (TupleFilter filter : this.children) {
if (isConstant(filter) == false) {
- filter.evaluate(tuple);
+ filter.evaluate(tuple, cs);
tupleValue = filter.getValues().iterator().next();
}
}
- // consider null string
- if (nullString != null && nullString.equals(tupleValue)) {
- tupleValue = null;
- }
- if (tupleValue == null) {
+ // consider null case
+ if (cs.isNull(tupleValue)) {
if (operator == FilterOperatorEnum.ISNULL)
return true;
else
return false;
}
-
- // always false if compare to null
- if (firstCondValue.equals(nullString))
+ if (cs.isNull(firstCondValue)) {
return false;
+ }
// tricky here -- order is ensured by string compare (even for number columns)
// because it's row key ID (not real value) being compared
- int comp = tupleValue.compareTo(firstCondValue);
+ int comp = cs.compare(tupleValue, firstCondValue);
boolean result;
switch (operator) {
@@ -212,31 +195,29 @@ public class CompareTupleFilter extends TupleFilter {
}
@Override
- public byte[] serialize() {
+ public byte[] serialize(ICodeSystem cs) {
ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
int size = this.dynamicVariables.size();
BytesUtil.writeVInt(size, buffer);
- for (Map.Entry<String, String> entry : this.dynamicVariables.entrySet()) {
+ for (Map.Entry<String, Object> entry : this.dynamicVariables.entrySet()) {
BytesUtil.writeUTFString(entry.getKey(), buffer);
- BytesUtil.writeUTFString(entry.getValue(), buffer);
+ cs.serialize(entry.getValue(), buffer);
}
- BytesUtil.writeAsciiString(nullString, buffer);
byte[] result = new byte[buffer.position()];
System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
return result;
}
@Override
- public void deserialize(byte[] bytes) {
+ public void deserialize(byte[] bytes, ICodeSystem cs) {
this.dynamicVariables.clear();
ByteBuffer buffer = ByteBuffer.wrap(bytes);
int size = BytesUtil.readVInt(buffer);
for (int i = 0; i < size; i++) {
- String nameString = BytesUtil.readUTFString(buffer);
- String valueString = BytesUtil.readUTFString(buffer);
- bindVariable(nameString, valueString);
+ String name = BytesUtil.readUTFString(buffer);
+ Object value = cs.deserialize(buffer);
+ bindVariable(name, value);
}
- this.nullString = BytesUtil.readAsciiString(buffer);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
index 1b21728..2280c6a 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
@@ -35,19 +35,19 @@ public class ConstantTupleFilter extends TupleFilter {
public static final ConstantTupleFilter FALSE = new ConstantTupleFilter();
public static final ConstantTupleFilter TRUE = new ConstantTupleFilter("TRUE");
- private Collection<String> constantValues;
+ private Collection<Object> constantValues;
public ConstantTupleFilter() {
super(Collections.<TupleFilter> emptyList(), FilterOperatorEnum.CONSTANT);
- this.constantValues = new HashSet<String>();
+ this.constantValues = new HashSet<Object>();
}
- public ConstantTupleFilter(String value) {
+ public ConstantTupleFilter(Object value) {
this();
this.constantValues.add(value);
}
- public ConstantTupleFilter(Collection<String> values) {
+ public ConstantTupleFilter(Collection<?> values) {
this();
this.constantValues.addAll(values);
}
@@ -63,7 +63,7 @@ public class ConstantTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple) {
+ public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs) {
return constantValues.size() > 0;
}
@@ -73,17 +73,17 @@ public class ConstantTupleFilter extends TupleFilter {
}
@Override
- public Collection<String> getValues() {
+ public Collection<?> getValues() {
return this.constantValues;
}
@Override
- public byte[] serialize() {
+ public byte[] serialize(ICodeSystem cs) {
ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
int size = this.constantValues.size();
BytesUtil.writeVInt(size, buffer);
- for (String val : this.constantValues) {
- BytesUtil.writeUTFString(val, buffer);
+ for (Object val : this.constantValues) {
+ cs.serialize(val, buffer);
}
byte[] result = new byte[buffer.position()];
System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
@@ -91,12 +91,12 @@ public class ConstantTupleFilter extends TupleFilter {
}
@Override
- public void deserialize(byte[] bytes) {
+ public void deserialize(byte[] bytes, ICodeSystem cs) {
this.constantValues.clear();
ByteBuffer buffer = ByteBuffer.wrap(bytes);
int size = BytesUtil.readVInt(buffer);
for (int i = 0; i < size; i++) {
- this.constantValues.add(BytesUtil.readUTFString(buffer));
+ this.constantValues.add(cs.deserialize(buffer));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
index 08426ba..64f12b8 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
@@ -53,7 +53,7 @@ public class DynamicTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple) {
+ public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs) {
return true;
}
@@ -63,12 +63,12 @@ public class DynamicTupleFilter extends TupleFilter {
}
@Override
- public Collection<String> getValues() {
+ public Collection<?> getValues() {
return Collections.emptyList();
}
@Override
- public byte[] serialize() {
+ public byte[] serialize(ICodeSystem cs) {
ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
BytesUtil.writeUTFString(variableName, buffer);
byte[] result = new byte[buffer.position()];
@@ -77,7 +77,7 @@ public class DynamicTupleFilter extends TupleFilter {
}
@Override
- public void deserialize(byte[] bytes) {
+ public void deserialize(byte[] bytes, ICodeSystem cs) {
ByteBuffer buffer = ByteBuffer.wrap(bytes);
this.variableName = BytesUtil.readUTFString(buffer);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
index a02f91a..929ab0f 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
@@ -31,12 +31,12 @@ import java.util.List;
public class ExtractTupleFilter extends TupleFilter {
private int date;
- private List<String> values;
+ private List<Object> values;
public ExtractTupleFilter(FilterOperatorEnum op) {
super(new ArrayList<TupleFilter>(3), op);
assert (op == FilterOperatorEnum.EXTRACT);
- this.values = new ArrayList<String>(1);
+ this.values = new ArrayList<Object>(1);
this.values.add(null);
this.date = 0;
}
@@ -52,16 +52,16 @@ public class ExtractTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple) {
+ public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs) {
// extract tuple value
String extractType = null;
String tupleValue = null;
for (TupleFilter filter : this.children) {
- filter.evaluate(tuple);
+ filter.evaluate(tuple, cs);
if (filter instanceof ConstantTupleFilter) {
- tupleValue = filter.getValues().iterator().next();
+ tupleValue = filter.getValues().iterator().next().toString();
} else if (filter instanceof CompareTupleFilter) {
- extractType = filter.getValues().iterator().next();
+ extractType = filter.getValues().iterator().next().toString();
}
}
@@ -106,18 +106,18 @@ public class ExtractTupleFilter extends TupleFilter {
}
@Override
- public Collection<String> getValues() {
+ public Collection<?> getValues() {
this.values.set(0, String.valueOf(this.date));
return this.values;
}
@Override
- public byte[] serialize() {
+ public byte[] serialize(ICodeSystem cs) {
return new byte[0];
}
@Override
- public void deserialize(byte[] bytes) {
+ public void deserialize(byte[] bytes, ICodeSystem cs) {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/filter/ICodeSystem.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ICodeSystem.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ICodeSystem.java
new file mode 100644
index 0000000..39dcd16
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ICodeSystem.java
@@ -0,0 +1,23 @@
+package org.apache.kylin.metadata.filter;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Decides how constant values in filter are coded and compared.
+ *
+ * TupleFilter are involved in both query engine and coprocessor. In query engine, the values are strings.
+ * In coprocessor, the values are dictionary IDs.
+ *
+ * @author yangli9
+ */
+public interface ICodeSystem {
+
+ boolean isNull(Object value);
+
+ int compare(Object tupleValue, Object constValue);
+
+ void serialize(Object value, ByteBuffer buffer);
+
+ Object deserialize(ByteBuffer buffer);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 2649674..6b77cb5 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -69,43 +69,43 @@ public class LogicalTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple) {
+ public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs) {
switch (this.operator) {
case AND:
- return evalAnd(tuple);
+ return evalAnd(tuple, cs);
case OR:
- return evalOr(tuple);
+ return evalOr(tuple, cs);
case NOT:
- return evalNot(tuple);
+ return evalNot(tuple, cs);
default:
return false;
}
}
- private boolean evalAnd(IEvaluatableTuple tuple) {
+ private boolean evalAnd(IEvaluatableTuple tuple, ICodeSystem cs) {
for (TupleFilter filter : this.children) {
- if (!filter.evaluate(tuple)) {
+ if (!filter.evaluate(tuple, cs)) {
return false;
}
}
return true;
}
- private boolean evalOr(IEvaluatableTuple tuple) {
+ private boolean evalOr(IEvaluatableTuple tuple, ICodeSystem cs) {
for (TupleFilter filter : this.children) {
- if (filter.evaluate(tuple)) {
+ if (filter.evaluate(tuple, cs)) {
return true;
}
}
return false;
}
- private boolean evalNot(IEvaluatableTuple tuple) {
- return !this.children.get(0).evaluate(tuple);
+ private boolean evalNot(IEvaluatableTuple tuple, ICodeSystem cs) {
+ return !this.children.get(0).evaluate(tuple, cs);
}
@Override
- public Collection<String> getValues() {
+ public Collection<?> getValues() {
return Collections.emptyList();
}
@@ -115,12 +115,12 @@ public class LogicalTupleFilter extends TupleFilter {
}
@Override
- public byte[] serialize() {
+ public byte[] serialize(ICodeSystem cs) {
return new byte[0];
}
@Override
- public void deserialize(byte[] bytes) {
+ public void deserialize(byte[] bytes, ICodeSystem cs) {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/filter/StringCodeSystem.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/StringCodeSystem.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/StringCodeSystem.java
new file mode 100644
index 0000000..b39b398
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/StringCodeSystem.java
@@ -0,0 +1,40 @@
+package org.apache.kylin.metadata.filter;
+
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.util.BytesUtil;
+
+/**
+ * A simple code system where all values are strings and conform to string comparison system.
+ *
+ * @author yangli9
+ */
+public class StringCodeSystem implements ICodeSystem {
+
+ public static final StringCodeSystem INSTANCE = new StringCodeSystem();
+
+ private StringCodeSystem() {
+ // singleton
+ }
+
+ @Override
+ public boolean isNull(Object value) {
+ return value == null;
+ }
+
+ @Override
+ public int compare(Object tupleValue, Object constValue) {
+ return ((String) tupleValue).compareTo((String) constValue);
+ }
+
+ @Override
+ public void serialize(Object value, ByteBuffer buffer) {
+ BytesUtil.writeUTFString((String) value, buffer);
+ }
+
+ @Override
+ public Object deserialize(ByteBuffer buffer) {
+ return BytesUtil.readUTFString(buffer);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 9a461c4..b87aae9 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -188,13 +188,13 @@ public abstract class TupleFilter {
public abstract boolean isEvaluable();
- public abstract boolean evaluate(IEvaluatableTuple tuple);
+ public abstract boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs);
- public abstract Collection<String> getValues();
+ public abstract Collection<?> getValues();
- public abstract byte[] serialize();
+ abstract byte[] serialize(ICodeSystem cs);
- public abstract void deserialize(byte[] bytes);
+ abstract void deserialize(byte[] bytes, ICodeSystem cs);
public static boolean isEvaluableRecursively(TupleFilter filter) {
if (filter == null)
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
index 04ffeb5..ba4b5e6 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -46,19 +46,19 @@ public class TupleFilterSerializer {
}
}
- public static byte[] serialize(TupleFilter rootFilter) {
- return serialize(rootFilter, null);
+ public static byte[] serialize(TupleFilter rootFilter, ICodeSystem cs) {
+ return serialize(rootFilter, null, cs);
}
- public static byte[] serialize(TupleFilter rootFilter, Decorator decorator) {
+ public static byte[] serialize(TupleFilter rootFilter, Decorator decorator, ICodeSystem cs) {
ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
- internalSerialize(rootFilter, decorator, buffer);
+ internalSerialize(rootFilter, decorator, buffer, cs);
byte[] result = new byte[buffer.position()];
System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
return result;
}
- private static void internalSerialize(TupleFilter filter, Decorator decorator, ByteBuffer buffer) {
+ private static void internalSerialize(TupleFilter filter, Decorator decorator, ByteBuffer buffer, ICodeSystem cs) {
if (decorator != null) { // give decorator a chance to manipulate the
// output filter
filter = decorator.onSerialize(filter);
@@ -70,24 +70,24 @@ public class TupleFilterSerializer {
if (filter.hasChildren()) {
// serialize filter+true
- serializeFilter(1, filter, decorator, buffer);
+ serializeFilter(1, filter, decorator, buffer, cs);
// serialize children
for (TupleFilter child : filter.getChildren()) {
- internalSerialize(child, decorator, buffer);
+ internalSerialize(child, decorator, buffer, cs);
}
// serialize none
- serializeFilter(-1, filter, decorator, buffer);
+ serializeFilter(-1, filter, decorator, buffer, cs);
} else {
// serialize filter+false
- serializeFilter(0, filter, decorator, buffer);
+ serializeFilter(0, filter, decorator, buffer, cs);
}
}
- private static void serializeFilter(int flag, TupleFilter filter, Decorator decorator, ByteBuffer buffer) {
+ private static void serializeFilter(int flag, TupleFilter filter, Decorator decorator, ByteBuffer buffer, ICodeSystem cs) {
if (flag < 0) {
BytesUtil.writeVInt(-1, buffer);
} else {
- byte[] bytes = filter.serialize();
+ byte[] bytes = filter.serialize(cs);
int opVal = filter.getOperator().getValue();
BytesUtil.writeVInt(opVal, buffer);
BytesUtil.writeByteArray(bytes, buffer);
@@ -95,7 +95,7 @@ public class TupleFilterSerializer {
}
}
- public static TupleFilter deserialize(byte[] bytes) {
+ public static TupleFilter deserialize(byte[] bytes, ICodeSystem cs) {
ByteBuffer buffer = ByteBuffer.wrap(bytes);
TupleFilter rootFilter = null;
Stack<TupleFilter> parentStack = new Stack<TupleFilter>();
@@ -109,7 +109,7 @@ public class TupleFilterSerializer {
// deserialize filter
TupleFilter filter = createTupleFilter(opVal);
byte[] filetrBytes = BytesUtil.readByteArray(buffer);
- filter.deserialize(filetrBytes);
+ filter.deserialize(filetrBytes, cs);
if (rootFilter == null) {
// push root to stack
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/model/DatabaseDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/DatabaseDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/DatabaseDesc.java
index 4b1f739..3269e43 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/DatabaseDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/DatabaseDesc.java
@@ -33,7 +33,7 @@ public class DatabaseDesc {
* @return the name
*/
public String getName() {
- return name.toUpperCase();
+ return name == null ? "null" : name.toUpperCase();
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index 14ea390..6db1202 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -41,7 +41,7 @@ public class TableDesc extends RootPersistentEntity {
@JsonProperty("columns")
private ColumnDesc[] columns;
- private DatabaseDesc database;
+ private DatabaseDesc database = new DatabaseDesc();
public ColumnDesc findColumnByName(String name) {
//ignore the db name and table name if exists
@@ -111,7 +111,6 @@ public class TableDesc extends RootPersistentEntity {
@JsonProperty("database")
public void setDatabase(String database) {
- this.database = new DatabaseDesc();
this.database.setName(database);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
index dafc960..9edd8cc 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
@@ -18,7 +18,6 @@
package org.apache.kylin.query.enumerator;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -27,17 +26,16 @@ import net.hydromatic.linq4j.Enumerator;
import net.hydromatic.optiq.DataContext;
import net.hydromatic.optiq.jdbc.OptiqConnection;
-import org.eigenbase.reltype.RelDataTypeField;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.storage.IStorageEngine;
-import org.apache.kylin.storage.StorageEngineFactory;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.query.relnode.OLAPContext;
+import org.apache.kylin.storage.IStorageEngine;
+import org.apache.kylin.storage.StorageEngineFactory;
+import org.eigenbase.reltype.RelDataTypeField;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* @author xjiang
@@ -169,7 +167,7 @@ public class CubeEnumerator implements Enumerator<Object[]> {
if (filter instanceof CompareTupleFilter && optiqContext != null) {
CompareTupleFilter compFilter = (CompareTupleFilter) filter;
- for (Map.Entry<String, String> entry : compFilter.getVariables().entrySet()) {
+ for (Map.Entry<String, Object> entry : compFilter.getVariables().entrySet()) {
String variable = entry.getKey();
Object value = optiqContext.get(variable);
if (value != null) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 0783e31..16f2562 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -156,7 +156,7 @@ public class OLAPFilterRel extends FilterRelBase implements OLAPRel, EnumerableR
private CompareTupleFilter mergeToInClause(TupleFilter filter) {
List<? extends TupleFilter> children = filter.getChildren();
TblColRef inColumn = null;
- List<String> inValues = new LinkedList<String>();
+ List<Object> inValues = new LinkedList<Object>();
for (TupleFilter child : children) {
if (child.getOperator() == FilterOperatorEnum.EQ) {
CompareTupleFilter compFilter = (CompareTupleFilter) child;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
index ace1dda..fa0ef5b 100644
--- a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
@@ -18,11 +18,17 @@
package org.apache.kylin.query.test;
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Properties;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.query.enumerator.OLAPQuery;
import org.apache.kylin.query.relnode.OLAPContext;
@@ -35,14 +41,6 @@ import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
-import java.io.File;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.util.List;
-import java.util.Properties;
-
-import static org.junit.Assert.assertTrue;
-
@Ignore("KylinQueryTest is contained by CombinationTest")
public class KylinQueryTest extends KylinTestBase {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/storage/src/main/java/org/apache/kylin/storage/filter/BitMapFilterEvaluator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/filter/BitMapFilterEvaluator.java b/storage/src/main/java/org/apache/kylin/storage/filter/BitMapFilterEvaluator.java
index 244662c..6bca948 100644
--- a/storage/src/main/java/org/apache/kylin/storage/filter/BitMapFilterEvaluator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/filter/BitMapFilterEvaluator.java
@@ -99,22 +99,22 @@ public class BitMapFilterEvaluator {
}
private ConciseSet evalCompareLT(CompareTupleFilter filter) {
- int id = Dictionary.stringToDictId(filter.getFirstValue());
+ int id = Dictionary.stringToDictId((String) filter.getFirstValue());
return collectRange(filter.getColumn(), null, id - 1);
}
private ConciseSet evalCompareLTE(CompareTupleFilter filter) {
- int id = Dictionary.stringToDictId(filter.getFirstValue());
+ int id = Dictionary.stringToDictId((String) filter.getFirstValue());
return collectRange(filter.getColumn(), null, id);
}
private ConciseSet evalCompareGT(CompareTupleFilter filter) {
- int id = Dictionary.stringToDictId(filter.getFirstValue());
+ int id = Dictionary.stringToDictId((String) filter.getFirstValue());
return collectRange(filter.getColumn(), id + 1, null);
}
private ConciseSet evalCompareGTE(CompareTupleFilter filter) {
- int id = Dictionary.stringToDictId(filter.getFirstValue());
+ int id = Dictionary.stringToDictId((String) filter.getFirstValue());
return collectRange(filter.getColumn(), id, null);
}
@@ -123,7 +123,7 @@ public class BitMapFilterEvaluator {
}
private ConciseSet evalCompareEqual(CompareTupleFilter filter) {
- int id = Dictionary.stringToDictId(filter.getFirstValue());
+ int id = Dictionary.stringToDictId((String) filter.getFirstValue());
ConciseSet bitMap = provider.getBitMap(filter.getColumn(), id, id);
if (bitMap == null)
return null;
@@ -139,8 +139,8 @@ public class BitMapFilterEvaluator {
private ConciseSet evalCompareIn(CompareTupleFilter filter) {
ConciseSet set = new ConciseSet();
- for (String value : filter.getValues()) {
- int id = Dictionary.stringToDictId(value);
+ for (Object value : filter.getValues()) {
+ int id = Dictionary.stringToDictId((String) value);
ConciseSet bitMap = provider.getBitMap(filter.getColumn(), id, id);
if (bitMap == null)
return null;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
index a4ecc2a..99a7613 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
@@ -26,7 +26,6 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
-
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@@ -37,6 +36,7 @@ import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+
import org.apache.kylin.common.persistence.HBaseConnection;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -469,7 +469,8 @@ public class CubeStorageEngine implements IStorageEngine {
continue;
}
- ColumnValueRange range = new ColumnValueRange(comp.getColumn(), comp.getValues(), comp.getOperator());
+ @SuppressWarnings("unchecked")
+ ColumnValueRange range = new ColumnValueRange(comp.getColumn(), (Collection<String>) comp.getValues(), comp.getOperator());
andMerge(range, rangeMap);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
index bfa9d3c..7fc0429 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
@@ -32,6 +32,7 @@ import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
import org.apache.kylin.metadata.filter.IEvaluatableTuple;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.StringCodeSystem;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.TblColRef;
@@ -75,7 +76,7 @@ public class DerivedFilterTranslator {
SingleColumnTuple tuple = new SingleColumnTuple(derivedCol);
for (String[] row : lookup.getAllRows()) {
tuple.value = row[di];
- if (compf.evaluate(tuple)) {
+ if (compf.evaluate(tuple, StringCodeSystem.INSTANCE)) {
collect(row, pi, satisfyingHostRecords);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
index aedce4d..74c7040 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
@@ -58,11 +58,12 @@ public class CoprocessorFilter {
}
@Override
+ @SuppressWarnings("unchecked")
public TupleFilter onSerialize(TupleFilter filter) {
if (filter == null)
return null;
- // In case of NOT(unEvaluatableFilter), we should immediatedly replace it as TRUE,
+ // In case of NOT(unEvaluatableFilter), we should immediately replace it as TRUE,
// Otherwise, unEvaluatableFilter will later be replace with TRUE and NOT(unEvaluatableFilter) will
// always return FALSE
if (filter.getOperator() == FilterOperatorEnum.NOT && !TupleFilter.isEvaluableRecursively(filter)) {
@@ -86,29 +87,25 @@ public class CoprocessorFilter {
return filter;
}
- String nullString = nullString(col);
- Collection<String> constValues = compf.getValues();
+ Collection<String> constValues = (Collection<String>) compf.getValues();
if (constValues == null || constValues.isEmpty()) {
- compf.setNullString(nullString); // maybe ISNULL
return filter;
}
TupleFilter result;
CompareTupleFilter newComp = new CompareTupleFilter(compf.getOperator());
- newComp.setNullString(nullString);
newComp.addChild(new ColumnTupleFilter(col));
String v;
- //TODO: seems not working when CompareTupleFilter has multiple values, like IN
- String firstValue = constValues.iterator().next();
// translate constant into rowkey ID
+ String firstValue = constValues.iterator().next();
switch (newComp.getOperator()) {
case EQ:
case IN:
Set<String> newValues = Sets.newHashSet();
for (String value : constValues) {
v = translate(col, value, 0);
- if (!nullString.equals(v))
+ if (!isDictNull(v))
newValues.add(v);
}
if (newValues.isEmpty()) {
@@ -120,7 +117,7 @@ public class CoprocessorFilter {
break;
case NEQ:
v = translate(col, firstValue, 0);
- if (nullString.equals(v)) {
+ if (isDictNull(v)) {
result = ConstantTupleFilter.TRUE;
} else {
newComp.addChild(new ConstantTupleFilter(v));
@@ -129,7 +126,7 @@ public class CoprocessorFilter {
break;
case LT:
v = translate(col, firstValue, 1);
- if (nullString.equals(v)) {
+ if (isDictNull(v)) {
result = ConstantTupleFilter.TRUE;
} else {
newComp.addChild(new ConstantTupleFilter(v));
@@ -138,7 +135,7 @@ public class CoprocessorFilter {
break;
case LTE:
v = translate(col, firstValue, -1);
- if (nullString.equals(v)) {
+ if (isDictNull(v)) {
result = ConstantTupleFilter.FALSE;
} else {
newComp.addChild(new ConstantTupleFilter(v));
@@ -147,7 +144,7 @@ public class CoprocessorFilter {
break;
case GT:
v = translate(col, firstValue, -1);
- if (nullString.equals(v)) {
+ if (isDictNull(v)) {
result = ConstantTupleFilter.TRUE;
} else {
newComp.addChild(new ConstantTupleFilter(v));
@@ -156,7 +153,7 @@ public class CoprocessorFilter {
break;
case GTE:
v = translate(col, firstValue, 1);
- if (nullString.equals(v)) {
+ if (isDictNull(v)) {
result = ConstantTupleFilter.FALSE;
} else {
newComp.addChild(new ConstantTupleFilter(v));
@@ -169,12 +166,8 @@ public class CoprocessorFilter {
return result;
}
- private String nullString(TblColRef column) {
- byte[] id = new byte[columnIO.getColumnLength(column)];
- for (int i = 0; i < id.length; i++) {
- id[i] = Dictionary.NULL;
- }
- return Dictionary.dictIdToString(id, 0, id.length);
+ private boolean isDictNull(String v) {
+ return DictCodeSystem.INSTANCE.isNull(v);
}
private String translate(TblColRef column, String v, int roundingFlag) {
@@ -188,17 +181,18 @@ public class CoprocessorFilter {
public static CoprocessorFilter fromFilter(final ISegment seg, TupleFilter rootFilter) {
// translate constants into dictionary IDs via a serialize copy
FilterDecorator filterDecorator = new FilterDecorator(seg);
- byte[] bytes = TupleFilterSerializer.serialize(rootFilter, filterDecorator);
- TupleFilter copy = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(rootFilter, filterDecorator, DictCodeSystem.INSTANCE);
+ TupleFilter copy = TupleFilterSerializer.deserialize(bytes, DictCodeSystem.INSTANCE);
return new CoprocessorFilter(copy, filterDecorator.getUnstrictlyFilteredColumns());
}
public static byte[] serialize(CoprocessorFilter o) {
- return (o.filter == null) ? BytesUtil.EMPTY_BYTE_ARRAY : TupleFilterSerializer.serialize(o.filter);
+ return (o.filter == null) ? BytesUtil.EMPTY_BYTE_ARRAY : TupleFilterSerializer.serialize(o.filter, DictCodeSystem.INSTANCE);
}
public static CoprocessorFilter deserialize(byte[] filterBytes) {
- TupleFilter filter = (filterBytes == null || filterBytes.length == 0) ? null : TupleFilterSerializer.deserialize(filterBytes);
+ TupleFilter filter = (filterBytes == null || filterBytes.length == 0) ? null //
+ : TupleFilterSerializer.deserialize(filterBytes, DictCodeSystem.INSTANCE);
return new CoprocessorFilter(filter, null);
}
@@ -224,7 +218,7 @@ public class CoprocessorFilter {
if (filter == null)
return true;
else
- return filter.evaluate(tuple);
+ return filter.evaluate(tuple, DictCodeSystem.INSTANCE);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
new file mode 100644
index 0000000..14497f4
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
@@ -0,0 +1,45 @@
+package org.apache.kylin.storage.hbase.coprocessor;
+
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.metadata.filter.ICodeSystem;
+
+public class DictCodeSystem implements ICodeSystem {
+
+ public static final DictCodeSystem INSTANCE = new DictCodeSystem();
+
+ private DictCodeSystem() {
+ // singleton
+ }
+
+ @Override
+ public boolean isNull(Object value) {
+ if (value == null)
+ return true;
+
+ String v = (String) value;
+ for (int i = 0, n = v.length(); i < n; i++) {
+ if ((byte) v.charAt(i) != Dictionary.NULL)
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int compare(Object tupleValue, Object constValue) {
+ return ((String) tupleValue).compareTo((String) constValue);
+ }
+
+ @Override
+ public void serialize(Object value, ByteBuffer buffer) {
+ BytesUtil.writeUTFString((String) value, buffer);
+ }
+
+ @Override
+ public Object deserialize(ByteBuffer buffer) {
+ return BytesUtil.readUTFString(buffer);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/storage/src/test/java/org/apache/kylin/storage/filter/BitMapFilterEvaluatorTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/BitMapFilterEvaluatorTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/BitMapFilterEvaluatorTest.java
index 47b519f..e116aae 100644
--- a/storage/src/test/java/org/apache/kylin/storage/filter/BitMapFilterEvaluatorTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/filter/BitMapFilterEvaluatorTest.java
@@ -196,7 +196,6 @@ public class BitMapFilterEvaluatorTest {
public static CompareTupleFilter compare(TblColRef col, TupleFilter.FilterOperatorEnum op, int... ids) {
CompareTupleFilter filter = new CompareTupleFilter(op);
- filter.setNullString(idToStr(Dictionary.NULL_ID[1]));
filter.addChild(columnFilter(col));
for (int i : ids) {
filter.addChild(constFilter(i));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
index 412bab6..97e0e33 100644
--- a/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
@@ -36,6 +36,8 @@ import org.apache.kylin.storage.tuple.TupleInfo;
*
*/
public class FilterBaseTest {
+
+ static final ICodeSystem CS = StringCodeSystem.INSTANCE;
protected List<TblColRef> buildGroups() {
List<TblColRef> groups = new ArrayList<TblColRef>();
@@ -200,7 +202,7 @@ public class FilterBaseTest {
protected int evaluateTuples(Collection<Tuple> tuples, TupleFilter filter) {
int match = 0;
for (Tuple t : tuples) {
- if (filter.evaluate(t)) {
+ if (filter.evaluate(t, CS)) {
match++;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/storage/src/test/java/org/apache/kylin/storage/filter/FilterEvaluateTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/FilterEvaluateTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/FilterEvaluateTest.java
index feaf98a..19ef2e7 100644
--- a/storage/src/test/java/org/apache/kylin/storage/filter/FilterEvaluateTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/filter/FilterEvaluateTest.java
@@ -41,8 +41,8 @@ public class FilterEvaluateTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildCompareFilter(groups, 0);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
int number = 10000;
int[] matcheCounts = new int[] { 0, 0, 0 };
@@ -57,8 +57,8 @@ public class FilterEvaluateTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildCompareFilter(groups, 1);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
int number = 10000;
int[] matcheCounts = new int[] { 0, 0, 0 };
@@ -73,8 +73,8 @@ public class FilterEvaluateTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildOrFilter(groups);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
int number = 10000;
int[] matcheCounts = new int[] { 0, 0, 0 };
@@ -89,8 +89,8 @@ public class FilterEvaluateTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildAndFilter(groups);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
int number = 10000;
int[] matcheCounts = new int[] { 0, 0, 0 };
@@ -105,8 +105,8 @@ public class FilterEvaluateTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildCompareCaseFilter(groups, "0");
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
int number = 10000;
int[] matcheCounts = new int[] { 0, 0, 0 };
@@ -121,8 +121,8 @@ public class FilterEvaluateTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildCompareCaseFilter(groups, "1");
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
int number = 10000;
int[] matcheCounts = new int[] { 0, 0, 0 };
@@ -137,8 +137,8 @@ public class FilterEvaluateTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildCompareCaseFilter(groups, "2");
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
int number = 1;
int[] matcheCounts = new int[] { 0, 0, 0 };
@@ -153,8 +153,8 @@ public class FilterEvaluateTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildCompareCaseFilter(groups, "3");
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
int number = 10000;
int[] matcheCounts = new int[] { 0, 0, 0 };
@@ -169,8 +169,8 @@ public class FilterEvaluateTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildCompareCaseFilter(groups, "4");
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
int number = 10000;
int[] matcheCounts = new int[] { 0, 0, 0 };
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/storage/src/test/java/org/apache/kylin/storage/filter/FilterSerializeTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/FilterSerializeTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/FilterSerializeTest.java
index 07dfc8e..a095012 100644
--- a/storage/src/test/java/org/apache/kylin/storage/filter/FilterSerializeTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/filter/FilterSerializeTest.java
@@ -42,8 +42,8 @@ public class FilterSerializeTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildCompareFilter(groups, 0);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(filter, newFilter);
}
@@ -53,8 +53,8 @@ public class FilterSerializeTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildCompareFilter(groups, 1);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(filter, newFilter);
}
@@ -64,8 +64,8 @@ public class FilterSerializeTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildAndFilter(groups);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(filter, newFilter);
}
@@ -75,8 +75,8 @@ public class FilterSerializeTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildOrFilter(groups);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(filter, newFilter);
}
@@ -90,8 +90,8 @@ public class FilterSerializeTest extends FilterBaseTest {
groups.add(colRef);
TupleFilter filter = buildCompareFilter(groups, 0);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(filter, newFilter);
}
@@ -105,8 +105,8 @@ public class FilterSerializeTest extends FilterBaseTest {
groups.add(colRef);
TupleFilter filter = buildCompareFilter(groups, 0);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(filter, newFilter);
}
@@ -124,8 +124,8 @@ public class FilterSerializeTest extends FilterBaseTest {
groups.add(colRef);
TupleFilter filter = buildCompareFilter(groups, 0);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(filter, newFilter);
}
@@ -142,8 +142,8 @@ public class FilterSerializeTest extends FilterBaseTest {
groups.add(colRef);
TupleFilter filter = buildCompareFilter(groups, 0);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(filter, newFilter);
}
@@ -158,8 +158,8 @@ public class FilterSerializeTest extends FilterBaseTest {
logicFilter.addChild(orFilter);
logicFilter.addChild(andFilter);
- byte[] bytes = TupleFilterSerializer.serialize(logicFilter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(logicFilter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(logicFilter, newFilter);
}
@@ -174,8 +174,8 @@ public class FilterSerializeTest extends FilterBaseTest {
logicFilter.addChild(orFilter);
logicFilter.addChild(andFilter);
- byte[] bytes = TupleFilterSerializer.serialize(logicFilter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(logicFilter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(logicFilter, newFilter);
}
@@ -185,8 +185,8 @@ public class FilterSerializeTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildCaseFilter(groups);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(filter, newFilter);
}
@@ -196,8 +196,8 @@ public class FilterSerializeTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildCompareCaseFilter(groups, "0");
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(filter, newFilter);
}