You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/01/15 13:26:24 UTC
[08/50] [abbrv] incubator-kylin git commit: put tuplefilter and
Ituple into medadata module
put tuplefilter and Ituple into medadata module
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/94352aec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/94352aec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/94352aec
Branch: refs/heads/inverted-index
Commit: 94352aec8b9c57366543b24e58e837b769d4f6d8
Parents: 41b327d
Author: honma <ho...@ebay.com>
Authored: Mon Jan 12 21:44:46 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Jan 13 10:07:15 2015 +0800
----------------------------------------------------------------------
.../java/com/kylinolap/cube/CubeInstance.java | 1 -
.../com/kylinolap/invertedindex/IIInstance.java | 4 -
.../job/coprocessor/IIEndpointTest.java | 10 +-
.../metadata/filter/CaseTupleFilter.java | 110 +++++++++
.../metadata/filter/ColumnTupleFilter.java | 116 +++++++++
.../metadata/filter/CompareTupleFilter.java | 240 +++++++++++++++++++
.../metadata/filter/ConstantTupleFilter.java | 101 ++++++++
.../metadata/filter/DynamicTupleFilter.java | 83 +++++++
.../metadata/filter/ExtractTupleFilter.java | 107 +++++++++
.../metadata/filter/LogicalTupleFilter.java | 125 ++++++++++
.../kylinolap/metadata/filter/TupleFilter.java | 207 ++++++++++++++++
.../metadata/filter/TupleFilterSerializer.java | 179 ++++++++++++++
.../metadata/realization/IRealization.java | 2 -
.../metadata/realization/SQLDigest.java | 2 +-
.../com/kylinolap/metadata/tuple/ITuple.java | 41 ++++
.../metadata/tuple/ITupleIterator.java | 44 ++++
.../query/enumerator/CubeEnumerator.java | 8 +-
.../kylinolap/query/relnode/OLAPContext.java | 5 +-
.../kylinolap/query/relnode/OLAPFilterRel.java | 18 +-
.../com/kylinolap/storage/IStorageEngine.java | 2 +-
.../storage/filter/BitMapFilterEvaluator.java | 3 +
.../storage/filter/CaseTupleFilter.java | 109 ---------
.../storage/filter/ColumnTupleFilter.java | 116 ---------
.../storage/filter/CompareTupleFilter.java | 240 -------------------
.../storage/filter/ConstantTupleFilter.java | 101 --------
.../storage/filter/DynamicTupleFilter.java | 83 -------
.../storage/filter/ExtractTupleFilter.java | 106 --------
.../storage/filter/LogicalTupleFilter.java | 124 ----------
.../kylinolap/storage/filter/TupleFilter.java | 207 ----------------
.../storage/filter/TupleFilterSerializer.java | 180 --------------
.../storage/hbase/ColumnValueRange.java | 2 +-
.../hbase/ConcurrentHBaseTupleIterator.java | 4 +-
.../storage/hbase/CubeSegmentTupleIterator.java | 4 +-
.../storage/hbase/CubeStorageEngine.java | 12 +-
.../storage/hbase/DerivedFilterTranslator.java | 14 +-
.../hbase/InvertedIndexStorageEngine.java | 2 +-
.../hbase/SerializedHBaseTupleIterator.java | 6 +-
.../hbase/coprocessor/CoprocessorFilter.java | 16 +-
.../endpoint/EndpointTupleIterator.java | 8 +-
.../coprocessor/observer/ObserverEnabler.java | 2 +-
.../coprocessor/observer/ObserverTuple.java | 2 +-
.../com/kylinolap/storage/tuple/ITuple.java | 41 ----
.../kylinolap/storage/tuple/ITupleIterator.java | 44 ----
.../java/com/kylinolap/storage/tuple/Tuple.java | 1 +
.../filter/BitMapFilterEvaluatorTest.java | 6 +-
.../storage/filter/FilterBaseTest.java | 3 +-
.../storage/filter/FilterEvaluateTest.java | 2 +
.../storage/filter/FilterSerializeTest.java | 5 +-
.../endpoint/EndpointAggregationTest.java | 8 +-
.../com/kylinolap/storage/test/StorageTest.java | 21 +-
50 files changed, 1442 insertions(+), 1435 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/cube/src/main/java/com/kylinolap/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/CubeInstance.java b/cube/src/main/java/com/kylinolap/cube/CubeInstance.java
index 3b38a14..f1d3953 100644
--- a/cube/src/main/java/com/kylinolap/cube/CubeInstance.java
+++ b/cube/src/main/java/com/kylinolap/cube/CubeInstance.java
@@ -417,7 +417,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization {
return Lists.newArrayList(getDescriptor().listAllColumns());
}
- @Override
public List<TblColRef> getDimensions() {
List<TblColRef> ret = Lists.newArrayList();
for (DimensionDesc dim : getDescriptor().getDimensions()) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/invertedindex/src/main/java/com/kylinolap/invertedindex/IIInstance.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/com/kylinolap/invertedindex/IIInstance.java b/invertedindex/src/main/java/com/kylinolap/invertedindex/IIInstance.java
index f77d5ef..53f5a0c 100644
--- a/invertedindex/src/main/java/com/kylinolap/invertedindex/IIInstance.java
+++ b/invertedindex/src/main/java/com/kylinolap/invertedindex/IIInstance.java
@@ -409,8 +409,4 @@ public class IIInstance extends RootPersistentEntity implements IRealization {
return getDescriptor().getMeasures();
}
- @Override
- public List<TblColRef> getDimensions() {
- throw new UnsupportedOperationException();
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/job/src/test/java/com/kylinolap/job/coprocessor/IIEndpointTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/coprocessor/IIEndpointTest.java b/job/src/test/java/com/kylinolap/job/coprocessor/IIEndpointTest.java
index 26ede05..adede2d 100644
--- a/job/src/test/java/com/kylinolap/job/coprocessor/IIEndpointTest.java
+++ b/job/src/test/java/com/kylinolap/job/coprocessor/IIEndpointTest.java
@@ -23,12 +23,12 @@ import com.kylinolap.invertedindex.model.IIKeyValueCodec;
import com.kylinolap.metadata.MetadataManager;
import com.kylinolap.metadata.model.*;
import com.kylinolap.storage.StorageContext;
-import com.kylinolap.storage.filter.ColumnTupleFilter;
-import com.kylinolap.storage.filter.CompareTupleFilter;
-import com.kylinolap.storage.filter.ConstantTupleFilter;
-import com.kylinolap.storage.filter.TupleFilter;
+import com.kylinolap.metadata.filter.ColumnTupleFilter;
+import com.kylinolap.metadata.filter.CompareTupleFilter;
+import com.kylinolap.metadata.filter.ConstantTupleFilter;
+import com.kylinolap.metadata.filter.TupleFilter;
import com.kylinolap.storage.hbase.coprocessor.endpoint.EndpointTupleIterator;
-import com.kylinolap.storage.tuple.ITuple;
+import com.kylinolap.metadata.tuple.ITuple;
/**
* Created by Hongbin Ma(Binmahone) on 11/14/14.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/filter/CaseTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/filter/CaseTupleFilter.java b/metadata/src/main/java/com/kylinolap/metadata/filter/CaseTupleFilter.java
new file mode 100644
index 0000000..d40dd4a
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/filter/CaseTupleFilter.java
@@ -0,0 +1,110 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.metadata.filter;
+
+import com.kylinolap.metadata.tuple.ITuple;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+
+/**
+ * @author xjiang
+ *
+ */
+public class CaseTupleFilter extends TupleFilter {
+
+ private List<TupleFilter> whenFilters;
+ private List<TupleFilter> thenFilters;
+ private TupleFilter elseFilter;
+ private Collection<String> values;
+ private int filterIndex;
+
+ public CaseTupleFilter() {
+ super(new ArrayList<TupleFilter>(), FilterOperatorEnum.CASE);
+ this.filterIndex = 0;
+ this.values = Collections.emptyList();
+ this.whenFilters = new ArrayList<TupleFilter>();
+ this.thenFilters = new ArrayList<TupleFilter>();
+ this.elseFilter = null;
+ }
+
+ @Override
+ public void addChild(TupleFilter child) {
+ super.addChild(child);
+ if (this.filterIndex % 2 == 0) {
+ this.whenFilters.add(child);
+ } else {
+ this.thenFilters.add(child);
+ }
+ this.filterIndex++;
+ }
+
+ @Override
+ public String toString() {
+ return "CaseTupleFilter [when=" + whenFilters + ", then=" + thenFilters + ", else=" + elseFilter + ", children=" + children + "]";
+ }
+
+ @Override
+ public boolean evaluate(ITuple tuple) {
+ if (whenFilters.size() != thenFilters.size()) {
+ elseFilter = whenFilters.remove(whenFilters.size() - 1);
+ }
+ boolean matched = false;
+ for (int i = 0; i < whenFilters.size(); i++) {
+ TupleFilter whenFilter = whenFilters.get(i);
+ if (whenFilter.evaluate(tuple)) {
+ TupleFilter thenFilter = thenFilters.get(i);
+ thenFilter.evaluate(tuple);
+ values = thenFilter.getValues();
+ matched = true;
+ break;
+ }
+ }
+ if (!matched) {
+ if (elseFilter != null) {
+ elseFilter.evaluate(tuple);
+ values = elseFilter.getValues();
+ } else {
+ values = Collections.emptyList();
+ }
+ }
+
+ return true;
+ }
+
+ @Override
+ public boolean isEvaluable() {
+ return false;
+ }
+
+ @Override
+ public Collection<String> getValues() {
+ return this.values;
+ }
+
+ @Override
+ public byte[] serialize() {
+ return new byte[0];
+ }
+
+ @Override
+ public void deserialize(byte[] bytes) {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/filter/ColumnTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/filter/ColumnTupleFilter.java b/metadata/src/main/java/com/kylinolap/metadata/filter/ColumnTupleFilter.java
new file mode 100644
index 0000000..3011425
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/filter/ColumnTupleFilter.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.metadata.filter;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import com.kylinolap.common.util.BytesUtil;
+import com.kylinolap.metadata.model.ColumnDesc;
+import com.kylinolap.metadata.model.TableDesc;
+import com.kylinolap.metadata.model.TblColRef;
+import com.kylinolap.metadata.tuple.ITuple;
+
+/**
+ *
+ * @author xjiang
+ *
+ */
+public class ColumnTupleFilter extends TupleFilter {
+
+ private TblColRef columnRef;
+ private Object tupleValue;
+ private List<String> values;
+
+ public ColumnTupleFilter(TblColRef column) {
+ super(Collections.<TupleFilter> emptyList(), FilterOperatorEnum.COLUMN);
+ this.columnRef = column;
+ this.values = new ArrayList<String>(1);
+ this.values.add(null);
+ }
+
+ public TblColRef getColumn() {
+ return columnRef;
+ }
+
+ public void setColumn(TblColRef col) {
+ this.columnRef = col;
+ }
+
+ @Override
+ public void addChild(TupleFilter child) {
+ throw new UnsupportedOperationException("This is " + this + " and child is " + child);
+ }
+
+ @Override
+ public String toString() {
+ return "ColumnFilter [column=" + columnRef + "]";
+ }
+
+ @Override
+ public boolean evaluate(ITuple tuple) {
+ this.tupleValue = tuple.getValue(columnRef);
+ return true;
+ }
+
+ @Override
+ public boolean isEvaluable() {
+ return true;
+ }
+
+ @Override
+ public Collection<String> getValues() {
+ this.values.set(0, (String) this.tupleValue);
+ return this.values;
+ }
+
+ @Override
+ public byte[] serialize() {
+ ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
+ String table = columnRef.getTable();
+ BytesUtil.writeUTFString(table, buffer);
+
+ String columnName = columnRef.getName();
+ BytesUtil.writeUTFString(columnName, buffer);
+
+ String dataType = columnRef.getDatatype();
+ BytesUtil.writeUTFString(dataType, buffer);
+
+ byte[] result = new byte[buffer.position()];
+ System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
+ return result;
+ }
+
+ @Override
+ public void deserialize(byte[] bytes) {
+ ColumnDesc column = new ColumnDesc();
+ ByteBuffer buffer = ByteBuffer.wrap(bytes);
+ String tableName = BytesUtil.readUTFString(buffer);
+ if (tableName != null) {
+ TableDesc table = new TableDesc();
+ table.setName(tableName);
+ column.setTable(table);
+ }
+
+ column.setName(BytesUtil.readUTFString(buffer));
+ column.setDatatype(BytesUtil.readUTFString(buffer));
+
+ this.columnRef = new TblColRef(column);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/filter/CompareTupleFilter.java b/metadata/src/main/java/com/kylinolap/metadata/filter/CompareTupleFilter.java
new file mode 100644
index 0000000..9c3c294
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/filter/CompareTupleFilter.java
@@ -0,0 +1,240 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.metadata.filter;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+
+import com.kylinolap.common.util.BytesUtil;
+import com.kylinolap.metadata.model.TblColRef;
+import com.kylinolap.metadata.tuple.ITuple;
+
+/**
+ * @author xjiang
+ */
+public class CompareTupleFilter extends TupleFilter {
+
+ private TblColRef column;
+ private Collection<String> conditionValues;
+ private String firstCondValue;
+ private Map<String, String> dynamicVariables;
+ private String nullString;
+
+ public CompareTupleFilter(FilterOperatorEnum op) {
+ super(new ArrayList<TupleFilter>(2), op);
+ this.conditionValues = new HashSet<String>();
+ this.dynamicVariables = new HashMap<String, String>();
+ boolean opGood = (op == FilterOperatorEnum.EQ || op == FilterOperatorEnum.NEQ //
+ || op == FilterOperatorEnum.LT || op == FilterOperatorEnum.LTE //
+ || op == FilterOperatorEnum.GT || op == FilterOperatorEnum.GTE //
+ || op == FilterOperatorEnum.IN || op == FilterOperatorEnum.NOTIN //
+ || op == FilterOperatorEnum.ISNULL || op == FilterOperatorEnum.ISNOTNULL);
+ if (opGood == false)
+ throw new IllegalArgumentException("Unsupported operator " + op);
+ }
+
+ private CompareTupleFilter(CompareTupleFilter another) {
+ super(new ArrayList<TupleFilter>(another.children), another.operator);
+ this.column = another.column;
+ this.conditionValues = new HashSet<String>();
+ this.conditionValues.addAll(another.conditionValues);
+ this.dynamicVariables = new HashMap<String, String>();
+ this.dynamicVariables.putAll(another.dynamicVariables);
+ }
+
+ @Override
+ public void addChild(TupleFilter child) {
+ super.addChild(child);
+ if (child instanceof ColumnTupleFilter) {
+ ColumnTupleFilter columnFilter = (ColumnTupleFilter) child;
+ if (this.column != null) {
+ throw new IllegalStateException("Duplicate columns! old is " + column.getName() + " and new is " + columnFilter.getColumn().getName());
+ }
+ this.column = columnFilter.getColumn();
+ // if value is before column, we need to reverse the operator. e.g. "1 >= c1" => "c1 <= 1"
+ if (!this.conditionValues.isEmpty() && needSwapOperator()) {
+ this.operator = SWAP_OP_MAP.get(this.operator);
+ }
+ } else if (child instanceof ConstantTupleFilter) {
+ this.conditionValues.addAll(child.getValues());
+ this.firstCondValue = this.conditionValues.iterator().next();
+ } else if (child instanceof DynamicTupleFilter) {
+ DynamicTupleFilter dynamicFilter = (DynamicTupleFilter) child;
+ this.dynamicVariables.put(dynamicFilter.getVariableName(), null);
+ }
+ //TODO
+ // else if (child instanceof ExtractTupleFilter) {
+ // } else if (child instanceof CaseTupleFilter) {
+ // }
+ }
+
+ private boolean needSwapOperator() {
+ return operator == FilterOperatorEnum.LT || operator == FilterOperatorEnum.GT || operator == FilterOperatorEnum.LTE || operator == FilterOperatorEnum.GTE;
+ }
+
+ @Override
+ public Collection<String> getValues() {
+ return conditionValues;
+ }
+
+ public String getFirstValue() {
+ return firstCondValue;
+ }
+
+ public TblColRef getColumn() {
+ return column;
+ }
+
+ public Map<String, String> getVariables() {
+ return dynamicVariables;
+ }
+
+ public void bindVariable(String variable, String value) {
+ this.dynamicVariables.put(variable, value);
+ this.conditionValues.add(value);
+ this.firstCondValue = this.conditionValues.iterator().next();
+ }
+
+ public String getNullString() {
+ return nullString;
+ }
+
+ public void setNullString(String nullString) {
+ this.nullString = nullString;
+ }
+
+ @Override
+ public TupleFilter copy() {
+ return new CompareTupleFilter(this);
+ }
+
+ @Override
+ public TupleFilter reverse() {
+ TupleFilter reverse = copy();
+ reverse.operator = REVERSE_OP_MAP.get(this.operator);
+ return reverse;
+ }
+
+ @Override
+ public String toString() {
+ return "CompareFilter [" + column + " " + operator + " " + conditionValues + ", children=" + children + "]";
+ }
+
+ // TODO requires generalize, currently only evaluates COLUMN {op} CONST
+ @Override
+ public boolean evaluate(ITuple tuple) {
+ // extract tuple value
+ String tupleValue = null;
+ for (TupleFilter filter : this.children) {
+ if (isConstant(filter) == false) {
+ filter.evaluate(tuple);
+ tupleValue = filter.getValues().iterator().next();
+ }
+ }
+
+ // consider null string
+ if (nullString != null && nullString.equals(tupleValue)) {
+ tupleValue = null;
+ }
+ if (tupleValue == null) {
+ if (operator == FilterOperatorEnum.ISNULL)
+ return true;
+ else
+ return false;
+ }
+
+ // always false if compare to null
+ if (firstCondValue.equals(nullString))
+ return false;
+
+ // tricky here -- order is ensured by string compare (even for number columns)
+ // because it's row key ID (not real value) being compared
+ int comp = tupleValue.compareTo(firstCondValue);
+
+ boolean result;
+ switch (operator) {
+ case EQ:
+ result = comp == 0;
+ break;
+ case NEQ:
+ result = comp != 0;
+ break;
+ case LT:
+ result = comp < 0;
+ break;
+ case LTE:
+ result = comp <= 0;
+ break;
+ case GT:
+ result = comp > 0;
+ break;
+ case GTE:
+ result = comp >= 0;
+ break;
+ case IN:
+ result = conditionValues.contains(tupleValue);
+ break;
+ case NOTIN:
+ result = !conditionValues.contains(tupleValue);
+ break;
+ default:
+ result = false;
+ }
+ return result;
+ }
+
+ private boolean isConstant(TupleFilter filter) {
+ return (filter instanceof ConstantTupleFilter) || (filter instanceof DynamicTupleFilter);
+ }
+
+ @Override
+ public boolean isEvaluable() {
+ return column != null && !conditionValues.isEmpty();
+ }
+
+ @Override
+ public byte[] serialize() {
+ ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
+ int size = this.dynamicVariables.size();
+ BytesUtil.writeVInt(size, buffer);
+ for (Map.Entry<String, String> entry : this.dynamicVariables.entrySet()) {
+ BytesUtil.writeUTFString(entry.getKey(), buffer);
+ BytesUtil.writeUTFString(entry.getValue(), buffer);
+ }
+ BytesUtil.writeAsciiString(nullString, buffer);
+ byte[] result = new byte[buffer.position()];
+ System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
+ return result;
+ }
+
+ @Override
+ public void deserialize(byte[] bytes) {
+ this.dynamicVariables.clear();
+ ByteBuffer buffer = ByteBuffer.wrap(bytes);
+ int size = BytesUtil.readVInt(buffer);
+ for (int i = 0; i < size; i++) {
+ String nameString = BytesUtil.readUTFString(buffer);
+ String valueString = BytesUtil.readUTFString(buffer);
+ bindVariable(nameString, valueString);
+ }
+ this.nullString = BytesUtil.readAsciiString(buffer);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/filter/ConstantTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/filter/ConstantTupleFilter.java b/metadata/src/main/java/com/kylinolap/metadata/filter/ConstantTupleFilter.java
new file mode 100644
index 0000000..b18a49c
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/filter/ConstantTupleFilter.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.metadata.filter;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+
+import com.kylinolap.common.util.BytesUtil;
+import com.kylinolap.metadata.tuple.ITuple;
+
+/**
+ *
+ * @author xjiang
+ *
+ */
+public class ConstantTupleFilter extends TupleFilter {
+
+ public static final ConstantTupleFilter FALSE = new ConstantTupleFilter();
+ public static final ConstantTupleFilter TRUE = new ConstantTupleFilter("TRUE");
+
+ private Collection<String> constantValues;
+
+ public ConstantTupleFilter() {
+ super(Collections.<TupleFilter> emptyList(), FilterOperatorEnum.CONSTANT);
+ this.constantValues = new HashSet<String>();
+ }
+
+ public ConstantTupleFilter(String value) {
+ this();
+ this.constantValues.add(value);
+ }
+
+ public ConstantTupleFilter(Collection<String> values) {
+ this();
+ this.constantValues.addAll(values);
+ }
+
+ @Override
+ public void addChild(TupleFilter child) {
+ throw new UnsupportedOperationException("This is " + this + " and child is " + child);
+ }
+
+ @Override
+ public String toString() {
+ return "ConstantFilter [constant=" + constantValues + "]";
+ }
+
+ @Override
+ public boolean evaluate(ITuple tuple) {
+ return constantValues.size() > 0;
+ }
+
+ @Override
+ public boolean isEvaluable() {
+ return true;
+ }
+
+ @Override
+ public Collection<String> getValues() {
+ return this.constantValues;
+ }
+
+ @Override
+ public byte[] serialize() {
+ ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
+ int size = this.constantValues.size();
+ BytesUtil.writeVInt(size, buffer);
+ for (String val : this.constantValues) {
+ BytesUtil.writeUTFString(val, buffer);
+ }
+ byte[] result = new byte[buffer.position()];
+ System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
+ return result;
+ }
+
+ @Override
+ public void deserialize(byte[] bytes) {
+ this.constantValues.clear();
+ ByteBuffer buffer = ByteBuffer.wrap(bytes);
+ int size = BytesUtil.readVInt(buffer);
+ for (int i = 0; i < size; i++) {
+ this.constantValues.add(BytesUtil.readUTFString(buffer));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/filter/DynamicTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/filter/DynamicTupleFilter.java b/metadata/src/main/java/com/kylinolap/metadata/filter/DynamicTupleFilter.java
new file mode 100644
index 0000000..3c2d395
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/filter/DynamicTupleFilter.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.metadata.filter;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Collections;
+
+import com.kylinolap.common.util.BytesUtil;
+import com.kylinolap.metadata.tuple.ITuple;
+
+/**
+ *
+ * @author xjiang
+ *
+ */
+public class DynamicTupleFilter extends TupleFilter {
+
+ private String variableName;
+
+ public DynamicTupleFilter(String name) {
+ super(Collections.<TupleFilter> emptyList(), FilterOperatorEnum.DYNAMIC);
+ this.variableName = name;
+ }
+
+ public String getVariableName() {
+ return variableName;
+ }
+
+ @Override
+ public void addChild(TupleFilter child) {
+ throw new UnsupportedOperationException("This is " + this + " and child is " + child);
+ }
+
+ @Override
+ public String toString() {
+ return "DynamicFilter [variableName=" + variableName + "]";
+ }
+
+ @Override
+ public boolean evaluate(ITuple tuple) {
+ return true;
+ }
+
+ @Override
+ public boolean isEvaluable() {
+ return true;
+ }
+
+ @Override
+ public Collection<String> getValues() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public byte[] serialize() {
+ ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
+ BytesUtil.writeUTFString(variableName, buffer);
+ byte[] result = new byte[buffer.position()];
+ System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
+ return result;
+ }
+
+ @Override
+ public void deserialize(byte[] bytes) {
+ ByteBuffer buffer = ByteBuffer.wrap(bytes);
+ this.variableName = BytesUtil.readUTFString(buffer);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/filter/ExtractTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/filter/ExtractTupleFilter.java b/metadata/src/main/java/com/kylinolap/metadata/filter/ExtractTupleFilter.java
new file mode 100644
index 0000000..9cfe08d
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/filter/ExtractTupleFilter.java
@@ -0,0 +1,107 @@
+package com.kylinolap.metadata.filter;
+
+import com.kylinolap.metadata.tuple.ITuple;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+
+/**
+ *
+ * @author xjiang
+ *
+ */
+public class ExtractTupleFilter extends TupleFilter {
+
+ private int date;
+ private List<String> values;
+
+ public ExtractTupleFilter(FilterOperatorEnum op) {
+ super(new ArrayList<TupleFilter>(3), op);
+ assert (op == FilterOperatorEnum.EXTRACT);
+ this.values = new ArrayList<String>(1);
+ this.values.add(null);
+ this.date = 0;
+ }
+
+ @Override
+ public String toString() {
+ return "ExtractTupleFilter=[children=" + this.children + "]";
+ }
+
+ @Override
+ public boolean isEvaluable() {
+ return false;
+ }
+
+ @Override
+ public boolean evaluate(ITuple tuple) {
+ // extract tuple value
+ String extractType = null;
+ String tupleValue = null;
+ for (TupleFilter filter : this.children) {
+ filter.evaluate(tuple);
+ if (filter instanceof ConstantTupleFilter) {
+ tupleValue = filter.getValues().iterator().next();
+ } else if (filter instanceof CompareTupleFilter) {
+ extractType = filter.getValues().iterator().next();
+ }
+ }
+
+ // extract date
+ this.date = extractDate(extractType, Integer.valueOf(tupleValue));
+ return true;
+ }
+
+ private int extractDate(String type, int inDate) {
+ // this shifts the epoch back to astronomical year -4800 instead of the
+ // start of the Christian era in year AD 1 of the proleptic Gregorian
+ // calendar.
+ int j = inDate + 32044;
+ int g = j / 146097;
+ int dg = j % 146097;
+ int c = (dg / 36524 + 1) * 3 / 4;
+ int dc = dg - c * 36524;
+ int b = dc / 1461;
+ int db = dc % 1461;
+ int a = (db / 365 + 1) * 3 / 4;
+ int da = db - a * 365;
+
+ // integer number of full years elapsed since March 1, 4801 BC
+ int y = g * 400 + c * 100 + b * 4 + a;
+ // integer number of full months elapsed since the last March 1
+ int m = (da * 5 + 308) / 153 - 2;
+ // number of days elapsed since day 1 of the month
+ int d = da - (m + 4) * 153 / 5 + 122;
+ int year = y - 4800 + (m + 2) / 12;
+ int month = (m + 2) % 12 + 1;
+ int day = d + 1;
+ if ("YEAR".equalsIgnoreCase(type)) {
+ return year;
+ }
+ if ("MONTH".equalsIgnoreCase(type)) {
+ return month;
+ }
+ if ("DAY".equalsIgnoreCase(type)) {
+ return day;
+ }
+ return -1;
+ }
+
+ @Override
+ public Collection<String> getValues() {
+ this.values.set(0, String.valueOf(this.date));
+ return this.values;
+ }
+
+ @Override
+ public byte[] serialize() {
+ return new byte[0];
+ }
+
+ @Override
+ public void deserialize(byte[] bytes) {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/filter/LogicalTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/filter/LogicalTupleFilter.java b/metadata/src/main/java/com/kylinolap/metadata/filter/LogicalTupleFilter.java
new file mode 100644
index 0000000..c868737
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/filter/LogicalTupleFilter.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.metadata.filter;
+
+import com.kylinolap.metadata.tuple.ITuple;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+
+
+public class LogicalTupleFilter extends TupleFilter {
+
+ public LogicalTupleFilter(FilterOperatorEnum op) {
+ super(new ArrayList<TupleFilter>(2), op);
+ boolean opGood = (op == FilterOperatorEnum.AND || op == FilterOperatorEnum.OR || op == FilterOperatorEnum.NOT);
+ if (opGood == false)
+ throw new IllegalArgumentException("Unsupported operator " + op);
+ }
+
+ private LogicalTupleFilter(List<TupleFilter> filters, FilterOperatorEnum op) {
+ super(filters, op);
+ }
+
+ @Override
+ public TupleFilter copy() {
+ List<TupleFilter> cloneChildren = new LinkedList<TupleFilter>(children);
+ TupleFilter cloneTuple = new LogicalTupleFilter(cloneChildren, operator);
+ return cloneTuple;
+ }
+
+ @Override
+ public TupleFilter reverse() {
+ switch (operator) {
+ case NOT:
+ assert (children.size() == 1);
+ return children.get(0);
+ case AND:
+ case OR:
+ LogicalTupleFilter reverse = new LogicalTupleFilter(REVERSE_OP_MAP.get(operator));
+ for (TupleFilter child : children) {
+ reverse.addChild(child.reverse());
+ }
+ return reverse;
+ default:
+ throw new IllegalStateException();
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "LogicalFilter [operator=" + operator + ", children=" + children + "]";
+ }
+
+ @Override
+ public boolean evaluate(ITuple tuple) {
+ switch (this.operator) {
+ case AND:
+ return evalAnd(tuple);
+ case OR:
+ return evalOr(tuple);
+ case NOT:
+ return evalNot(tuple);
+ default:
+ return false;
+ }
+ }
+
+ private boolean evalAnd(ITuple tuple) {
+ for (TupleFilter filter : this.children) {
+ if (!filter.evaluate(tuple)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private boolean evalOr(ITuple tuple) {
+ for (TupleFilter filter : this.children) {
+ if (filter.evaluate(tuple)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean evalNot(ITuple tuple) {
+ return !this.children.get(0).evaluate(tuple);
+ }
+
+ @Override
+ public Collection<String> getValues() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public boolean isEvaluable() {
+ return true;
+ }
+
+ @Override
+ public byte[] serialize() {
+ return new byte[0];
+ }
+
+ @Override
+ public void deserialize(byte[] bytes) {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/filter/TupleFilter.java b/metadata/src/main/java/com/kylinolap/metadata/filter/TupleFilter.java
new file mode 100644
index 0000000..fa5e624
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/filter/TupleFilter.java
@@ -0,0 +1,207 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.metadata.filter;
+
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+import com.kylinolap.metadata.tuple.ITuple;
+
+/**
+ *
+ * @author xjiang
+ *
+ */
+public abstract class TupleFilter {
+
+ public enum FilterOperatorEnum {
+ EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8), IN(9), NOTIN(10), AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31), DYNAMIC(32), EXTRACT(33), CASE(34);
+
+ private final int value;
+
+ private FilterOperatorEnum(int v) {
+ this.value = v;
+ }
+
+ public int getValue() {
+ return this.value;
+ }
+ }
+
+ public static final int BUFFER_SIZE = 10240;
+
+ protected static final Map<FilterOperatorEnum, FilterOperatorEnum> REVERSE_OP_MAP = Maps.newHashMap();
+ protected static final Map<FilterOperatorEnum, FilterOperatorEnum> SWAP_OP_MAP = Maps.newHashMap();
+
+ static {
+ REVERSE_OP_MAP.put(FilterOperatorEnum.EQ, FilterOperatorEnum.NEQ);
+ REVERSE_OP_MAP.put(FilterOperatorEnum.NEQ, FilterOperatorEnum.EQ);
+ REVERSE_OP_MAP.put(FilterOperatorEnum.GT, FilterOperatorEnum.LTE);
+ REVERSE_OP_MAP.put(FilterOperatorEnum.LTE, FilterOperatorEnum.GT);
+ REVERSE_OP_MAP.put(FilterOperatorEnum.LT, FilterOperatorEnum.GTE);
+ REVERSE_OP_MAP.put(FilterOperatorEnum.GTE, FilterOperatorEnum.LT);
+ REVERSE_OP_MAP.put(FilterOperatorEnum.IN, FilterOperatorEnum.NOTIN);
+ REVERSE_OP_MAP.put(FilterOperatorEnum.NOTIN, FilterOperatorEnum.IN);
+ REVERSE_OP_MAP.put(FilterOperatorEnum.ISNULL, FilterOperatorEnum.ISNOTNULL);
+ REVERSE_OP_MAP.put(FilterOperatorEnum.ISNOTNULL, FilterOperatorEnum.ISNULL);
+ REVERSE_OP_MAP.put(FilterOperatorEnum.AND, FilterOperatorEnum.OR);
+ REVERSE_OP_MAP.put(FilterOperatorEnum.OR, FilterOperatorEnum.AND);
+
+ SWAP_OP_MAP.put(FilterOperatorEnum.EQ, FilterOperatorEnum.EQ);
+ SWAP_OP_MAP.put(FilterOperatorEnum.NEQ, FilterOperatorEnum.NEQ);
+ SWAP_OP_MAP.put(FilterOperatorEnum.GT, FilterOperatorEnum.LT);
+ SWAP_OP_MAP.put(FilterOperatorEnum.LTE, FilterOperatorEnum.GTE);
+ SWAP_OP_MAP.put(FilterOperatorEnum.LT, FilterOperatorEnum.GT);
+ SWAP_OP_MAP.put(FilterOperatorEnum.GTE, FilterOperatorEnum.LTE);
+ }
+
+ protected final List<TupleFilter> children;
+ protected FilterOperatorEnum operator;
+ protected boolean hasChildren;
+
+ protected TupleFilter(List<TupleFilter> filters, FilterOperatorEnum op) {
+ this.children = filters;
+ this.operator = op;
+ }
+
+ public void addChild(TupleFilter child) {
+ children.add(child);
+ }
+
+ final public void addChildren(List<? extends TupleFilter> children) {
+ for (TupleFilter c : children)
+ addChild(c); // subclass overrides addChild()
+ }
+
+ public List<? extends TupleFilter> getChildren() {
+ return children;
+ }
+
+ public boolean hasChildren() {
+ return children != null && !children.isEmpty();
+ }
+
+ public FilterOperatorEnum getOperator() {
+ return operator;
+ }
+
+ public TupleFilter copy() {
+ throw new UnsupportedOperationException();
+ }
+
+ public TupleFilter reverse() {
+ throw new UnsupportedOperationException();
+ }
+
+ public TupleFilter flatFilter() {
+ return flattenInternal(this);
+ }
+
+ private TupleFilter flattenInternal(TupleFilter filter) {
+ TupleFilter flatFilter = null;
+ if (!(filter instanceof LogicalTupleFilter)) {
+ flatFilter = new LogicalTupleFilter(FilterOperatorEnum.AND);
+ flatFilter.addChild(filter);
+ return flatFilter;
+ }
+
+ // post-order recursive travel
+ FilterOperatorEnum op = filter.getOperator();
+ List<TupleFilter> andChildren = new LinkedList<TupleFilter>();
+ List<TupleFilter> orChildren = new LinkedList<TupleFilter>();
+ for (TupleFilter child : filter.getChildren()) {
+ TupleFilter flatChild = flattenInternal(child);
+ FilterOperatorEnum childOp = flatChild.getOperator();
+ if (childOp == FilterOperatorEnum.AND) {
+ andChildren.add(flatChild);
+ } else if (childOp == FilterOperatorEnum.OR) {
+ orChildren.add(flatChild);
+ } else {
+ throw new IllegalStateException("Filter is " + filter + " and child is " + flatChild);
+ }
+ }
+
+ // boolean algebra flatten
+ if (op == FilterOperatorEnum.AND) {
+ flatFilter = new LogicalTupleFilter(FilterOperatorEnum.AND);
+ for (TupleFilter andChild : andChildren) {
+ flatFilter.addChildren(andChild.getChildren());
+ }
+ if (!orChildren.isEmpty()) {
+ List<TupleFilter> fullAndFilters = cartesianProduct(orChildren, flatFilter);
+ flatFilter = new LogicalTupleFilter(FilterOperatorEnum.OR);
+ flatFilter.addChildren(fullAndFilters);
+ }
+ } else if (op == FilterOperatorEnum.OR) {
+ flatFilter = new LogicalTupleFilter(FilterOperatorEnum.OR);
+ for (TupleFilter orChild : orChildren) {
+ flatFilter.addChildren(orChild.getChildren());
+ }
+ flatFilter.addChildren(andChildren);
+ } else if (op == FilterOperatorEnum.NOT) {
+ assert (filter.children.size() == 1);
+ TupleFilter reverse = filter.children.get(0).reverse();
+ flatFilter = flattenInternal(reverse);
+ } else {
+ throw new IllegalStateException("Filter is " + filter);
+ }
+ return flatFilter;
+ }
+
+ private List<TupleFilter> cartesianProduct(List<TupleFilter> leftOrFilters, TupleFilter partialAndFilter) {
+ List<TupleFilter> oldProductFilters = new LinkedList<TupleFilter>();
+ oldProductFilters.add(partialAndFilter);
+ for (TupleFilter orFilter : leftOrFilters) {
+ List<TupleFilter> newProductFilters = new LinkedList<TupleFilter>();
+ for (TupleFilter orChildFilter : orFilter.getChildren()) {
+ for (TupleFilter productFilter : oldProductFilters) {
+ TupleFilter fullAndFilter = productFilter.copy();
+ fullAndFilter.addChildren(orChildFilter.getChildren());
+ newProductFilters.add(fullAndFilter);
+ }
+ }
+ oldProductFilters = newProductFilters;
+ }
+ return oldProductFilters;
+ }
+
+ public abstract boolean isEvaluable();
+
+ public abstract boolean evaluate(ITuple tuple);
+
+ public abstract Collection<String> getValues();
+
+ public abstract byte[] serialize();
+
+ public abstract void deserialize(byte[] bytes);
+
+ public static boolean isEvaluableRecursively(TupleFilter filter) {
+ if (filter == null)
+ return true;
+
+ if (filter.isEvaluable() == false)
+ return false;
+
+ for (TupleFilter child : filter.getChildren()) {
+ if (isEvaluableRecursively(child) == false)
+ return false;
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/filter/TupleFilterSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/filter/TupleFilterSerializer.java b/metadata/src/main/java/com/kylinolap/metadata/filter/TupleFilterSerializer.java
new file mode 100644
index 0000000..2ca9356
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/filter/TupleFilterSerializer.java
@@ -0,0 +1,179 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.metadata.filter;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Stack;
+
+import com.kylinolap.common.util.BytesUtil;
+
+/**
+ * http://eli.thegreenplace.net/2011/09/29/an-interesting-tree-serialization-algorithm-from-dwarf
+ *
+ * @author xjiang
+ *
+ */
+public class TupleFilterSerializer {
+
+ public static interface Decorator {
+ TupleFilter onSerialize(TupleFilter filter);
+ }
+
+ private static final int BUFFER_SIZE = 65536;
+ private static final Map<Integer, TupleFilter.FilterOperatorEnum> ID_OP_MAP = new HashMap<Integer, TupleFilter.FilterOperatorEnum>();
+
+ static {
+ for (TupleFilter.FilterOperatorEnum op : TupleFilter.FilterOperatorEnum.values()) {
+ ID_OP_MAP.put(op.getValue(), op);
+ }
+ }
+
+ public static byte[] serialize(TupleFilter rootFilter) {
+ return serialize(rootFilter, null);
+ }
+
+ public static byte[] serialize(TupleFilter rootFilter, Decorator decorator) {
+ ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
+ internalSerialize(rootFilter, decorator, buffer);
+ byte[] result = new byte[buffer.position()];
+ System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
+ return result;
+ }
+
+ private static void internalSerialize(TupleFilter filter, Decorator decorator, ByteBuffer buffer) {
+ if (decorator != null) { // give decorator a chance to manipulate the
+ // output filter
+ filter = decorator.onSerialize(filter);
+ }
+
+ if (filter == null) {
+ return;
+ }
+
+ if (filter.hasChildren()) {
+ // serialize filter+true
+ serializeFilter(1, filter, decorator, buffer);
+ // serialize children
+ for (TupleFilter child : filter.getChildren()) {
+ internalSerialize(child, decorator, buffer);
+ }
+ // serialize none
+ serializeFilter(-1, filter, decorator, buffer);
+ } else {
+ // serialize filter+false
+ serializeFilter(0, filter, decorator, buffer);
+ }
+ }
+
+ private static void serializeFilter(int flag, TupleFilter filter, Decorator decorator, ByteBuffer buffer) {
+ if (flag < 0) {
+ BytesUtil.writeVInt(-1, buffer);
+ } else {
+ byte[] bytes = filter.serialize();
+ int opVal = filter.getOperator().getValue();
+ BytesUtil.writeVInt(opVal, buffer);
+ BytesUtil.writeByteArray(bytes, buffer);
+ BytesUtil.writeVInt(flag, buffer);
+ }
+ }
+
+ public static TupleFilter deserialize(byte[] bytes) {
+ ByteBuffer buffer = ByteBuffer.wrap(bytes);
+ TupleFilter rootFilter = null;
+ Stack<TupleFilter> parentStack = new Stack<TupleFilter>();
+ while (buffer.hasRemaining()) {
+ int opVal = BytesUtil.readVInt(buffer);
+ if (opVal < 0) {
+ parentStack.pop();
+ continue;
+ }
+
+ // deserialize filter
+ TupleFilter filter = createTupleFilter(opVal);
+ byte[] filetrBytes = BytesUtil.readByteArray(buffer);
+ filter.deserialize(filetrBytes);
+
+ if (rootFilter == null) {
+ // push root to stack
+ rootFilter = filter;
+ parentStack.push(filter);
+ BytesUtil.readVInt(buffer);
+ continue;
+ }
+
+ // add filter to parent
+ TupleFilter parentFilter = parentStack.peek();
+ if (parentFilter != null) {
+ parentFilter.addChild(filter);
+ }
+
+ // push filter to stack or not based on having children or not
+ int hasChild = BytesUtil.readVInt(buffer);
+ if (hasChild == 1) {
+ parentStack.push(filter);
+ }
+ }
+ return rootFilter;
+ }
+
+ private static TupleFilter createTupleFilter(int opVal) {
+ TupleFilter.FilterOperatorEnum op = ID_OP_MAP.get(opVal);
+ if (op == null) {
+ throw new IllegalStateException("operator value is " + opVal);
+ }
+ TupleFilter filter = null;
+ switch (op) {
+ case AND:
+ case OR:
+ case NOT:
+ filter = new LogicalTupleFilter(op);
+ break;
+ case EQ:
+ case NEQ:
+ case LT:
+ case LTE:
+ case GT:
+ case GTE:
+ case IN:
+ case ISNULL:
+ case ISNOTNULL:
+ filter = new CompareTupleFilter(op);
+ break;
+ case EXTRACT:
+ filter = new ExtractTupleFilter(op);
+ break;
+ case CASE:
+ filter = new CaseTupleFilter();
+ break;
+ case COLUMN:
+ filter = new ColumnTupleFilter(null);
+ break;
+ case CONSTANT:
+ filter = new ConstantTupleFilter();
+ break;
+ case DYNAMIC:
+ filter = new DynamicTupleFilter(null);
+ break;
+ default:
+ throw new IllegalStateException("Error FilterOperatorEnum: " + op.getValue());
+ }
+
+ return filter;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/realization/IRealization.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/realization/IRealization.java b/metadata/src/main/java/com/kylinolap/metadata/realization/IRealization.java
index 57651d0..7169e6b 100644
--- a/metadata/src/main/java/com/kylinolap/metadata/realization/IRealization.java
+++ b/metadata/src/main/java/com/kylinolap/metadata/realization/IRealization.java
@@ -35,8 +35,6 @@ public interface IRealization {
public List<MeasureDesc> getMeasures();
- public List<TblColRef> getDimensions();
-
public boolean isReady();
public String getName();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/realization/SQLDigest.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/realization/SQLDigest.java b/metadata/src/main/java/com/kylinolap/metadata/realization/SQLDigest.java
index 5922519..f7b3434 100644
--- a/metadata/src/main/java/com/kylinolap/metadata/realization/SQLDigest.java
+++ b/metadata/src/main/java/com/kylinolap/metadata/realization/SQLDigest.java
@@ -2,10 +2,10 @@ package com.kylinolap.metadata.realization;
import java.util.Collection;
+import com.kylinolap.metadata.filter.TupleFilter;
import com.kylinolap.metadata.model.FunctionDesc;
import com.kylinolap.metadata.model.JoinDesc;
import com.kylinolap.metadata.model.TblColRef;
-import com.kylinolap.storage.filter.TupleFilter;
/**
* Created by Hongbin Ma(Binmahone) on 1/8/15.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/tuple/ITuple.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/tuple/ITuple.java b/metadata/src/main/java/com/kylinolap/metadata/tuple/ITuple.java
new file mode 100644
index 0000000..e5a7c39
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/tuple/ITuple.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.metadata.tuple;
+
+import java.util.List;
+
+import com.kylinolap.metadata.model.TblColRef;
+
+/**
+ * Tuple is a record row, contains multiple values being lookup by either field
+ * (optiq notion) or column (kylin notion).
+ *
+ * @author yangli9
+ */
+public interface ITuple {
+
+ public List<String> getAllFields();
+
+ public List<TblColRef> getAllColumns();
+
+ public Object[] getAllValues();
+
+ public Object getValue(TblColRef col);
+
+ public Object getValue(String field);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/tuple/ITupleIterator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/tuple/ITupleIterator.java b/metadata/src/main/java/com/kylinolap/metadata/tuple/ITupleIterator.java
new file mode 100644
index 0000000..862649d
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/tuple/ITupleIterator.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.metadata.tuple;
+
+/**
+ * @author xjiang
+ */
+public interface ITupleIterator {
+ public static final ITupleIterator EMPTY_TUPLE_ITERATOR = new ITupleIterator() {
+ @Override
+ public boolean hasNext() {
+ return false;
+ }
+
+ @Override
+ public ITuple next() {
+ return null;
+ }
+
+ @Override
+ public void close() {
+ }
+ };
+
+ public boolean hasNext();
+
+ public ITuple next();
+
+ public void close();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/query/src/main/java/com/kylinolap/query/enumerator/CubeEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/com/kylinolap/query/enumerator/CubeEnumerator.java b/query/src/main/java/com/kylinolap/query/enumerator/CubeEnumerator.java
index d3adca5..82f2101 100644
--- a/query/src/main/java/com/kylinolap/query/enumerator/CubeEnumerator.java
+++ b/query/src/main/java/com/kylinolap/query/enumerator/CubeEnumerator.java
@@ -30,10 +30,10 @@ import org.slf4j.LoggerFactory;
import com.kylinolap.query.relnode.OLAPContext;
import com.kylinolap.storage.IStorageEngine;
import com.kylinolap.storage.StorageEngineFactory;
-import com.kylinolap.storage.filter.CompareTupleFilter;
-import com.kylinolap.storage.filter.TupleFilter;
-import com.kylinolap.storage.tuple.ITuple;
-import com.kylinolap.storage.tuple.ITupleIterator;
+import com.kylinolap.metadata.filter.CompareTupleFilter;
+import com.kylinolap.metadata.filter.TupleFilter;
+import com.kylinolap.metadata.tuple.ITuple;
+import com.kylinolap.metadata.tuple.ITupleIterator;
/**
* @author xjiang
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/query/src/main/java/com/kylinolap/query/relnode/OLAPContext.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/com/kylinolap/query/relnode/OLAPContext.java b/query/src/main/java/com/kylinolap/query/relnode/OLAPContext.java
index f8ec2d9..3ea622a 100644
--- a/query/src/main/java/com/kylinolap/query/relnode/OLAPContext.java
+++ b/query/src/main/java/com/kylinolap/query/relnode/OLAPContext.java
@@ -33,7 +33,7 @@ import com.kylinolap.metadata.model.TblColRef;
import com.kylinolap.metadata.realization.IRealization;
import com.kylinolap.query.schema.OLAPSchema;
import com.kylinolap.storage.StorageContext;
-import com.kylinolap.storage.filter.TupleFilter;
+import com.kylinolap.metadata.filter.TupleFilter;
/**
* @author xjiang
@@ -103,7 +103,6 @@ public class OLAPContext {
// cube metadata
public IRealization realization;
-
public Collection<TblColRef> allColumns = new HashSet<TblColRef>();
public Collection<TblColRef> groupByColumns = new ArrayList<TblColRef>();
public Collection<TblColRef> metricsColumns = new HashSet<TblColRef>();
@@ -126,7 +125,7 @@ public class OLAPContext {
public SQLDigest getSQLDigest() {
if (sqlDigest == null)
- sqlDigest = new SQLDigest(firstTableScan.getTableName(), joins, allColumns, groupByColumns, filterColumns, metricsColumns, aggregations);
+ sqlDigest = new SQLDigest(firstTableScan.getTableName(), filter, joins, allColumns, groupByColumns, filterColumns, metricsColumns, aggregations);
return sqlDigest;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/query/src/main/java/com/kylinolap/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/com/kylinolap/query/relnode/OLAPFilterRel.java b/query/src/main/java/com/kylinolap/query/relnode/OLAPFilterRel.java
index 66265c2..550d154 100644
--- a/query/src/main/java/com/kylinolap/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/com/kylinolap/query/relnode/OLAPFilterRel.java
@@ -50,15 +50,15 @@ import org.eigenbase.util.NlsString;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.kylinolap.metadata.model.TblColRef;
-import com.kylinolap.storage.filter.CaseTupleFilter;
-import com.kylinolap.storage.filter.ColumnTupleFilter;
-import com.kylinolap.storage.filter.CompareTupleFilter;
-import com.kylinolap.storage.filter.ConstantTupleFilter;
-import com.kylinolap.storage.filter.DynamicTupleFilter;
-import com.kylinolap.storage.filter.ExtractTupleFilter;
-import com.kylinolap.storage.filter.LogicalTupleFilter;
-import com.kylinolap.storage.filter.TupleFilter;
-import com.kylinolap.storage.filter.TupleFilter.FilterOperatorEnum;
+import com.kylinolap.metadata.filter.CaseTupleFilter;
+import com.kylinolap.metadata.filter.ColumnTupleFilter;
+import com.kylinolap.metadata.filter.CompareTupleFilter;
+import com.kylinolap.metadata.filter.ConstantTupleFilter;
+import com.kylinolap.metadata.filter.DynamicTupleFilter;
+import com.kylinolap.metadata.filter.ExtractTupleFilter;
+import com.kylinolap.metadata.filter.LogicalTupleFilter;
+import com.kylinolap.metadata.filter.TupleFilter;
+import com.kylinolap.metadata.filter.TupleFilter.FilterOperatorEnum;
/**
* @author xjiang
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/storage/src/main/java/com/kylinolap/storage/IStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/IStorageEngine.java b/storage/src/main/java/com/kylinolap/storage/IStorageEngine.java
index a131752..2352831 100644
--- a/storage/src/main/java/com/kylinolap/storage/IStorageEngine.java
+++ b/storage/src/main/java/com/kylinolap/storage/IStorageEngine.java
@@ -16,7 +16,7 @@
package com.kylinolap.storage;
import com.kylinolap.metadata.realization.SQLDigest;
-import com.kylinolap.storage.tuple.ITupleIterator;
+import com.kylinolap.metadata.tuple.ITupleIterator;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/storage/src/main/java/com/kylinolap/storage/filter/BitMapFilterEvaluator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/filter/BitMapFilterEvaluator.java b/storage/src/main/java/com/kylinolap/storage/filter/BitMapFilterEvaluator.java
index 22fc57f..8f249e6 100644
--- a/storage/src/main/java/com/kylinolap/storage/filter/BitMapFilterEvaluator.java
+++ b/storage/src/main/java/com/kylinolap/storage/filter/BitMapFilterEvaluator.java
@@ -3,6 +3,9 @@ package com.kylinolap.storage.filter;
import java.util.List;
import com.kylinolap.dict.Dictionary;
+import com.kylinolap.metadata.filter.CompareTupleFilter;
+import com.kylinolap.metadata.filter.LogicalTupleFilter;
+import com.kylinolap.metadata.filter.TupleFilter;
import com.kylinolap.metadata.model.TblColRef;
import it.uniroma3.mat.extendedset.intset.ConciseSet;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/storage/src/main/java/com/kylinolap/storage/filter/CaseTupleFilter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/filter/CaseTupleFilter.java b/storage/src/main/java/com/kylinolap/storage/filter/CaseTupleFilter.java
deleted file mode 100644
index 3629742..0000000
--- a/storage/src/main/java/com/kylinolap/storage/filter/CaseTupleFilter.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.storage.filter;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-import com.kylinolap.storage.tuple.ITuple;
-
-/**
- * @author xjiang
- *
- */
-public class CaseTupleFilter extends TupleFilter {
-
- private List<TupleFilter> whenFilters;
- private List<TupleFilter> thenFilters;
- private TupleFilter elseFilter;
- private Collection<String> values;
- private int filterIndex;
-
- public CaseTupleFilter() {
- super(new ArrayList<TupleFilter>(), FilterOperatorEnum.CASE);
- this.filterIndex = 0;
- this.values = Collections.emptyList();
- this.whenFilters = new ArrayList<TupleFilter>();
- this.thenFilters = new ArrayList<TupleFilter>();
- this.elseFilter = null;
- }
-
- @Override
- public void addChild(TupleFilter child) {
- super.addChild(child);
- if (this.filterIndex % 2 == 0) {
- this.whenFilters.add(child);
- } else {
- this.thenFilters.add(child);
- }
- this.filterIndex++;
- }
-
- @Override
- public String toString() {
- return "CaseTupleFilter [when=" + whenFilters + ", then=" + thenFilters + ", else=" + elseFilter + ", children=" + children + "]";
- }
-
- @Override
- public boolean evaluate(ITuple tuple) {
- if (whenFilters.size() != thenFilters.size()) {
- elseFilter = whenFilters.remove(whenFilters.size() - 1);
- }
- boolean matched = false;
- for (int i = 0; i < whenFilters.size(); i++) {
- TupleFilter whenFilter = whenFilters.get(i);
- if (whenFilter.evaluate(tuple)) {
- TupleFilter thenFilter = thenFilters.get(i);
- thenFilter.evaluate(tuple);
- values = thenFilter.getValues();
- matched = true;
- break;
- }
- }
- if (!matched) {
- if (elseFilter != null) {
- elseFilter.evaluate(tuple);
- values = elseFilter.getValues();
- } else {
- values = Collections.emptyList();
- }
- }
-
- return true;
- }
-
- @Override
- public boolean isEvaluable() {
- return false;
- }
-
- @Override
- public Collection<String> getValues() {
- return this.values;
- }
-
- @Override
- public byte[] serialize() {
- return new byte[0];
- }
-
- @Override
- public void deserialize(byte[] bytes) {
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/storage/src/main/java/com/kylinolap/storage/filter/ColumnTupleFilter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/filter/ColumnTupleFilter.java b/storage/src/main/java/com/kylinolap/storage/filter/ColumnTupleFilter.java
deleted file mode 100644
index 746cc96..0000000
--- a/storage/src/main/java/com/kylinolap/storage/filter/ColumnTupleFilter.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.storage.filter;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-import com.kylinolap.common.util.BytesUtil;
-import com.kylinolap.metadata.model.ColumnDesc;
-import com.kylinolap.metadata.model.TableDesc;
-import com.kylinolap.metadata.model.TblColRef;
-import com.kylinolap.storage.tuple.ITuple;
-
-/**
- *
- * @author xjiang
- *
- */
-public class ColumnTupleFilter extends TupleFilter {
-
- private TblColRef columnRef;
- private Object tupleValue;
- private List<String> values;
-
- public ColumnTupleFilter(TblColRef column) {
- super(Collections.<TupleFilter> emptyList(), FilterOperatorEnum.COLUMN);
- this.columnRef = column;
- this.values = new ArrayList<String>(1);
- this.values.add(null);
- }
-
- public TblColRef getColumn() {
- return columnRef;
- }
-
- public void setColumn(TblColRef col) {
- this.columnRef = col;
- }
-
- @Override
- public void addChild(TupleFilter child) {
- throw new UnsupportedOperationException("This is " + this + " and child is " + child);
- }
-
- @Override
- public String toString() {
- return "ColumnFilter [column=" + columnRef + "]";
- }
-
- @Override
- public boolean evaluate(ITuple tuple) {
- this.tupleValue = tuple.getValue(columnRef);
- return true;
- }
-
- @Override
- public boolean isEvaluable() {
- return true;
- }
-
- @Override
- public Collection<String> getValues() {
- this.values.set(0, (String) this.tupleValue);
- return this.values;
- }
-
- @Override
- public byte[] serialize() {
- ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
- String table = columnRef.getTable();
- BytesUtil.writeUTFString(table, buffer);
-
- String columnName = columnRef.getName();
- BytesUtil.writeUTFString(columnName, buffer);
-
- String dataType = columnRef.getDatatype();
- BytesUtil.writeUTFString(dataType, buffer);
-
- byte[] result = new byte[buffer.position()];
- System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
- return result;
- }
-
- @Override
- public void deserialize(byte[] bytes) {
- ColumnDesc column = new ColumnDesc();
- ByteBuffer buffer = ByteBuffer.wrap(bytes);
- String tableName = BytesUtil.readUTFString(buffer);
- if (tableName != null) {
- TableDesc table = new TableDesc();
- table.setName(tableName);
- column.setTable(table);
- }
-
- column.setName(BytesUtil.readUTFString(buffer));
- column.setDatatype(BytesUtil.readUTFString(buffer));
-
- this.columnRef = new TblColRef(column);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/storage/src/main/java/com/kylinolap/storage/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/filter/CompareTupleFilter.java b/storage/src/main/java/com/kylinolap/storage/filter/CompareTupleFilter.java
deleted file mode 100644
index c7cc0c2..0000000
--- a/storage/src/main/java/com/kylinolap/storage/filter/CompareTupleFilter.java
+++ /dev/null
@@ -1,240 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.storage.filter;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-
-import com.kylinolap.common.util.BytesUtil;
-import com.kylinolap.metadata.model.TblColRef;
-import com.kylinolap.storage.tuple.ITuple;
-
-/**
- * @author xjiang
- */
-public class CompareTupleFilter extends TupleFilter {
-
- private TblColRef column;
- private Collection<String> conditionValues;
- private String firstCondValue;
- private Map<String, String> dynamicVariables;
- private String nullString;
-
- public CompareTupleFilter(FilterOperatorEnum op) {
- super(new ArrayList<TupleFilter>(2), op);
- this.conditionValues = new HashSet<String>();
- this.dynamicVariables = new HashMap<String, String>();
- boolean opGood = (op == FilterOperatorEnum.EQ || op == FilterOperatorEnum.NEQ //
- || op == FilterOperatorEnum.LT || op == FilterOperatorEnum.LTE //
- || op == FilterOperatorEnum.GT || op == FilterOperatorEnum.GTE //
- || op == FilterOperatorEnum.IN || op == FilterOperatorEnum.NOTIN //
- || op == FilterOperatorEnum.ISNULL || op == FilterOperatorEnum.ISNOTNULL);
- if (opGood == false)
- throw new IllegalArgumentException("Unsupported operator " + op);
- }
-
- private CompareTupleFilter(CompareTupleFilter another) {
- super(new ArrayList<TupleFilter>(another.children), another.operator);
- this.column = another.column;
- this.conditionValues = new HashSet<String>();
- this.conditionValues.addAll(another.conditionValues);
- this.dynamicVariables = new HashMap<String, String>();
- this.dynamicVariables.putAll(another.dynamicVariables);
- }
-
- @Override
- public void addChild(TupleFilter child) {
- super.addChild(child);
- if (child instanceof ColumnTupleFilter) {
- ColumnTupleFilter columnFilter = (ColumnTupleFilter) child;
- if (this.column != null) {
- throw new IllegalStateException("Duplicate columns! old is " + column.getName() + " and new is " + columnFilter.getColumn().getName());
- }
- this.column = columnFilter.getColumn();
- // if value is before column, we need to reverse the operator. e.g. "1 >= c1" => "c1 <= 1"
- if (!this.conditionValues.isEmpty() && needSwapOperator()) {
- this.operator = SWAP_OP_MAP.get(this.operator);
- }
- } else if (child instanceof ConstantTupleFilter) {
- this.conditionValues.addAll(child.getValues());
- this.firstCondValue = this.conditionValues.iterator().next();
- } else if (child instanceof DynamicTupleFilter) {
- DynamicTupleFilter dynamicFilter = (DynamicTupleFilter) child;
- this.dynamicVariables.put(dynamicFilter.getVariableName(), null);
- }
- //TODO
- // else if (child instanceof ExtractTupleFilter) {
- // } else if (child instanceof CaseTupleFilter) {
- // }
- }
-
- private boolean needSwapOperator() {
- return operator == FilterOperatorEnum.LT || operator == FilterOperatorEnum.GT || operator == FilterOperatorEnum.LTE || operator == FilterOperatorEnum.GTE;
- }
-
- @Override
- public Collection<String> getValues() {
- return conditionValues;
- }
-
- public String getFirstValue() {
- return firstCondValue;
- }
-
- public TblColRef getColumn() {
- return column;
- }
-
- public Map<String, String> getVariables() {
- return dynamicVariables;
- }
-
- public void bindVariable(String variable, String value) {
- this.dynamicVariables.put(variable, value);
- this.conditionValues.add(value);
- this.firstCondValue = this.conditionValues.iterator().next();
- }
-
- public String getNullString() {
- return nullString;
- }
-
- public void setNullString(String nullString) {
- this.nullString = nullString;
- }
-
- @Override
- public TupleFilter copy() {
- return new CompareTupleFilter(this);
- }
-
- @Override
- public TupleFilter reverse() {
- TupleFilter reverse = copy();
- reverse.operator = REVERSE_OP_MAP.get(this.operator);
- return reverse;
- }
-
- @Override
- public String toString() {
- return "CompareFilter [" + column + " " + operator + " " + conditionValues + ", children=" + children + "]";
- }
-
- // TODO requires generalize, currently only evaluates COLUMN {op} CONST
- @Override
- public boolean evaluate(ITuple tuple) {
- // extract tuple value
- String tupleValue = null;
- for (TupleFilter filter : this.children) {
- if (isConstant(filter) == false) {
- filter.evaluate(tuple);
- tupleValue = filter.getValues().iterator().next();
- }
- }
-
- // consider null string
- if (nullString != null && nullString.equals(tupleValue)) {
- tupleValue = null;
- }
- if (tupleValue == null) {
- if (operator == FilterOperatorEnum.ISNULL)
- return true;
- else
- return false;
- }
-
- // always false if compare to null
- if (firstCondValue.equals(nullString))
- return false;
-
- // tricky here -- order is ensured by string compare (even for number columns)
- // because it's row key ID (not real value) being compared
- int comp = tupleValue.compareTo(firstCondValue);
-
- boolean result;
- switch (operator) {
- case EQ:
- result = comp == 0;
- break;
- case NEQ:
- result = comp != 0;
- break;
- case LT:
- result = comp < 0;
- break;
- case LTE:
- result = comp <= 0;
- break;
- case GT:
- result = comp > 0;
- break;
- case GTE:
- result = comp >= 0;
- break;
- case IN:
- result = conditionValues.contains(tupleValue);
- break;
- case NOTIN:
- result = !conditionValues.contains(tupleValue);
- break;
- default:
- result = false;
- }
- return result;
- }
-
- private boolean isConstant(TupleFilter filter) {
- return (filter instanceof ConstantTupleFilter) || (filter instanceof DynamicTupleFilter);
- }
-
- @Override
- public boolean isEvaluable() {
- return column != null && !conditionValues.isEmpty();
- }
-
- @Override
- public byte[] serialize() {
- ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
- int size = this.dynamicVariables.size();
- BytesUtil.writeVInt(size, buffer);
- for (Map.Entry<String, String> entry : this.dynamicVariables.entrySet()) {
- BytesUtil.writeUTFString(entry.getKey(), buffer);
- BytesUtil.writeUTFString(entry.getValue(), buffer);
- }
- BytesUtil.writeAsciiString(nullString, buffer);
- byte[] result = new byte[buffer.position()];
- System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
- return result;
- }
-
- @Override
- public void deserialize(byte[] bytes) {
- this.dynamicVariables.clear();
- ByteBuffer buffer = ByteBuffer.wrap(bytes);
- int size = BytesUtil.readVInt(buffer);
- for (int i = 0; i < size; i++) {
- String nameString = BytesUtil.readUTFString(buffer);
- String valueString = BytesUtil.readUTFString(buffer);
- bindVariable(nameString, valueString);
- }
- this.nullString = BytesUtil.readAsciiString(buffer);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/storage/src/main/java/com/kylinolap/storage/filter/ConstantTupleFilter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/filter/ConstantTupleFilter.java b/storage/src/main/java/com/kylinolap/storage/filter/ConstantTupleFilter.java
deleted file mode 100644
index 9457d03..0000000
--- a/storage/src/main/java/com/kylinolap/storage/filter/ConstantTupleFilter.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.storage.filter;
-
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-
-import com.kylinolap.common.util.BytesUtil;
-import com.kylinolap.storage.tuple.ITuple;
-
-/**
- *
- * @author xjiang
- *
- */
-public class ConstantTupleFilter extends TupleFilter {
-
- public static final ConstantTupleFilter FALSE = new ConstantTupleFilter();
- public static final ConstantTupleFilter TRUE = new ConstantTupleFilter("TRUE");
-
- private Collection<String> constantValues;
-
- public ConstantTupleFilter() {
- super(Collections.<TupleFilter> emptyList(), FilterOperatorEnum.CONSTANT);
- this.constantValues = new HashSet<String>();
- }
-
- public ConstantTupleFilter(String value) {
- this();
- this.constantValues.add(value);
- }
-
- public ConstantTupleFilter(Collection<String> values) {
- this();
- this.constantValues.addAll(values);
- }
-
- @Override
- public void addChild(TupleFilter child) {
- throw new UnsupportedOperationException("This is " + this + " and child is " + child);
- }
-
- @Override
- public String toString() {
- return "ConstantFilter [constant=" + constantValues + "]";
- }
-
- @Override
- public boolean evaluate(ITuple tuple) {
- return constantValues.size() > 0;
- }
-
- @Override
- public boolean isEvaluable() {
- return true;
- }
-
- @Override
- public Collection<String> getValues() {
- return this.constantValues;
- }
-
- @Override
- public byte[] serialize() {
- ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
- int size = this.constantValues.size();
- BytesUtil.writeVInt(size, buffer);
- for (String val : this.constantValues) {
- BytesUtil.writeUTFString(val, buffer);
- }
- byte[] result = new byte[buffer.position()];
- System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
- return result;
- }
-
- @Override
- public void deserialize(byte[] bytes) {
- this.constantValues.clear();
- ByteBuffer buffer = ByteBuffer.wrap(bytes);
- int size = BytesUtil.readVInt(buffer);
- for (int i = 0; i < size; i++) {
- this.constantValues.add(BytesUtil.readUTFString(buffer));
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/storage/src/main/java/com/kylinolap/storage/filter/DynamicTupleFilter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/filter/DynamicTupleFilter.java b/storage/src/main/java/com/kylinolap/storage/filter/DynamicTupleFilter.java
deleted file mode 100644
index b2f85d5..0000000
--- a/storage/src/main/java/com/kylinolap/storage/filter/DynamicTupleFilter.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.storage.filter;
-
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.Collections;
-
-import com.kylinolap.common.util.BytesUtil;
-import com.kylinolap.storage.tuple.ITuple;
-
-/**
- *
- * @author xjiang
- *
- */
-public class DynamicTupleFilter extends TupleFilter {
-
- private String variableName;
-
- public DynamicTupleFilter(String name) {
- super(Collections.<TupleFilter> emptyList(), FilterOperatorEnum.DYNAMIC);
- this.variableName = name;
- }
-
- public String getVariableName() {
- return variableName;
- }
-
- @Override
- public void addChild(TupleFilter child) {
- throw new UnsupportedOperationException("This is " + this + " and child is " + child);
- }
-
- @Override
- public String toString() {
- return "DynamicFilter [variableName=" + variableName + "]";
- }
-
- @Override
- public boolean evaluate(ITuple tuple) {
- return true;
- }
-
- @Override
- public boolean isEvaluable() {
- return true;
- }
-
- @Override
- public Collection<String> getValues() {
- return Collections.emptyList();
- }
-
- @Override
- public byte[] serialize() {
- ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
- BytesUtil.writeUTFString(variableName, buffer);
- byte[] result = new byte[buffer.position()];
- System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
- return result;
- }
-
- @Override
- public void deserialize(byte[] bytes) {
- ByteBuffer buffer = ByteBuffer.wrap(bytes);
- this.variableName = BytesUtil.readUTFString(buffer);
- }
-
-}