You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/03/10 08:50:09 UTC
[06/50] [abbrv] kylin git commit: KYLIN-1453 cuboid sharding based on
specific column
KYLIN-1453 cuboid sharding based on specific column
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/294fc707
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/294fc707
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/294fc707
Branch: refs/heads/master
Commit: 294fc70785ab009e6b4d8a12cbeb609d46f89a93
Parents: 3f5074e
Author: honma <ho...@ebay.com>
Authored: Thu Feb 25 19:02:28 2016 +0800
Committer: honma <ho...@ebay.com>
Committed: Fri Feb 26 17:54:37 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/BytesUtil.java | 18 +-
.../dict/TupleFilterFunctionTransformer.java | 170 +++++++++++++++++++
.../dict/TupleFilterFunctionTranslator.java | 166 ------------------
.../kylin/metadata/filter/CaseTupleFilter.java | 7 +-
.../metadata/filter/ColumnTupleFilter.java | 11 +-
.../metadata/filter/CompareTupleFilter.java | 10 +-
.../metadata/filter/ConstantTupleFilter.java | 16 +-
.../metadata/filter/DynamicTupleFilter.java | 9 +-
.../metadata/filter/ExtractTupleFilter.java | 7 +-
.../metadata/filter/FunctionTupleFilter.java | 16 +-
.../filter/ITupleFilterTransformer.java | 23 +++
.../metadata/filter/ITupleFilterTranslator.java | 26 ---
.../metadata/filter/LogicalTupleFilter.java | 10 +-
.../kylin/metadata/filter/TupleFilter.java | 5 +-
.../metadata/filter/TupleFilterSerializer.java | 25 ++-
.../common/coprocessor/FilterDecorator.java | 8 +-
.../hbase/cube/v2/CubeSegmentScanner.java | 10 +-
.../common/coprocessor/FilterBaseTest.java | 46 ++++-
.../common/coprocessor/FilterEvaluateTest.java | 4 +-
.../common/coprocessor/FilterSerializeTest.java | 26 ++-
20 files changed, 325 insertions(+), 288 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
index e01ce4f..4e0701c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
@@ -57,7 +57,6 @@ public class BytesUtil {
return integer;
}
-
public static long readLong(ByteBuffer buffer, int size) {
long integer = 0;
for (int i = 0; i < size; i++) {
@@ -133,11 +132,15 @@ public class BytesUtil {
// from WritableUtils
// ============================================================================
+
public static void writeVInt(int i, ByteBuffer out) {
+
writeVLong(i, out);
+
}
public static void writeVLong(long i, ByteBuffer out) {
+
if (i >= -112 && i <= 127) {
out.put((byte) i);
return;
@@ -203,6 +206,8 @@ public class BytesUtil {
}
public static void writeUnsigned(int num, int size, ByteBuffer out) {
+
+
int mask = 0xff << ((size - 1) * 8);
for (int i = size; i > 0; i--) {
int v = (num & mask) >> (i - 1) * 8;
@@ -222,6 +227,7 @@ public class BytesUtil {
}
public static void writeLong(long num, ByteBuffer out) {
+
for (int i = 0; i < 8; i++) {
out.put((byte) num);
num >>>= 8;
@@ -257,6 +263,8 @@ public class BytesUtil {
}
int len = str.length();
BytesUtil.writeVInt(len, out);
+
+
for (int i = 0; i < len; i++) {
out.put((byte) str.charAt(i));
}
@@ -335,7 +343,7 @@ public class BytesUtil {
writeVInt(-1, out);
return;
}
- writeVInt(array.length, out);
+ writeVInt(length, out);
out.put(array, offset, length);
}
@@ -348,7 +356,7 @@ public class BytesUtil {
in.get(array);
return array;
}
-
+
public static int peekByteArrayLength(ByteBuffer in) {
int start = in.position();
int arrayLen = readVInt(in);
@@ -369,6 +377,7 @@ public class BytesUtil {
writeVInt(array.length, out);
byte b_true = (byte) 1;
byte b_false = (byte) 0;
+
for (int i = 0; i < array.length; i++) {
if (array[i])
out.put(b_true);
@@ -428,7 +437,4 @@ public class BytesUtil {
return sb.toString();
}
- public static void main(String[] args) throws Exception {
- }
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTransformer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTransformer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTransformer.java
new file mode 100644
index 0000000..096a28d
--- /dev/null
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTransformer.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.dict;
+
+import java.util.Collection;
+import java.util.ListIterator;
+
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.FunctionTupleFilter;
+import org.apache.kylin.metadata.filter.ITupleFilterTransformer;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+import com.google.common.primitives.Primitives;
+
+/**
+ * only take effect when the compare filter has function
+ */
+public class TupleFilterFunctionTransformer implements ITupleFilterTransformer {
+ public static final Logger logger = LoggerFactory.getLogger(TupleFilterFunctionTransformer.class);
+
+ private IDictionaryAware dictionaryAware;
+
+ public TupleFilterFunctionTransformer(IDictionaryAware dictionaryAware) {
+ this.dictionaryAware = dictionaryAware;
+ }
+
+ @Override
+ public TupleFilter transform(TupleFilter tupleFilter) {
+ TupleFilter translated = null;
+ if (tupleFilter instanceof CompareTupleFilter) {
+ translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter);
+ if (translated != null) {
+ logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}");
+ }
+ } else if (tupleFilter instanceof FunctionTupleFilter) {
+ translated = translateFunctionTupleFilter((FunctionTupleFilter) tupleFilter);
+ if (translated != null) {
+ logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}");
+ }
+ } else if (tupleFilter instanceof LogicalTupleFilter) {
+ ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator();
+ while (childIterator.hasNext()) {
+ TupleFilter transformed = transform(childIterator.next());
+ if (transformed != null)
+ childIterator.set(transformed);
+ }
+ }
+ return translated == null ? tupleFilter : translated;
+ }
+
+ private TupleFilter translateFunctionTupleFilter(FunctionTupleFilter functionTupleFilter) {
+ if (!functionTupleFilter.isValid())
+ return null;
+
+ TblColRef columnRef = functionTupleFilter.getColumn();
+ Dictionary<?> dict = dictionaryAware.getDictionary(columnRef);
+ if (dict == null)
+ return null;
+
+ CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN);
+ translated.addChild(new ColumnTupleFilter(columnRef));
+
+ try {
+ for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
+ Object dictVal = dict.getValueFromId(i);
+ if ((Boolean) functionTupleFilter.invokeFunction(dictVal)) {
+ translated.addChild(new ConstantTupleFilter(dictVal));
+ }
+ }
+ } catch (Exception e) {
+ logger.debug(e.getMessage());
+ return null;
+ }
+ return translated;
+ }
+
+ @SuppressWarnings("unchecked")
+ private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter) {
+ if (compTupleFilter.getFunction() == null)
+ return null;
+
+ FunctionTupleFilter functionTupleFilter = compTupleFilter.getFunction();
+ if (!functionTupleFilter.isValid())
+ return null;
+
+ TblColRef columnRef = functionTupleFilter.getColumn();
+ Dictionary<?> dict = dictionaryAware.getDictionary(columnRef);
+ if (dict == null)
+ return null;
+
+ CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN);
+ translated.addChild(new ColumnTupleFilter(columnRef));
+
+ try {
+ Collection<Object> inValues = Lists.newArrayList();
+ for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
+ Object dictVal = dict.getValueFromId(i);
+ Object computedVal = functionTupleFilter.invokeFunction(dictVal);
+ Class clazz = Primitives.wrap(computedVal.getClass());
+ Object targetVal = compTupleFilter.getFirstValue();
+ if (Primitives.isWrapperType(clazz))
+ targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, compTupleFilter.getFirstValue()));
+
+ int comp = ((Comparable) computedVal).compareTo(targetVal);
+ boolean compResult = false;
+ switch (compTupleFilter.getOperator()) {
+ case EQ:
+ compResult = comp == 0;
+ break;
+ case NEQ:
+ compResult = comp != 0;
+ break;
+ case LT:
+ compResult = comp < 0;
+ break;
+ case LTE:
+ compResult = comp <= 0;
+ break;
+ case GT:
+ compResult = comp > 0;
+ break;
+ case GTE:
+ compResult = comp >= 0;
+ break;
+ case IN:
+ compResult = compTupleFilter.getValues().contains(computedVal.toString());
+ break;
+ case NOTIN:
+ compResult = !compTupleFilter.getValues().contains(computedVal.toString());
+ break;
+ default:
+ break;
+ }
+ if (compResult) {
+ inValues.add(dictVal);
+ }
+ }
+ translated.addChild(new ConstantTupleFilter(inValues));
+ } catch (Exception e) {
+ logger.debug(e.getMessage());
+ return null;
+ }
+ return translated;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTranslator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTranslator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTranslator.java
deleted file mode 100644
index 1c96dd4..0000000
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTranslator.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import java.util.ListIterator;
-
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.FunctionTupleFilter;
-import org.apache.kylin.metadata.filter.ITupleFilterTranslator;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.primitives.Primitives;
-
-/**
- * only take effect when the compare filter has function
- */
-public class TupleFilterFunctionTranslator implements ITupleFilterTranslator {
- public static final Logger logger = LoggerFactory.getLogger(TupleFilterFunctionTranslator.class);
-
- private IDictionaryAware dictionaryAware;
-
- public TupleFilterFunctionTranslator(IDictionaryAware dictionaryAware) {
- this.dictionaryAware = dictionaryAware;
- }
-
- @Override
- public TupleFilter translate(TupleFilter tupleFilter) {
- TupleFilter translated = null;
- if (tupleFilter instanceof CompareTupleFilter) {
- translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter);
- if (translated != null) {
- logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}");
- }
- } else if (tupleFilter instanceof FunctionTupleFilter) {
- translated = translateFunctionTupleFilter((FunctionTupleFilter) tupleFilter);
- if (translated != null) {
- logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}");
- }
- } else if (tupleFilter instanceof LogicalTupleFilter) {
- ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator();
- while (childIterator.hasNext()) {
- TupleFilter tempTranslated = translate(childIterator.next());
- if (tempTranslated != null)
- childIterator.set(tempTranslated);
- }
- }
- return translated == null ? tupleFilter : translated;
- }
-
- private TupleFilter translateFunctionTupleFilter(FunctionTupleFilter functionTupleFilter) {
- if (!functionTupleFilter.isValid())
- return null;
-
- TblColRef columnRef = functionTupleFilter.getColumn();
- Dictionary<?> dict = dictionaryAware.getDictionary(columnRef);
- if (dict == null)
- return null;
-
- CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN);
- translated.addChild(new ColumnTupleFilter(columnRef));
-
- try {
- for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
- Object dictVal = dict.getValueFromId(i);
- if ((Boolean) functionTupleFilter.invokeFunction(dictVal)) {
- translated.addChild(new ConstantTupleFilter(dictVal));
- }
- }
- } catch (Exception e) {
- logger.debug(e.getMessage());
- return null;
- }
- return translated;
- }
-
- @SuppressWarnings("unchecked")
- private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter) {
- if (compTupleFilter.getFunction() == null)
- return null;
-
- FunctionTupleFilter functionTupleFilter = compTupleFilter.getFunction();
- if (!functionTupleFilter.isValid())
- return null;
-
- TblColRef columnRef = functionTupleFilter.getColumn();
- Dictionary<?> dict = dictionaryAware.getDictionary(columnRef);
- if (dict == null)
- return null;
-
- CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN);
- translated.addChild(new ColumnTupleFilter(columnRef));
-
- try {
- for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
- Object dictVal = dict.getValueFromId(i);
- Object computedVal = functionTupleFilter.invokeFunction(dictVal);
- Class clazz = Primitives.wrap(computedVal.getClass());
- Object targetVal = compTupleFilter.getFirstValue();
- if (Primitives.isWrapperType(clazz))
- targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, compTupleFilter.getFirstValue()));
-
- int comp = ((Comparable) computedVal).compareTo(targetVal);
- boolean compResult = false;
- switch (compTupleFilter.getOperator()) {
- case EQ:
- compResult = comp == 0;
- break;
- case NEQ:
- compResult = comp != 0;
- break;
- case LT:
- compResult = comp < 0;
- break;
- case LTE:
- compResult = comp <= 0;
- break;
- case GT:
- compResult = comp > 0;
- break;
- case GTE:
- compResult = comp >= 0;
- break;
- case IN:
- compResult = compTupleFilter.getValues().contains(computedVal.toString());
- break;
- case NOTIN:
- compResult = !compTupleFilter.getValues().contains(computedVal.toString());
- break;
- default:
- break;
- }
- if (compResult) {
- translated.addChild(new ConstantTupleFilter(dictVal));
- }
- }
- } catch (Exception e) {
- logger.debug(e.getMessage());
- return null;
- }
- return translated;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
index 6dbc614..2b00d69 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
@@ -18,6 +18,7 @@
package org.apache.kylin.metadata.filter;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -101,12 +102,12 @@ public class CaseTupleFilter extends TupleFilter {
}
@Override
- public byte[] serialize(IFilterCodeSystem<?> cs) {
- return new byte[0];
+ public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+ //serialize nothing
}
@Override
- public void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
+ public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
index 0d2a73d..029233d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
@@ -84,8 +84,7 @@ public class ColumnTupleFilter extends TupleFilter {
}
@Override
- public byte[] serialize(IFilterCodeSystem<?> cs) {
- ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
+ public void serialize(IFilterCodeSystem<?> cs,ByteBuffer buffer) {
String table = columnRef.getTable();
BytesUtil.writeUTFString(table, buffer);
@@ -97,17 +96,13 @@ public class ColumnTupleFilter extends TupleFilter {
String dataType = columnRef.getDatatype();
BytesUtil.writeUTFString(dataType, buffer);
-
- byte[] result = new byte[buffer.position()];
- System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
- return result;
}
@Override
- public void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
+ public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+
TableDesc table = null;
ColumnDesc column = new ColumnDesc();
- ByteBuffer buffer = ByteBuffer.wrap(bytes);
String tableName = BytesUtil.readUTFString(buffer);
if (tableName != null) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index 248ab3b..fc0bab7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -214,23 +214,19 @@ public class CompareTupleFilter extends TupleFilter {
@SuppressWarnings({ "unchecked", "rawtypes" })
@Override
- public byte[] serialize(IFilterCodeSystem cs) {
- ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
+ public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) {
int size = this.dynamicVariables.size();
BytesUtil.writeVInt(size, buffer);
for (Map.Entry<String, Object> entry : this.dynamicVariables.entrySet()) {
BytesUtil.writeUTFString(entry.getKey(), buffer);
cs.serialize(entry.getValue(), buffer);
}
- byte[] result = new byte[buffer.position()];
- System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
- return result;
}
@Override
- public void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
+ public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+
this.dynamicVariables.clear();
- ByteBuffer buffer = ByteBuffer.wrap(bytes);
int size = BytesUtil.readVInt(buffer);
for (int i = 0; i < size; i++) {
String name = BytesUtil.readUTFString(buffer);
http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
index 3056a9c..db3eb4f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
@@ -21,8 +21,10 @@ package org.apache.kylin.metadata.filter;
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.Collections;
-import java.util.HashSet;
+import java.util.TreeSet;
+import com.google.common.collect.Lists;
+import org.apache.commons.collections.comparators.NullComparator;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
@@ -40,7 +42,7 @@ public class ConstantTupleFilter extends TupleFilter {
public ConstantTupleFilter() {
super(Collections.<TupleFilter> emptyList(), FilterOperatorEnum.CONSTANT);
- this.constantValues = new HashSet<Object>();
+ this.constantValues = Lists.newArrayList();
}
public ConstantTupleFilter(Object value) {
@@ -89,22 +91,18 @@ public class ConstantTupleFilter extends TupleFilter {
@SuppressWarnings({ "unchecked", "rawtypes" })
@Override
- public byte[] serialize(IFilterCodeSystem cs) {
- ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
+ public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) {
int size = this.constantValues.size();
BytesUtil.writeVInt(size, buffer);
for (Object val : this.constantValues) {
cs.serialize(val, buffer);
}
- byte[] result = new byte[buffer.position()];
- System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
- return result;
}
@Override
- public void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
+ public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+
this.constantValues.clear();
- ByteBuffer buffer = ByteBuffer.wrap(bytes);
int size = BytesUtil.readVInt(buffer);
for (int i = 0; i < size; i++) {
this.constantValues.add(cs.deserialize(buffer));
http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
index a482519..d9dc52a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
@@ -69,17 +69,12 @@ public class DynamicTupleFilter extends TupleFilter {
}
@Override
- public byte[] serialize(IFilterCodeSystem<?> cs) {
- ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
+ public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
BytesUtil.writeUTFString(variableName, buffer);
- byte[] result = new byte[buffer.position()];
- System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
- return result;
}
@Override
- public void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
- ByteBuffer buffer = ByteBuffer.wrap(bytes);
+ public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
this.variableName = BytesUtil.readUTFString(buffer);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
index 6f7dfaf..591e64b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
@@ -18,6 +18,7 @@
package org.apache.kylin.metadata.filter;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -113,12 +114,12 @@ public class ExtractTupleFilter extends TupleFilter {
}
@Override
- public byte[] serialize(IFilterCodeSystem<?> cs) {
- return new byte[0];
+ public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+ //do nothing
}
@Override
- public void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
+ public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java
index 30bef97..2a08728 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java
@@ -123,32 +123,20 @@ public class FunctionTupleFilter extends TupleFilter {
}
@Override
- byte[] serialize(IFilterCodeSystem<?> cs) {
- ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
+ void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
BytesUtil.writeUTFString(name, buffer);
BytesUtil.writeVInt(colPosition, buffer);
BytesUtil.writeVInt(isValid ? 1 : 0, buffer);
- BytesUtil.writeByteArray(TupleFilterSerializer.serialize(columnContainerFilter, cs), buffer);
-
- byte[] result = new byte[buffer.position()];
- System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
- return result;
}
@Override
- void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
- ByteBuffer buffer = ByteBuffer.wrap(bytes);
+ public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
this.name = BytesUtil.readUTFString(buffer);
this.initMethod();
this.colPosition = BytesUtil.readVInt(buffer);
this.isValid = BytesUtil.readVInt(buffer) == 1;
-
- byte[] columnFilter = BytesUtil.readByteArray(buffer);
- if (columnFilter != null) {
- this.columnContainerFilter = TupleFilterSerializer.deserialize(columnFilter, cs);
- }
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java
new file mode 100644
index 0000000..d3d5076
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+public interface ITupleFilterTransformer {
+ TupleFilter transform(TupleFilter tupleFilter);
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java
deleted file mode 100644
index aed284c..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.filter;
-
-/**
- * Created by dongli on 1/7/16.
- */
-public interface ITupleFilterTranslator {
- TupleFilter translate(TupleFilter tupleFilter);
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 0929cf1..61657fb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -18,6 +18,7 @@
package org.apache.kylin.metadata.filter;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -46,7 +47,6 @@ public class LogicalTupleFilter extends TupleFilter {
return cloneTuple;
}
-
// private TupleFilter reverseNestedNots(TupleFilter filter, int depth) {
// if ((filter instanceof LogicalTupleFilter) && (filter.operator == FilterOperatorEnum.NOT)) {
// assert (filter.children.size() == 1);
@@ -60,7 +60,6 @@ public class LogicalTupleFilter extends TupleFilter {
// }
// }
-
@Override
public TupleFilter reverse() {
switch (operator) {
@@ -151,12 +150,13 @@ public class LogicalTupleFilter extends TupleFilter {
}
@Override
- public byte[] serialize(IFilterCodeSystem<?> cs) {
- return new byte[0];
+ public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+ //do nothing
}
@Override
- public void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
+ public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 944ddd0..1e23499 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -18,6 +18,7 @@
package org.apache.kylin.metadata.filter;
+import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
@@ -204,9 +205,9 @@ public abstract class TupleFilter {
public abstract Collection<?> getValues();
- abstract byte[] serialize(IFilterCodeSystem<?> cs);
+ abstract void serialize(IFilterCodeSystem<?> cs,ByteBuffer buffer);
- abstract void deserialize(byte[] bytes, IFilterCodeSystem<?> cs);
+ abstract void deserialize(IFilterCodeSystem<?> cs,ByteBuffer buffer);
public static boolean isEvaluableRecursively(TupleFilter filter) {
if (filter == null)
http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
index a394a51..39ccb15 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -18,12 +18,15 @@
package org.apache.kylin.metadata.filter;
+import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.Map;
import java.util.Stack;
import org.apache.kylin.common.util.BytesUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* http://eli.thegreenplace.net/2011/09/29/an-interesting-tree-serialization-algorithm-from-dwarf
@@ -33,6 +36,8 @@ import org.apache.kylin.common.util.BytesUtil;
*/
public class TupleFilterSerializer {
+ private static final Logger logger = LoggerFactory.getLogger(TupleFilterSerializer.class);
+
public interface Decorator {
TupleFilter onSerialize(TupleFilter filter);
}
@@ -51,8 +56,18 @@ public class TupleFilterSerializer {
}
public static byte[] serialize(TupleFilter rootFilter, Decorator decorator, IFilterCodeSystem<?> cs) {
- ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
- internalSerialize(rootFilter, decorator, buffer, cs);
+ ByteBuffer buffer;
+ int bufferSize = BUFFER_SIZE;
+ while (true) {
+ try {
+ buffer = ByteBuffer.allocate(bufferSize);
+ internalSerialize(rootFilter, decorator, buffer, cs);
+ break;
+ } catch (BufferOverflowException e) {
+ logger.info("Buffer size {} cannot hold the filter, resizing to 4 times", bufferSize);
+ bufferSize *= 4;
+ }
+ }
byte[] result = new byte[buffer.position()];
System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
return result;
@@ -86,10 +101,9 @@ public class TupleFilterSerializer {
if (flag < 0) {
BytesUtil.writeVInt(-1, buffer);
} else {
- byte[] bytes = filter.serialize(cs);
int opVal = filter.getOperator().getValue();
BytesUtil.writeVInt(opVal, buffer);
- BytesUtil.writeByteArray(bytes, buffer);
+ filter.serialize(cs, buffer);
BytesUtil.writeVInt(flag, buffer);
}
}
@@ -107,8 +121,7 @@ public class TupleFilterSerializer {
// deserialize filter
TupleFilter filter = createTupleFilter(opVal);
- byte[] filterBytes = BytesUtil.readByteArray(buffer);
- filter.deserialize(filterBytes, cs);
+ filter.deserialize(cs, buffer);
if (rootFilter == null) {
// push root to stack
http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
index 01d3041..5208ba7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
@@ -8,11 +8,11 @@ import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.kv.RowKeyColumnIO;
import org.apache.kylin.dict.DictCodeSystem;
import org.apache.kylin.dict.IDictionaryAware;
-import org.apache.kylin.dict.TupleFilterFunctionTranslator;
+import org.apache.kylin.dict.TupleFilterFunctionTransformer;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.ITupleFilterTranslator;
+import org.apache.kylin.metadata.filter.ITupleFilterTransformer;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilterSerializer;
import org.apache.kylin.metadata.model.TblColRef;
@@ -131,8 +131,8 @@ public class FilterDecorator implements TupleFilterSerializer.Decorator {
if (filter == null)
return null;
- ITupleFilterTranslator translator = new TupleFilterFunctionTranslator(columnIO.getIDictionaryAware());
- filter = translator.translate(filter);
+ ITupleFilterTransformer translator = new TupleFilterFunctionTransformer(columnIO.getIDictionaryAware());
+ filter = translator.transform(filter);
// un-evaluatable filter is replaced with TRUE
if (!filter.isEvaluable()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
index e96c602..3f00566 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
@@ -23,7 +23,7 @@ import org.apache.kylin.cube.gridtable.CubeGridTable;
import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
import org.apache.kylin.cube.gridtable.NotEnoughGTInfoException;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.dict.TupleFilterFunctionTranslator;
+import org.apache.kylin.dict.TupleFilterFunctionTransformer;
import org.apache.kylin.gridtable.EmptyGTScanner;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
@@ -33,7 +33,7 @@ import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.GTUtil;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.filter.ITupleFilterTranslator;
+import org.apache.kylin.metadata.filter.ITupleFilterTransformer;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
@@ -63,9 +63,9 @@ public class CubeSegmentScanner implements IGTScanner {
CuboidToGridTableMapping mapping = cuboid.getCuboidToGridTableMapping();
- // translate FunctionTupleFilter to IN clause
- ITupleFilterTranslator translator = new TupleFilterFunctionTranslator(this.cubeSeg);
- filter = translator.translate(filter);
+ // transform FunctionTupleFilter to equivalent IN clause
+ ITupleFilterTransformer translator = new TupleFilterFunctionTransformer(this.cubeSeg);
+ filter = translator.transform(filter);
//replace the constant values in filter to dictionary codes
TupleFilter gtFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, mapping.getCuboidDimensionsInGTOrder(), groups);
http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
index 92e9699..e7ed1a8 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
@@ -18,8 +18,12 @@
package org.apache.kylin.storage.hbase.common.coprocessor;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
+import java.util.Calendar;
import java.util.Collection;
+import java.util.Date;
import java.util.List;
import java.util.Random;
@@ -39,6 +43,8 @@ import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.Tuple;
import org.apache.kylin.metadata.tuple.TupleInfo;
+import com.google.common.collect.Lists;
+
/**
* @author xjiang
*
@@ -64,7 +70,7 @@ public class FilterBaseTest {
return groups;
}
- protected CompareTupleFilter buildCompareFilter(List<TblColRef> groups, int index) {
+ protected CompareTupleFilter buildEQCompareFilter(List<TblColRef> groups, int index) {
TblColRef column = groups.get(index);
CompareTupleFilter compareFilter = new CompareTupleFilter(FilterOperatorEnum.EQ);
ColumnTupleFilter columnFilter = new ColumnTupleFilter(column);
@@ -79,9 +85,31 @@ public class FilterBaseTest {
return compareFilter;
}
+ protected CompareTupleFilter buildINCompareFilter(TblColRef dateColumn) throws ParseException {
+ CompareTupleFilter compareFilter = new CompareTupleFilter(FilterOperatorEnum.IN);
+ ColumnTupleFilter columnFilter = new ColumnTupleFilter(dateColumn);
+ compareFilter.addChild(columnFilter);
+
+ List<String> inValues = Lists.newArrayList();
+ SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
+ Date startDate = simpleDateFormat.parse("1970-01-01");
+ Date endDate = simpleDateFormat.parse("2100-01-01");
+ Calendar start = Calendar.getInstance();
+ start.setTime(startDate);
+ Calendar end = Calendar.getInstance();
+ end.setTime(endDate);
+ for (Date date = start.getTime(); start.before(end); start.add(Calendar.DATE, 1), date = start.getTime()) {
+ inValues.add(simpleDateFormat.format(date));
+ }
+
+ ConstantTupleFilter constantFilter = new ConstantTupleFilter(inValues);
+ compareFilter.addChild(constantFilter);
+ return compareFilter;
+ }
+
protected TupleFilter buildAndFilter(List<TblColRef> columns) {
- CompareTupleFilter compareFilter1 = buildCompareFilter(columns, 0);
- CompareTupleFilter compareFilter2 = buildCompareFilter(columns, 1);
+ CompareTupleFilter compareFilter1 = buildEQCompareFilter(columns, 0);
+ CompareTupleFilter compareFilter2 = buildEQCompareFilter(columns, 1);
LogicalTupleFilter andFilter = new LogicalTupleFilter(FilterOperatorEnum.AND);
andFilter.addChild(compareFilter1);
andFilter.addChild(compareFilter2);
@@ -89,8 +117,8 @@ public class FilterBaseTest {
}
protected TupleFilter buildOrFilter(List<TblColRef> columns) {
- CompareTupleFilter compareFilter1 = buildCompareFilter(columns, 0);
- CompareTupleFilter compareFilter2 = buildCompareFilter(columns, 1);
+ CompareTupleFilter compareFilter1 = buildEQCompareFilter(columns, 0);
+ CompareTupleFilter compareFilter2 = buildEQCompareFilter(columns, 1);
LogicalTupleFilter logicFilter = new LogicalTupleFilter(FilterOperatorEnum.OR);
logicFilter.addChild(compareFilter1);
logicFilter.addChild(compareFilter2);
@@ -105,12 +133,12 @@ public class FilterBaseTest {
TupleFilter then0 = new ConstantTupleFilter("0");
caseFilter.addChild(then0);
- TupleFilter when1 = buildCompareFilter(groups, 0);
+ TupleFilter when1 = buildEQCompareFilter(groups, 0);
caseFilter.addChild(when1);
TupleFilter then1 = new ConstantTupleFilter("1");
caseFilter.addChild(then1);
- TupleFilter when2 = buildCompareFilter(groups, 1);
+ TupleFilter when2 = buildEQCompareFilter(groups, 1);
caseFilter.addChild(when2);
TupleFilter then2 = new ConstantTupleFilter("2");
caseFilter.addChild(then2);
@@ -153,9 +181,9 @@ public class FilterBaseTest {
}
String str1 = f1.toString();
- System.out.println("f1=" + str1);
+ //System.out.println("f1=" + str1);
String str2 = f2.toString();
- System.out.println("f2=" + str2);
+ //System.out.println("f2=" + str2);
if (!str1.equals(str2)) {
throw new IllegalStateException("f1=" + str1 + ", f2=" + str2);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
index bde8dd2..aac09b7 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
@@ -38,7 +38,7 @@ public class FilterEvaluateTest extends FilterBaseTest {
@Test
public void testEvaluate00() {
List<TblColRef> groups = buildGroups();
- TupleFilter filter = buildCompareFilter(groups, 0);
+ TupleFilter filter = buildEQCompareFilter(groups, 0);
byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
@@ -54,7 +54,7 @@ public class FilterEvaluateTest extends FilterBaseTest {
@Test
public void testEvaluate01() {
List<TblColRef> groups = buildGroups();
- TupleFilter filter = buildCompareFilter(groups, 1);
+ TupleFilter filter = buildEQCompareFilter(groups, 1);
byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java
index 8bf8ecb..0a21598 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java
@@ -18,6 +18,7 @@
package org.apache.kylin.storage.hbase.common.coprocessor;
+import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
@@ -40,7 +41,7 @@ public class FilterSerializeTest extends FilterBaseTest {
@Test
public void testSerialize01() {
List<TblColRef> groups = buildGroups();
- TupleFilter filter = buildCompareFilter(groups, 0);
+ TupleFilter filter = buildEQCompareFilter(groups, 0);
byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
@@ -51,7 +52,7 @@ public class FilterSerializeTest extends FilterBaseTest {
@Test
public void testSerialize02() {
List<TblColRef> groups = buildGroups();
- TupleFilter filter = buildCompareFilter(groups, 1);
+ TupleFilter filter = buildEQCompareFilter(groups, 1);
byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
@@ -88,7 +89,7 @@ public class FilterSerializeTest extends FilterBaseTest {
TblColRef colRef = new TblColRef(column);
List<TblColRef> groups = new ArrayList<TblColRef>();
groups.add(colRef);
- TupleFilter filter = buildCompareFilter(groups, 0);
+ TupleFilter filter = buildEQCompareFilter(groups, 0);
byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
@@ -103,7 +104,7 @@ public class FilterSerializeTest extends FilterBaseTest {
TblColRef colRef = new TblColRef(column);
List<TblColRef> groups = new ArrayList<TblColRef>();
groups.add(colRef);
- TupleFilter filter = buildCompareFilter(groups, 0);
+ TupleFilter filter = buildEQCompareFilter(groups, 0);
byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
@@ -122,7 +123,7 @@ public class FilterSerializeTest extends FilterBaseTest {
TblColRef colRef = new TblColRef(column);
List<TblColRef> groups = new ArrayList<TblColRef>();
groups.add(colRef);
- TupleFilter filter = buildCompareFilter(groups, 0);
+ TupleFilter filter = buildEQCompareFilter(groups, 0);
byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
@@ -140,7 +141,7 @@ public class FilterSerializeTest extends FilterBaseTest {
TblColRef colRef = new TblColRef(column);
List<TblColRef> groups = new ArrayList<TblColRef>();
groups.add(colRef);
- TupleFilter filter = buildCompareFilter(groups, 0);
+ TupleFilter filter = buildEQCompareFilter(groups, 0);
byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
@@ -203,6 +204,19 @@ public class FilterSerializeTest extends FilterBaseTest {
}
@Test
+ public void testSerialize14() throws ParseException {
+ List<TblColRef> groups = buildGroups();
+ TupleFilter filter = buildINCompareFilter(groups.get(0));
+
+ long start = System.currentTimeMillis();
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ System.out.println("Size of serialized filter " + bytes.length + ", serialize time: " + (System.currentTimeMillis() - start));
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ compareFilter(filter, newFilter);
+ }
+
+ @Test
public void testDynamic() {
final CompareTupleFilter compareDynamicFilter = buildCompareDynamicFilter(buildGroups());