You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/08/30 11:08:44 UTC
kylin git commit: minor refactor
Repository: kylin
Updated Branches:
refs/heads/master 27426aef2 -> 6c23313e9
minor refactor
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6c23313e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6c23313e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6c23313e
Branch: refs/heads/master
Commit: 6c23313e9d14f8eabf165566ae0a8156b25a47e8
Parents: 27426ae
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Aug 30 16:15:28 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Aug 30 19:08:32 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/ByteArray.java | 4 +
.../kylin/dict/BuildInFunctionTransformer.java | 178 -------------------
.../kylin/dict/BuiltInFunctionTransformer.java | 178 +++++++++++++++++++
.../filter/BuildInFunctionTupleFilter.java | 165 -----------------
.../filter/BuiltInFunctionTupleFilter.java | 157 ++++++++++++++++
.../kylin/metadata/filter/TupleFilter.java | 1 -
.../metadata/filter/TupleFilterSerializer.java | 2 +-
.../metadata/filter/function/Functions.java | 4 +-
.../storage/gtrecord/CubeSegmentScanner.java | 4 +-
.../test/resources/query/sql_like/query03.sql | 33 ++++
.../common/coprocessor/FilterDecorator.java | 4 +-
11 files changed, 379 insertions(+), 351 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java b/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
index a72425b..db02f34 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
@@ -136,6 +136,10 @@ public class ByteArray implements Comparable<ByteArray>, Serializable {
return ByteBuffer.wrap(data, offset, length).slice();
}
+ public byte[] toBytes() {
+ return Bytes.copy(this.array(), this.offset(), this.length());
+ }
+
@Override
public int hashCode() {
if (data == null) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java
deleted file mode 100644
index 665a7e6..0000000
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import java.util.Collection;
-import java.util.ListIterator;
-
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dimension.IDimensionEncodingMap;
-import org.apache.kylin.metadata.filter.BuildInFunctionTupleFilter;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.ITupleFilterTransformer;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-import com.google.common.primitives.Primitives;
-
-/**
- * only take effect when the compare filter has function
- */
-public class BuildInFunctionTransformer implements ITupleFilterTransformer {
- public static final Logger logger = LoggerFactory.getLogger(BuildInFunctionTransformer.class);
-
- private IDimensionEncodingMap dimEncMap;
-
- public BuildInFunctionTransformer(IDimensionEncodingMap dimEncMap) {
- this.dimEncMap = dimEncMap;
- }
-
- @Override
- public TupleFilter transform(TupleFilter tupleFilter) {
- TupleFilter translated = null;
- if (tupleFilter instanceof CompareTupleFilter) {
- //normal case
- translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter);
- if (translated != null) {
- logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}");
- }
- } else if (tupleFilter instanceof BuildInFunctionTupleFilter) {
- //like case
- translated = translateFunctionTupleFilter((BuildInFunctionTupleFilter) tupleFilter);
- if (translated != null) {
- logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}");
- }
- } else if (tupleFilter instanceof LogicalTupleFilter) {
- @SuppressWarnings("unchecked")
- ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator();
- while (childIterator.hasNext()) {
- TupleFilter transformed = transform(childIterator.next());
- if (transformed != null) {
- childIterator.set(transformed);
- } else {
- throw new IllegalStateException("Should not be null");
- }
- }
- }
- return translated == null ? tupleFilter : translated;
- }
-
- private TupleFilter translateFunctionTupleFilter(BuildInFunctionTupleFilter buildInFunctionTupleFilter) {
- if (!buildInFunctionTupleFilter.isValid())
- return null;
-
- TblColRef columnRef = buildInFunctionTupleFilter.getColumn();
- Dictionary<?> dict = dimEncMap.getDictionary(columnRef);
- if (dict == null)
- return null;
-
- CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN);
- translated.addChild(new ColumnTupleFilter(columnRef));
-
- try {
- for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
- Object dictVal = dict.getValueFromId(i);
- if ((Boolean) buildInFunctionTupleFilter.invokeFunction(dictVal)) {
- translated.addChild(new ConstantTupleFilter(dictVal));
- }
- }
- } catch (Exception e) {
- logger.debug(e.getMessage());
- return null;
- }
- return translated;
- }
-
- @SuppressWarnings({ "unchecked", "rawtypes" })
- private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter) {
- if (compTupleFilter.getFunction() == null || (!(compTupleFilter.getFunction() instanceof BuildInFunctionTupleFilter)))
- return null;
-
- BuildInFunctionTupleFilter buildInFunctionTupleFilter = (BuildInFunctionTupleFilter) compTupleFilter.getFunction();
-
- if (!buildInFunctionTupleFilter.isValid())
- return null;
-
- TblColRef columnRef = buildInFunctionTupleFilter.getColumn();
- Dictionary<?> dict = dimEncMap.getDictionary(columnRef);
- if (dict == null)
- return null;
-
- CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN);
- translated.addChild(new ColumnTupleFilter(columnRef));
-
- try {
- Collection<Object> inValues = Lists.newArrayList();
- for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
- Object dictVal = dict.getValueFromId(i);
- Object computedVal = buildInFunctionTupleFilter.invokeFunction(dictVal);
- Class clazz = Primitives.wrap(computedVal.getClass());
- Object targetVal = compTupleFilter.getFirstValue();
- if (Primitives.isWrapperType(clazz))
- targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, compTupleFilter.getFirstValue()));
-
- int comp = ((Comparable) computedVal).compareTo(targetVal);
- boolean compResult = false;
- switch (compTupleFilter.getOperator()) {
- case EQ:
- compResult = comp == 0;
- break;
- case NEQ:
- compResult = comp != 0;
- break;
- case LT:
- compResult = comp < 0;
- break;
- case LTE:
- compResult = comp <= 0;
- break;
- case GT:
- compResult = comp > 0;
- break;
- case GTE:
- compResult = comp >= 0;
- break;
- case IN:
- compResult = compTupleFilter.getValues().contains(computedVal.toString());
- break;
- case NOTIN:
- compResult = !compTupleFilter.getValues().contains(computedVal.toString());
- break;
- default:
- break;
- }
- if (compResult) {
- inValues.add(dictVal);
- }
- }
- translated.addChild(new ConstantTupleFilter(inValues));
- } catch (Exception e) {
- logger.debug(e.getMessage());
- return null;
- }
- return translated;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
new file mode 100644
index 0000000..9707f8f
--- /dev/null
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.dict;
+
+import java.util.Collection;
+import java.util.ListIterator;
+
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.dimension.IDimensionEncodingMap;
+import org.apache.kylin.metadata.filter.BuiltInFunctionTupleFilter;
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.ITupleFilterTransformer;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+import com.google.common.primitives.Primitives;
+
+/**
+ * only take effect when the compare filter has function
+ */
+public class BuiltInFunctionTransformer implements ITupleFilterTransformer {
+ public static final Logger logger = LoggerFactory.getLogger(BuiltInFunctionTransformer.class);
+
+ private IDimensionEncodingMap dimEncMap;
+
+ public BuiltInFunctionTransformer(IDimensionEncodingMap dimEncMap) {
+ this.dimEncMap = dimEncMap;
+ }
+
+ @Override
+ public TupleFilter transform(TupleFilter tupleFilter) {
+ TupleFilter translated = null;
+ if (tupleFilter instanceof CompareTupleFilter) {
+ //normal case
+ translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter);
+ if (translated != null) {
+ logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}");
+ }
+ } else if (tupleFilter instanceof BuiltInFunctionTupleFilter) {
+ //like case
+ translated = translateFunctionTupleFilter((BuiltInFunctionTupleFilter) tupleFilter);
+ if (translated != null) {
+ logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}");
+ }
+ } else if (tupleFilter instanceof LogicalTupleFilter) {
+ @SuppressWarnings("unchecked")
+ ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator();
+ while (childIterator.hasNext()) {
+ TupleFilter transformed = transform(childIterator.next());
+ if (transformed != null) {
+ childIterator.set(transformed);
+ } else {
+ throw new IllegalStateException("Should not be null");
+ }
+ }
+ }
+ return translated == null ? tupleFilter : translated;
+ }
+
+ private TupleFilter translateFunctionTupleFilter(BuiltInFunctionTupleFilter builtInFunctionTupleFilter) {
+ if (!builtInFunctionTupleFilter.isValid())
+ return null;
+
+ TblColRef columnRef = builtInFunctionTupleFilter.getColumn();
+ Dictionary<?> dict = dimEncMap.getDictionary(columnRef);
+ if (dict == null)
+ return null;
+
+ CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN);
+ translated.addChild(new ColumnTupleFilter(columnRef));
+
+ try {
+ for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
+ Object dictVal = dict.getValueFromId(i);
+ if ((Boolean) builtInFunctionTupleFilter.invokeFunction(dictVal)) {
+ translated.addChild(new ConstantTupleFilter(dictVal));
+ }
+ }
+ } catch (Exception e) {
+ logger.debug(e.getMessage());
+ return null;
+ }
+ return translated;
+ }
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter) {
+ if (compTupleFilter.getFunction() == null || (!(compTupleFilter.getFunction() instanceof BuiltInFunctionTupleFilter)))
+ return null;
+
+ BuiltInFunctionTupleFilter builtInFunctionTupleFilter = (BuiltInFunctionTupleFilter) compTupleFilter.getFunction();
+
+ if (!builtInFunctionTupleFilter.isValid())
+ return null;
+
+ TblColRef columnRef = builtInFunctionTupleFilter.getColumn();
+ Dictionary<?> dict = dimEncMap.getDictionary(columnRef);
+ if (dict == null)
+ return null;
+
+ CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN);
+ translated.addChild(new ColumnTupleFilter(columnRef));
+
+ try {
+ Collection<Object> inValues = Lists.newArrayList();
+ for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
+ Object dictVal = dict.getValueFromId(i);
+ Object computedVal = builtInFunctionTupleFilter.invokeFunction(dictVal);
+ Class clazz = Primitives.wrap(computedVal.getClass());
+ Object targetVal = compTupleFilter.getFirstValue();
+ if (Primitives.isWrapperType(clazz))
+ targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, compTupleFilter.getFirstValue()));
+
+ int comp = ((Comparable) computedVal).compareTo(targetVal);
+ boolean compResult = false;
+ switch (compTupleFilter.getOperator()) {
+ case EQ:
+ compResult = comp == 0;
+ break;
+ case NEQ:
+ compResult = comp != 0;
+ break;
+ case LT:
+ compResult = comp < 0;
+ break;
+ case LTE:
+ compResult = comp <= 0;
+ break;
+ case GT:
+ compResult = comp > 0;
+ break;
+ case GTE:
+ compResult = comp >= 0;
+ break;
+ case IN:
+ compResult = compTupleFilter.getValues().contains(computedVal.toString());
+ break;
+ case NOTIN:
+ compResult = !compTupleFilter.getValues().contains(computedVal.toString());
+ break;
+ default:
+ break;
+ }
+ if (compResult) {
+ inValues.add(dictVal);
+ }
+ }
+ translated.addChild(new ConstantTupleFilter(inValues));
+ } catch (Exception e) {
+ logger.debug(e.getMessage());
+ return null;
+ }
+ return translated;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuildInFunctionTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuildInFunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuildInFunctionTupleFilter.java
deleted file mode 100644
index c15b083..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuildInFunctionTupleFilter.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.filter;
-
-import java.io.Serializable;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.filter.function.BuiltInMethod;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-import com.google.common.primitives.Primitives;
-
-public class BuildInFunctionTupleFilter extends FunctionTupleFilter {
- public static final Logger logger = LoggerFactory.getLogger(BuildInFunctionTupleFilter.class);
-
- private String name;
- // FIXME Only supports single parameter functions currently
- private TupleFilter columnContainerFilter;//might be a ColumnTupleFilter(simple case) or FunctionTupleFilter(complex case like substr(lower()))
- private int colPosition;
- private Method method;
- private List<Serializable> methodParams;
- private boolean isValid = false;
-
- public BuildInFunctionTupleFilter(String name) {
- super(Lists.<TupleFilter> newArrayList(), FilterOperatorEnum.FUNCTION);
- this.methodParams = Lists.newArrayList();
-
- if (name != null) {
- this.name = name.toUpperCase();
- initMethod();
- }
- }
-
- public String getName() {
- return name;
- }
-
- public TblColRef getColumn() {
- if (columnContainerFilter == null)
- return null;
-
- if (columnContainerFilter instanceof ColumnTupleFilter)
- return ((ColumnTupleFilter) columnContainerFilter).getColumn();
- else if (columnContainerFilter instanceof BuildInFunctionTupleFilter)
- return ((BuildInFunctionTupleFilter) columnContainerFilter).getColumn();
-
- throw new UnsupportedOperationException("Wrong type TupleFilter in FunctionTupleFilter.");
- }
-
- public Object invokeFunction(Object input) throws InvocationTargetException, IllegalAccessException {
- if (columnContainerFilter instanceof ColumnTupleFilter)
- methodParams.set(colPosition, (Serializable) input);
- else if (columnContainerFilter instanceof BuildInFunctionTupleFilter)
- methodParams.set(colPosition, (Serializable) ((BuildInFunctionTupleFilter) columnContainerFilter).invokeFunction(input));
- return method.invoke(null, (Object[]) (methodParams.toArray()));
- }
-
- public boolean isValid() {
- return isValid && method != null && methodParams.size() == children.size();
- }
-
- @Override
- public void addChild(TupleFilter child) {
- if (child instanceof ColumnTupleFilter || child instanceof BuildInFunctionTupleFilter) {
- columnContainerFilter = child;
- colPosition = methodParams.size();
- methodParams.add(null);
- } else if (child instanceof ConstantTupleFilter) {
- Serializable constVal = (Serializable) child.getValues().iterator().next();
- try {
- Class<?> clazz = Primitives.wrap(method.getParameterTypes()[methodParams.size()]);
- if (!Primitives.isWrapperType(clazz))
- methodParams.add(constVal);
- else
- methodParams.add((Serializable) clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, constVal)));
- } catch (Exception e) {
- logger.warn(e.getMessage());
- isValid = false;
- }
- }
- super.addChild(child);
- }
-
- @Override
- public boolean isEvaluable() {
- return false;
- }
-
- @Override
- public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem<?> cs) {
- throw new UnsupportedOperationException("Function filter cannot be evaluated immediately");
- }
-
- @Override
- public Collection<String> getValues() {
- throw new UnsupportedOperationException("Function filter cannot be evaluated immediately");
- }
-
- @Override
- public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
- BytesUtil.writeUTFString(name, buffer);
- BytesUtil.writeVInt(colPosition, buffer);
- BytesUtil.writeVInt(isValid ? 1 : 0, buffer);
- }
-
- @Override
- public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
-
- this.name = BytesUtil.readUTFString(buffer);
- this.initMethod();
-
- this.colPosition = BytesUtil.readVInt(buffer);
- this.isValid = BytesUtil.readVInt(buffer) == 1;
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append(name);
- sb.append("(");
- for (int i = 0; i < methodParams.size(); i++) {
- if (colPosition == i) {
- sb.append(columnContainerFilter);
- } else {
- sb.append(methodParams.get(i));
- }
- if (i < methodParams.size() - 1)
- sb.append(",");
- }
- sb.append(")");
- return sb.toString();
- }
-
- private void initMethod() {
- if (BuiltInMethod.MAP.containsKey(name)) {
- this.method = BuiltInMethod.MAP.get(name).method;
- isValid = true;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
new file mode 100644
index 0000000..5a10371
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+import java.io.Serializable;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.kylin.metadata.filter.function.BuiltInMethod;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+import com.google.common.primitives.Primitives;
+
+public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
+ public static final Logger logger = LoggerFactory.getLogger(BuiltInFunctionTupleFilter.class);
+
+ protected String name;
+ // FIXME Only supports single parameter functions currently
+ protected TupleFilter columnContainerFilter;//might be a ColumnTupleFilter(simple case) or FunctionTupleFilter(complex case like substr(lower()))
+ protected int colPosition;
+ protected Method method;
+ protected List<Serializable> methodParams;
+ protected boolean isValidFunc = false;
+
+ public BuiltInFunctionTupleFilter(String name) {
+ super(Lists.<TupleFilter> newArrayList(), FilterOperatorEnum.FUNCTION);
+ this.methodParams = Lists.newArrayList();
+
+ if (name != null) {
+ this.name = name.toUpperCase();
+ initMethod();
+ }
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public TblColRef getColumn() {
+ if (columnContainerFilter == null)
+ return null;
+
+ if (columnContainerFilter instanceof ColumnTupleFilter)
+ return ((ColumnTupleFilter) columnContainerFilter).getColumn();
+ else if (columnContainerFilter instanceof BuiltInFunctionTupleFilter)
+ return ((BuiltInFunctionTupleFilter) columnContainerFilter).getColumn();
+
+ throw new UnsupportedOperationException("Wrong type TupleFilter in FunctionTupleFilter.");
+ }
+
+ public Object invokeFunction(Object input) throws InvocationTargetException, IllegalAccessException {
+ if (columnContainerFilter instanceof ColumnTupleFilter)
+ methodParams.set(colPosition, (Serializable) input);
+ else if (columnContainerFilter instanceof BuiltInFunctionTupleFilter)
+ methodParams.set(colPosition, (Serializable) ((BuiltInFunctionTupleFilter) columnContainerFilter).invokeFunction(input));
+ return method.invoke(null, (Object[]) (methodParams.toArray()));
+ }
+
+ public boolean isValid() {
+ return isValidFunc && method != null && methodParams.size() == children.size();
+ }
+
+ @Override
+ public void addChild(TupleFilter child) {
+ if (child instanceof ColumnTupleFilter || child instanceof BuiltInFunctionTupleFilter) {
+ columnContainerFilter = child;
+ colPosition = methodParams.size();
+ methodParams.add(null);
+ } else if (child instanceof ConstantTupleFilter) {
+ Serializable constVal = (Serializable) child.getValues().iterator().next();
+ try {
+ Class<?> clazz = Primitives.wrap(method.getParameterTypes()[methodParams.size()]);
+ if (!Primitives.isWrapperType(clazz))
+ methodParams.add(constVal);
+ else
+ methodParams.add((Serializable) clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, constVal)));
+ } catch (Exception e) {
+ logger.warn(e.getMessage());
+ isValidFunc = false;
+ }
+ }
+ super.addChild(child);
+ }
+
+ @Override
+ public boolean isEvaluable() {
+ return false;
+ }
+
+ @Override
+ public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem<?> cs) {
+ throw new UnsupportedOperationException("Function filter cannot be evaluated immediately");
+ }
+
+ @Override
+ public Collection<String> getValues() {
+ throw new UnsupportedOperationException("Function filter cannot be evaluated immediately");
+ }
+
+ @Override
+ public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+ throw new UnsupportedOperationException("Function filter cannot serialized");
+ }
+
+ @Override
+ public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+ throw new UnsupportedOperationException("Function filter cannot serialized");
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(name);
+ sb.append("(");
+ for (int i = 0; i < methodParams.size(); i++) {
+ if (colPosition == i) {
+ sb.append(columnContainerFilter);
+ } else {
+ sb.append(methodParams.get(i));
+ }
+ if (i < methodParams.size() - 1)
+ sb.append(",");
+ }
+ sb.append(")");
+ return sb.toString();
+ }
+
+ protected void initMethod() {
+ if (BuiltInMethod.MAP.containsKey(name)) {
+ this.method = BuiltInMethod.MAP.get(name).method;
+ isValidFunc = true;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 4647c51..3250640 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -80,7 +80,6 @@ public abstract class TupleFilter {
protected final List<TupleFilter> children;
protected FilterOperatorEnum operator;
- protected boolean hasChildren;
protected TupleFilter(List<TupleFilter> filters, FilterOperatorEnum op) {
this.children = filters;
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
index bcb005f..f6c8bc4 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -186,7 +186,7 @@ public class TupleFilterSerializer {
filter = new DynamicTupleFilter(null);
break;
case FUNCTION:
- filter = new BuildInFunctionTupleFilter(null);
+ filter = new BuiltInFunctionTupleFilter(null);
break;
case MASSIN:
filter = new MassInTupleFilter();
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/Functions.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/Functions.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/Functions.java
index 401c90b..861e530 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/Functions.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/Functions.java
@@ -21,7 +21,7 @@ package org.apache.kylin.metadata.filter.function;
import java.lang.reflect.InvocationTargetException;
import java.util.Map;
-import org.apache.kylin.metadata.filter.BuildInFunctionTupleFilter;
+import org.apache.kylin.metadata.filter.BuiltInFunctionTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
@@ -54,7 +54,7 @@ public class Functions {
}
}
- return new BuildInFunctionTupleFilter(name);
+ return new BuiltInFunctionTupleFilter(name);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index 6ed7d3b..3b9d9c6 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -27,7 +27,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.gridtable.CubeScanRangePlanner;
-import org.apache.kylin.dict.BuildInFunctionTransformer;
+import org.apache.kylin.dict.BuiltInFunctionTransformer;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequest;
@@ -64,7 +64,7 @@ public class CubeSegmentScanner implements IGTScanner {
byte[] serialize = TupleFilterSerializer.serialize(originalfilter, StringCodeSystem.INSTANCE);
TupleFilter filter = TupleFilterSerializer.deserialize(serialize, StringCodeSystem.INSTANCE);
// translate FunctionTupleFilter to IN clause
- ITupleFilterTransformer translator = new BuildInFunctionTransformer(cubeSeg.getDimensionEncodingMap());
+ ITupleFilterTransformer translator = new BuiltInFunctionTransformer(cubeSeg.getDimensionEncodingMap());
filter = translator.transform(filter);
String plannerName = KylinConfig.getInstanceFromEnv().getQueryStorageVisitPlanner();
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/kylin-it/src/test/resources/query/sql_like/query03.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query03.sql b/kylin-it/src/test/resources/query/sql_like/query03.sql
new file mode 100644
index 0000000..ae095fc
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query03.sql
@@ -0,0 +1,33 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+-- test case for KYLIN-1954
+
+select lstg_format_name as lstg_format_name, count(*) as cnt
+
+ from test_kylin_fact
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+
+where lstg_format_name like '%BIN%' and lstg_format_name > 'A'
+group by lstg_format_name
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
index 01bd4e6..ea4b504 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
@@ -24,7 +24,7 @@ import java.util.Set;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.kv.RowKeyColumnIO;
-import org.apache.kylin.dict.BuildInFunctionTransformer;
+import org.apache.kylin.dict.BuiltInFunctionTransformer;
import org.apache.kylin.dict.DictCodeSystem;
import org.apache.kylin.dimension.DimensionEncoding;
import org.apache.kylin.dimension.IDimensionEncodingMap;
@@ -151,7 +151,7 @@ public class FilterDecorator implements TupleFilterSerializer.Decorator {
if (filter == null)
return null;
- BuildInFunctionTransformer translator = new BuildInFunctionTransformer(dimEncMap);
+ BuiltInFunctionTransformer translator = new BuiltInFunctionTransformer(dimEncMap);
filter = translator.transform(filter);
// un-evaluatable filter is replaced with TRUE