You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/11/18 03:39:19 UTC
[25/30] incubator-kylin git commit: KYLIN-993 Support functions in
where clause
KYLIN-993 Support functions in where clause
Signed-off-by: Li, Yang <ya...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/bed15abb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/bed15abb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/bed15abb
Branch: refs/heads/1.x-HBase1.1.3
Commit: bed15abbdd70305f5bd998eb7fb1775e6900fcee
Parents: e9219d7
Author: lidongsjtu <do...@ebay.com>
Authored: Mon Nov 16 22:39:56 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Tue Nov 17 16:51:12 2015 +0800
----------------------------------------------------------------------
.../metadata/filter/CompareTupleFilter.java | 23 +--
.../metadata/filter/FunctionTupleFilter.java | 154 +++++++++++++++++
.../metadata/filter/ITupleFilterTranslator.java | 26 +++
.../kylin/metadata/filter/TupleFilter.java | 2 +-
.../metadata/filter/util/BuiltInMethod.java | 56 +++++++
.../kylin/query/relnode/OLAPFilterRel.java | 45 ++---
query/src/test/resources/query/sql/query85.sql | 22 +++
.../kylin/storage/hbase/CubeStorageEngine.java | 2 +
.../hbase/coprocessor/CoprocessorFilter.java | 9 +-
.../CoprocessorTupleFilterTranslator.java | 166 +++++++++++++++++++
10 files changed, 457 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bed15abb/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index d6368dd..6754ff7 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -19,12 +19,9 @@
package org.apache.kylin.metadata.filter;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
+import java.util.*;
+import org.apache.calcite.sql.SqlFunction;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
@@ -39,6 +36,7 @@ public class CompareTupleFilter extends TupleFilter {
private String firstCondValue;
private Map<String, String> dynamicVariables;
private String nullString;
+ private FunctionTupleFilter functionTupleFilter;
public CompareTupleFilter(FilterOperatorEnum op) {
super(new ArrayList<TupleFilter>(2), op);
@@ -49,7 +47,7 @@ public class CompareTupleFilter extends TupleFilter {
|| op == FilterOperatorEnum.GT || op == FilterOperatorEnum.GTE //
|| op == FilterOperatorEnum.IN || op == FilterOperatorEnum.NOTIN //
|| op == FilterOperatorEnum.ISNULL || op == FilterOperatorEnum.ISNOTNULL);
- if (opGood == false)
+ if (!opGood)
throw new IllegalArgumentException("Unsupported operator " + op);
}
@@ -81,6 +79,8 @@ public class CompareTupleFilter extends TupleFilter {
} else if (child instanceof DynamicTupleFilter) {
DynamicTupleFilter dynamicFilter = (DynamicTupleFilter) child;
this.dynamicVariables.put(dynamicFilter.getVariableName(), null);
+ } else if (child instanceof FunctionTupleFilter) {
+ this.functionTupleFilter = (FunctionTupleFilter)child;
}
//TODO
// else if (child instanceof ExtractTupleFilter) {
@@ -105,6 +105,10 @@ public class CompareTupleFilter extends TupleFilter {
return column;
}
+ public FunctionTupleFilter getFunctionTupleFilter() {
+ return functionTupleFilter;
+ }
+
public Map<String, String> getVariables() {
return dynamicVariables;
}
@@ -137,7 +141,7 @@ public class CompareTupleFilter extends TupleFilter {
@Override
public String toString() {
- return "CompareFilter [" + column + " " + operator + " " + conditionValues + ", children=" + children + "]";
+ return "CompareFilter [" + (functionTupleFilter == null ? column : functionTupleFilter) + " " + operator + " " + conditionValues + ", children=" + children + "]";
}
// TODO requires generalize, currently only evaluates COLUMN {op} CONST
@@ -146,7 +150,7 @@ public class CompareTupleFilter extends TupleFilter {
// extract tuple value
String tupleValue = null;
for (TupleFilter filter : this.children) {
- if (isConstant(filter) == false) {
+ if (!isConstant(filter)) {
filter.evaluate(tuple);
tupleValue = filter.getValues().iterator().next();
}
@@ -209,7 +213,7 @@ public class CompareTupleFilter extends TupleFilter {
@Override
public boolean isEvaluable() {
- return column != null && !conditionValues.isEmpty();
+ return (functionTupleFilter != null || column != null) && !conditionValues.isEmpty();
}
@Override
@@ -239,5 +243,4 @@ public class CompareTupleFilter extends TupleFilter {
}
this.nullString = BytesUtil.readAsciiString(buffer);
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bed15abb/metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java
new file mode 100644
index 0000000..62ab42f
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+import com.google.common.collect.Lists;
+import com.google.common.primitives.Primitives;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.kylin.metadata.filter.util.BuiltInMethod;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.tuple.ITuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Created by dongli on 11/11/15.
+ */
+public class FunctionTupleFilter extends TupleFilter {
+ public static final Logger logger = LoggerFactory.getLogger(FunctionTupleFilter.class);
+
+ private SqlOperator sqlOperator;
+ // FIXME Only supports single parameter functions currently
+ private TupleFilter columnContainerFilter;
+ private int colPosition;
+ private Method method;
+ private List<Object> methodParams;
+ private boolean isValid = false;
+
+ public FunctionTupleFilter(SqlOperator sqlOperator) {
+ super(Lists.<TupleFilter>newArrayList(), FilterOperatorEnum.FUNCTION);
+ this.methodParams = Lists.newArrayList();
+ this.sqlOperator = sqlOperator;
+
+ String opName = sqlOperator.getName().toUpperCase();
+ if (BuiltInMethod.MAP.containsKey(opName)) {
+ this.method = BuiltInMethod.MAP.get(opName).method;
+ isValid = true;
+ }
+ }
+
+ public SqlOperator getSqlOperator() {
+ return sqlOperator;
+ }
+
+ public TblColRef getColumn() {
+ if (columnContainerFilter == null)
+ return null;
+
+ if (columnContainerFilter instanceof ColumnTupleFilter)
+ return ((ColumnTupleFilter)columnContainerFilter).getColumn();
+ else if (columnContainerFilter instanceof FunctionTupleFilter)
+ return ((FunctionTupleFilter)columnContainerFilter).getColumn();
+
+ throw new UnsupportedOperationException("Wrong type TupleFilter in FunctionTupleFilter.");
+ }
+
+ public Object invokeFunction(Object input) throws InvocationTargetException, IllegalAccessException {
+ if (columnContainerFilter instanceof ColumnTupleFilter)
+ methodParams.set(colPosition, input);
+ else if (columnContainerFilter instanceof FunctionTupleFilter)
+ methodParams.set(colPosition, ((FunctionTupleFilter) columnContainerFilter).invokeFunction(input));
+ return method.invoke(null, (Object[])(methodParams.toArray()));
+ }
+
+ public boolean isValid() {
+ return isValid && method != null && methodParams.size() == children.size();
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void addChild(TupleFilter child) {
+ if (child instanceof ColumnTupleFilter || child instanceof FunctionTupleFilter) {
+ columnContainerFilter = child;
+ colPosition = methodParams.size();
+ methodParams.add(null);
+ } else if (child instanceof ConstantTupleFilter) {
+ String constVal = child.getValues().iterator().next();
+ try {
+ Class clazz = Primitives.wrap(method.getParameterTypes()[methodParams.size()]);
+ if (!Primitives.isWrapperType(clazz))
+ methodParams.add(constVal);
+ else
+ methodParams.add(clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, constVal)));
+ } catch (Exception e) {
+ logger.debug(e.getMessage());
+ isValid = false;
+ }
+ }
+ super.addChild(child);
+ }
+
+ @Override
+ public boolean isEvaluable() {
+ return false;
+ }
+
+ @Override
+ public boolean evaluate(ITuple tuple) {
+ throw new UnsupportedOperationException("Function filter cannot be evaluated immediately");
+ }
+
+ @Override
+ public Collection<String> getValues() {
+ throw new UnsupportedOperationException("Function filter cannot be evaluated immediately");
+ }
+
+ @Override
+ public String toString(){
+ StringBuilder sb = new StringBuilder();
+ sb.append(sqlOperator.getName());
+ sb.append("(");
+ for (int i = 0; i < methodParams.size(); i++) {
+ if (colPosition == i) {
+ sb.append(columnContainerFilter);
+ } else {
+ sb.append(methodParams.get(i));
+ }
+ if (i < methodParams.size() - 1)
+ sb.append(",");
+ }
+ sb.append(")");
+ return sb.toString();
+ }
+
+ @Override
+ public byte[] serialize() {
+ throw new UnsupportedOperationException("Method serialize() is not supported for FunctionTupleFilter.");
+ }
+
+ @Override
+ public void deserialize(byte[] bytes) {
+ throw new UnsupportedOperationException("Method deserialize(byte[] bytes) is not supported for FunctionTupleFilter.");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bed15abb/metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java
new file mode 100644
index 0000000..7f1aed6
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+/**
+ * Created by dongli on 11/11/15.
+ */
+public interface ITupleFilterTranslator {
+ TupleFilter translate(TupleFilter tupleFilter);
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bed15abb/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 66dc0db..25cae20 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -37,7 +37,7 @@ import com.google.common.collect.Maps;
public abstract class TupleFilter {
public enum FilterOperatorEnum {
- EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8), IN(9), NOTIN(10), AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31), DYNAMIC(32), EXTRACT(33), CASE(34);
+ EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8), IN(9), NOTIN(10), AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31), DYNAMIC(32), EXTRACT(33), CASE(34), FUNCTION(35);
private final int value;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bed15abb/metadata/src/main/java/org/apache/kylin/metadata/filter/util/BuiltInMethod.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/util/BuiltInMethod.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/util/BuiltInMethod.java
new file mode 100644
index 0000000..1f15c9c
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/util/BuiltInMethod.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter.util;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.calcite.avatica.util.DateTimeUtils;
+import org.apache.calcite.linq4j.tree.Types;
+import org.apache.calcite.runtime.SqlFunctions;
+
+import java.lang.reflect.Method;
+
+/**
+ * Created by dongli on 11/13/15.
+ */
+public enum BuiltInMethod {
+ UPPER(SqlFunctions.class, "upper", String.class),
+ LOWER(SqlFunctions.class, "lower", String.class),
+ SUBSTRING(SqlFunctions.class, "substring", String.class, int.class, int.class),
+ CHAR_LENGTH(SqlFunctions.class, "charLength", String.class),
+ LIKE(SqlFunctions.class, "like", String.class, String.class),
+ INITCAP(SqlFunctions.class, "initcap", String.class);
+ public final Method method;
+
+ public static final ImmutableMap<String, BuiltInMethod> MAP;
+
+ static {
+ final ImmutableMap.Builder<String, BuiltInMethod> builder =
+ ImmutableMap.builder();
+ for (BuiltInMethod value : BuiltInMethod.values()) {
+ if (value.method != null) {
+ builder.put(value.name(), value);
+ }
+ }
+ MAP = builder.build();
+ }
+
+ BuiltInMethod(Class clazz, String methodName, Class... argumentTypes) {
+ this.method = Types.lookupMethod(clazz, methodName, argumentTypes);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bed15abb/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index cfd5e3b..583af9a 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -18,50 +18,25 @@
package org.apache.kylin.query.relnode;
-import java.util.Calendar;
-import java.util.GregorianCalendar;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
import org.apache.calcite.adapter.enumerable.EnumerableCalc;
import org.apache.calcite.adapter.enumerable.EnumerableConvention;
import org.apache.calcite.adapter.enumerable.EnumerableRel;
import org.apache.calcite.avatica.util.TimeUnitRange;
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptCost;
-import org.apache.calcite.plan.RelOptPlanner;
-import org.apache.calcite.plan.RelTrait;
-import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.plan.*;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rex.RexBuilder;
-import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexDynamicParam;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexLiteral;
-import org.apache.calcite.rex.RexLocalRef;
-import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.rex.RexProgram;
-import org.apache.calcite.rex.RexProgramBuilder;
-import org.apache.calcite.rex.RexVisitorImpl;
+import org.apache.calcite.rex.*;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.util.NlsString;
-import org.apache.kylin.metadata.filter.CaseTupleFilter;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.DynamicTupleFilter;
-import org.apache.kylin.metadata.filter.ExtractTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.*;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.TblColRef;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Sets;
+import java.util.*;
/**
*/
@@ -127,9 +102,13 @@ public class OLAPFilterRel extends Filter implements OLAPRel {
if (op.getName().equalsIgnoreCase("extract_date")) {
filter = new ExtractTupleFilter(FilterOperatorEnum.EXTRACT);
} else {
- throw new UnsupportedOperationException(op.getName());
+ filter = new FunctionTupleFilter(op);
}
break;
+ case LIKE:
+ case OTHER_FUNCTION:
+ filter = new FunctionTupleFilter(op);
+ break;
default:
throw new UnsupportedOperationException(op.getName());
}
@@ -286,6 +265,8 @@ public class OLAPFilterRel extends Filter implements OLAPRel {
}
private void collectColumnsRecursively(TupleFilter filter, Set<TblColRef> collector) {
+ if (filter == null) return;
+
if (filter instanceof ColumnTupleFilter) {
collector.add(((ColumnTupleFilter) filter).getColumn());
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bed15abb/query/src/test/resources/query/sql/query85.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query85.sql b/query/src/test/resources/query/sql/query85.sql
new file mode 100644
index 0000000..4e80d59
--- /dev/null
+++ b/query/src/test/resources/query/sql/query85.sql
@@ -0,0 +1,22 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select upper(lstg_format_name) as lstg_format_name, count(*) as cnt from test_kylin_fact
+where lower(lstg_format_name)='abin' and substring(lstg_format_name,1,3) in ('ABI') and upper(lstg_format_name) > 'AAAA' and
+lower(lstg_format_name) like '%b%' and char_length(lstg_format_name) < 10 and char_length(lstg_format_name) > 3 and lstg_format_name||'a'='ABINa'
+group by lstg_format_name
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bed15abb/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
index 8eb7bcb..fdb8986 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
@@ -267,6 +267,8 @@ public class CubeStorageEngine implements IStorageEngine {
}
private void collectColumnsRecursively(TupleFilter filter, Set<TblColRef> collector) {
+ if (filter == null) return;
+
if (filter instanceof ColumnTupleFilter) {
collectColumns(((ColumnTupleFilter) filter).getColumn(), collector);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bed15abb/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
index 65fddd2..b3e2d31 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
@@ -26,11 +26,7 @@ import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.cube.kv.RowKeyColumnIO;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.ISegment;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilterSerializer;
+import org.apache.kylin.metadata.filter.*;
import org.apache.kylin.metadata.filter.TupleFilterSerializer.Decorator;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
@@ -61,6 +57,9 @@ public class CoprocessorFilter {
if (filter == null)
return null;
+ ITupleFilterTranslator translator = new CoprocessorTupleFilterTranslator(columnIO);
+ filter = translator.translate(filter);
+
// un-evaluatable filter is replaced with TRUE
if (!filter.isEvaluable()) {
TupleFilter.collectColumns(filter, unstrictlyFilteredColumns);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bed15abb/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorTupleFilterTranslator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorTupleFilterTranslator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorTupleFilterTranslator.java
new file mode 100644
index 0000000..aae945d
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorTupleFilterTranslator.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.hbase.coprocessor;
+
+import com.google.common.primitives.Primitives;
+import org.apache.kylin.cube.kv.RowKeyColumnIO;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.metadata.filter.*;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationTargetException;
+
+/**
+ * Created by dongli on 11/11/15.
+ */
+public class CoprocessorTupleFilterTranslator implements ITupleFilterTranslator {
+ public static final Logger logger = LoggerFactory.getLogger(CoprocessorTupleFilterTranslator.class);
+
+ private RowKeyColumnIO rowKeyColumnIO;
+
+ public CoprocessorTupleFilterTranslator(RowKeyColumnIO rowKeyColumnIO) {
+ this.rowKeyColumnIO = rowKeyColumnIO;
+ }
+
+ @Override
+ public TupleFilter translate(TupleFilter tupleFilter) {
+ TupleFilter translated = null;
+ if (tupleFilter instanceof CompareTupleFilter) {
+ logger.info("Translation to IN clause: " + tupleFilter);
+ translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter);
+ logger.info(translated == null ? "Failed, will use Calcite to handle computed comparison." : "Succeed: " + translated);
+ } else if (tupleFilter instanceof FunctionTupleFilter) {
+ logger.info("Translation to IN clause: " + tupleFilter);
+ translated = translateFunctionTupleFilter((FunctionTupleFilter) tupleFilter);
+ logger.info(translated == null ? "Failed, will use Calcite to handle computed column." : "Succeed: " + translated);
+ }
+ return translated == null ? tupleFilter : translated;
+ }
+
+ private TupleFilter translateFunctionTupleFilter(FunctionTupleFilter functionTupleFilter) {
+ if (!functionTupleFilter.isValid())
+ return null;
+
+ TblColRef columnRef = functionTupleFilter.getColumn();
+ Dictionary<String> dict = rowKeyColumnIO.getDictionary(columnRef);
+ if (dict == null)
+ return null;
+
+ CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN);
+ translated.addChild(new ColumnTupleFilter(columnRef));
+
+ try {
+ for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
+ String dictVal = dict.getValueFromId(i);
+ if ((Boolean)functionTupleFilter.invokeFunction(dictVal)) {
+ translated.addChild(new ConstantTupleFilter(dictVal));
+ }
+ }
+ } catch (IllegalAccessException e) {
+ logger.debug(e.getMessage());
+ return null;
+ } catch (InvocationTargetException e) {
+ logger.debug(e.getMessage());
+ return null;
+ } catch (IllegalArgumentException e) {
+ logger.debug(e.getMessage());
+ return null;
+ }
+ return translated;
+ }
+
+ @SuppressWarnings("unchecked")
+ private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter){
+ if (compTupleFilter.getFunctionTupleFilter() == null)
+ return null;
+
+ FunctionTupleFilter functionTupleFilter = compTupleFilter.getFunctionTupleFilter();
+ if (!functionTupleFilter.isValid())
+ return null;
+
+ TblColRef columnRef = functionTupleFilter.getColumn();
+ Dictionary<String> dict = rowKeyColumnIO.getDictionary(columnRef);
+ if (dict == null)
+ return null;
+
+ CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN);
+ translated.addChild(new ColumnTupleFilter(columnRef));
+
+ try {
+ for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
+ String dictVal = dict.getValueFromId(i);
+ Object computedVal = functionTupleFilter.invokeFunction(dictVal);
+ Class clazz = Primitives.wrap(computedVal.getClass());
+ Object targetVal = compTupleFilter.getFirstValue();
+ if (Primitives.isWrapperType(clazz))
+ targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, compTupleFilter.getFirstValue()));
+
+ int comp = ((Comparable)computedVal).compareTo(targetVal);
+ boolean compResult = false;
+ switch (compTupleFilter.getOperator()) {
+ case EQ:
+ compResult = comp == 0;
+ break;
+ case NEQ:
+ compResult = comp != 0;
+ break;
+ case LT:
+ compResult = comp < 0;
+ break;
+ case LTE:
+ compResult = comp <= 0;
+ break;
+ case GT:
+ compResult = comp > 0;
+ break;
+ case GTE:
+ compResult = comp >= 0;
+ break;
+ case IN:
+ compResult = compTupleFilter.getValues().contains(computedVal.toString());
+ break;
+ case NOTIN:
+ compResult = !compTupleFilter.getValues().contains(computedVal.toString());
+ break;
+ default:
+ break;
+ }
+ if (compResult) {
+ translated.addChild(new ConstantTupleFilter(dictVal));
+ }
+ }
+ } catch (IllegalAccessException e) {
+ logger.debug(e.getMessage());
+ return null;
+ } catch (InvocationTargetException e) {
+ logger.debug(e.getMessage());
+ return null;
+ } catch (IllegalArgumentException e) {
+ logger.debug(e.getMessage());
+ return null;
+ } catch (NoSuchMethodException e) {
+ logger.debug(e.getMessage());
+ return null;
+ }
+ return translated;
+ }
+}