You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/09/03 06:37:36 UTC
[29/50] [abbrv] kylin git commit: minor,
add LIKE FilterOperatorEnum in case it will be pushed down(it won't
for now)
minor, add LIKE FilterOperatorEnum in case it will be pushed down(it won't for now)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9e0b8d80
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9e0b8d80
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9e0b8d80
Branch: refs/heads/1.5.x-HBase1.x
Commit: 9e0b8d8033dbe4e86d299adbd102cf52792cd5fe
Parents: c5fdacd
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Aug 31 11:47:37 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Aug 31 11:47:42 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/gridtable/GTUtil.java | 53 +++++++++++
.../filter/BuiltInFunctionTupleFilter.java | 22 ++++-
.../filter/EvaluatableLikeFunction.java | 96 ++++++++++++++++++++
.../EvaluatableLikeFunctionTransformer.java | 76 ++++++++++++++++
.../kylin/metadata/filter/TupleFilter.java | 2 +-
.../metadata/filter/TupleFilterSerializer.java | 3 +
.../org/apache/kylin/query/KylinTestBase.java | 8 +-
7 files changed, 254 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
index 4b50176..ce0f016 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
@@ -18,6 +18,7 @@
package org.apache.kylin.gridtable;
+import java.lang.reflect.InvocationTargetException;
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.List;
@@ -25,6 +26,7 @@ import java.util.Set;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.filter.BuiltInFunctionTupleFilter;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
@@ -104,6 +106,13 @@ public class GTUtil {
if (encodeConstants && filter instanceof CompareTupleFilter) {
return encodeConstants((CompareTupleFilter) filter);
}
+ if (encodeConstants && filter instanceof BuiltInFunctionTupleFilter) {
+ if (!((BuiltInFunctionTupleFilter) filter).hasNested()) {
+ return encodeConstants((BuiltInFunctionTupleFilter) filter);
+ } else {
+ throw new IllegalStateException("Nested BuiltInFunctionTupleFilter is not supported to be pushed down");
+ }
+ }
return filter;
}
@@ -122,9 +131,13 @@ public class GTUtil {
return oldCompareFilter;
}
+ //CompareTupleFilter containing BuiltInFunctionTupleFilter will not reach here caz it will be transformed by BuiltInFunctionTransformer
CompareTupleFilter newCompareFilter = new CompareTupleFilter(oldCompareFilter.getOperator());
newCompareFilter.addChild(new ColumnTupleFilter(externalCol));
+ //for CompareTupleFilter containing dynamicVariables, the below codes will actually replace dynamicVariables
+ //with normal ConstantTupleFilter
+
Object firstValue = constValues.iterator().next();
int col = colMapping == null ? externalCol.getColumnDesc().getZeroBasedIndex() : colMapping.indexOf(externalCol);
@@ -199,6 +212,46 @@ public class GTUtil {
return result;
}
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ private TupleFilter encodeConstants(BuiltInFunctionTupleFilter funcFilter) {
+ // extract ColumnFilter & ConstantFilter
+ TblColRef externalCol = funcFilter.getColumn();
+
+ if (externalCol == null) {
+ return funcFilter;
+ }
+
+ Collection constValues = funcFilter.getConstantTupleFilter().getValues();
+ if (constValues == null || constValues.isEmpty()) {
+ return funcFilter;
+ }
+
+ BuiltInFunctionTupleFilter newFuncFilter;
+ try {
+ newFuncFilter = funcFilter.getClass().getConstructor(String.class).newInstance(funcFilter.getName());
+ } catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
+ throw new RuntimeException(e);
+ }
+ newFuncFilter.addChild(new ColumnTupleFilter(externalCol));
+
+ int col = colMapping == null ? externalCol.getColumnDesc().getZeroBasedIndex() : colMapping.indexOf(externalCol);
+
+ ByteArray code;
+
+ // translate constant into code
+ Set newValues = Sets.newHashSet();
+ for (Object value : constValues) {
+ code = translate(col, value, 0);
+ if (code == null) {
+ throw new IllegalStateException("Cannot serialize BuiltInFunctionTupleFilter");
+ }
+ newValues.add(code);
+ }
+ newFuncFilter.addChild(new ConstantTupleFilter(newValues));
+
+ return newFuncFilter;
+ }
+
transient ByteBuffer buf = ByteBuffer.allocate(info.getMaxColumnLength());
private ByteArray translate(int col, Object value, int roundingFlag) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
index 5a10371..40afb18 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
@@ -25,6 +25,7 @@ import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.List;
+import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.filter.function.BuiltInMethod;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
@@ -40,13 +41,18 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
protected String name;
// FIXME Only supports single parameter functions currently
protected TupleFilter columnContainerFilter;//might be a ColumnTupleFilter(simple case) or FunctionTupleFilter(complex case like substr(lower()))
+ protected ConstantTupleFilter constantTupleFilter;
protected int colPosition;
protected Method method;
protected List<Serializable> methodParams;
protected boolean isValidFunc = false;
public BuiltInFunctionTupleFilter(String name) {
- super(Lists.<TupleFilter> newArrayList(), FilterOperatorEnum.FUNCTION);
+ this(name, null);
+ }
+
+ public BuiltInFunctionTupleFilter(String name, FilterOperatorEnum filterOperatorEnum) {
+ super(Lists.<TupleFilter> newArrayList(), filterOperatorEnum == null ? FilterOperatorEnum.FUNCTION : filterOperatorEnum);
this.methodParams = Lists.newArrayList();
if (name != null) {
@@ -59,6 +65,10 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
return name;
}
+ public ConstantTupleFilter getConstantTupleFilter() {
+ return constantTupleFilter;
+ }
+
public TblColRef getColumn() {
if (columnContainerFilter == null)
return null;
@@ -71,6 +81,10 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
throw new UnsupportedOperationException("Wrong type TupleFilter in FunctionTupleFilter.");
}
+ public boolean hasNested() {
+ return (columnContainerFilter != null && columnContainerFilter instanceof BuiltInFunctionTupleFilter);
+ }
+
public Object invokeFunction(Object input) throws InvocationTargetException, IllegalAccessException {
if (columnContainerFilter instanceof ColumnTupleFilter)
methodParams.set(colPosition, (Serializable) input);
@@ -90,6 +104,7 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
colPosition = methodParams.size();
methodParams.add(null);
} else if (child instanceof ConstantTupleFilter) {
+ this.constantTupleFilter = (ConstantTupleFilter) child;
Serializable constVal = (Serializable) child.getValues().iterator().next();
try {
Class<?> clazz = Primitives.wrap(method.getParameterTypes()[methodParams.size()]);
@@ -122,12 +137,13 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
@Override
public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
- throw new UnsupportedOperationException("Function filter cannot serialized");
+ BytesUtil.writeUTFString(name, buffer);
}
@Override
public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
- throw new UnsupportedOperationException("Function filter cannot serialized");
+ this.name = BytesUtil.readUTFString(buffer);
+ this.initMethod();
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java
new file mode 100644
index 0000000..59961ac
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+import java.lang.reflect.InvocationTargetException;
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.StringSerializer;
+import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+
+public class EvaluatableLikeFunction extends BuiltInFunctionTupleFilter {
+
+ public EvaluatableLikeFunction(String name) {
+ super(name, FilterOperatorEnum.LIKE);
+ }
+
+ @Override
+ public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem cs) {
+
+ // extract tuple value
+ Object tupleValue = null;
+ for (TupleFilter filter : this.children) {
+ if (!isConstant(filter)) {
+ filter.evaluate(tuple, cs);
+ tupleValue = filter.getValues().iterator().next();
+ }
+ }
+
+ // consider null case
+ if (cs.isNull(tupleValue)) {
+ return false;
+ }
+
+ ByteArray valueByteArray = (ByteArray) tupleValue;
+ StringSerializer serializer = new StringSerializer(DataType.getType("string"));
+ String value = serializer.deserialize(ByteBuffer.wrap(valueByteArray.array(), valueByteArray.offset(), valueByteArray.length()));
+ try {
+ return (Boolean) invokeFunction(value);
+ } catch (InvocationTargetException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+ if (!isValid()) {
+ throw new IllegalStateException("must be valid");
+ }
+ if (methodParams.size() != 2 || methodParams.get(0) != null || methodParams.get(1) == null) {
+ throw new IllegalArgumentException("bad methodParams: " + methodParams);
+ }
+ BytesUtil.writeUTFString(name, buffer);
+ }
+
+ @Override
+ public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+ this.name = BytesUtil.readUTFString(buffer);
+ this.initMethod();
+ }
+
+ @Override
+ public boolean isEvaluable() {
+ return true;
+ }
+
+ private boolean isConstant(TupleFilter filter) {
+ return (filter instanceof ConstantTupleFilter) || (filter instanceof DynamicTupleFilter);
+ }
+
+ public String getLikePattern() {
+ ByteArray byteArray = (ByteArray) methodParams.get(1);
+ StringSerializer s = new StringSerializer(DataType.getType("string"));
+ String pattern = s.deserialize(ByteBuffer.wrap(byteArray.array(), byteArray.offset(), byteArray.length()));
+ return pattern;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunctionTransformer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunctionTransformer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunctionTransformer.java
new file mode 100644
index 0000000..7295533
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunctionTransformer.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+import java.util.ListIterator;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class EvaluatableLikeFunctionTransformer {
+ public static final Logger logger = LoggerFactory.getLogger(EvaluatableLikeFunctionTransformer.class);
+
+ public static TupleFilter transform(TupleFilter tupleFilter) {
+ TupleFilter translated = null;
+ if (tupleFilter instanceof CompareTupleFilter) {
+ CompareTupleFilter compTupleFilter = (CompareTupleFilter) tupleFilter;
+ if (compTupleFilter.getFunction() != null && (compTupleFilter.getFunction() instanceof BuiltInFunctionTupleFilter)) {
+ throw new IllegalArgumentException("BuiltInFunctionTupleFilter not supported :" + ((BuiltInFunctionTupleFilter) compTupleFilter.getFunction()).getName());
+ }
+ } else if (tupleFilter instanceof BuiltInFunctionTupleFilter) {
+ BuiltInFunctionTupleFilter builtInFunctionTupleFilter = (BuiltInFunctionTupleFilter) tupleFilter;
+ if (isLikeFunction(builtInFunctionTupleFilter)) {
+ for (TupleFilter child : builtInFunctionTupleFilter.getChildren()) {
+ if (!(child instanceof ColumnTupleFilter) && !(child instanceof ConstantTupleFilter)) {
+ throw new IllegalArgumentException("Only simple like clause is supported");
+ }
+ }
+
+ translated = new EvaluatableLikeFunction(builtInFunctionTupleFilter.getName());
+ for (TupleFilter child : builtInFunctionTupleFilter.getChildren()) {
+ translated.addChild(child);
+ }
+
+ } else {
+ throw new IllegalArgumentException("BuiltInFunctionTupleFilter not supported: " + builtInFunctionTupleFilter.getName());
+ }
+
+ } else if (tupleFilter instanceof LogicalTupleFilter) {
+ @SuppressWarnings("unchecked")
+ ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator();
+ while (childIterator.hasNext()) {
+ TupleFilter transformed = transform(childIterator.next());
+ if (transformed != null) {
+ childIterator.set(transformed);
+ } else {
+ throw new IllegalStateException("Should not be null");
+ }
+ }
+ }
+ return translated == null ? tupleFilter : translated;
+ }
+
+ private static boolean isLikeFunction(BuiltInFunctionTupleFilter builtInFunctionTupleFilter) {
+ return "like".equalsIgnoreCase(builtInFunctionTupleFilter.getName());
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 3250640..2fb4e1f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -38,7 +38,7 @@ import com.google.common.collect.Maps;
public abstract class TupleFilter {
public enum FilterOperatorEnum {
- EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8), IN(9), NOTIN(10), AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31), DYNAMIC(32), EXTRACT(33), CASE(34), FUNCTION(35), MASSIN(36);
+ EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8), IN(9), NOTIN(10), AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31), DYNAMIC(32), EXTRACT(33), CASE(34), FUNCTION(35), MASSIN(36), LIKE(37);
private final int value;
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
index f6c8bc4..0aca9a1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -188,6 +188,9 @@ public class TupleFilterSerializer {
case FUNCTION:
filter = new BuiltInFunctionTupleFilter(null);
break;
+ case LIKE:
+ filter = new EvaluatableLikeFunction(null);
+ break;
case MASSIN:
filter = new MassInTupleFilter();
break;
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index 4e59815..2ad1105 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -63,6 +63,8 @@ import com.google.common.io.Files;
*/
public class KylinTestBase {
+ public static boolean PRINT_RESULT = false;
+
class ObjectArray {
Object[] data;
@@ -224,7 +226,8 @@ public class KylinTestBase {
if (needSort) {
queryTable = new SortedTable(queryTable, columnNames);
}
- //printResult(queryTable);
+ if (PRINT_RESULT)
+ printResult(queryTable);
return queryTable;
}
@@ -280,7 +283,8 @@ public class KylinTestBase {
if (needSort) {
queryTable = new SortedTable(queryTable, columnNames);
}
- //printResult(queryTable);
+ if (PRINT_RESULT)
+ printResult(queryTable);
return queryTable;
}