You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2020/06/15 02:51:31 UTC
[kylin] 15/15: KYLIN-4429 Auto convert filter date values to date
format string when the related column type is string
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit b5ef6dd7ddd587e4eccdebd2ab3efb74e2745571
Author: Zhong, Yanghong <nj...@apache.org>
AuthorDate: Fri Jun 12 13:40:04 2020 +0800
KYLIN-4429 Auto convert filter date values to date format string when the related column type is string
---
.../storage/translate/DerivedFilterTranslator.java | 27 +++++++++++++++++++++-
.../query/relnode/visitor/TupleFilterVisitor.java | 22 ++++++++++++++++++
2 files changed, 48 insertions(+), 1 deletion(-)
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
index 3aeabba..9bfdd76 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
@@ -23,10 +23,12 @@ import java.util.Set;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Array;
+import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.cube.model.CubeDesc.DeriveType;
import org.apache.kylin.dict.lookup.ILookupTable;
+import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.datatype.DataTypeOrder;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
@@ -60,7 +62,20 @@ public class DerivedFilterTranslator {
assert hostCols.length == 1;
CompareTupleFilter newComp = new CompareTupleFilter(compf.getOperator());
newComp.addChild(new ColumnTupleFilter(hostCols[0]));
- newComp.addChild(new ConstantTupleFilter(compf.getValues()));
+ Set<?> values = compf.getValues();
+ DataType pkDataType = compf.getColumn().getType();
+ if (pkDataType.isDateTimeFamily() && hostCols[0].getType().isStringFamily()) {
+ Set<String> newValues = Sets.newHashSetWithExpectedSize(values.size());
+ for (Object entry : values) {
+ long ts = DateFormat.stringToMillis((String) entry);
+ String newEntry = pkDataType.isDate() ? DateFormat.formatToDateStr(ts)
+ : DateFormat.formatToTimeWithoutMilliStr(ts);
+ newValues.add(newEntry);
+ }
+ newComp.addChild(new ConstantTupleFilter(newValues));
+ } else {
+ newComp.addChild(new ConstantTupleFilter(values));
+ }
return new Pair<TupleFilter, Boolean>(newComp, false);
}
@@ -83,6 +98,16 @@ public class DerivedFilterTranslator {
}
}
+ for (Array<String> entry : satisfyingHostRecords) {
+ for (int i = 0; i < pkCols.length; i++) {
+ if (pkCols[i].getType().isDateTimeFamily() && hostCols[i].getType().isStringFamily()) {
+ long ts = DateFormat.stringToMillis(entry.data[i]);
+ entry.data[i] = pkCols[i].getType().isDate() ? DateFormat.formatToDateStr(ts)
+ : DateFormat.formatToTimeWithoutMilliStr(ts);
+ }
+ }
+ }
+
TupleFilter translated;
boolean loosened;
if (satisfyingHostRecords.size() > KylinConfig.getInstanceFromEnv().getDerivedInThreshold()) {
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/visitor/TupleFilterVisitor.java b/query/src/main/java/org/apache/kylin/query/relnode/visitor/TupleFilterVisitor.java
index e0792a0..ffff10b 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/visitor/TupleFilterVisitor.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/visitor/TupleFilterVisitor.java
@@ -149,6 +149,7 @@ public class TupleFilterVisitor extends RexVisitorImpl<TupleFilter> {
filter = new UnsupportedTupleFilter(TupleFilter.FilterOperatorEnum.UNSUPPORTED);
}
+ boolean isChildValueDateTimeType = false;
for (RexNode operand : call.operands) {
TupleFilter childFilter = operand.accept(this);
if (filter == null) {
@@ -156,6 +157,27 @@ public class TupleFilterVisitor extends RexVisitorImpl<TupleFilter> {
} else {
filter.addChild(childFilter);
}
+ if (operand instanceof RexLiteral && ((RexLiteral) operand).getValue() instanceof GregorianCalendar) {
+ isChildValueDateTimeType = true;
+ }
+ }
+ if (filter instanceof CompareTupleFilter) {
+ CompareTupleFilter compFilter = (CompareTupleFilter) filter;
+ if (compFilter.getChildren().size() == 2 && compFilter.getChildren().get(0) instanceof ColumnTupleFilter
+ && compFilter.getChildren().get(1) instanceof ConstantTupleFilter) {
+ ColumnTupleFilter colFilter = (ColumnTupleFilter) compFilter.getChildren().get(0);
+ ConstantTupleFilter constFilter = (ConstantTupleFilter) compFilter.getChildren().get(1);
+ if (isChildValueDateTimeType && colFilter.getColumn().getType().isStringFamily()) {
+ Set<Object> newValues = Sets.newHashSet();
+ for (Object v : constFilter.getValues()) {
+ newValues.add(DateFormat.formatToDateStr(DateFormat.stringToMillis(v.toString())));
+ }
+ ConstantTupleFilter newConstFilter = new ConstantTupleFilter(newValues);
+ filter = new CompareTupleFilter(filter.getOperator());
+ filter.addChild(colFilter);
+ filter.addChild(newConstFilter);
+ }
+ }
}
if (op.getKind() == SqlKind.OR) {