You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/12/02 10:34:43 UTC
[1/2] kylin git commit: KYLIN-2212 'NOT' operator in filter on
derived column may get incorrect result
Repository: kylin
Updated Branches:
refs/heads/master 0eebd7d8a -> eb686a942
KYLIN-2212 'NOT' operator in filter on derived column may get incorrect result
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0fd1ed6f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0fd1ed6f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0fd1ed6f
Branch: refs/heads/master
Commit: 0fd1ed6f14758bb38f1abf7a7ed9e24ed818f50f
Parents: 0eebd7d
Author: zhengdong <zh...@outlook.com>
Authored: Wed Nov 23 16:13:59 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Dec 2 18:32:24 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/gridtable/GTUtil.java | 14 ++++++++++++
.../metadata/filter/CompareTupleFilter.java | 2 ++
.../metadata/filter/TupleFilterSerializer.java | 1 +
.../resources/query/sql_derived/query12.sql | 24 ++++++++++++++++++++
.../kylin/query/relnode/OLAPFilterRel.java | 3 +++
.../common/coprocessor/FilterDecorator.java | 14 ++++++++++++
6 files changed, 58 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
old mode 100644
new mode 100755
index a4e574c..7496778
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
@@ -196,6 +196,20 @@ public class GTUtil {
result = newCompareFilter;
}
break;
+ case NOTIN:
+ Set notInValues = Sets.newHashSet();
+ for (Object value : constValues) {
+ code = translate(col, value, 0);
+ if (code != null)
+ notInValues.add(code);
+ }
+ if (notInValues.isEmpty()) {
+ result = ConstantTupleFilter.TRUE;
+ } else {
+ newCompareFilter.addChild(new ConstantTupleFilter(notInValues));
+ result = newCompareFilter;
+ }
+ break;
case NEQ:
code = translate(col, firstValue, 0);
if (code == null) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
old mode 100644
new mode 100755
index c7a3721..f2af735
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -59,6 +59,8 @@ public class CompareTupleFilter extends TupleFilter {
private CompareTupleFilter(CompareTupleFilter another) {
super(new ArrayList<TupleFilter>(another.children), another.operator);
this.column = another.column;
+ this.firstCondValue = another.getFirstValue();
+ this.function = another.getFunction();
this.conditionValues = new HashSet<Object>();
this.conditionValues.addAll(another.conditionValues);
this.dynamicVariables = new HashMap<String, Object>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
old mode 100644
new mode 100755
index 2df474e..63153ef
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -170,6 +170,7 @@ public class TupleFilterSerializer {
case GT:
case GTE:
case IN:
+ case NOTIN:
case ISNULL:
case ISNOTNULL:
filter = new CompareTupleFilter(op);
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/kylin-it/src/test/resources/query/sql_derived/query12.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_derived/query12.sql b/kylin-it/src/test/resources/query/sql_derived/query12.sql
new file mode 100755
index 0000000..959a59c
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_derived/query12.sql
@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select UPD_USER,count(1) as CNT
+from TEST_KYLIN_FACT as TEST_KYLIN_FACT
+JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
+ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
+where UPD_USER not in ('USER_Y')
+group by UPD_USER
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
old mode 100644
new mode 100755
index 8b2035b..411142d
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -160,6 +160,9 @@ public class OLAPFilterRel extends Filter implements OLAPRel {
if (inFilter != null) {
filter = inFilter;
}
+ } else if (op.getKind() == SqlKind.NOT) {
+ assert (filter.getChildren().size() == 1);
+ filter = filter.getChildren().get(0).reverse();
}
return filter;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
old mode 100644
new mode 100755
index 5ab4117..2b2e490
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
@@ -90,6 +90,20 @@ public class FilterDecorator implements TupleFilterSerializer.Decorator {
result = newCompareFilter;
}
break;
+ case NOTIN:
+ Set<String> notInValues = Sets.newHashSet();
+ for (String value : constValues) {
+ v = translate(col, value, 0);
+ if (!isDictNull(v))
+ notInValues.add(v);
+ }
+ if (notInValues.isEmpty()) {
+ result = ConstantTupleFilter.TRUE;
+ } else {
+ newCompareFilter.addChild(new ConstantTupleFilter(notInValues));
+ result = newCompareFilter;
+ }
+ break;
case NEQ:
v = translate(col, firstValue, 0);
if (isDictNull(v)) {
[2/2] kylin git commit: KYLIN-2212 bug fix in
BuiltInFunctionTupleFilter serialization
Posted by li...@apache.org.
KYLIN-2212 bug fix in BuiltInFunctionTupleFilter serialization
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eb686a94
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eb686a94
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eb686a94
Branch: refs/heads/master
Commit: eb686a9428d930a60ec87883a2c253fb07f80a72
Parents: 0fd1ed6
Author: Li Yang <li...@apache.org>
Authored: Fri Dec 2 18:34:29 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Dec 2 18:34:29 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/eb686a94/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
index 1e836b7..f6e687b 100755
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
@@ -151,17 +151,21 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
@Override
public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
BytesUtil.writeUTFString(name, buffer);
+ buffer.put((byte) (isReversed ? 1 : 0));
}
@Override
public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
this.name = BytesUtil.readUTFString(buffer);
+ this.isReversed = buffer.get() != 0;
this.initMethod();
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
+ if (isReversed)
+ sb.append("NOT ");
sb.append(name);
sb.append("(");
for (int i = 0; i < methodParams.size(); i++) {