You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2022/12/28 08:09:37 UTC
[kylin] branch kylin5 updated: KYLIN-5380 fix npe when sql filter having null <> 0 (#2057)
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new 2bcfea7d78 KYLIN-5380 fix npe when sql filter having null <> 0 (#2057)
2bcfea7d78 is described below
commit 2bcfea7d782fead9360e005cb256216e21344200
Author: Jiawei Li <ji...@kyligence.io>
AuthorDate: Wed Dec 28 16:09:32 2022 +0800
KYLIN-5380 fix npe when sql filter having null <> 0 (#2057)
* KYLIN-5380 fix npe when sql filter having null <> 0
* minor fix license
Co-authored-by: fanshu.kong <17...@qq.com>
---
.../test/resources/query/sql_boolean/query04.sql | 23 ++++++++++++++++++++++
.../test/resources/query/sql_boolean/query05.sql | 23 ++++++++++++++++++++++
.../test/resources/query/sql_boolean/query06.sql | 23 ++++++++++++++++++++++
.../test/resources/query/sql_boolean/query07.sql | 23 ++++++++++++++++++++++
.../kylin/query/runtime/SparderRexVisitor.scala | 18 ++++++++++++-----
5 files changed, 105 insertions(+), 5 deletions(-)
diff --git a/src/kylin-it/src/test/resources/query/sql_boolean/query04.sql b/src/kylin-it/src/test/resources/query/sql_boolean/query04.sql
new file mode 100644
index 0000000000..74dd23eac7
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_boolean/query04.sql
@@ -0,0 +1,23 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+select ID1,sum(ID1)
+from TEST_MEASURE
+where null <> 0 or null <> 0 and ID1 != 3
+group by ID1
diff --git a/src/kylin-it/src/test/resources/query/sql_boolean/query05.sql b/src/kylin-it/src/test/resources/query/sql_boolean/query05.sql
new file mode 100644
index 0000000000..dd01192eb0
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_boolean/query05.sql
@@ -0,0 +1,23 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+select ID1,sum(ID1)
+from TEST_MEASURE
+where ID1 != 3 and null <> 0 or null <> 0 and ID1 != 6
+group by ID1
diff --git a/src/kylin-it/src/test/resources/query/sql_boolean/query06.sql b/src/kylin-it/src/test/resources/query/sql_boolean/query06.sql
new file mode 100644
index 0000000000..148bbdea7b
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_boolean/query06.sql
@@ -0,0 +1,23 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+select ID1,sum(ID1)
+from TEST_MEASURE
+where ID1 != 3 and null <> 0 and ID1 != 10 and ID1 != 2
+group by ID1
diff --git a/src/kylin-it/src/test/resources/query/sql_boolean/query07.sql b/src/kylin-it/src/test/resources/query/sql_boolean/query07.sql
new file mode 100644
index 0000000000..45223bc178
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_boolean/query07.sql
@@ -0,0 +1,23 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+select ID1,sum(ID1)
+from TEST_MEASURE
+where ID1 != 3 or ID1 != 10 or null <> 0 or ID1 != 9
+group by ID1
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/SparderRexVisitor.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/SparderRexVisitor.scala
index a308459066..881e7ebee0 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/SparderRexVisitor.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/SparderRexVisitor.scala
@@ -31,9 +31,10 @@ import org.apache.calcite.sql.fun.SqlDatetimeSubtractionOperator
import org.apache.calcite.sql.fun.SqlDateTimeDivisionOperator
import org.apache.kylin.common.util.DateFormat
import org.apache.spark.sql.KapFunctions._
+import org.apache.spark.sql.catalyst.expressions.Literal
import org.apache.spark.sql.catalyst.util.DateTimeUtils._
import org.apache.spark.sql.functions._
-import org.apache.spark.sql.types.{DateType, LongType, TimestampType}
+import org.apache.spark.sql.types.{DataTypes, DateType, LongType, TimestampType}
import org.apache.spark.sql.{Column, DataFrame}
import org.apache.spark.sql.util.SparderTypeUtil
import org.apache.spark.unsafe.types.UTF8String
@@ -80,6 +81,15 @@ class SparderRexVisitor(val inputFieldNames: Array[String],
children += childFilter
}
+ def getColumns: ListBuffer[Column] = {
+ children.map {
+ case null => new Column(Literal(null, DataTypes.BooleanType))
+ case child =>
+ assert(child.isInstanceOf[Column])
+ child.asInstanceOf[Column]
+ }
+ }
+
def getOperands: (Column, Column) = {
var left = k_lit(children.head)
var right = k_lit(children.last)
@@ -112,13 +122,11 @@ class SparderRexVisitor(val inputFieldNames: Array[String],
val op = call.getOperator
op.getKind match {
case AND =>
- children.foreach(filter => if (filter != null) assert(filter.isInstanceOf[Column]))
- children.map(_.asInstanceOf[Column]).reduce {
+ getColumns.reduce {
_.and(_)
}
case OR =>
- children.foreach(filter => if (filter != null) assert(filter.isInstanceOf[Column]))
- children.map(_.asInstanceOf[Column]).reduce {
+ getColumns.reduce {
_.or(_)
}