You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2016/05/20 05:14:13 UTC
spark git commit: [SPARK-14990][SQL] Fix checkForSameTypeInputExpr
(ignore nullability)
Repository: spark
Updated Branches:
refs/heads/master f2ee0ed4b -> 3ba34d435
[SPARK-14990][SQL] Fix checkForSameTypeInputExpr (ignore nullability)
## What changes were proposed in this pull request?
This patch fixes a bug in TypeUtils.checkForSameTypeInputExpr. Previously the code was testing on strict equality, which does not taking nullability into account.
This is based on https://github.com/apache/spark/pull/12768. This patch fixed a bug there (with empty expression) and added a test case.
## How was this patch tested?
Added a new test suite and test case.
Closes #12768.
Author: Reynold Xin <rx...@databricks.com>
Author: Oleg Danilov <ol...@wandisco.com>
Closes #13208 from rxin/SPARK-14990.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3ba34d43
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3ba34d43
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3ba34d43
Branch: refs/heads/master
Commit: 3ba34d435c1f61435c2dddc28650cd111e7c1f33
Parents: f2ee0ed
Author: Reynold Xin <rx...@databricks.com>
Authored: Thu May 19 22:14:10 2016 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Thu May 19 22:14:10 2016 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/util/TypeUtils.scala | 14 ++++--
.../sql/catalyst/util/TypeUtilsSuite.scala | 46 ++++++++++++++++++++
2 files changed, 56 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/3ba34d43/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TypeUtils.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TypeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TypeUtils.scala
index f603cbf..7101ca5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TypeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TypeUtils.scala
@@ -42,11 +42,17 @@ object TypeUtils {
}
def checkForSameTypeInputExpr(types: Seq[DataType], caller: String): TypeCheckResult = {
- if (types.distinct.size > 1) {
- TypeCheckResult.TypeCheckFailure(
- s"input to $caller should all be the same type, but it's " +
- types.map(_.simpleString).mkString("[", ", ", "]"))
+ if (types.size <= 1) {
+ TypeCheckResult.TypeCheckSuccess
} else {
+ val firstType = types.head
+ types.foreach { t =>
+ if (!t.sameType(firstType)) {
+ return TypeCheckResult.TypeCheckFailure(
+ s"input to $caller should all be the same type, but it's " +
+ types.map(_.simpleString).mkString("[", ", ", "]"))
+ }
+ }
TypeCheckResult.TypeCheckSuccess
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/3ba34d43/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TypeUtilsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TypeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TypeUtilsSuite.scala
new file mode 100644
index 0000000..bc6852c
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TypeUtilsSuite.scala
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
+import org.apache.spark.sql.types._
+
+class TypeUtilsSuite extends SparkFunSuite {
+
+ private def typeCheckPass(types: Seq[DataType]): Unit = {
+ assert(TypeUtils.checkForSameTypeInputExpr(types, "a") == TypeCheckSuccess)
+ }
+
+ private def typeCheckFail(types: Seq[DataType]): Unit = {
+ assert(TypeUtils.checkForSameTypeInputExpr(types, "a").isInstanceOf[TypeCheckFailure])
+ }
+
+ test("checkForSameTypeInputExpr") {
+ typeCheckPass(Nil)
+ typeCheckPass(StringType :: Nil)
+ typeCheckPass(StringType :: StringType :: Nil)
+
+ typeCheckFail(StringType :: IntegerType :: Nil)
+ typeCheckFail(StringType :: IntegerType :: Nil)
+
+ // Should also work on arrays. See SPARK-14990
+ typeCheckPass(ArrayType(StringType, containsNull = true) ::
+ ArrayType(StringType, containsNull = false) :: Nil)
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org