You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2016/05/20 05:14:13 UTC

spark git commit: [SPARK-14990][SQL] Fix checkForSameTypeInputExpr (ignore nullability)

Repository: spark
Updated Branches:
  refs/heads/master f2ee0ed4b -> 3ba34d435


[SPARK-14990][SQL] Fix checkForSameTypeInputExpr (ignore nullability)

## What changes were proposed in this pull request?
This patch fixes a bug in TypeUtils.checkForSameTypeInputExpr. Previously the code was testing on strict equality, which does not taking nullability into account.

This is based on https://github.com/apache/spark/pull/12768. This patch fixed a bug there (with empty expression) and added a test case.

## How was this patch tested?
Added a new test suite and test case.

Closes #12768.

Author: Reynold Xin <rx...@databricks.com>
Author: Oleg Danilov <ol...@wandisco.com>

Closes #13208 from rxin/SPARK-14990.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3ba34d43
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3ba34d43
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3ba34d43

Branch: refs/heads/master
Commit: 3ba34d435c1f61435c2dddc28650cd111e7c1f33
Parents: f2ee0ed
Author: Reynold Xin <rx...@databricks.com>
Authored: Thu May 19 22:14:10 2016 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Thu May 19 22:14:10 2016 -0700

----------------------------------------------------------------------
 .../spark/sql/catalyst/util/TypeUtils.scala     | 14 ++++--
 .../sql/catalyst/util/TypeUtilsSuite.scala      | 46 ++++++++++++++++++++
 2 files changed, 56 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/3ba34d43/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TypeUtils.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TypeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TypeUtils.scala
index f603cbf..7101ca5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TypeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TypeUtils.scala
@@ -42,11 +42,17 @@ object TypeUtils {
   }
 
   def checkForSameTypeInputExpr(types: Seq[DataType], caller: String): TypeCheckResult = {
-    if (types.distinct.size > 1) {
-      TypeCheckResult.TypeCheckFailure(
-        s"input to $caller should all be the same type, but it's " +
-          types.map(_.simpleString).mkString("[", ", ", "]"))
+    if (types.size <= 1) {
+      TypeCheckResult.TypeCheckSuccess
     } else {
+      val firstType = types.head
+      types.foreach { t =>
+        if (!t.sameType(firstType)) {
+          return TypeCheckResult.TypeCheckFailure(
+            s"input to $caller should all be the same type, but it's " +
+              types.map(_.simpleString).mkString("[", ", ", "]"))
+        }
+      }
       TypeCheckResult.TypeCheckSuccess
     }
   }

http://git-wip-us.apache.org/repos/asf/spark/blob/3ba34d43/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TypeUtilsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TypeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TypeUtilsSuite.scala
new file mode 100644
index 0000000..bc6852c
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TypeUtilsSuite.scala
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
+import org.apache.spark.sql.types._
+
+class TypeUtilsSuite extends SparkFunSuite {
+
+  private def typeCheckPass(types: Seq[DataType]): Unit = {
+    assert(TypeUtils.checkForSameTypeInputExpr(types, "a") == TypeCheckSuccess)
+  }
+
+  private def typeCheckFail(types: Seq[DataType]): Unit = {
+    assert(TypeUtils.checkForSameTypeInputExpr(types, "a").isInstanceOf[TypeCheckFailure])
+  }
+
+  test("checkForSameTypeInputExpr") {
+    typeCheckPass(Nil)
+    typeCheckPass(StringType :: Nil)
+    typeCheckPass(StringType :: StringType :: Nil)
+
+    typeCheckFail(StringType :: IntegerType :: Nil)
+    typeCheckFail(StringType :: IntegerType :: Nil)
+
+    // Should also work on arrays. See SPARK-14990
+    typeCheckPass(ArrayType(StringType, containsNull = true) ::
+      ArrayType(StringType, containsNull = false) :: Nil)
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org