[spark] branch master updated: [SPARK-28497][SQL] Disallow upcasting complex data types to string type

[gu...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new b367b32  [SPARK-28497][SQL] Disallow upcasting complex data types to string type
b367b32 is described below

commit b367b323d2d69c6fcb8d4c83d87813e3924b7dc8
Author: Gengliang Wang <ge...@databricks.com>
AuthorDate: Thu Jul 25 20:55:01 2019 +0900

    [SPARK-28497][SQL] Disallow upcasting complex data types to string type
    
    ## What changes were proposed in this pull request?
    
    In the current implementation. complex types like Array/Map/StructType are allowed to upcast as StringType.
    This is not safe casting. We should disallow it.
    
    ## How was this patch tested?
    
    Update the existing test case
    
    Closes #25242 from gengliangwang/fixUpCastStringType.
    
    Authored-by: Gengliang Wang <ge...@databricks.com>
    Signed-off-by: HyukjinKwon <gu...@apache.org>
---
 .../org/apache/spark/sql/catalyst/expressions/Cast.scala  |  3 ++-
 .../sql/catalyst/encoders/EncoderResolutionSuite.scala    | 15 +++++++++++++++
 .../apache/spark/sql/catalyst/expressions/CastSuite.scala |  5 +++++
 .../scala/org/apache/spark/sql/UserDefinedTypeSuite.scala |  5 +++++
 4 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index 9691288..85bad74 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -131,7 +131,8 @@ object Cast {
     case (from: DecimalType, to: NumericType) if from.isTighterThan(to) => true
     case (f, t) if legalNumericPrecedence(f, t) => true
     case (DateType, TimestampType) => true
-    case (_, StringType) => true
+    case (_: AtomicType, StringType) => true
+    case (_: CalendarIntervalType, StringType) => true
 
     // Spark supports casting between long and timestamp, please see `longToTimestamp` and
     // `timestampToLong` for details.
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala
index da1b695..53cb8bc 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala
@@ -196,6 +196,21 @@ class EncoderResolutionSuite extends PlanTest {
     encoder.resolveAndBind(attrs)
   }
 
+  test("SPARK-28497: complex type is not compatible with string encoder schema") {
+    val encoder = ExpressionEncoder[String]
+
+    Seq('a.struct('x.long), 'a.array(StringType), 'a.map(StringType, StringType)).foreach { attr =>
+      val attrs = Seq(attr)
+      assert(intercept[AnalysisException](encoder.resolveAndBind(attrs)).message ==
+        s"""
+           |Cannot up cast `a` from ${attr.dataType.catalogString} to string.
+           |The type path of the target object is:
+           |- root class: "java.lang.String"
+           |You can either add an explicit cast to the input data or choose a higher precision type
+        """.stripMargin.trim + " of the field in the target object")
+    }
+  }
+
   test("throw exception if real type is not compatible with encoder schema") {
     val msg1 = intercept[AnalysisException] {
       ExpressionEncoder[StringIntClass].resolveAndBind(Seq('a.string, 'b.long))
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
index 4d667fd..44825c7 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
@@ -1002,6 +1002,11 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper {
         }
       }
     }
+    numericTypes.foreach { dt =>
+      makeComplexTypes(dt, true).foreach { complexType =>
+        assert(!Cast.canUpCast(complexType, StringType))
+      }
+    }
   }
 
   test("SPARK-27671: cast from nested null type in struct") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
index 6628d36..49f0000 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
@@ -272,4 +272,9 @@ class UserDefinedTypeSuite extends QueryTest with SharedSQLContext with ParquetT
     val ret = Cast(Literal(data, udt), StringType, None)
     checkEvaluation(ret, "(1.0, 3.0, 5.0, 7.0, 9.0)")
   }
+
+  test("SPARK-28497 Can't up cast UserDefinedType to string") {
+    val udt = new TestUDT.MyDenseVectorUDT()
+    assert(!Cast.canUpCast(udt, StringType))
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org