You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by GitBox <gi...@apache.org> on 2020/12/07 00:08:49 UTC

[GitHub] [spark] maropu commented on a change in pull request #30586: [SPARK-33641][SQL] Invalidate new char/varchar types in public APIs that produce incorrect results

maropu commented on a change in pull request #30586:
URL: https://github.com/apache/spark/pull/30586#discussion_r537154858



##########
File path: sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
##########
@@ -2954,6 +2954,15 @@ object SQLConf {
       .booleanConf
       .createWithDefault(true)
 
+  val LEGACY_CHAR_VARCHAR_AS_STRING =
+    buildConf("spark.sql.legacy.charVarcharAsString")
+      .internal()
+      .doc("When true, the parser will not fail to parse char and varchar type but treat it as" +

Review comment:
       It seems this config is not related only to the parser because `failIfHasCharVarchar` is used in many places. Could you update it accordingly?

##########
File path: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/CharVarcharUtils.scala
##########
@@ -52,6 +55,17 @@ object CharVarcharUtils {
     dt.existsRecursively(f => f.isInstanceOf[CharType] || f.isInstanceOf[VarcharType])
   }
 
+  /**
+   * Validate the given [[DataType]] to fail if it is char or varchar types or contains nested ones
+   */
+  def failIfHasCharVarchar(dt: DataType): DataType = {
+    if (!SQLConf.get.charVarcharAsString && hasCharVarchar(dt)) {
+      throw new AnalysisException(s"char/varchar type can only be used in the table schema")

Review comment:
       nit: remove `s` at the beginning.

##########
File path: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/CharVarcharUtils.scala
##########
@@ -69,6 +83,20 @@ object CharVarcharUtils {
     case _ => dt
   }
 
+  /**
+   * Replaces CharType/VarcharType with StringType recursively in the given data type, with a
+   * warning message if it has char or varchar types
+   */
+  def replaceCharVarcharWithStringForCast(dt: DataType): DataType = {
+    if (hasCharVarchar(dt)) {
+      logWarning("The Spark cast operator does not support char/varchar type and simply treat" +
+        " them as string type. Please use string type directly to avoid confusion.")

Review comment:
       Could you add tests for this warning msg by using `LogAppender`?

##########
File path: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/CharVarcharUtils.scala
##########
@@ -69,6 +83,20 @@ object CharVarcharUtils {
     case _ => dt
   }
 
+  /**
+   * Replaces CharType/VarcharType with StringType recursively in the given data type, with a
+   * warning message if it has char or varchar types
+   */
+  def replaceCharVarcharWithStringForCast(dt: DataType): DataType = {
+    if (hasCharVarchar(dt)) {
+      logWarning("The Spark cast operator does not support char/varchar type and simply treat" +

Review comment:
       nit: `treat` -> `treats`




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org