You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2025/01/16 17:20:49 UTC
(spark) branch branch-4.0 updated: [SPARK-50829][SQL] Add flag to disable session collation by default
This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 165e413d8b11 [SPARK-50829][SQL] Add flag to disable session collation by default
165e413d8b11 is described below
commit 165e413d8b11edfa89a7b2ddf39d2de8ecad51d4
Author: Stefan Kandic <15...@users.noreply.github.com>
AuthorDate: Thu Jan 16 20:20:14 2025 +0300
[SPARK-50829][SQL] Add flag to disable session collation by default
### What changes were proposed in this pull request?
Putting session level collation under a flag which is disabled by default.
### Why are the changes needed?
To not have feature on by default which is not 100% ready.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
New unit tests.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #49505 from stefankandic/sessionCollationFlag.
Lead-authored-by: Stefan Kandic <15...@users.noreply.github.com>
Co-authored-by: Stefan Kandic <st...@databricks.com>
Signed-off-by: Max Gekk <ma...@gmail.com>
(cherry picked from commit e2cff2264d3f6ed2ecab19016d5c299d5d8e04fd)
Signed-off-by: Max Gekk <ma...@gmail.com>
---
.../src/main/resources/error/error-conditions.json | 5 +++++
.../org/apache/spark/sql/internal/SQLConf.scala | 14 ++++++++++++
.../org/apache/spark/sql/CollationSuite.scala | 26 +++++++++++++++++++++-
3 files changed, 44 insertions(+), 1 deletion(-)
diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json
index 3b4e49999569..48a71cfbf615 100644
--- a/common/utils/src/main/resources/error/error-conditions.json
+++ b/common/utils/src/main/resources/error/error-conditions.json
@@ -2277,6 +2277,11 @@
"Cannot resolve the given default collation. Suggested valid collation names: ['<proposals>']?"
]
},
+ "DEFAULT_COLLATION_NOT_SUPPORTED" : {
+ "message" : [
+ "Setting default session collation other than UTF8_BINARY is currently not supported."
+ ]
+ },
"TIME_ZONE" : {
"message" : [
"Cannot resolve the given timezone."
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 4ae360b69641..24b66bdac260 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -894,12 +894,26 @@ object SQLConf {
.booleanConf
.createWithDefault(Utils.isTesting)
+ lazy val DEFAULT_COLLATION_ENABLED =
+ buildConf("spark.sql.sessionDefaultCollation.enabled")
+ .internal()
+ .doc("Session default collation feature is under development and its use should be done " +
+ "under this feature flag.")
+ .version("4.0.0")
+ .booleanConf
+ .createWithDefault(Utils.isTesting)
+
val DEFAULT_COLLATION =
buildConf(SqlApiConfHelper.DEFAULT_COLLATION)
+ .internal()
.doc("Sets default collation to use for string literals, parameter markers or the string" +
" produced by a builtin function such as to_char or CAST")
.version("4.0.0")
.stringConf
+ .checkValue(
+ value => value == CollationNames.UTF8_BINARY || get.getConf(DEFAULT_COLLATION_ENABLED),
+ errorClass = "DEFAULT_COLLATION_NOT_SUPPORTED",
+ parameters = _ => Map.empty)
.checkValue(
collationName => {
try {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala
index fd52142c4bd8..89c205d4f13f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql
import scala.jdk.CollectionConverters.MapHasAsJava
-import org.apache.spark.SparkException
+import org.apache.spark.{SparkException, SparkThrowable}
import org.apache.spark.sql.catalyst.ExtendedAnalysisException
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.util.CollationFactory
@@ -2083,4 +2083,28 @@ class CollationSuite extends DatasourceV2SQLBase with AdaptiveSparkPlanHelper {
sql(s"CREATE TABLE t (c STRING COLLATE system.builtin.UTF8_LCASE)")
}
}
+
+ test("flag for enabling session default collation") {
+ withSQLConf(SQLConf.DEFAULT_COLLATION_ENABLED.key -> "false") {
+ checkError(
+ exception = intercept[SparkThrowable] {
+ sql("SET COLLATION UNICODE_CI")
+ },
+ condition = "INVALID_CONF_VALUE.DEFAULT_COLLATION_NOT_SUPPORTED",
+ sqlState = "22022",
+ parameters = Map("confValue" -> "UNICODE_CI",
+ "confName" -> "spark.sql.session.collation.default"))
+
+ checkAnswer(
+ sql("SELECT 'a' = 'A'"),
+ Row(false))
+ }
+
+ withSQLConf(SQLConf.DEFAULT_COLLATION_ENABLED.key -> "true") {
+ sql("SET COLLATION UNICODE_CI")
+ checkAnswer(
+ sql("SELECT 'a' = 'A'"),
+ Row(true))
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org