You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2024/10/26 20:40:50 UTC
(spark) branch master updated: [SPARK-48912][SQL][TESTS] Improve collation support testing for various collations
This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 9909817aef91 [SPARK-48912][SQL][TESTS] Improve collation support testing for various collations
9909817aef91 is described below
commit 9909817aef9198fd88058b4c8fec292de2797b8d
Author: Dejan Krakovic <de...@databricks.com>
AuthorDate: Sat Oct 26 22:40:36 2024 +0200
[SPARK-48912][SQL][TESTS] Improve collation support testing for various collations
### What changes were proposed in this pull request?
Extend collation-related unit and e2e sql tests for various collations in addition to the 4 common collations already used.
### Why are the changes needed?
Expand collation testing coverage for various collations, incorporating different languages, scripts, case/accent sensitivity, etc.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Extending existing collation-related unit and e2e sql tests.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #48608 from dejankrak-db/collation-tests.
Authored-by: Dejan Krakovic <de...@databricks.com>
Signed-off-by: Max Gekk <ma...@gmail.com>
---
.../spark/unsafe/types/CollationFactorySuite.scala | 21 +-
.../expressions/CollationExpressionSuite.scala | 5 +-
.../spark/sql/CollationSQLExpressionsSuite.scala | 969 +++++++++------------
.../sql/CollationStringExpressionsSuite.scala | 116 ++-
.../org/apache/spark/sql/CollationSuite.scala | 2 +-
5 files changed, 505 insertions(+), 608 deletions(-)
diff --git a/common/unsafe/src/test/scala/org/apache/spark/unsafe/types/CollationFactorySuite.scala b/common/unsafe/src/test/scala/org/apache/spark/unsafe/types/CollationFactorySuite.scala
index 6daaf2a4c675..1b16432e6378 100644
--- a/common/unsafe/src/test/scala/org/apache/spark/unsafe/types/CollationFactorySuite.scala
+++ b/common/unsafe/src/test/scala/org/apache/spark/unsafe/types/CollationFactorySuite.scala
@@ -169,7 +169,13 @@ class CollationFactorySuite extends AnyFunSuite with Matchers { // scalastyle:ig
CollationTestCase("UNICODE_CI_RTRIM", "aaa", "AaA ", true),
CollationTestCase("UNICODE_CI_RTRIM", "aaa ", "AaA ", true),
CollationTestCase("UNICODE_CI_RTRIM", "aaa", " AaA ", false),
- CollationTestCase("UNICODE_RTRIM", " ", " ", true)
+ CollationTestCase("UNICODE_RTRIM", " ", " ", true),
+ CollationTestCase("SR_CI", "cČć", "CčĆ", true),
+ CollationTestCase("SR_CI", "cCc", "CčĆ", false),
+ CollationTestCase("SR_CI_AI", "cCc", "CčĆ", true),
+ CollationTestCase("sr_Cyrl_CI", "цЧћ", "ЦчЋ", true),
+ CollationTestCase("sr_Cyrl_CI", "цЦц", "ЦчЋ", false),
+ CollationTestCase("sr_Cyrl_CI_AI", "цЦц", "ЦчЋ", false)
)
checks.foreach(testCase => {
@@ -229,7 +235,9 @@ class CollationFactorySuite extends AnyFunSuite with Matchers { // scalastyle:ig
CollationTestCase("UNICODE_CI_RTRIM", "aaa ", "bbb ", -1),
CollationTestCase("UNICODE_CI_RTRIM", "aaa ", "aa", 1),
CollationTestCase("UNICODE_CI_RTRIM", "aaa ", "aa ", 1),
- CollationTestCase("UNICODE_CI_RTRIM", " ", " ", 0)
+ CollationTestCase("UNICODE_CI_RTRIM", " ", " ", 0),
+ CollationTestCase("SR_CI_AI", "cČć", "ČćC", 0),
+ CollationTestCase("SR_CI", "cČć", "ČćC", -1)
)
checks.foreach(testCase => {
@@ -248,7 +256,10 @@ class CollationFactorySuite extends AnyFunSuite with Matchers { // scalastyle:ig
CollationTestCase("UNICODE_CI", "abcde", "abcde", 5),
CollationTestCase("UNICODE_CI", "abcde", "ABCDE", 5),
CollationTestCase("UNICODE_CI", "abcde", "fgh", 0),
- CollationTestCase("UNICODE_CI", "abcde", "FGH", 0)
+ CollationTestCase("UNICODE_CI", "abcde", "FGH", 0),
+ CollationTestCase("SR_CI_AI", "abcčċ", "CCC", 3),
+ CollationTestCase("SR_CI", "abcčċ", "C", 1),
+ CollationTestCase("SR", "abcčċ", "CCC", 0)
)
checks.foreach(testCase => {
@@ -285,7 +296,9 @@ class CollationFactorySuite extends AnyFunSuite with Matchers { // scalastyle:ig
"UNICODE_CI",
"UNICODE_AI",
"UNICODE_CI_AI",
- "UNICODE_AI_CI"
+ "UNICODE_AI_CI",
+ "DE_CI_AI",
+ "MT_CI"
).foreach(collationId => {
val col1 = fetchCollation(collationId)
val col2 = fetchCollation(collationId)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollationExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollationExpressionSuite.scala
index 2634008cdd9e..a30f604550a3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollationExpressionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollationExpressionSuite.scala
@@ -95,7 +95,10 @@ class CollationExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
(Seq("a"), Seq("A"), true, "UTF8_LCASE"),
(Seq("a", "B"), Seq("A", "b"), true, "UTF8_LCASE"),
(Seq("a"), Seq("A"), false, "UNICODE"),
- (Seq("a", "B"), Seq("A", "b"), true, "UNICODE_CI")
+ (Seq("a", "B"), Seq("A", "b"), true, "UNICODE_CI"),
+ (Seq("c"), Seq("C"), false, "SR"),
+ (Seq("c"), Seq("C"), true, "SR_CI"),
+ (Seq("a", "c"), Seq("b", "C"), true, "SR_CI_AI")
)
for ((inLeft, inRight, out, collName) <- overlap) {
val left = arrayLiteral(inLeft, collName)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CollationSQLExpressionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CollationSQLExpressionsSuite.scala
index daa9e6cf9e0a..adfc5b703da4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CollationSQLExpressionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CollationSQLExpressionsSuite.scala
@@ -39,6 +39,7 @@ class CollationSQLExpressionsSuite
with ExpressionEvalHelper {
private val testSuppCollations = Seq("UTF8_BINARY", "UTF8_LCASE", "UNICODE", "UNICODE_CI")
+ private val testAdditionalCollations = Seq("UNICODE", "SR", "SR_CI", "SR_AI", "SR_CI_AI")
test("Support Md5 hash expression with collation") {
case class Md5TestCase(
@@ -55,7 +56,8 @@ class CollationSQLExpressionsSuite
Md5TestCase("SQL", "UNICODE", "9778840a0100cb30c982876741b0b5a2"),
Md5TestCase("SQL", "UNICODE_RTRIM", "9778840a0100cb30c982876741b0b5a2"),
Md5TestCase("SQL", "UNICODE_CI", "9778840a0100cb30c982876741b0b5a2"),
- Md5TestCase("SQL", "UNICODE_CI_RTRIM", "9778840a0100cb30c982876741b0b5a2")
+ Md5TestCase("SQL", "UNICODE_CI_RTRIM", "9778840a0100cb30c982876741b0b5a2"),
+ Md5TestCase("SQL", "SR_CI_AI", "9778840a0100cb30c982876741b0b5a2")
)
// Supported collations
@@ -98,6 +100,8 @@ class CollationSQLExpressionsSuite
Sha2TestCase("SQL", "UNICODE_CI", 256,
"a7056a455639d1c7deec82ee787db24a0c1878e2792b4597709f0facf7cc7b35"),
Sha2TestCase("SQL", "UNICODE_CI_RTRIM", 256,
+ "a7056a455639d1c7deec82ee787db24a0c1878e2792b4597709f0facf7cc7b35"),
+ Sha2TestCase("SQL", "SR_AI", 256,
"a7056a455639d1c7deec82ee787db24a0c1878e2792b4597709f0facf7cc7b35")
)
@@ -132,7 +136,8 @@ class CollationSQLExpressionsSuite
Sha1TestCase("SQL", "UNICODE", "2064cb643caa8d9e1de12eea7f3e143ca9f8680d"),
Sha1TestCase("SQL", "UNICODE_RTRIM", "2064cb643caa8d9e1de12eea7f3e143ca9f8680d"),
Sha1TestCase("SQL", "UNICODE_CI", "2064cb643caa8d9e1de12eea7f3e143ca9f8680d"),
- Sha1TestCase("SQL", "UNICODE_CI_RTRIM", "2064cb643caa8d9e1de12eea7f3e143ca9f8680d")
+ Sha1TestCase("SQL", "UNICODE_CI_RTRIM", "2064cb643caa8d9e1de12eea7f3e143ca9f8680d"),
+ Sha1TestCase("Spark", "SR_CI", "85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c")
)
// Supported collations
@@ -549,7 +554,8 @@ class CollationSQLExpressionsSuite
HexTestCase("Spark SQL", "UTF8_BINARY", "537061726B2053514C"),
HexTestCase("Spark SQL", "UTF8_LCASE", "537061726B2053514C"),
HexTestCase("Spark SQL", "UNICODE", "537061726B2053514C"),
- HexTestCase("Spark SQL", "UNICODE_CI", "537061726B2053514C")
+ HexTestCase("Spark SQL", "UNICODE_CI", "537061726B2053514C"),
+ HexTestCase("Spark SQL", "DE_CI_AI", "537061726B2053514C")
)
testCases.foreach(t => {
val query =
@@ -572,7 +578,8 @@ class CollationSQLExpressionsSuite
UnHexTestCase("537061726B2053514C", "UTF8_BINARY", "Spark SQL"),
UnHexTestCase("537061726B2053514C", "UTF8_LCASE", "Spark SQL"),
UnHexTestCase("537061726B2053514C", "UNICODE", "Spark SQL"),
- UnHexTestCase("537061726B2053514C", "UNICODE_CI", "Spark SQL")
+ UnHexTestCase("537061726B2053514C", "UNICODE_CI", "Spark SQL"),
+ UnHexTestCase("537061726B2053514C", "DE", "Spark SQL")
)
testCases.foreach(t => {
val query =
@@ -640,7 +647,8 @@ class CollationSQLExpressionsSuite
StringSpaceTestCase(1, "UTF8_BINARY", " "),
StringSpaceTestCase(2, "UTF8_LCASE", " "),
StringSpaceTestCase(3, "UNICODE", " "),
- StringSpaceTestCase(4, "UNICODE_CI", " ")
+ StringSpaceTestCase(4, "UNICODE_CI", " "),
+ StringSpaceTestCase(5, "AF_CI_AI", " ")
)
// Supported collations
@@ -1008,7 +1016,11 @@ class CollationSQLExpressionsSuite
StringToMapTestCase("1:ax2:bx3:c", "x", ":", "UNICODE",
Map("1" -> "a", "2" -> "b", "3" -> "c")),
StringToMapTestCase("1/AX2/BX3/C", "x", "/", "UNICODE_CI",
- Map("1" -> "A", "2" -> "B", "3" -> "C"))
+ Map("1" -> "A", "2" -> "B", "3" -> "C")),
+ StringToMapTestCase("1:cx2:čx3:ć", "x", ":", "SR_CI_AI",
+ Map("1" -> "c", "2" -> "č", "3" -> "ć")),
+ StringToMapTestCase("c:1,č:2,ć:3", ",", ":", "SR_CI",
+ Map("c" -> "1", "č" -> "2", "ć" -> "3"))
)
val unsupportedTestCases = Seq(
StringToMapTestCase("a:1,b:2,c:3", "?", "?", "UNICODE_AI", null),
@@ -1082,7 +1094,7 @@ class CollationSQLExpressionsSuite
test("Support CurrentDatabase/Catalog/User expressions with collation") {
// Supported collations
- Seq("UTF8_LCASE", "UNICODE", "UNICODE_CI").foreach(collationName =>
+ Seq("UTF8_LCASE", "UNICODE", "UNICODE_CI", "SR_CI_AI").foreach(collationName =>
withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collationName) {
val queryDatabase = sql("SELECT current_schema()")
val queryCatalog = sql("SELECT current_catalog()")
@@ -1098,7 +1110,7 @@ class CollationSQLExpressionsSuite
test("Support Uuid misc expression with collation") {
// Supported collations
- Seq("UTF8_LCASE", "UNICODE", "UNICODE_CI").foreach(collationName =>
+ Seq("UTF8_LCASE", "UNICODE", "UNICODE_CI", "NO_CI_AI").foreach(collationName =>
withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collationName) {
val query = s"SELECT uuid()"
// Result & data type
@@ -1114,7 +1126,7 @@ class CollationSQLExpressionsSuite
test("Support SparkVersion misc expression with collation") {
// Supported collations
- Seq("UTF8_BINARY", "UTF8_LCASE", "UNICODE", "UNICODE_CI").foreach(collationName =>
+ Seq("UTF8_BINARY", "UTF8_LCASE", "UNICODE", "UNICODE_CI", "DE").foreach(collationName =>
withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collationName) {
val query = s"SELECT version()"
// Result & data type
@@ -1689,7 +1701,7 @@ class CollationSQLExpressionsSuite
test("Support InputFileName expression with collation") {
// Supported collations
- Seq("UTF8_BINARY", "UTF8_LCASE", "UNICODE", "UNICODE_CI").foreach(collationName => {
+ Seq("UTF8_BINARY", "UTF8_LCASE", "UNICODE", "UNICODE_CI", "MT_CI_AI").foreach(collationName => {
val query =
s"""
|select input_file_name()
@@ -1737,7 +1749,7 @@ class CollationSQLExpressionsSuite
}
test("Support mode for string expression with collation - Basic Test") {
- Seq("utf8_binary", "UTF8_LCASE", "unicode_ci", "unicode").foreach { collationId =>
+ Seq("utf8_binary", "UTF8_LCASE", "unicode_ci", "unicode", "NL_AI").foreach { collationId =>
val query = s"SELECT mode(collate('abc', '${collationId}'))"
checkAnswer(sql(query), Row("abc"))
assert(sql(query).schema.fields.head.dataType.sameType(StringType(collationId)))
@@ -1750,7 +1762,8 @@ class CollationSQLExpressionsSuite
ModeTestCase("utf8_binary", Map("a" -> 3L, "b" -> 2L, "B" -> 2L), "a"),
ModeTestCase("UTF8_LCASE", Map("a" -> 3L, "b" -> 2L, "B" -> 2L), "b"),
ModeTestCase("unicode_ci", Map("a" -> 3L, "b" -> 2L, "B" -> 2L), "b"),
- ModeTestCase("unicode", Map("a" -> 3L, "b" -> 2L, "B" -> 2L), "a")
+ ModeTestCase("unicode", Map("a" -> 3L, "b" -> 2L, "B" -> 2L), "a"),
+ ModeTestCase("SR", Map("c" -> 3L, "č" -> 2L, "Č" -> 2L), "c")
)
testCases.foreach(t => {
val valuesToAdd = t.bufferValues.map { case (elt, numRepeats) =>
@@ -2434,829 +2447,637 @@ class CollationSQLExpressionsSuite
)
}
- test("min_by supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT min_by(x, y) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y);"
+ // common method for subsequent tests verifying various SQL expressions with collations
+ private def testCollationSqlExpressionCommon(
+ query: String,
+ collation: String,
+ result: Row,
+ expectedType: DataType): Unit = {
+ testCollationSqlExpressionCommon(query, collation, Seq(result), Seq(expectedType))
+ }
+
+ // common method for subsequent tests verifying various SQL expressions with collations
+ private def testCollationSqlExpressionCommon(
+ query: String,
+ collation: String,
+ result: Seq[Row],
+ expectedTypes: Seq[DataType]): Unit = {
withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row("a")
- )
+ // check result correctness
+ checkAnswer(sql(query), result)
+ // check result rows data types
+ for (i <- 0 until expectedTypes.length)
+ assert(sql(query).schema(i).dataType == expectedTypes(i))
+ }
+ }
+
+ test("min_by supports collation") {
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT min_by(x, y) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y)",
+ collation,
+ result = Row("a"),
+ expectedType = StringType(collation)
)
- // check result row data type
- val dataType = StringType(collation)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("max_by supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT max_by(x, y) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y);"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row("b")
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT max_by(x, y) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y)",
+ collation,
+ result = Row("b"),
+ expectedType = StringType(collation)
)
- // check result row data type
- val dataType = StringType(collation)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("array supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT array('a', 'b', 'c');"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq("a", "b", "c"))
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT array('a', 'b', 'c')",
+ collation,
+ result = Row(Seq("a", "b", "c")),
+ expectedType = ArrayType(StringType(collation), false)
)
- // check result row data type
- val dataType = ArrayType(StringType(collation), false)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("array_agg supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT array_agg(col) FROM VALUES ('a'), ('b'), ('c') AS tab(col);"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq("a", "b", "c"))
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT array_agg(col) FROM VALUES ('a'), ('b'), ('c') AS tab(col)",
+ collation,
+ result = Row(Seq("a", "b", "c")),
+ expectedType = ArrayType(StringType(collation), false)
)
- // check result row data type
- val dataType = ArrayType(StringType(collation), false)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("array_contains supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT array_contains(array('a', 'b', 'c'), 'b');"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(true)
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT array_contains(array('a', 'b', 'c'), 'b')",
+ collation,
+ result = Row(true),
+ expectedType = BooleanType
)
- // check result row data type
- val dataType = BooleanType
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("arrays_overlap supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT arrays_overlap(array('a', 'b', 'c'), array('c', 'd', 'e'));"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(true)
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT arrays_overlap(array('a', 'b', 'c'), array('c', 'd', 'e'))",
+ collation,
+ result = Row(true),
+ expectedType = BooleanType
)
- // check result row data type
- val dataType = BooleanType
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("array_insert supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT array_insert(array('a', 'b', 'c', 'd'), 5, 'e');"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq("a", "b", "c", "d", "e"))
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT array_insert(array('a', 'b', 'c', 'd'), 5, 'e')",
+ collation,
+ result = Row(Seq("a", "b", "c", "d", "e")),
+ expectedType = ArrayType(StringType(collation), true)
)
- // check result row data type
- val dataType = ArrayType(StringType(collation), true)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("array_intersect supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT array_intersect(array('a', 'b', 'c'), array('b', 'c', 'd'));"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq("b", "c"))
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT array_intersect(array('a', 'b', 'c'), array('b', 'c', 'd'))",
+ collation,
+ result = Row(Seq("b", "c")),
+ expectedType = ArrayType(StringType(collation), false)
)
- // check result row data type
- val dataType = ArrayType(StringType(collation), false)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("array_join supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT array_join(array('hello', 'world'), ' ');"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row("hello world")
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT array_join(array('hello', 'world'), ' ')",
+ collation,
+ result = Row("hello world"),
+ expectedType = StringType(collation)
)
- // check result row data type
- val dataType = StringType(collation)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("array_position supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT array_position(array('a', 'b', 'c', 'c'), 'c');"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(3)
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT array_position(array('a', 'b', 'c', 'c'), 'c')",
+ collation,
+ result = Row(3),
+ expectedType = LongType
)
- // check result row data type
- val dataType = LongType
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("array_size supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT array_size(array('a', 'b', 'c', 'c'));"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(4)
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT array_size(array('a', 'b', 'c', 'c'))",
+ collation,
+ result = Row(4),
+ expectedType = IntegerType
)
- // check result row data type
- val dataType = IntegerType
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("array_sort supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT array_sort(array('b', null, 'A'));"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq("A", "b", null))
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT array_sort(array('b', null, 'A'))",
+ collation,
+ result = Row(Seq("A", "b", null)),
+ expectedType = ArrayType(StringType(collation), true)
)
- // check result row data type
- val dataType = ArrayType(StringType(collation), true)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("array_except supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT array_except(array('a', 'b', 'c'), array('c', 'd', 'e'));"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq("a", "b"))
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT array_except(array('a', 'b', 'c'), array('c', 'd', 'e'))",
+ collation,
+ result = Row(Seq("a", "b")),
+ expectedType = ArrayType(StringType(collation), false)
)
- // check result row data type
- val dataType = ArrayType(StringType(collation), false)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("array_union supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT array_union(array('a', 'b', 'c'), array('a', 'c', 'd'));"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq("a", "b", "c", "d"))
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT array_union(array('a', 'b', 'c'), array('a', 'c', 'd'))",
+ collation,
+ result = Row(Seq("a", "b", "c", "d")),
+ expectedType = ArrayType(StringType(collation), false)
)
- // check result row data type
- val dataType = ArrayType(StringType(collation), false)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("array_compact supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT array_compact(array('a', 'b', null, 'c'));"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq("a", "b", "c"))
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT array_compact(array('a', 'b', null, 'c'))",
+ collation,
+ result = Row(Seq("a", "b", "c")),
+ expectedType = ArrayType(StringType(collation), false)
)
- // check result row data type
- val dataType = ArrayType(StringType(collation), false)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("arrays_zip supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT arrays_zip(array('a', 'b', 'c'), array(1, 2, 3));"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq(Row("a", 1), Row("b", 2), Row("c", 3)))
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT arrays_zip(array('a', 'b', 'c'), array(1, 2, 3))",
+ collation,
+ result = Row(Seq(Row("a", 1), Row("b", 2), Row("c", 3))),
+ expectedType = ArrayType(StructType(
+ StructField("0", StringType(collation), true) ::
+ StructField("1", IntegerType, true) :: Nil
+ ), false)
)
- // check result row data type
- val dataType = ArrayType(StructType(
- StructField("0", StringType(collation), true) ::
- StructField("1", IntegerType, true) :: Nil
- ), false)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("array_min supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT array_min(array('a', 'b', null, 'c'));"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row("a")
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT array_min(array('a', 'b', null, 'c'))",
+ collation,
+ result = Row("a"),
+ expectedType = StringType(collation)
)
- // check result row data type
- val dataType = StringType(collation)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("array_max supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT array_max(array('a', 'b', null, 'c'));"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row("c")
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT array_max(array('a', 'b', null, 'c'))",
+ collation,
+ result = Row("c"),
+ expectedType = StringType(collation)
)
- // check result row data type
- val dataType = StringType(collation)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("array_append supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT array_append(array('b', 'd', 'c', 'a'), 'e');"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq("b", "d", "c", "a", "e"))
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT array_append(array('b', 'd', 'c', 'a'), 'e')",
+ collation,
+ result = Row(Seq("b", "d", "c", "a", "e")),
+ expectedType = ArrayType(StringType(collation), true)
)
- // check result row data type
- val dataType = ArrayType(StringType(collation), true)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("array_repeat supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT array_repeat('abc', 2);"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq("abc", "abc"))
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT array_repeat('abc', 2)",
+ collation,
+ result = Row(Seq("abc", "abc")),
+ expectedType = ArrayType(StringType(collation), false)
)
- // check result row data type
- val dataType = ArrayType(StringType(collation), false)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("array_remove supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT array_remove(array('a', 'b', null, 'c'), 'b');"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq("a", null, "c"))
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT array_remove(array('a', 'b', null, 'c'), 'b')",
+ collation,
+ result = Row(Seq("a", null, "c")),
+ expectedType = ArrayType(StringType(collation), true)
)
- // check result row data type
- val dataType = ArrayType(StringType(collation), true)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("array_prepend supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT array_prepend(array('b', 'd', 'c', 'a'), 'd');"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq("d", "b", "d", "c", "a"))
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT array_prepend(array('b', 'd', 'c', 'a'), 'd')",
+ collation,
+ result = Row(Seq("d", "b", "d", "c", "a")),
+ expectedType = ArrayType(StringType(collation), true)
)
- // check result row data type
- val dataType = ArrayType(StringType(collation), true)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("array_distinct supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT array_distinct(array('a', 'b', 'c', null, 'c'));"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq("a", "b", "c", null))
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT array_distinct(array('a', 'b', 'c', null, 'c'))",
+ collation,
+ result = Row(Seq("a", "b", "c", null)),
+ expectedType = ArrayType(StringType(collation), true)
)
- // check result row data type
- val dataType = ArrayType(StringType(collation), true)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("collect_list supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT collect_list(col) FROM VALUES ('a'), ('b'), ('c') AS tab(col);"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq("a", "b", "c"))
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT collect_list(col) FROM VALUES ('a'), ('b'), ('c') AS tab(col)",
+ collation,
+ result = Row(Seq("a", "b", "c")),
+ expectedType = ArrayType(StringType(collation), false)
)
- // check result row data type
- val dataType = ArrayType(StringType(collation), false)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("collect_set does not support collation") {
- val collation = "UNICODE"
- val query = s"SELECT collect_set(col) FROM VALUES ('a'), ('b'), ('a') AS tab(col);"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkError(
- exception = intercept[AnalysisException] {
- sql(query)
- },
- condition = "DATATYPE_MISMATCH.UNSUPPORTED_INPUT_TYPE",
- sqlState = Some("42K09"),
- parameters = Map(
- "functionName" -> "`collect_set`",
- "dataType" -> "\"MAP\" or \"COLLATED STRING\"",
- "sqlExpr" -> "\"collect_set(col)\""),
- context = ExpectedContext(
- fragment = "collect_set(col)",
- start = 7,
- stop = 22))
+ testAdditionalCollations.foreach { collation =>
+ val query = "SELECT collect_set(col) FROM VALUES ('a'), ('b'), ('a') AS tab(col);"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql(query)
+ },
+ condition = "DATATYPE_MISMATCH.UNSUPPORTED_INPUT_TYPE",
+ sqlState = Some("42K09"),
+ parameters = Map(
+ "functionName" -> "`collect_set`",
+ "dataType" -> "\"MAP\" or \"COLLATED STRING\"",
+ "sqlExpr" -> "\"collect_set(col)\""),
+ context = ExpectedContext(
+ fragment = "collect_set(col)",
+ start = 7,
+ stop = 22))
+ }
}
}
test("element_at supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT element_at(array('a', 'b', 'c'), 2);"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row("b")
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT element_at(array('a', 'b', 'c'), 2)",
+ collation,
+ result = Row("b"),
+ expectedType = StringType(collation)
)
- // check result row data type
- val dataType = StringType(collation)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("aggregate supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT aggregate(array('a', 'b', 'c'), '', (acc, x) -> concat(acc, x));"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row("abc")
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT aggregate(array('a', 'b', 'c'), '', (acc, x) -> concat(acc, x))",
+ collation,
+ result = Row("abc"),
+ expectedType = StringType(collation)
)
- // check result row data type
- val dataType = StringType(collation)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("explode supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT explode(array('a', 'b'));"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT explode(array('a', 'b'))",
+ collation,
+ result = Seq(
Row("a"),
Row("b")
+ ),
+ expectedTypes = Seq(
+ StringType(collation)
)
)
- // check result row data type
- val dataType = StringType(collation)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("posexplode supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT posexplode(array('a', 'b'));"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT posexplode(array('a', 'b'))",
+ collation,
+ result = Seq(
Row(0, "a"),
Row(1, "b")
+ ),
+ expectedTypes = Seq(
+ IntegerType,
+ StringType(collation)
)
)
- // check result row data type
- val dataType = StringType(collation)
- assert(sql(query).schema.head.dataType == IntegerType)
- assert(sql(query).schema(1).dataType == dataType)
}
}
test("filter supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT filter(array('a', 'b', 'c'), x -> x < 'b');"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq("a"))
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT filter(array('a', 'b', 'c'), x -> x < 'b')",
+ collation,
+ result = Row(Seq("a")),
+ expectedType = ArrayType(StringType(collation), false)
)
- // check result row data type
- val dataType = ArrayType(StringType(collation), false)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("flatten supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT flatten(array(array('a', 'b'), array('c', 'd')));"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq("a", "b", "c", "d"))
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT flatten(array(array('a', 'b'), array('c', 'd')))",
+ collation,
+ result = Row(Seq("a", "b", "c", "d")),
+ expectedType = ArrayType(StringType(collation), false)
)
- // check result row data type
- val dataType = ArrayType(StringType(collation), false)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("inline supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT inline(array(struct(1, 'a'), struct(2, 'b')));"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT inline(array(struct(1, 'a'), struct(2, 'b')))",
+ collation,
Seq(
Row(1, "a"),
Row(2, "b")
+ ),
+ expectedTypes = Seq(
+ IntegerType,
+ StringType(collation)
)
)
- // check result row data type
- val dataType = StringType(collation)
- assert(sql(query).schema.head.dataType == IntegerType)
- assert(sql(query).schema(1).dataType == dataType)
}
}
test("shuffle supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT shuffle(array('a', 'b', 'c', 'd'));"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- // check result row data type
- val dataType = ArrayType(StringType(collation), false)
- assert(sql(query).schema.head.dataType == dataType)
+ testAdditionalCollations.foreach { collation =>
+ val query = "SELECT shuffle(array('a', 'b', 'c', 'd'));"
+ withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
+ // check result row data type
+ val dataType = ArrayType(StringType(collation), false)
+ assert(sql(query).schema.head.dataType == dataType)
+ }
}
}
test("slice supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT slice(array('a', 'b', 'c', 'd'), 2, 2);"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq("b", "c"))
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT slice(array('a', 'b', 'c', 'd'), 2, 2)",
+ collation,
+ result = Row(Seq("b", "c")),
+ expectedType = ArrayType(StringType(collation), false)
)
- // check result row data type
- val dataType = ArrayType(StringType(collation), false)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("sort_array supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT sort_array(array('b', 'd', null, 'c', 'a'), true);"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq(null, "a", "b", "c", "d"))
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT sort_array(array('b', 'd', null, 'c', 'a'), true)",
+ collation,
+ result = Row(Seq(null, "a", "b", "c", "d")),
+ expectedType = ArrayType(StringType(collation), true)
)
- // check result row data type
- val dataType = ArrayType(StringType(collation), true)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("zip_with supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT zip_with(array('a', 'b'), array('x', 'y'), (x, y) -> concat(x, y));"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq("ax", "by"))
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT zip_with(array('a', 'b'), array('x', 'y'), (x, y) -> concat(x, y))",
+ collation,
+ result = Row(Seq("ax", "by")),
+ expectedType = ArrayType(
+ StringType(collation),
+ containsNull = true
)
)
- // check result row data type
- val dataType = ArrayType(
- StringType(collation),
- containsNull = true
- )
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("map_contains_key supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT map_contains_key(map('a', 1, 'b', 2), 'a')"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(true)
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT map_contains_key(map('a', 1, 'b', 2), 'a')",
+ collation,
+ result = Row(true),
+ expectedType = BooleanType
)
- // check result row data type
- val dataType = BooleanType
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("map_from_arrays supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT map_from_arrays(array('a','b','c'), array(1,2,3))"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Map("a" -> 1, "b" -> 2, "c" -> 3))
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT map_from_arrays(array('a','b','c'), array(1,2,3))",
+ collation,
+ result = Row(Map("a" -> 1, "b" -> 2, "c" -> 3)),
+ expectedType = MapType(
+ StringType(collation),
+ IntegerType, false
)
)
- // check result row data type
- val dataType = MapType(
- StringType(collation),
- IntegerType, false
- )
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("map_keys supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT map_keys(map('a', 1, 'b', 2))"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq("a", "b"))
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT map_keys(map('a', 1, 'b', 2))",
+ collation,
+ result = Row(Seq("a", "b")),
+ expectedType = ArrayType(StringType(collation), true)
)
- // check result row data type
- val dataType = ArrayType(
- StringType(collation), true
- )
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("map_values supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT map_values(map(1, 'a', 2, 'b'))"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq("a", "b"))
- )
- )
- // check result row data type
- val dataType = ArrayType(
- StringType(collation), true
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT map_values(map(1, 'a', 2, 'b'))",
+ collation,
+ result = Row(Seq("a", "b")),
+ expectedType = ArrayType(StringType(collation), true)
)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("map_entries supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT map_entries(map('a', 1, 'b', 2))"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq(Row("a", 1), Row("b", 2)))
- )
- )
- // check result row data type
- val dataType = ArrayType(StructType(
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT map_entries(map('a', 1, 'b', 2))",
+ collation,
+ result = Row(Seq(Row("a", 1), Row("b", 2))),
+ expectedType = ArrayType(StructType(
StructField("key", StringType(collation), false) ::
- StructField("value", IntegerType, false) :: Nil
- ), false)
- assert(sql(query).schema.head.dataType == dataType)
+ StructField("value", IntegerType, false) :: Nil
+ ), false)
+ )
}
}
test("map_from_entries supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT map_from_entries(array(struct(1, 'a'), struct(2, 'b')))"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Map(1 -> "a", 2 -> "b"))
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT map_from_entries(array(struct(1, 'a'), struct(2, 'b')))",
+ collation,
+ result = Row(Map(1 -> "a", 2 -> "b")),
+ expectedType = MapType(
+ IntegerType,
+ StringType(collation),
+ valueContainsNull = false
)
)
- // check result row data type
- val dataType = MapType(
- IntegerType,
- StringType(collation),
- valueContainsNull = false
- )
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("map_concat supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT map_concat(map(1, 'a'), map(2, 'b'))"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Map(1 -> "a", 2 -> "b"))
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT map_concat(map(1, 'a'), map(2, 'b'))",
+ collation,
+ result = Row(Map(1 -> "a", 2 -> "b")),
+ expectedType = MapType(
+ IntegerType,
+ StringType(collation),
+ valueContainsNull = false
)
)
- // check result row data type
- val dataType = MapType(
- IntegerType,
- StringType(collation),
- valueContainsNull = false
- )
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("map_filter supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT map_filter(map('a', 1, 'b', 2, 'c', 3), (k, v) -> k < 'c')"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Map("a" -> 1, "b" -> 2))
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT map_filter(map('a', 1, 'b', 2, 'c', 3), (k, v) -> k < 'c')",
+ collation,
+ result = Row(Map("a" -> 1, "b" -> 2)),
+ expectedType = MapType(
+ StringType(collation),
+ IntegerType,
+ valueContainsNull = false
)
)
- // check result row data type
- val dataType = MapType(
- StringType(collation),
- IntegerType,
- valueContainsNull = false
- )
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("map_zip_with supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT map_zip_with(map(1, 'a'), map(1, 'x'), (k, v1, v2) -> concat(v1, v2))"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Map(1 -> "ax"))
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT map_zip_with(map(1, 'a'), map(1, 'x'), (k, v1, v2) -> concat(v1, v2))",
+ collation,
+ result = Row(Map(1 -> "ax")),
+ expectedType = MapType(
+ IntegerType,
+ StringType(collation),
+ valueContainsNull = true
)
)
- // check result row data type
- val dataType = MapType(
- IntegerType,
- StringType(collation),
- valueContainsNull = true
- )
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("transform supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT transform(array('aa', 'bb', 'cc'), x -> substring(x, 2))"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Seq("a", "b", "c"))
- )
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT transform(array('aa', 'bb', 'cc'), x -> substring(x, 2))",
+ collation,
+ result = Row(Seq("a", "b", "c")),
+ expectedType = ArrayType(StringType(collation), false)
)
- // check result row data type
- val dataType = ArrayType(StringType(collation), false)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("transform_values supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT transform_values(map_from_arrays(array(1, 2, 3)," +
- s"array('aa', 'bb', 'cc')), (k, v) -> substring(v, 2))"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Map(1 -> "a", 2 -> "b", 3 -> "c"))
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT transform_values(map_from_arrays(array(1, 2, 3)," +
+ "array('aa', 'bb', 'cc')), (k, v) -> substring(v, 2))",
+ collation,
+ result = Row(Map(1 -> "a", 2 -> "b", 3 -> "c")),
+ expectedType = MapType(
+ IntegerType,
+ StringType(collation),
+ false
)
)
- // check result row data type
- val dataType = MapType(IntegerType,
- StringType(collation), false)
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("transform_keys supports collation") {
- val collation = "UNICODE"
- val query = s"SELECT transform_keys(map_from_arrays(array('aa', 'bb', 'cc')," +
- s"array(1, 2, 3)), (k, v) -> substring(k, 2))"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
- Row(Map("a" -> 1, "b" -> 2, "c" -> 3))
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT transform_keys(map_from_arrays(array('aa', 'bb', 'cc')," +
+ "array(1, 2, 3)), (k, v) -> substring(k, 2))",
+ collation,
+ result = Row(Map("a" -> 1, "b" -> 2, "c" -> 3)),
+ expectedType = MapType(
+ StringType(collation),
+ IntegerType,
+ false
)
)
- // check result row data type
- val dataType = MapType(
- StringType(collation), IntegerType, false
- )
- assert(sql(query).schema.head.dataType == dataType)
}
}
test("stack supports collation") {
- val query = s"SELECT stack(2, 'a', 'b', 'c')"
- val collation = "UNICODE"
- withSQLConf(SqlApiConf.DEFAULT_COLLATION -> collation) {
- checkAnswer(
- sql(query),
- Seq(
+ testAdditionalCollations.foreach { collation =>
+ testCollationSqlExpressionCommon(
+ query = "SELECT stack(2, 'a', 'b', 'c')",
+ collation,
+ result = Seq(
Row("a", "b"),
Row("c", null)
+ ),
+ expectedTypes = Seq(
+ StringType(collation)
)
)
- // check result row data type
- val dataType = StringType(collation)
- assert(sql(query).schema.head.dataType == dataType)
}
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CollationStringExpressionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CollationStringExpressionsSuite.scala
index 6db30a7ed0c6..2e942ef442ef 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CollationStringExpressionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CollationStringExpressionsSuite.scala
@@ -40,7 +40,8 @@ class CollationStringExpressionsSuite
ConcatWsTestCase(" ", Array("Spark", "SQL"), "UTF8_BINARY", "Spark SQL"),
ConcatWsTestCase(" ", Array("Spark", "SQL"), "UTF8_LCASE", "Spark SQL"),
ConcatWsTestCase(" ", Array("Spark", "SQL"), "UNICODE", "Spark SQL"),
- ConcatWsTestCase(" ", Array("Spark", "SQL"), "UNICODE_CI", "Spark SQL")
+ ConcatWsTestCase(" ", Array("Spark", "SQL"), "UNICODE_CI", "Spark SQL"),
+ ConcatWsTestCase(" ", Array("Spark", "Unterstützung"), "DE_CI_AI", "Spark Unterstützung")
)
testCases.foreach(t => {
// Unit test.
@@ -66,7 +67,8 @@ class CollationStringExpressionsSuite
EltTestCase(1, Array("Spark", "SQL"), "UTF8_BINARY", "Spark"),
EltTestCase(1, Array("Spark", "SQL"), "UTF8_LCASE", "Spark"),
EltTestCase(2, Array("Spark", "SQL"), "UNICODE", "SQL"),
- EltTestCase(2, Array("Spark", "SQL"), "UNICODE_CI", "SQL")
+ EltTestCase(2, Array("Spark", "SQL"), "UNICODE_CI", "SQL"),
+ EltTestCase(2, Array("Spark", "Unterstützung"), "DE_CI", "Unterstützung")
)
testCases.foreach(t => {
// Unit test.
@@ -96,7 +98,8 @@ class CollationStringExpressionsSuite
SplitPartTestCase("1a2", "a", 2, "UTF8_BINARY", "2"),
SplitPartTestCase("1a2", "a", 2, "UNICODE", "2"),
SplitPartTestCase("1a2", "A", 2, "UTF8_LCASE", "2"),
- SplitPartTestCase("1a2", "A", 2, "UNICODE_CI", "2")
+ SplitPartTestCase("1a2", "A", 2, "UNICODE_CI", "2"),
+ SplitPartTestCase("1ö2", "O", 2, "DE_CI_AI", "2")
)
val unsupportedTestCase = SplitPartTestCase("1a2", "a", 2, "UNICODE_AI", "2")
testCases.foreach(t => {
@@ -144,7 +147,9 @@ class CollationStringExpressionsSuite
StringSplitSQLTestCase("1a2", "a", "UTF8_BINARY", Array("1", "2")),
StringSplitSQLTestCase("1a2", "a", "UNICODE", Array("1", "2")),
StringSplitSQLTestCase("1a2", "A", "UTF8_LCASE", Array("1", "2")),
- StringSplitSQLTestCase("1a2", "A", "UNICODE_CI", Array("1", "2"))
+ StringSplitSQLTestCase("1a2", "A", "UNICODE_CI", Array("1", "2")),
+ StringSplitSQLTestCase("1ä2", "Ä", "DE_CI", Array("1", "2")),
+ StringSplitSQLTestCase("1ä2", "A", "DE_CI_AI", Array("1", "2"))
)
testCases.foreach(t => {
// Unit test.
@@ -189,7 +194,13 @@ class CollationStringExpressionsSuite
ContainsTestCase("", "", "UTF8_BINARY", true),
ContainsTestCase("abcde", "C", "UNICODE", false),
ContainsTestCase("abcde", "FGH", "UTF8_LCASE", false),
- ContainsTestCase("abcde", "BCD", "UNICODE_CI", true)
+ ContainsTestCase("abcde", "BCD", "UNICODE_CI", true),
+ ContainsTestCase("Priča o Maču u kamenu", "MAC", "SR_CI_AI", true),
+ ContainsTestCase("Priča o Maču u kamenu", "MAC", "SR_CI", false),
+ ContainsTestCase("Priča o Maču u kamenu", "MAČ", "SR", false),
+ ContainsTestCase("Priča o Maču u kamenu", "Mač", "SR", true),
+ ContainsTestCase("Прича о Мачу у камену", "мач", "sr_Cyrl_CI_AI", true),
+ ContainsTestCase("Прича о Мачу у камену", "мац", "sr_Cyrl_CI_AI", false)
)
val unsupportedTestCase = ContainsTestCase("abcde", "A", "UNICODE_AI", false)
testCases.foreach(t => {
@@ -237,7 +248,8 @@ class CollationStringExpressionsSuite
SubstringIndexTestCase("www||apache||org", "||", 2, "UTF8_BINARY", "www||apache"),
SubstringIndexTestCase("wwwXapacheXorg", "x", 2, "UTF8_LCASE", "wwwXapache"),
SubstringIndexTestCase("aaaaaaaaaa", "aa", 2, "UNICODE", "a"),
- SubstringIndexTestCase("wwwmapacheMorg", "M", -2, "UNICODE_CI", "apacheMorg")
+ SubstringIndexTestCase("wwwmapacheMorg", "M", -2, "UNICODE_CI", "apacheMorg"),
+ SubstringIndexTestCase("wwwüapacheüorg", "U", 2, "DE_CI_AI", "wwwüapache")
)
val unsupportedTestCase = SubstringIndexTestCase("abacde", "a", 2, "UNICODE_AI", "cde")
testCases.foreach(t => {
@@ -351,7 +363,9 @@ class CollationStringExpressionsSuite
StartsWithTestCase("", "", "UTF8_BINARY", true),
StartsWithTestCase("abcde", "A", "UNICODE", false),
StartsWithTestCase("abcde", "FGH", "UTF8_LCASE", false),
- StartsWithTestCase("abcde", "ABC", "UNICODE_CI", true)
+ StartsWithTestCase("abcde", "ABC", "UNICODE_CI", true),
+ StartsWithTestCase("Šuma", "šum", "SR_CI_AI", true),
+ StartsWithTestCase("Šuma", "šum", "SR", false)
)
val unsupportedTestCase = StartsWithTestCase("abcde", "A", "UNICODE_AI", false)
testCases.foreach(t => {
@@ -398,7 +412,8 @@ class CollationStringExpressionsSuite
StringTranslateTestCase("Translate", "Rnlt", "12", "UTF8_BINARY", "Tra2sae"),
StringTranslateTestCase("Translate", "Rnlt", "1234", "UTF8_LCASE", "41a2s3a4e"),
StringTranslateTestCase("Translate", "Rn", "\u0000\u0000", "UNICODE", "Traslate"),
- StringTranslateTestCase("Translate", "Rn", "1234", "UNICODE_CI", "T1a2slate")
+ StringTranslateTestCase("Translate", "Rn", "1234", "UNICODE_CI", "T1a2slate"),
+ StringTranslateTestCase("Êtèréêë", "te", "12", "AF_CI_AI", "212r222")
)
val unsupportedTestCase = StringTranslateTestCase("ABC", "AB", "12", "UNICODE_AI", "12C")
testCases.foreach(t => {
@@ -450,7 +465,8 @@ class CollationStringExpressionsSuite
StringReplaceTestCase("abcdabcd", "bc", "", "UNICODE", "adad"),
StringReplaceTestCase("aBc世abc", "b", "12", "UNICODE_CI", "a12c世a12c"),
StringReplaceTestCase("abi̇o12i̇o", "İo", "yy", "UNICODE_CI", "abyy12yy"),
- StringReplaceTestCase("abİo12i̇o", "i̇o", "xx", "UNICODE_CI", "abxx12xx")
+ StringReplaceTestCase("abİo12i̇o", "i̇o", "xx", "UNICODE_CI", "abxx12xx"),
+ StringReplaceTestCase("češalj", "eSal", "A", "SR_CI_AI", "čAj")
)
val unsupportedTestCase = StringReplaceTestCase("abcde", "A", "B", "UNICODE_AI", "abcde")
testCases.foreach(t => {
@@ -495,7 +511,13 @@ class CollationStringExpressionsSuite
EndsWithTestCase("", "", "UTF8_BINARY", true),
EndsWithTestCase("abcde", "E", "UNICODE", false),
EndsWithTestCase("abcde", "FGH", "UTF8_LCASE", false),
- EndsWithTestCase("abcde", "CDE", "UNICODE_CI", true)
+ EndsWithTestCase("abcde", "CDE", "UNICODE_CI", true),
+ EndsWithTestCase("xnigħat", "għat", "MT", true),
+ // The following two test cases showcase different behavior based on collation.
+ EndsWithTestCase("xnigħat", "ħat", "MT_CI", false),
+ EndsWithTestCase("muljavo", "javo", "SR_CI", true),
+ EndsWithTestCase("xnigħat", "GĦat", "MT_CI", true),
+ EndsWithTestCase("xnigħat", "Għat", "MT_CI", true)
)
val unsupportedTestCase = EndsWithTestCase("abcde", "A", "UNICODE_AI", false)
testCases.foreach(t => {
@@ -640,7 +662,8 @@ class CollationStringExpressionsSuite
FormatNumberTestCase(123.123, "###.###", "UTF8_BINARY", "123.123"),
FormatNumberTestCase(99.99, "##.##", "UTF8_LCASE", "99.99"),
FormatNumberTestCase(123.123, "###.###", "UNICODE", "123.123"),
- FormatNumberTestCase(99.99, "##.##", "UNICODE_CI", "99.99")
+ FormatNumberTestCase(99.99, "##.##", "UNICODE_CI", "99.99"),
+ FormatNumberTestCase(99.999, "##.###", "AF_CI_AI", "99.999")
)
testCases.foreach(t => {
// Unit test.
@@ -751,6 +774,11 @@ class CollationStringExpressionsSuite
"Something else. Nothing here.",
"UNICODE_CI",
Seq(Seq("Something", "else"), Seq("Nothing", "here"))
+ ),
+ SentencesTestCase(
+ "Hello, dinja! Ġurnata sabiħa.",
+ "MT_AI",
+ Seq(Seq("Hello", "dinja"), Seq("Ġurnata", "sabiħa"))
)
)
testCases.foreach(t => {
@@ -772,7 +800,8 @@ class CollationStringExpressionsSuite
UpperTestCase("aBc", "UTF8_BINARY", "ABC"),
UpperTestCase("aBc", "UTF8_LCASE", "ABC"),
UpperTestCase("aBc", "UNICODE", "ABC"),
- UpperTestCase("aBc", "UNICODE_CI", "ABC")
+ UpperTestCase("aBc", "UNICODE_CI", "ABC"),
+ UpperTestCase("xnìgħat", "MT_CI_AI", "XNÌGĦAT")
)
testCases.foreach(t => {
// Unit test.
@@ -792,7 +821,8 @@ class CollationStringExpressionsSuite
LowerTestCase("aBc", "UTF8_BINARY", "abc"),
LowerTestCase("aBc", "UTF8_LCASE", "abc"),
LowerTestCase("aBc", "UNICODE", "abc"),
- LowerTestCase("aBc", "UNICODE_CI", "abc")
+ LowerTestCase("aBc", "UNICODE_CI", "abc"),
+ LowerTestCase("VeRGrÖßeRn", "DE_CI", "vergrößern")
)
testCases.foreach(t => {
// Unit test.
@@ -812,7 +842,8 @@ class CollationStringExpressionsSuite
InitCapTestCase("aBc ABc", "UTF8_BINARY", "Abc Abc"),
InitCapTestCase("aBc ABc", "UTF8_LCASE", "Abc Abc"),
InitCapTestCase("aBc ABc", "UNICODE", "Abc Abc"),
- InitCapTestCase("aBc ABc", "UNICODE_CI", "Abc Abc")
+ InitCapTestCase("aBc ABc", "UNICODE_CI", "Abc Abc"),
+ InitCapTestCase("æØÅ ÆøÅ", "NO", "Æøå Æøå")
)
testCases.foreach(t => {
// Unit test.
@@ -866,7 +897,8 @@ class CollationStringExpressionsSuite
FormatStringTestCase("%s%s", Seq("a", "b"), "UTF8_BINARY", "ab"),
FormatStringTestCase("%d", Seq(123), "UTF8_LCASE", "123"),
FormatStringTestCase("%s%d", Seq("A", 0), "UNICODE", "A0"),
- FormatStringTestCase("%s%s", Seq("Hello", "!!!"), "UNICODE_CI", "Hello!!!")
+ FormatStringTestCase("%s%s", Seq("Hello", "!!!"), "UNICODE_CI", "Hello!!!"),
+ FormatStringTestCase("%s%s", Seq("Storslått", ".?!"), "NN_AI", "Storslått.?!")
)
testCases.foreach(t => {
// Unit test.
@@ -897,7 +929,8 @@ class CollationStringExpressionsSuite
SoundExTestCase("A", "UTF8_BINARY", "A000"),
SoundExTestCase("!", "UTF8_LCASE", "!"),
SoundExTestCase("$", "UNICODE", "$"),
- SoundExTestCase("X", "UNICODE_CI", "X000")
+ SoundExTestCase("X", "UNICODE_CI", "X000"),
+ SoundExTestCase("ß", "DE", "ß")
)
testCases.foreach(t => {
// Unit test.
@@ -917,7 +950,9 @@ class CollationStringExpressionsSuite
LengthTestCase("", "UTF8_BINARY", 0),
LengthTestCase("abc", "UTF8_LCASE", 3),
LengthTestCase("hello", "UNICODE", 5),
- LengthTestCase("ff", "UNICODE_CI", 1)
+ LengthTestCase("ff", "UNICODE_CI", 1),
+ LengthTestCase("groß", "DE_CI_AI", 4),
+ LengthTestCase("gross", "DE_AI", 5)
)
testCases.foreach(t => {
// Unit test.
@@ -937,7 +972,8 @@ class CollationStringExpressionsSuite
BitLengthTestCase("", "UTF8_BINARY", 0),
BitLengthTestCase("abc", "UTF8_LCASE", 24),
BitLengthTestCase("hello", "UNICODE", 40),
- BitLengthTestCase("ff", "UNICODE_CI", 24)
+ BitLengthTestCase("ff", "UNICODE_CI", 24),
+ BitLengthTestCase("GROß", "DE", 40)
)
testCases.foreach(t => {
// Unit test.
@@ -1002,7 +1038,10 @@ class CollationStringExpressionsSuite
LevenshteinTestCase("kitten", "sitTing", "UTF8_BINARY", None, 4),
LevenshteinTestCase("kitten", "sitTing", "UTF8_LCASE", None, 4),
LevenshteinTestCase("kitten", "sitTing", "UNICODE", Some(3), -1),
- LevenshteinTestCase("kitten", "sitTing", "UNICODE_CI", Some(3), -1)
+ LevenshteinTestCase("kitten", "sitTing", "UNICODE_CI", Some(3), -1),
+ // Levenshtein function is currently not collation-aware (not considering case or accent).
+ LevenshteinTestCase("gr", "GR", "UNICODE_CI_AI", None, 2),
+ LevenshteinTestCase("groß", "Größer", "UNICODE_CI_AI", None, 4)
)
testCases.foreach(t => {
// Unit test.
@@ -1026,7 +1065,8 @@ class CollationStringExpressionsSuite
IsValidUTF8TestCase(null, "UTF8_BINARY", null),
IsValidUTF8TestCase("", "UTF8_LCASE", true),
IsValidUTF8TestCase("abc", "UNICODE", true),
- IsValidUTF8TestCase("hello", "UNICODE_CI", true)
+ IsValidUTF8TestCase("hello", "UNICODE_CI", true),
+ IsValidUTF8TestCase("ćao", "SR_CI_AI", true)
)
testCases.foreach(t => {
// Unit test.
@@ -1124,7 +1164,8 @@ class CollationStringExpressionsSuite
SubstringTestCase(null, null, Some(null), "UTF8_BINARY", null),
SubstringTestCase(null, null, None, "UNICODE_CI", null),
SubstringTestCase("ÀÃÂĀĂȦÄäåäáâãȻȻȻȻȻǢǼÆ", null, None, "UTF8_BINARY", null),
- SubstringTestCase("", null, None, "UNICODE_CI", null)
+ SubstringTestCase("", null, None, "UNICODE_CI", null),
+ SubstringTestCase("xnigħat", 4, Some(2), "MT_CI_AI", "għ")
)
testCases.foreach(t => {
// Unit test.
@@ -1149,7 +1190,8 @@ class CollationStringExpressionsSuite
LeftTestCase(null, null, "UTF8_BINARY", null),
LeftTestCase(" a世a ", 3, "UTF8_LCASE", " a世"),
LeftTestCase("", 1, "UNICODE", ""),
- LeftTestCase("ÀÃÂĀĂȦÄäåäáâãȻȻȻȻȻǢǼÆ", 3, "UNICODE", "ÀÃÂ")
+ LeftTestCase("ÀÃÂĀĂȦÄäåäáâãȻȻȻȻȻǢǼÆ", 3, "UNICODE", "ÀÃÂ"),
+ LeftTestCase("ÀÃÂĀĂȦÄäåäáâãȻȻȻȻȻǢǼÆ", 7, "NO_AI", "ÀÃÂĀĂȦÄ")
)
testCases.foreach(t => {
// Unit test.
@@ -1172,7 +1214,8 @@ class CollationStringExpressionsSuite
RightTestCase(null, null, "UTF8_BINARY", null),
RightTestCase(" a世a ", 3, "UTF8_LCASE", "世a "),
RightTestCase("", 1, "UNICODE", ""),
- RightTestCase("ÀÃÂĀĂȦÄäåäáâãȻȻȻȻȻǢǼÆ", 3, "UNICODE", "ǢǼÆ")
+ RightTestCase("ÀÃÂĀĂȦÄäåäáâãȻȻȻȻȻǢǼÆ", 3, "UNICODE", "ǢǼÆ"),
+ RightTestCase("ÀÃÂĀĂȦÄäåäáâãȻȻȻȻȻǢǼÆ", 5, "NO_CI_AI", "ȻȻǢǼÆ")
)
testCases.foreach(t => {
// Unit test.
@@ -1268,7 +1311,10 @@ class CollationStringExpressionsSuite
StringLocateTestCase("aa", "Aaads", 0, "UNICODE", 0),
StringLocateTestCase("abC", "abCabC", 2, "UNICODE", 4),
StringLocateTestCase("aa", "Aaads", 0, "UNICODE_CI", 0),
- StringLocateTestCase("界x", "test大千世界X大千世界", 1, "UNICODE_CI", 8)
+ StringLocateTestCase("界x", "test大千世界X大千世界", 1, "UNICODE_CI", 8),
+ StringLocateTestCase("oa", "TÖäöäoAoa", 1, "DE", 8),
+ StringLocateTestCase("oa", "TÖäöäoAoa", 1, "DE_CI", 6),
+ StringLocateTestCase("oa", "TÖäöäoAoa", 1, "DE_CI_AI", 2)
)
val unsupportedTestCase = StringLocateTestCase("aa", "Aaads", 0, "UNICODE_AI", 1)
testCases.foreach(t => {
@@ -1316,7 +1362,10 @@ class CollationStringExpressionsSuite
StringTrimLeftTestCase("xxasdxx", Some("x"), "UTF8_BINARY", "asdxx"),
StringTrimLeftTestCase("xxasdxx", Some("X"), "UTF8_LCASE", "asdxx"),
StringTrimLeftTestCase("xxasdxx", Some("y"), "UNICODE", "xxasdxx"),
- StringTrimLeftTestCase(" asd ", None, "UNICODE_CI", "asd ")
+ StringTrimLeftTestCase(" asd ", None, "UNICODE_CI", "asd "),
+ StringTrimLeftTestCase("ćĆčČcCabCcČčĆć", Some("ć"), "SR", "ĆčČcCabCcČčĆć"),
+ StringTrimLeftTestCase("ćĆčČcCabCcČčĆć", Some("Ć"), "SR_CI", "čČcCabCcČčĆć"),
+ StringTrimLeftTestCase("ćĆčČcCabCcČčĆć", Some("ć"), "SR_CI_AI", "abCcČčĆć")
)
val unsupportedTestCase = StringTrimLeftTestCase("xxasdxx", Some("x"), "UNICODE_AI", null)
testCases.foreach(t => {
@@ -1363,7 +1412,10 @@ class CollationStringExpressionsSuite
StringTrimRightTestCase("xxasdxx", Some("x"), "UTF8_BINARY", "xxasd"),
StringTrimRightTestCase("xxasdxx", Some("X"), "UTF8_LCASE", "xxasd"),
StringTrimRightTestCase("xxasdxx", Some("y"), "UNICODE", "xxasdxx"),
- StringTrimRightTestCase(" asd ", None, "UNICODE_CI", " asd")
+ StringTrimRightTestCase(" asd ", None, "UNICODE_CI", " asd"),
+ StringTrimRightTestCase("ćĆčČcCabCcČčĆć", Some("ć"), "SR", "ćĆčČcCabCcČčĆ"),
+ StringTrimRightTestCase("ćĆčČcCabCcČčĆć", Some("Ć"), "SR_CI", "ćĆčČcCabCcČč"),
+ StringTrimRightTestCase("ćĆčČcCabCcČčĆć", Some("ć"), "SR_CI_AI", "ćĆčČcCab")
)
val unsupportedTestCase = StringTrimRightTestCase("xxasdxx", Some("x"), "UNICODE_AI", "xxasd")
testCases.foreach(t => {
@@ -1411,7 +1463,11 @@ class CollationStringExpressionsSuite
StringTrimTestCase("xxasdxx", Some("x"), "UTF8_BINARY", "asd"),
StringTrimTestCase("xxasdxx", Some("X"), "UTF8_LCASE", "asd"),
StringTrimTestCase("xxasdxx", Some("y"), "UNICODE", "xxasdxx"),
- StringTrimTestCase(" asd ", None, "UNICODE_CI", "asd")
+ StringTrimTestCase(" asd ", None, "UNICODE_CI", "asd"),
+ StringTrimTestCase("ćĆčČcCabCcČčĆć", Some("ć"), "SR", "ĆčČcCabCcČčĆ"),
+ StringTrimTestCase("ćĆčČcCabCcČčĆć", Some("Ć"), "SR_CI", "čČcCabCcČč"),
+ StringTrimTestCase("ćĆčČcCabCcČčĆć", Some("ć"), "SR_CI_AI", "ab"),
+ StringTrimTestCase(" ćĆčČcCabCcČčĆć ", None, "SR_CI_AI", "ćĆčČcCabCcČčĆć")
)
val unsupportedTestCase = StringTrimTestCase("xxasdxx", Some("x"), "UNICODE_AI", "asd")
testCases.foreach(t => {
@@ -1458,7 +1514,11 @@ class CollationStringExpressionsSuite
StringTrimBothTestCase("xxasdxx", Some("x"), "UTF8_BINARY", "asd"),
StringTrimBothTestCase("xxasdxx", Some("X"), "UTF8_LCASE", "asd"),
StringTrimBothTestCase("xxasdxx", Some("y"), "UNICODE", "xxasdxx"),
- StringTrimBothTestCase(" asd ", None, "UNICODE_CI", "asd")
+ StringTrimBothTestCase(" asd ", None, "UNICODE_CI", "asd"),
+ StringTrimBothTestCase("ćĆčČcCabCcČčĆć", Some("ć"), "SR", "ĆčČcCabCcČčĆ"),
+ StringTrimBothTestCase("ćĆčČcCabCcČčĆć", Some("Ć"), "SR_CI", "čČcCabCcČč"),
+ StringTrimBothTestCase("ćĆčČcCabCcČčĆć", Some("ć"), "SR_CI_AI", "ab"),
+ StringTrimBothTestCase(" ćĆčČcCabCcČčĆć ", None, "SR_CI_AI", "ćĆčČcCabCcČčĆć")
)
testCases.foreach(t => {
// Unit test.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala
index f2444df38b3b..3a5a650e5a0c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala
@@ -1941,7 +1941,7 @@ class CollationSuite extends DatasourceV2SQLBase with AdaptiveSparkPlanHelper {
}
test("cache table with collated columns") {
- val collations = Seq("UTF8_BINARY", "UTF8_LCASE", "UNICODE", "UNICODE_CI")
+ val collations = Seq("UTF8_BINARY", "UTF8_LCASE", "UNICODE", "UNICODE_CI", "SR_CI_AI")
val lazyOptions = Seq(false, true)
for (
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org