You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2024/10/27 09:28:54 UTC
(spark) branch master updated: [SPARK-49665][SQL] Trim collation support for string functions
This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new eed1530790b6 [SPARK-49665][SQL] Trim collation support for string functions
eed1530790b6 is described below
commit eed1530790b6597163cc9d021d4729e48d24e9ed
Author: Jovan Pavlovic <jo...@databricks.com>
AuthorDate: Sun Oct 27 10:28:33 2024 +0100
[SPARK-49665][SQL] Trim collation support for string functions
### What changes were proposed in this pull request?
Implement trim support for string functions. Not all functions are covered in this pr, follow up: https://issues.apache.org/jira/browse/SPARK-50012
### Why are the changes needed?
Implement trim support for string functions. Not all functions are covered in this pr, follow up:
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Tests added in CollationStringExpressionsSuite.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #48521 from jovanpavl-db/trim-collation-string-functions-implementation.
Authored-by: Jovan Pavlovic <jo...@databricks.com>
Signed-off-by: Max Gekk <ma...@gmail.com>
---
.../catalyst/util/CollationAwareUTF8String.java | 187 +++++-
.../spark/sql/catalyst/util/CollationFactory.java | 35 ++
.../spark/sql/catalyst/util/CollationSupport.java | 169 +++---
.../org/apache/spark/unsafe/types/UTF8String.java | 1 +
.../catalyst/expressions/stringExpressions.scala | 231 ++++++--
.../sql-tests/analyzer-results/collations.sql.out | 136 +++++
.../test/resources/sql-tests/inputs/collations.sql | 18 +-
.../resources/sql-tests/results/collations.sql.out | 632 +++++++++++++++++++++
.../sql/CollationStringExpressionsSuite.scala | 285 +++++++++-
9 files changed, 1540 insertions(+), 154 deletions(-)
diff --git a/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationAwareUTF8String.java b/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationAwareUTF8String.java
index d67697eaea38..97c8bbe562af 100644
--- a/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationAwareUTF8String.java
+++ b/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationAwareUTF8String.java
@@ -1080,6 +1080,24 @@ public class CollationAwareUTF8String {
return UTF8String.fromString(sb.toString());
}
+ /**
+ * Trims the `srcString` string from both ends of the string using the specified `trimString`
+ * characters, with respect to the UTF8_BINARY trim collation. String trimming is performed by
+ * first trimming the left side of the string, and then trimming the right side of the string.
+ * The method returns the trimmed string. If the `trimString` is null, the method returns null.
+ *
+ * @param srcString the input string to be trimmed from both ends of the string
+ * @param trimString the trim string characters to trim
+ * @param collationId the collation ID to use for string trim
+ * @return the trimmed string (for UTF8_BINARY collation)
+ */
+ public static UTF8String binaryTrim(
+ final UTF8String srcString,
+ final UTF8String trimString,
+ final int collationId) {
+ return binaryTrimRight(srcString.trimLeft(trimString), trimString, collationId);
+ }
+
/**
* Trims the `srcString` string from both ends of the string using the specified `trimString`
* characters, with respect to the UTF8_LCASE collation. String trimming is performed by
@@ -1088,12 +1106,14 @@ public class CollationAwareUTF8String {
*
* @param srcString the input string to be trimmed from both ends of the string
* @param trimString the trim string characters to trim
+ * @param collationId the collation ID to use for string trim
* @return the trimmed string (for UTF8_LCASE collation)
*/
public static UTF8String lowercaseTrim(
final UTF8String srcString,
- final UTF8String trimString) {
- return lowercaseTrimRight(lowercaseTrimLeft(srcString, trimString), trimString);
+ final UTF8String trimString,
+ final int collationId) {
+ return lowercaseTrimRight(lowercaseTrimLeft(srcString, trimString), trimString, collationId);
}
/**
@@ -1121,7 +1141,8 @@ public class CollationAwareUTF8String {
* the left side, until reaching a character whose lowercased code point is not in the hash set.
* Finally, the method returns the substring from that position to the end of `srcString`.
* If `trimString` is null, null is returned. If `trimString` is empty, `srcString` is returned.
- *
+ * Note: as currently only trimming collation supported is RTRIM, trimLeft is not modified
+ * to support other trim collations, this should be done in case of adding TRIM and LTRIM.
* @param srcString the input string to be trimmed from the left end of the string
* @param trimString the trim string characters to trim
* @return the trimmed string (for UTF8_LCASE collation)
@@ -1184,7 +1205,9 @@ public class CollationAwareUTF8String {
* character in `trimString`, until reaching a character that is not found in `trimString`.
* Finally, the method returns the substring from that position to the end of `srcString`.
* If `trimString` is null, null is returned. If `trimString` is empty, `srcString` is returned.
- *
+ * Note: as currently only trimming collation supported is RTRIM, trimLeft is not modified
+ * to support other trim collations, this should be done in case of adding TRIM and LTRIM
+ * collation.
* @param srcString the input string to be trimmed from the left end of the string
* @param trimString the trim string characters to trim
* @param collationId the collation ID to use for string trimming
@@ -1232,22 +1255,103 @@ public class CollationAwareUTF8String {
// Return the substring from the calculated position until the end of the string.
return UTF8String.fromString(src.substring(charIndex));
}
+ /**
+ * Trims the `srcString` string from the right side using the specified `trimString` characters,
+ * with respect to the UTF8_BINARY trim collation. For UTF8_BINARY trim collation, the method has
+ * one special case to cover with respect to trimRight function for regular UTF8_Binary collation.
+ * Trailing spaces should be ignored in case of trim collation (rtrim for example) and if
+ * trimString does not contain spaces. In this case, the method trims the string from the right
+ * and after call of regular trim functions returns back trimmed spaces as those should not get
+ * removed.
+ * @param srcString the input string to be trimmed from the right end of the string
+ * @param trimString the trim string characters to trim
+ * @param collationId the collation ID to use for string trim
+ * @return the trimmed string (for UTF_BINARY collation)
+ */
+ public static UTF8String binaryTrimRight(
+ final UTF8String srcString,
+ final UTF8String trimString,
+ final int collationId) {
+ // Matching the default UTF8String behavior for null `trimString`.
+ if (trimString == null) {
+ return null;
+ }
+
+ // Create a hash set of code points for all characters of `trimString`.
+ HashSet<Integer> trimChars = new HashSet<>();
+ Iterator<Integer> trimIter = trimString.codePointIterator();
+ while (trimIter.hasNext()) trimChars.add(trimIter.next());
+
+ // Iterate over `srcString` from the right to find the first character that is not in the set.
+ int searchIndex = srcString.numChars(), codePoint, codePointBuffer = -1;
+
+ // In cases of trim collation (rtrim for example) trailing spaces should be ignored.
+ // If trimString contains spaces this behaviour is not important as they would get trimmed
+ // anyway. However, if it is not the case they should be ignored and then appended after
+ // trimming other characters.
+ int lastNonSpaceByteIdx = srcString.numBytes(), lastNonSpaceCharacterIdx = srcString.numChars();
+ if (!trimChars.contains(SpecialCodePointConstants.ASCII_SPACE) &&
+ CollationFactory.ignoresSpacesInTrimFunctions(
+ collationId, /*isLTrim=*/ false, /*isRTrim=*/true)) {
+ while (lastNonSpaceByteIdx > 0 &&
+ srcString.getByte(lastNonSpaceByteIdx - 1) == ' ') {
+ --lastNonSpaceByteIdx;
+ }
+ // In case of src string contains only spaces there is no need to do any trimming, since it's
+ // already checked that trim string does not contain any spaces.
+ if (lastNonSpaceByteIdx == 0) {
+ return srcString;
+ }
+ searchIndex = lastNonSpaceCharacterIdx =
+ srcString.numChars() - (srcString.numBytes() - lastNonSpaceByteIdx);
+ }
+ Iterator<Integer> srcIter = srcString.reverseCodePointIterator();
+ for (int i = lastNonSpaceCharacterIdx; i < srcString.numChars(); i++) {
+ srcIter.next();
+ }
+
+ while (srcIter.hasNext()) {
+ codePoint = srcIter.next();
+ if (trimChars.contains(codePoint)) {
+ --searchIndex;
+ }
+ else {
+ break;
+ }
+ }
+
+ // Return the substring from the start of the string to the calculated position and append
+ // trailing spaces if they were ignored
+ if (searchIndex == srcString.numChars()) {
+ return srcString;
+ }
+ if (lastNonSpaceCharacterIdx == srcString.numChars()) {
+ return srcString.substring(0, searchIndex);
+ }
+ return UTF8String.concat(
+ srcString.substring(0, searchIndex),
+ srcString.substring(lastNonSpaceCharacterIdx, srcString.numChars()));
+ }
/**
* Trims the `srcString` string from the right side using the specified `trimString` characters,
* with respect to the UTF8_LCASE collation. For UTF8_LCASE, the method first creates a hash
* set of lowercased code points in `trimString`, and then iterates over the `srcString` from
* the right side, until reaching a character whose lowercased code point is not in the hash set.
+ * In case of UTF8_LCASE trim collation and when trimString does not contain spaces, trailing
+ * spaces should be ignored. However, after trimming function call they should be appended back.
* Finally, the method returns the substring from the start of `srcString` until that position.
* If `trimString` is null, null is returned. If `trimString` is empty, `srcString` is returned.
*
* @param srcString the input string to be trimmed from the right end of the string
* @param trimString the trim string characters to trim
+ * @param collationId the collation ID to use for string trim
* @return the trimmed string (for UTF8_LCASE collation)
*/
public static UTF8String lowercaseTrimRight(
final UTF8String srcString,
- final UTF8String trimString) {
+ final UTF8String trimString,
+ final int collationId) {
// Matching the default UTF8String behavior for null `trimString`.
if (trimString == null) {
return null;
@@ -1260,7 +1364,32 @@ public class CollationAwareUTF8String {
// Iterate over `srcString` from the right to find the first character that is not in the set.
int searchIndex = srcString.numChars(), codePoint, codePointBuffer = -1;
+
+ // In cases of trim collation (rtrim for example) trailing spaces should be ignored.
+ // If trimString contains spaces this behaviour is not important as they would get trimmed
+ // anyway. However, if it is not the case they should be ignored and then appended after
+ // trimming other characters.
+ int lastNonSpaceByteIdx = srcString.numBytes(), lastNonSpaceCharacterIdx = srcString.numChars();
+ if (!trimChars.contains(SpecialCodePointConstants.ASCII_SPACE) &&
+ CollationFactory.ignoresSpacesInTrimFunctions(
+ collationId, /*isLTrim=*/ false, /*isRTrim=*/true)) {
+ while (lastNonSpaceByteIdx > 0 &&
+ srcString.getByte(lastNonSpaceByteIdx - 1) == ' ') {
+ --lastNonSpaceByteIdx;
+ }
+ // In case of src string contains only spaces there is no need to do any trimming, since it's
+ // already checked that trim string does not contain any spaces.
+ if (lastNonSpaceByteIdx == 0) {
+ return srcString;
+ }
+ searchIndex = lastNonSpaceCharacterIdx =
+ srcString.numChars() - (srcString.numBytes() - lastNonSpaceByteIdx);
+ }
Iterator<Integer> srcIter = srcString.reverseCodePointIterator();
+ for (int i = lastNonSpaceCharacterIdx; i < srcString.numChars(); i++) {
+ srcIter.next();
+ }
+
while (srcIter.hasNext()) {
if (codePointBuffer != -1) {
codePoint = codePointBuffer;
@@ -1291,8 +1420,17 @@ public class CollationAwareUTF8String {
}
}
- // Return the substring from the start of the string to the calculated position.
- return searchIndex == srcString.numChars() ? srcString : srcString.substring(0, searchIndex);
+ // Return the substring from the start of the string to the calculated position and append
+ // trailing spaces if they were ignored
+ if (searchIndex == srcString.numChars()) {
+ return srcString;
+ }
+ if (lastNonSpaceCharacterIdx == srcString.numChars()) {
+ return srcString.substring(0, searchIndex);
+ }
+ return UTF8String.concat(
+ srcString.substring(0, searchIndex),
+ srcString.substring(lastNonSpaceCharacterIdx, srcString.numChars()));
}
/**
@@ -1329,7 +1467,26 @@ public class CollationAwareUTF8String {
String src = srcString.toValidString();
CharacterIterator target = new StringCharacterIterator(src);
Collator collator = CollationFactory.fetchCollation(collationId).collator;
- int charIndex = src.length(), longestMatchLen;
+ int charIndex = src.length(), longestMatchLen, lastNonSpacePosition = src.length();
+
+ // In cases of trim collation (rtrim for example) trailing spaces should be ignored.
+ // If trimString contains spaces this behaviour is not important as they would get trimmed
+ // anyway. However, if it is not the case they should be ignored and then appended after
+ // trimming other characters.
+ if (!trimChars.containsKey(SpecialCodePointConstants.ASCII_SPACE) &&
+ CollationFactory.ignoresSpacesInTrimFunctions(
+ collationId, /*isLTrim=*/ false, /*isRTrim=*/true)) {
+ while (lastNonSpacePosition > 0 && src.charAt(lastNonSpacePosition - 1) == ' ') {
+ --lastNonSpacePosition;
+ }
+ // In case of src string contains only spaces there is no need to do any trimming, since it's
+ // already checked that trim string does not contain any spaces.
+ if (lastNonSpacePosition == 0) {
+ return UTF8String.fromString(src);
+ }
+ charIndex = lastNonSpacePosition;
+ }
+
while (charIndex >= 0) {
longestMatchLen = 0;
for (String trim : trimChars.values()) {
@@ -1357,8 +1514,18 @@ public class CollationAwareUTF8String {
else charIndex -= longestMatchLen;
}
- // Return the substring from the start of the string until that position.
- return UTF8String.fromString(src.substring(0, charIndex));
+ // Return the substring from the start of the string until that position and append
+ // trailing spaces if they were ignored
+ if (charIndex == src.length()) {
+ return srcString;
+ }
+ if (lastNonSpacePosition == srcString.numChars()) {
+ return UTF8String.fromString(src.substring(0, charIndex));
+ }
+ return UTF8String.fromString(
+ src.substring(0, charIndex) +
+ src.substring(lastNonSpacePosition)
+ );
}
public static UTF8String[] splitSQL(final UTF8String input, final UTF8String delim,
diff --git a/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationFactory.java b/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationFactory.java
index 1305d82bcd78..3117854a432b 100644
--- a/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationFactory.java
+++ b/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationFactory.java
@@ -359,6 +359,23 @@ public final class CollationFactory {
return applyTrimmingPolicy(s, getSpaceTrimming(collationId));
}
+ /**
+ * Returns if leading/trailing spaces should be ignored in trim string expressions. This is
+ * needed because space trimming collation directly changes behaviour of trim functions.
+ */
+ protected static boolean ignoresSpacesInTrimFunctions(
+ int collationId,
+ boolean isLTrim,
+ boolean isRTrim) {
+ if (isRTrim && getSpaceTrimming(collationId) == SpaceTrimming.RTRIM) {
+ return true;
+ }
+
+ // In case of adding new trimming collations in the future (LTRIM and TRIM) here logic
+ // should be added.
+ return false;
+ }
+
/**
* Utility function to trim spaces when collation uses space trimming.
*/
@@ -1200,6 +1217,24 @@ public final class CollationFactory {
return Collation.CollationSpecICU.ICULocaleNames;
}
+ /**
+ * Applies trimming policy depending up on trim collation type.
+ */
+ public static UTF8String applyTrimmingPolicy(UTF8String input, int collationId) {
+ return Collation.CollationSpec.applyTrimmingPolicy(input, collationId);
+ }
+
+ /**
+ * Returns if leading/trailing spaces should be ignored in trim string expressions. This is needed
+ * because space trimming collation directly changes behaviour of trim functions.
+ */
+ public static boolean ignoresSpacesInTrimFunctions(
+ int collationId,
+ boolean isLTrim,
+ boolean isRTrim) {
+ return Collation.CollationSpec.ignoresSpacesInTrimFunctions(collationId, isLTrim, isRTrim);
+ }
+
public static UTF8String getCollationKey(UTF8String input, int collationId) {
Collation collation = fetchCollation(collationId);
if (collation.supportsSpaceTrimming) {
diff --git a/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationSupport.java b/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationSupport.java
index 978b663cc25c..135250e482b1 100644
--- a/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationSupport.java
+++ b/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationSupport.java
@@ -35,8 +35,11 @@ public final class CollationSupport {
*/
public static class StringSplitSQL {
- public static UTF8String[] exec(final UTF8String s, final UTF8String d, final int collationId) {
+ public static UTF8String[] exec(final UTF8String s, UTF8String d, final int collationId) {
CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
+ if (collation.supportsSpaceTrimming) {
+ d = CollationFactory.applyTrimmingPolicy(d, collationId);
+ }
if (collation.isUtf8BinaryType) {
return execBinary(s, d);
} else if (collation.isUtf8LcaseType) {
@@ -46,14 +49,11 @@ public final class CollationSupport {
}
}
public static String genCode(final String s, final String d, final int collationId) {
- CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
String expr = "CollationSupport.StringSplitSQL.exec";
- if (collation.isUtf8BinaryType) {
+ if (collationId == CollationFactory.UTF8_BINARY_COLLATION_ID) {
return String.format(expr + "Binary(%s, %s)", s, d);
- } else if (collation.isUtf8LcaseType) {
- return String.format(expr + "Lowercase(%s, %s)", s, d);
} else {
- return String.format(expr + "ICU(%s, %s, %d)", s, d, collationId);
+ return String.format(expr + "(%s, %s, %d)", s, d, collationId);
}
}
public static UTF8String[] execBinary(final UTF8String string, final UTF8String delimiter) {
@@ -69,8 +69,12 @@ public final class CollationSupport {
}
public static class Contains {
- public static boolean exec(final UTF8String l, final UTF8String r, final int collationId) {
+ public static boolean exec(UTF8String l, UTF8String r, final int collationId) {
CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
+ if (collation.supportsSpaceTrimming) {
+ l = CollationFactory.applyTrimmingPolicy(l, collationId);
+ r = CollationFactory.applyTrimmingPolicy(r, collationId);
+ }
if (collation.isUtf8BinaryType) {
return execBinary(l, r);
} else if (collation.isUtf8LcaseType) {
@@ -80,14 +84,11 @@ public final class CollationSupport {
}
}
public static String genCode(final String l, final String r, final int collationId) {
- CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
String expr = "CollationSupport.Contains.exec";
- if (collation.isUtf8BinaryType) {
+ if (collationId == CollationFactory.UTF8_BINARY_COLLATION_ID) {
return String.format(expr + "Binary(%s, %s)", l, r);
- } else if (collation.isUtf8LcaseType) {
- return String.format(expr + "Lowercase(%s, %s)", l, r);
} else {
- return String.format(expr + "ICU(%s, %s, %d)", l, r, collationId);
+ return String.format(expr + "(%s, %s, %d)", l, r, collationId);
}
}
public static boolean execBinary(final UTF8String l, final UTF8String r) {
@@ -106,9 +107,14 @@ public final class CollationSupport {
}
public static class StartsWith {
- public static boolean exec(final UTF8String l, final UTF8String r,
+ public static boolean exec(UTF8String l, UTF8String r,
final int collationId) {
CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
+ if (collation.supportsSpaceTrimming) {
+ l = CollationFactory.applyTrimmingPolicy(l, collationId);
+ r = CollationFactory.applyTrimmingPolicy(r, collationId);
+ }
+
if (collation.isUtf8BinaryType) {
return execBinary(l, r);
} else if (collation.isUtf8LcaseType) {
@@ -118,14 +124,11 @@ public final class CollationSupport {
}
}
public static String genCode(final String l, final String r, final int collationId) {
- CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
String expr = "CollationSupport.StartsWith.exec";
- if (collation.isUtf8BinaryType) {
+ if (collationId == CollationFactory.UTF8_BINARY_COLLATION_ID) {
return String.format(expr + "Binary(%s, %s)", l, r);
- } else if (collation.isUtf8LcaseType) {
- return String.format(expr + "Lowercase(%s, %s)", l, r);
} else {
- return String.format(expr + "ICU(%s, %s, %d)", l, r, collationId);
+ return String.format(expr + "(%s, %s, %d)", l, r, collationId);
}
}
public static boolean execBinary(final UTF8String l, final UTF8String r) {
@@ -144,8 +147,12 @@ public final class CollationSupport {
}
public static class EndsWith {
- public static boolean exec(final UTF8String l, final UTF8String r, final int collationId) {
+ public static boolean exec(UTF8String l, UTF8String r, final int collationId) {
CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
+ if (collation.supportsSpaceTrimming) {
+ l = CollationFactory.applyTrimmingPolicy(l, collationId);
+ r = CollationFactory.applyTrimmingPolicy(r, collationId);
+ }
if (collation.isUtf8BinaryType) {
return execBinary(l, r);
} else if (collation.isUtf8LcaseType) {
@@ -155,14 +162,11 @@ public final class CollationSupport {
}
}
public static String genCode(final String l, final String r, final int collationId) {
- CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
String expr = "CollationSupport.EndsWith.exec";
- if (collation.isUtf8BinaryType) {
+ if (collationId == CollationFactory.UTF8_BINARY_COLLATION_ID) {
return String.format(expr + "Binary(%s, %s)", l, r);
- } else if (collation.isUtf8LcaseType) {
- return String.format(expr + "Lowercase(%s, %s)", l, r);
} else {
- return String.format(expr + "ICU(%s, %s, %d)", l, r, collationId);
+ return String.format(expr + "(%s, %s, %d)", l, r, collationId);
}
}
public static boolean execBinary(final UTF8String l, final UTF8String r) {
@@ -184,6 +188,7 @@ public final class CollationSupport {
public static class Upper {
public static UTF8String exec(final UTF8String v, final int collationId, boolean useICU) {
CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
+ // Space trimming does not affect the output of this expression.
if (collation.isUtf8BinaryType) {
return useICU ? execBinaryICU(v) : execBinary(v);
} else if (collation.isUtf8LcaseType) {
@@ -221,6 +226,7 @@ public final class CollationSupport {
public static class Lower {
public static UTF8String exec(final UTF8String v, final int collationId, boolean useICU) {
CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
+ // Space trimming does not affect the output of this expression.
if (collation.isUtf8BinaryType) {
return useICU ? execBinaryICU(v) : execBinary(v);
} else if (collation.isUtf8LcaseType) {
@@ -258,6 +264,7 @@ public final class CollationSupport {
public static class InitCap {
public static UTF8String exec(final UTF8String v, final int collationId, boolean useICU) {
CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
+ // Space trimming does not affect the output of this expression.
if (collation.isUtf8BinaryType) {
return useICU ? execBinaryICU(v) : execBinary(v);
} else if (collation.isUtf8LcaseType) {
@@ -295,17 +302,16 @@ public final class CollationSupport {
public static class FindInSet {
public static int exec(final UTF8String word, final UTF8String set, final int collationId) {
- CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
- if (collation.isUtf8BinaryType) {
+ // FindInSet does space trimming collation as comparison is space trimming collation aware
+ if (collationId == CollationFactory.UTF8_BINARY_COLLATION_ID) {
return execBinary(word, set);
} else {
return execCollationAware(word, set, collationId);
}
}
public static String genCode(final String word, final String set, final int collationId) {
- CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
String expr = "CollationSupport.FindInSet.exec";
- if (collation.isUtf8BinaryType) {
+ if (collationId == CollationFactory.UTF8_BINARY_COLLATION_ID) {
return String.format(expr + "Binary(%s, %s)", word, set);
} else {
return String.format(expr + "CollationAware(%s, %s, %d)", word, set, collationId);
@@ -321,9 +327,12 @@ public final class CollationSupport {
}
public static class StringInstr {
- public static int exec(final UTF8String string, final UTF8String substring,
+ public static int exec(final UTF8String string, UTF8String substring,
final int collationId) {
CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
+ if (collation.supportsSpaceTrimming) {
+ substring = CollationFactory.applyTrimmingPolicy(substring, collationId);
+ }
if (collation.isUtf8BinaryType) {
return execBinary(string, substring);
} else if (collation.isUtf8LcaseType) {
@@ -334,14 +343,11 @@ public final class CollationSupport {
}
public static String genCode(final String string, final String substring,
final int collationId) {
- CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
String expr = "CollationSupport.StringInstr.exec";
- if (collation.isUtf8BinaryType) {
+ if (collationId == CollationFactory.UTF8_BINARY_COLLATION_ID) {
return String.format(expr + "Binary(%s, %s)", string, substring);
- } else if (collation.isUtf8LcaseType) {
- return String.format(expr + "Lowercase(%s, %s)", string, substring);
} else {
- return String.format(expr + "ICU(%s, %s, %d)", string, substring, collationId);
+ return String.format(expr + "(%s, %s, %d)", string, substring, collationId);
}
}
public static int execBinary(final UTF8String string, final UTF8String substring) {
@@ -360,6 +366,7 @@ public final class CollationSupport {
public static UTF8String exec(final UTF8String src, final UTF8String search,
final UTF8String replace, final int collationId) {
CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
+ // Space trimming does not affect the output of this expression.
if (collation.isUtf8BinaryType) {
return execBinary(src, search, replace);
} else if (collation.isUtf8LcaseType) {
@@ -395,9 +402,12 @@ public final class CollationSupport {
}
public static class StringLocate {
- public static int exec(final UTF8String string, final UTF8String substring, final int start,
+ public static int exec(final UTF8String string, UTF8String substring, final int start,
final int collationId) {
CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
+ if (collation.supportsSpaceTrimming) {
+ substring = CollationFactory.applyTrimmingPolicy(substring, collationId);
+ }
if (collation.isUtf8BinaryType) {
return execBinary(string, substring, start);
} else if (collation.isUtf8LcaseType) {
@@ -408,14 +418,11 @@ public final class CollationSupport {
}
public static String genCode(final String string, final String substring, final int start,
final int collationId) {
- CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
String expr = "CollationSupport.StringLocate.exec";
- if (collation.isUtf8BinaryType) {
+ if (collationId == CollationFactory.UTF8_BINARY_COLLATION_ID) {
return String.format(expr + "Binary(%s, %s, %d)", string, substring, start);
- } else if (collation.isUtf8LcaseType) {
- return String.format(expr + "Lowercase(%s, %s, %d)", string, substring, start);
} else {
- return String.format(expr + "ICU(%s, %s, %d, %d)", string, substring, start, collationId);
+ return String.format(expr + "(%s, %s, %d, %d)", string, substring, start, collationId);
}
}
public static int execBinary(final UTF8String string, final UTF8String substring,
@@ -433,9 +440,12 @@ public final class CollationSupport {
}
public static class SubstringIndex {
- public static UTF8String exec(final UTF8String string, final UTF8String delimiter,
+ public static UTF8String exec(final UTF8String string, UTF8String delimiter,
final int count, final int collationId) {
CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
+ if (collation.supportsSpaceTrimming) {
+ delimiter = CollationFactory.applyTrimmingPolicy(delimiter, collationId);
+ }
if (collation.isUtf8BinaryType) {
return execBinary(string, delimiter, count);
} else if (collation.isUtf8LcaseType) {
@@ -446,14 +456,11 @@ public final class CollationSupport {
}
public static String genCode(final String string, final String delimiter,
final String count, final int collationId) {
- CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
String expr = "CollationSupport.SubstringIndex.exec";
- if (collation.isUtf8BinaryType) {
+ if (collationId == CollationFactory.UTF8_BINARY_COLLATION_ID) {
return String.format(expr + "Binary(%s, %s, %s)", string, delimiter, count);
- } else if (collation.isUtf8LcaseType) {
- return String.format(expr + "Lowercase(%s, %s, %s)", string, delimiter, count);
} else {
- return String.format(expr + "ICU(%s, %s, %s, %d)", string, delimiter, count, collationId);
+ return String.format(expr + "(%s, %s, %s, %d)", string, delimiter, count, collationId);
}
}
public static UTF8String execBinary(final UTF8String string, final UTF8String delimiter,
@@ -474,6 +481,7 @@ public final class CollationSupport {
public static UTF8String exec(final UTF8String source, Map<String, String> dict,
final int collationId) {
CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
+ // Space trimming does not affect the output of this expression.
if (collation.isUtf8BinaryType) {
return execBinary(source, dict);
} else if (collation.isUtf8LcaseType) {
@@ -503,10 +511,15 @@ public final class CollationSupport {
final UTF8String trimString,
final int collationId) {
CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
- if (collation.isUtf8BinaryType) {
+ if (collation.isUtf8BinaryType && !collation.supportsSpaceTrimming) {
return execBinary(srcString, trimString);
+ }
+
+ if (collation.isUtf8BinaryType) {
+ // special handling needed for utf8_binary_rtrim collation.
+ return execBinaryTrim(srcString, trimString, collationId);
} else if (collation.isUtf8LcaseType) {
- return execLowercase(srcString, trimString);
+ return execLowercase(srcString, trimString, collationId);
} else {
return execICU(srcString, trimString, collationId);
}
@@ -518,14 +531,11 @@ public final class CollationSupport {
final String srcString,
final String trimString,
final int collationId) {
- CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
String expr = "CollationSupport.StringTrim.exec";
- if (collation.isUtf8BinaryType) {
+ if (collationId == CollationFactory.UTF8_BINARY_COLLATION_ID) {
return String.format(expr + "Binary(%s, %s)", srcString, trimString);
- } else if (collation.isUtf8LcaseType) {
- return String.format(expr + "Lowercase(%s, %s)", srcString, trimString);
} else {
- return String.format(expr + "ICU(%s, %s, %d)", srcString, trimString, collationId);
+ return String.format(expr + "(%s, %s, %d)", srcString, trimString, collationId);
}
}
public static UTF8String execBinary(
@@ -539,8 +549,9 @@ public final class CollationSupport {
}
public static UTF8String execLowercase(
final UTF8String srcString,
- final UTF8String trimString) {
- return CollationAwareUTF8String.lowercaseTrim(srcString, trimString);
+ final UTF8String trimString,
+ final int collationId) {
+ return CollationAwareUTF8String.lowercaseTrim(srcString, trimString, collationId);
}
public static UTF8String execICU(
final UTF8String srcString,
@@ -548,6 +559,12 @@ public final class CollationSupport {
final int collationId) {
return CollationAwareUTF8String.trim(srcString, trimString, collationId);
}
+ public static UTF8String execBinaryTrim(
+ final UTF8String srcString,
+ final UTF8String trimString,
+ final int collationId) {
+ return CollationAwareUTF8String.binaryTrim(srcString, trimString, collationId);
+ }
}
public static class StringTrimLeft {
@@ -555,10 +572,12 @@ public final class CollationSupport {
return execBinary(srcString);
}
public static UTF8String exec(
- final UTF8String srcString,
- final UTF8String trimString,
- final int collationId) {
+ final UTF8String srcString,
+ UTF8String trimString,
+ final int collationId) {
CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
+ // Space trimming does not affect the output of this expression as currently only supported
+ // space trimming is RTRIM.
if (collation.isUtf8BinaryType) {
return execBinary(srcString, trimString);
} else if (collation.isUtf8LcaseType) {
@@ -574,14 +593,11 @@ public final class CollationSupport {
final String srcString,
final String trimString,
final int collationId) {
- CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
String expr = "CollationSupport.StringTrimLeft.exec";
- if (collation.isUtf8BinaryType) {
+ if (collationId == CollationFactory.UTF8_BINARY_COLLATION_ID) {
return String.format(expr + "Binary(%s, %s)", srcString, trimString);
- } else if (collation.isUtf8LcaseType) {
- return String.format(expr + "Lowercase(%s, %s)", srcString, trimString);
} else {
- return String.format(expr + "ICU(%s, %s, %d)", srcString, trimString, collationId);
+ return String.format(expr + "(%s, %s, %d)", srcString, trimString, collationId);
}
}
public static UTF8String execBinary(final UTF8String srcString) {
@@ -614,10 +630,15 @@ public final class CollationSupport {
final UTF8String trimString,
final int collationId) {
CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
- if (collation.isUtf8BinaryType) {
+ if (collation.isUtf8BinaryType && !collation.supportsSpaceTrimming) {
return execBinary(srcString, trimString);
+ }
+
+ if (collation.isUtf8BinaryType) {
+ // special handling needed for utf8_binary_rtrim collation.
+ return execBinaryTrim(srcString, trimString, collationId);
} else if (collation.isUtf8LcaseType) {
- return execLowercase(srcString, trimString);
+ return execLowercase(srcString, trimString, collationId);
} else {
return execICU(srcString, trimString, collationId);
}
@@ -629,14 +650,11 @@ public final class CollationSupport {
final String srcString,
final String trimString,
final int collationId) {
- CollationFactory.Collation collation = CollationFactory.fetchCollation(collationId);
String expr = "CollationSupport.StringTrimRight.exec";
- if (collation.isUtf8BinaryType) {
+ if (collationId == CollationFactory.UTF8_BINARY_COLLATION_ID) {
return String.format(expr + "Binary(%s, %s)", srcString, trimString);
- } else if (collation.isUtf8LcaseType) {
- return String.format(expr + "Lowercase(%s, %s)", srcString, trimString);
} else {
- return String.format(expr + "ICU(%s, %s, %d)", srcString, trimString, collationId);
+ return String.format(expr + "(%s, %s, %d)", srcString, trimString, collationId);
}
}
public static UTF8String execBinary(final UTF8String srcString) {
@@ -649,8 +667,9 @@ public final class CollationSupport {
}
public static UTF8String execLowercase(
final UTF8String srcString,
- final UTF8String trimString) {
- return CollationAwareUTF8String.lowercaseTrimRight(srcString, trimString);
+ final UTF8String trimString,
+ final int collationId) {
+ return CollationAwareUTF8String.lowercaseTrimRight(srcString, trimString, collationId);
}
public static UTF8String execICU(
final UTF8String srcString,
@@ -658,6 +677,12 @@ public final class CollationSupport {
final int collationId) {
return CollationAwareUTF8String.trimRight(srcString, trimString, collationId);
}
+ public static UTF8String execBinaryTrim(
+ final UTF8String srcString,
+ final UTF8String trimString,
+ final int collationId) {
+ return CollationAwareUTF8String.binaryTrimRight(srcString, trimString, collationId);
+ }
}
// TODO: Add more collation-aware string expressions.
diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
index 50a3cc6049ea..caf8461b0b5d 100644
--- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
+++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
@@ -141,6 +141,7 @@ public final class UTF8String implements Comparable<UTF8String>, Externalizable,
private static final UTF8String COMMA_UTF8 = UTF8String.fromString(",");
public static final UTF8String EMPTY_UTF8 = UTF8String.fromString("");
public static final UTF8String ZERO_UTF8 = UTF8String.fromString("0");
+ public static final UTF8String SPACE_UTF8 = UTF8String.fromString(" ");
/**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
index 4367920f939e..2452da5d6968 100755
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
@@ -82,10 +82,12 @@ case class ConcatWs(children: Seq[Expression])
/** The 1st child (separator) is str, and rest are either str or array of str. */
override def inputTypes: Seq[AbstractDataType] = {
val arrayOrStr =
- TypeCollection(AbstractArrayType(StringTypeWithCollation),
- StringTypeWithCollation
+ TypeCollection(AbstractArrayType(
+ StringTypeWithCollation(supportsTrimCollation = true)),
+ StringTypeWithCollation(supportsTrimCollation = true)
)
- StringTypeWithCollation +: Seq.fill(children.size - 1)(arrayOrStr)
+ StringTypeWithCollation(supportsTrimCollation = true) +:
+ Seq.fill(children.size - 1)(arrayOrStr)
}
override def dataType: DataType = children.head.dataType
@@ -436,7 +438,8 @@ trait String2StringExpression extends ImplicitCastInputTypes {
def convert(v: UTF8String): UTF8String
override def dataType: DataType = child.dataType
- override def inputTypes: Seq[AbstractDataType] = Seq(StringTypeWithCollation)
+ override def inputTypes: Seq[AbstractDataType] =
+ Seq(StringTypeWithCollation(supportsTrimCollation = true))
protected override def nullSafeEval(input: Any): Any =
convert(input.asInstanceOf[UTF8String])
@@ -518,7 +521,8 @@ abstract class StringPredicate extends BinaryExpression
def compare(l: UTF8String, r: UTF8String): Boolean
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeWithCollation, StringTypeWithCollation)
+ Seq(StringTypeWithCollation(supportsTrimCollation = true),
+ StringTypeWithCollation(supportsTrimCollation = true))
protected override def nullSafeEval(input1: Any, input2: Any): Any =
compare(input1.asInstanceOf[UTF8String], input2.asInstanceOf[UTF8String])
@@ -613,7 +617,9 @@ case class Contains(left: Expression, right: Expression) extends StringPredicate
CollationSupport.Contains.genCode(c1, c2, collationId))
}
override def inputTypes : Seq[AbstractDataType] =
- Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation)
+ Seq(StringTypeNonCSAICollation(supportsTrimCollation = true),
+ StringTypeNonCSAICollation(supportsTrimCollation = true)
+ )
override protected def withNewChildrenInternal(
newLeft: Expression, newRight: Expression): Contains = copy(left = newLeft, right = newRight)
}
@@ -657,7 +663,11 @@ case class StartsWith(left: Expression, right: Expression) extends StringPredica
}
override def inputTypes : Seq[AbstractDataType] =
- Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation, StringTypeNonCSAICollation)
+ Seq(
+ StringTypeNonCSAICollation(supportsTrimCollation = true),
+ StringTypeNonCSAICollation(supportsTrimCollation = true),
+ StringTypeNonCSAICollation(supportsTrimCollation = true)
+ )
override protected def withNewChildrenInternal(
newLeft: Expression, newRight: Expression): StartsWith = copy(left = newLeft, right = newRight)
@@ -702,7 +712,11 @@ case class EndsWith(left: Expression, right: Expression) extends StringPredicate
}
override def inputTypes : Seq[AbstractDataType] =
- Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation, StringTypeNonCSAICollation)
+ Seq(
+ StringTypeNonCSAICollation(supportsTrimCollation = true),
+ StringTypeNonCSAICollation(supportsTrimCollation = true),
+ StringTypeNonCSAICollation(supportsTrimCollation = true)
+ )
override protected def withNewChildrenInternal(
newLeft: Expression, newRight: Expression): EndsWith = copy(left = newLeft, right = newRight)
@@ -735,7 +749,8 @@ case class IsValidUTF8(input: Expression) extends RuntimeReplaceable with Implic
override lazy val replacement: Expression = Invoke(input, "isValid", BooleanType)
- override def inputTypes: Seq[AbstractDataType] = Seq(StringTypeWithCollation)
+ override def inputTypes: Seq[AbstractDataType] =
+ Seq(StringTypeWithCollation(supportsTrimCollation = true))
override def nodeName: String = "is_valid_utf8"
@@ -782,7 +797,8 @@ case class MakeValidUTF8(input: Expression) extends RuntimeReplaceable with Impl
override lazy val replacement: Expression = Invoke(input, "makeValid", input.dataType)
- override def inputTypes: Seq[AbstractDataType] = Seq(StringTypeWithCollation)
+ override def inputTypes: Seq[AbstractDataType] =
+ Seq(StringTypeWithCollation(supportsTrimCollation = true))
override def nodeName: String = "make_valid_utf8"
@@ -827,7 +843,8 @@ case class ValidateUTF8(input: Expression) extends RuntimeReplaceable with Impli
Seq(input),
inputTypes)
- override def inputTypes: Seq[AbstractDataType] = Seq(StringTypeWithCollation)
+ override def inputTypes: Seq[AbstractDataType] =
+ Seq(StringTypeWithCollation(supportsTrimCollation = true))
override def nodeName: String = "validate_utf8"
@@ -876,7 +893,8 @@ case class TryValidateUTF8(input: Expression) extends RuntimeReplaceable with Im
Seq(input),
inputTypes)
- override def inputTypes: Seq[AbstractDataType] = Seq(StringTypeWithCollation)
+ override def inputTypes: Seq[AbstractDataType] =
+ Seq(StringTypeWithCollation(supportsTrimCollation = true))
override def nodeName: String = "try_validate_utf8"
@@ -932,7 +950,11 @@ case class StringReplace(srcExpr: Expression, searchExpr: Expression, replaceExp
override def dataType: DataType = srcExpr.dataType
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation, StringTypeNonCSAICollation)
+ Seq(
+ StringTypeNonCSAICollation(supportsTrimCollation = true),
+ StringTypeNonCSAICollation(supportsTrimCollation = true),
+ StringTypeNonCSAICollation(supportsTrimCollation = true)
+ )
override def first: Expression = srcExpr
override def second: Expression = searchExpr
override def third: Expression = replaceExpr
@@ -1011,8 +1033,14 @@ case class Overlay(input: Expression, replace: Expression, pos: Expression, len:
override def dataType: DataType = input.dataType
override def inputTypes: Seq[AbstractDataType] = Seq(
- TypeCollection(StringTypeWithCollation, BinaryType),
- TypeCollection(StringTypeWithCollation, BinaryType), IntegerType, IntegerType)
+ TypeCollection(
+ StringTypeWithCollation(supportsTrimCollation = true), BinaryType
+ ),
+ TypeCollection(
+ StringTypeWithCollation(supportsTrimCollation = true), BinaryType
+ ),
+ IntegerType,
+ IntegerType)
override def checkInputDataTypes(): TypeCheckResult = {
val inputTypeCheck = super.checkInputDataTypes()
@@ -1180,7 +1208,10 @@ case class StringTranslate(srcExpr: Expression, matchingExpr: Expression, replac
override def dataType: DataType = srcExpr.dataType
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation, StringTypeNonCSAICollation)
+ Seq(
+ StringTypeNonCSAICollation(supportsTrimCollation = true),
+ StringTypeNonCSAICollation(supportsTrimCollation = true),
+ StringTypeNonCSAICollation(supportsTrimCollation = true))
override def first: Expression = srcExpr
override def second: Expression = matchingExpr
override def third: Expression = replaceExpr
@@ -1216,7 +1247,10 @@ case class FindInSet(left: Expression, right: Expression) extends BinaryExpressi
final lazy val collationId: Int = left.dataType.asInstanceOf[StringType].collationId
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeWithCollation, StringTypeWithCollation)
+ Seq(
+ StringTypeWithCollation(supportsTrimCollation = true),
+ StringTypeWithCollation(supportsTrimCollation = true)
+ )
override protected def nullSafeEval(word: Any, set: Any): Any = {
CollationSupport.FindInSet.
@@ -1245,7 +1279,7 @@ trait String2TrimExpression extends Expression with ImplicitCastInputTypes {
override def children: Seq[Expression] = srcStr +: trimStr.toSeq
override def dataType: DataType = srcStr.dataType
override def inputTypes: Seq[AbstractDataType] =
- Seq.fill(children.size)(StringTypeWithCollation)
+ Seq.fill(children.size)(StringTypeWithCollation(supportsTrimCollation = true))
final lazy val collationId: Int = srcStr.dataType.asInstanceOf[StringType].collationId
@@ -1409,7 +1443,10 @@ case class StringTrim(srcStr: Expression, trimStr: Option[Expression] = None)
CollationSupport.StringTrim.exec(srcString, trimString, collationId)
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation)
+ Seq(
+ StringTypeNonCSAICollation(supportsTrimCollation = true),
+ StringTypeNonCSAICollation(supportsTrimCollation = true)
+ )
override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
copy(
@@ -1519,7 +1556,10 @@ case class StringTrimLeft(srcStr: Expression, trimStr: Option[Expression] = None
CollationSupport.StringTrimLeft.exec(srcString, trimString, collationId)
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation)
+ Seq(
+ StringTypeNonCSAICollation(supportsTrimCollation = true),
+ StringTypeNonCSAICollation(supportsTrimCollation = true)
+ )
override protected def withNewChildrenInternal(
newChildren: IndexedSeq[Expression]): StringTrimLeft =
@@ -1582,7 +1622,10 @@ case class StringTrimRight(srcStr: Expression, trimStr: Option[Expression] = Non
CollationSupport.StringTrimRight.exec(srcString, trimString, collationId)
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation)
+ Seq(
+ StringTypeNonCSAICollation(supportsTrimCollation = true),
+ StringTypeNonCSAICollation(supportsTrimCollation = true)
+ )
override protected def withNewChildrenInternal(
newChildren: IndexedSeq[Expression]): StringTrimRight =
@@ -1618,7 +1661,10 @@ case class StringInstr(str: Expression, substr: Expression)
override def right: Expression = substr
override def dataType: DataType = IntegerType
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation)
+ Seq(
+ StringTypeNonCSAICollation(supportsTrimCollation = true),
+ StringTypeNonCSAICollation(supportsTrimCollation = true)
+ )
override def nullSafeEval(string: Any, sub: Any): Any = {
CollationSupport.StringInstr.
@@ -1666,7 +1712,11 @@ case class SubstringIndex(strExpr: Expression, delimExpr: Expression, countExpr:
override def dataType: DataType = strExpr.dataType
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation, IntegerType)
+ Seq(
+ StringTypeNonCSAICollation(supportsTrimCollation = true),
+ StringTypeNonCSAICollation(supportsTrimCollation = true),
+ IntegerType
+ )
override def first: Expression = strExpr
override def second: Expression = delimExpr
override def third: Expression = countExpr
@@ -1724,7 +1774,11 @@ case class StringLocate(substr: Expression, str: Expression, start: Expression)
override def nullable: Boolean = substr.nullable || str.nullable
override def dataType: DataType = IntegerType
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation, IntegerType)
+ Seq(
+ StringTypeNonCSAICollation(supportsTrimCollation = true),
+ StringTypeNonCSAICollation(supportsTrimCollation = true),
+ IntegerType
+ )
override def eval(input: InternalRow): Any = {
val s = start.eval(input)
@@ -1850,7 +1904,11 @@ case class StringLPad(str: Expression, len: Expression, pad: Expression)
override def dataType: DataType = str.dataType
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeWithCollation, IntegerType, StringTypeWithCollation)
+ Seq(
+ StringTypeWithCollation(supportsTrimCollation = true),
+ IntegerType,
+ StringTypeWithCollation(supportsTrimCollation = true)
+ )
override def nullSafeEval(string: Any, len: Any, pad: Any): Any = {
string.asInstanceOf[UTF8String].lpad(len.asInstanceOf[Int], pad.asInstanceOf[UTF8String])
@@ -1930,7 +1988,11 @@ case class StringRPad(str: Expression, len: Expression, pad: Expression = Litera
override def dataType: DataType = str.dataType
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeWithCollation, IntegerType, StringTypeWithCollation)
+ Seq(
+ StringTypeWithCollation(supportsTrimCollation = true),
+ IntegerType,
+ StringTypeWithCollation(supportsTrimCollation = true)
+ )
override def nullSafeEval(string: Any, len: Any, pad: Any): Any = {
string.asInstanceOf[UTF8String].rpad(len.asInstanceOf[Int], pad.asInstanceOf[UTF8String])
@@ -1975,7 +2037,8 @@ case class FormatString(children: Expression*) extends Expression with ImplicitC
override def dataType: DataType = children(0).dataType
override def inputTypes: Seq[AbstractDataType] =
- StringTypeWithCollation :: List.fill(children.size - 1)(AnyDataType)
+ StringTypeWithCollation(supportsTrimCollation = true) ::
+ List.fill(children.size - 1)(AnyDataType)
override def checkInputDataTypes(): TypeCheckResult = {
if (children.isEmpty) {
@@ -2086,7 +2149,8 @@ case class InitCap(child: Expression)
// Flag to indicate whether to use ICU instead of JVM case mappings for UTF8_BINARY collation.
private final lazy val useICU = SQLConf.get.getConf(SQLConf.ICU_CASE_MAPPINGS_ENABLED)
- override def inputTypes: Seq[AbstractDataType] = Seq(StringTypeWithCollation)
+ override def inputTypes: Seq[AbstractDataType] =
+ Seq(StringTypeWithCollation(supportsTrimCollation = true))
override def dataType: DataType = child.dataType
override def nullSafeEval(string: Any): Any = {
@@ -2119,7 +2183,10 @@ case class StringRepeat(str: Expression, times: Expression)
override def right: Expression = times
override def dataType: DataType = str.dataType
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeWithCollation, IntegerType)
+ Seq(
+ StringTypeWithCollation(supportsTrimCollation = true),
+ IntegerType
+ )
override def nullSafeEval(string: Any, n: Any): Any = {
string.asInstanceOf[UTF8String].repeat(n.asInstanceOf[Integer])
@@ -2212,7 +2279,11 @@ case class Substring(str: Expression, pos: Expression, len: Expression)
override def dataType: DataType = str.dataType
override def inputTypes: Seq[AbstractDataType] =
- Seq(TypeCollection(StringTypeWithCollation, BinaryType), IntegerType, IntegerType)
+ Seq(
+ TypeCollection(StringTypeWithCollation(supportsTrimCollation = true), BinaryType),
+ IntegerType,
+ IntegerType
+ )
override def first: Expression = str
override def second: Expression = pos
@@ -2271,7 +2342,10 @@ case class Right(str: Expression, len: Expression) extends RuntimeReplaceable
)
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeWithCollation, IntegerType)
+ Seq(
+ StringTypeWithCollation(supportsTrimCollation = true),
+ IntegerType
+ )
override def left: Expression = str
override def right: Expression = len
override protected def withNewChildrenInternal(
@@ -2302,7 +2376,12 @@ case class Left(str: Expression, len: Expression) extends RuntimeReplaceable
override lazy val replacement: Expression = Substring(str, Literal(1), len)
override def inputTypes: Seq[AbstractDataType] = {
- Seq(TypeCollection(StringTypeWithCollation, BinaryType), IntegerType)
+ Seq(
+ TypeCollection(
+ StringTypeWithCollation(supportsTrimCollation = true),
+ BinaryType)
+ , IntegerType
+ )
}
override def left: Expression = str
@@ -2338,7 +2417,12 @@ case class Length(child: Expression)
extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
override def dataType: DataType = IntegerType
override def inputTypes: Seq[AbstractDataType] =
- Seq(TypeCollection(StringTypeWithCollation, BinaryType))
+ Seq(
+ TypeCollection(
+ StringTypeWithCollation(supportsTrimCollation = true),
+ BinaryType
+ )
+ )
protected override def nullSafeEval(value: Any): Any = child.dataType match {
case _: StringType => value.asInstanceOf[UTF8String].numChars
@@ -2373,8 +2457,12 @@ case class BitLength(child: Expression)
extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
override def dataType: DataType = IntegerType
override def inputTypes: Seq[AbstractDataType] =
- Seq(TypeCollection(StringTypeWithCollation, BinaryType))
-
+ Seq(
+ TypeCollection(
+ StringTypeWithCollation(supportsTrimCollation = true),
+ BinaryType
+ )
+ )
protected override def nullSafeEval(value: Any): Any = child.dataType match {
case _: StringType => value.asInstanceOf[UTF8String].numBytes * 8
case BinaryType => value.asInstanceOf[Array[Byte]].length * 8
@@ -2412,7 +2500,12 @@ case class OctetLength(child: Expression)
extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
override def dataType: DataType = IntegerType
override def inputTypes: Seq[AbstractDataType] =
- Seq(TypeCollection(StringTypeWithCollation, BinaryType))
+ Seq(
+ TypeCollection(
+ StringTypeWithCollation(supportsTrimCollation = true),
+ BinaryType
+ )
+ )
protected override def nullSafeEval(value: Any): Any = child.dataType match {
case _: StringType => value.asInstanceOf[UTF8String].numBytes
@@ -2473,8 +2566,16 @@ case class Levenshtein(
override def inputTypes: Seq[AbstractDataType] = threshold match {
case Some(_) =>
- Seq(StringTypeWithCollation, StringTypeWithCollation, IntegerType)
- case _ => Seq(StringTypeWithCollation, StringTypeWithCollation)
+ Seq(
+ StringTypeWithCollation(supportsTrimCollation = true),
+ StringTypeWithCollation(supportsTrimCollation = true),
+ IntegerType
+ )
+ case _ =>
+ Seq(
+ StringTypeWithCollation(supportsTrimCollation = true),
+ StringTypeWithCollation(supportsTrimCollation = true)
+ )
}
override def children: Seq[Expression] = threshold match {
@@ -2599,7 +2700,8 @@ case class SoundEx(child: Expression)
override def dataType: DataType = SQLConf.get.defaultStringType
- override def inputTypes: Seq[AbstractDataType] = Seq(StringTypeWithCollation)
+ override def inputTypes: Seq[AbstractDataType] =
+ Seq(StringTypeWithCollation(supportsTrimCollation = true))
override def nullSafeEval(input: Any): Any = input.asInstanceOf[UTF8String].soundex()
@@ -2629,7 +2731,8 @@ case class Ascii(child: Expression)
extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
override def dataType: DataType = IntegerType
- override def inputTypes: Seq[AbstractDataType] = Seq(StringTypeWithCollation)
+ override def inputTypes: Seq[AbstractDataType] =
+ Seq(StringTypeWithCollation(supportsTrimCollation = true))
protected override def nullSafeEval(string: Any): Any = {
// only pick the first character to reduce the `toString` cost
@@ -2774,7 +2877,8 @@ case class UnBase64(child: Expression, failOnError: Boolean = false)
extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
override def dataType: DataType = BinaryType
- override def inputTypes: Seq[AbstractDataType] = Seq(StringTypeWithCollation)
+ override def inputTypes: Seq[AbstractDataType] =
+ Seq(StringTypeWithCollation(supportsTrimCollation = true))
def this(expr: Expression) = this(expr, false)
@@ -2953,8 +3057,10 @@ case class StringDecode(
this(bin, charset, SQLConf.get.legacyJavaCharsets, SQLConf.get.legacyCodingErrorAction)
override val dataType: DataType = SQLConf.get.defaultStringType
- override def inputTypes: Seq[AbstractDataType] =
- Seq(BinaryType, StringTypeWithCollation)
+ override def inputTypes: Seq[AbstractDataType] = Seq(
+ BinaryType,
+ StringTypeWithCollation(supportsTrimCollation = true)
+ )
override def prettyName: String = "decode"
override def toString: String = s"$prettyName($bin, $charset)"
@@ -2963,7 +3069,13 @@ case class StringDecode(
SQLConf.get.defaultStringType,
"decode",
Seq(bin, charset, Literal(legacyCharsets), Literal(legacyErrorAction)),
- Seq(BinaryType, StringTypeWithCollation, BooleanType, BooleanType))
+ Seq(
+ BinaryType,
+ StringTypeWithCollation(supportsTrimCollation = true),
+ BooleanType,
+ BooleanType
+ )
+ )
override def children: Seq[Expression] = Seq(bin, charset)
override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
@@ -3020,7 +3132,10 @@ case class Encode(
override def dataType: DataType = BinaryType
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeWithCollation, StringTypeWithCollation)
+ Seq(
+ StringTypeWithCollation(supportsTrimCollation = true),
+ StringTypeWithCollation(supportsTrimCollation = true)
+ )
override lazy val replacement: Expression = StaticInvoke(
classOf[Encode],
@@ -3030,8 +3145,8 @@ case class Encode(
str, charset, Literal(legacyCharsets, BooleanType), Literal(legacyErrorAction, BooleanType)
),
Seq(
- StringTypeWithCollation,
- StringTypeWithCollation,
+ StringTypeWithCollation(supportsTrimCollation = true),
+ StringTypeWithCollation(supportsTrimCollation = true),
BooleanType,
BooleanType))
@@ -3118,7 +3233,7 @@ case class ToBinary(
override def children: Seq[Expression] = expr +: format.toSeq
override def inputTypes: Seq[AbstractDataType] =
- children.map(_ => StringTypeWithCollation)
+ children.map(_ => StringTypeWithCollation(supportsTrimCollation = true))
override def checkInputDataTypes(): TypeCheckResult = {
def isValidFormat: Boolean = {
@@ -3205,7 +3320,12 @@ case class FormatNumber(x: Expression, d: Expression)
override def dataType: DataType = SQLConf.get.defaultStringType
override def nullable: Boolean = true
override def inputTypes: Seq[AbstractDataType] =
- Seq(NumericType, TypeCollection(IntegerType, StringTypeWithCollation))
+ Seq(
+ NumericType,
+ TypeCollection(IntegerType,
+ StringTypeWithCollation(supportsTrimCollation = true)
+ )
+ )
private val defaultFormat = "#,###,###,###,###,###,##0"
@@ -3412,7 +3532,9 @@ case class Sentences(
override def inputTypes: Seq[AbstractDataType] =
Seq(
StringTypeWithCollation,
- StringTypeWithCollation, StringTypeWithCollation)
+ StringTypeWithCollation,
+ StringTypeWithCollation
+ )
override def first: Expression = str
override def second: Expression = language
override def third: Expression = country
@@ -3499,7 +3621,11 @@ case class SplitPart (
false)
override def nodeName: String = "split_part"
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation, IntegerType)
+ Seq(
+ StringTypeNonCSAICollation(supportsTrimCollation = true),
+ StringTypeNonCSAICollation(supportsTrimCollation = true),
+ IntegerType
+ )
def children: Seq[Expression] = Seq(str, delimiter, partNum)
protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression = {
copy(str = newChildren.apply(0), delimiter = newChildren.apply(1),
@@ -3560,7 +3686,8 @@ case class Luhncheck(input: Expression) extends RuntimeReplaceable with Implicit
"isLuhnNumber",
Seq(input), inputTypes)
- override def inputTypes: Seq[AbstractDataType] = Seq(StringTypeWithCollation)
+ override def inputTypes: Seq[AbstractDataType] =
+ Seq(StringTypeWithCollation(supportsTrimCollation = true))
override def prettyName: String = "luhn_check"
diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/collations.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/collations.sql.out
index 883acd3ca966..c8e28c2cfafc 100644
--- a/sql/core/src/test/resources/sql-tests/analyzer-results/collations.sql.out
+++ b/sql/core/src/test/resources/sql-tests/analyzer-results/collations.sql.out
@@ -649,6 +649,22 @@ InsertIntoHadoopFsRelationCommand file:[not included in comparison]/{warehouse_d
+- LocalRelation [col1#x, col2#x, col3#x]
+-- !query
+insert into t5 values ('İo', 'İo', 'İo ')
+-- !query analysis
+InsertIntoHadoopFsRelationCommand file:[not included in comparison]/{warehouse_dir}/t5, false, Parquet, [path=file:[not included in comparison]/{warehouse_dir}/t5], Append, `spark_catalog`.`default`.`t5`, org.apache.spark.sql.execution.datasources.InMemoryFileIndex(file:[not included in comparison]/{warehouse_dir}/t5), [s, utf8_binary, utf8_lcase]
++- Project [cast(col1#x as string) AS s#x, cast(col2#x as string) AS utf8_binary#x, cast(col3#x as string collate UTF8_LCASE) AS utf8_lcase#x]
+ +- LocalRelation [col1#x, col2#x, col3#x]
+
+
+-- !query
+insert into t5 values ('İo', 'İo ', 'İo')
+-- !query analysis
+InsertIntoHadoopFsRelationCommand file:[not included in comparison]/{warehouse_dir}/t5, false, Parquet, [path=file:[not included in comparison]/{warehouse_dir}/t5], Append, `spark_catalog`.`default`.`t5`, org.apache.spark.sql.execution.datasources.InMemoryFileIndex(file:[not included in comparison]/{warehouse_dir}/t5), [s, utf8_binary, utf8_lcase]
++- Project [cast(col1#x as string) AS s#x, cast(col2#x as string) AS utf8_binary#x, cast(col3#x as string collate UTF8_LCASE) AS utf8_lcase#x]
+ +- LocalRelation [col1#x, col2#x, col3#x]
+
+
-- !query
insert into t5 values ('İo', 'İo', 'i̇o')
-- !query analysis
@@ -1021,6 +1037,14 @@ Project [split_part(cast(utf8_binary#x as string collate UTF8_LCASE), collate(a,
+- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+-- !query
+select split_part(utf8_binary, 'a ' collate utf8_lcase_rtrim, 3), split_part(utf8_lcase, 'a' collate utf8_binary, 3) from t5
+-- !query analysis
+Project [split_part(cast(utf8_binary#x as string collate UTF8_LCASE_RTRIM), collate(a , utf8_lcase_rtrim), 3) AS split_part(utf8_binary, collate(a , utf8_lcase_rtrim), 3)#x, split_part(cast(utf8_lcase#x as string), collate(a, utf8_binary), 3) AS split_part(utf8_lcase, collate(a, utf8_binary), 3)#x]
++- SubqueryAlias spark_catalog.default.t5
+ +- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+
+
-- !query
select contains(utf8_binary, utf8_lcase) from t5
-- !query analysis
@@ -1111,6 +1135,14 @@ Project [Contains(cast(utf8_binary#x as string collate UTF8_LCASE), collate(AaAA
+- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+-- !query
+select contains(utf8_binary, 'AaAA ' collate utf8_lcase_rtrim), contains(utf8_lcase, 'AAa ' collate utf8_binary_rtrim) from t5
+-- !query analysis
+Project [Contains(cast(utf8_binary#x as string collate UTF8_LCASE_RTRIM), collate(AaAA , utf8_lcase_rtrim)) AS contains(utf8_binary, collate(AaAA , utf8_lcase_rtrim))#x, Contains(cast(utf8_lcase#x as string collate UTF8_BINARY_RTRIM), collate(AAa , utf8_binary_rtrim)) AS contains(utf8_lcase, collate(AAa , utf8_binary_rtrim))#x]
++- SubqueryAlias spark_catalog.default.t5
+ +- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+
+
-- !query
select substring_index(utf8_binary, utf8_lcase, 2) from t5
-- !query analysis
@@ -1201,6 +1233,14 @@ Project [substring_index(cast(utf8_binary#x as string collate UTF8_LCASE), colla
+- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+-- !query
+select substring_index(utf8_binary, 'AaAA ' collate utf8_lcase_rtrim, 2), substring_index(utf8_lcase, 'AAa' collate utf8_binary, 2) from t5
+-- !query analysis
+Project [substring_index(cast(utf8_binary#x as string collate UTF8_LCASE_RTRIM), collate(AaAA , utf8_lcase_rtrim), 2) AS substring_index(utf8_binary, collate(AaAA , utf8_lcase_rtrim), 2)#x, substring_index(cast(utf8_lcase#x as string), collate(AAa, utf8_binary), 2) AS substring_index(utf8_lcase, collate(AAa, utf8_binary), 2)#x]
++- SubqueryAlias spark_catalog.default.t5
+ +- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+
+
-- !query
select instr(utf8_binary, utf8_lcase) from t5
-- !query analysis
@@ -1357,6 +1397,14 @@ Project [find_in_set(cast(utf8_binary#x as string collate UTF8_LCASE), collate(a
+- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+-- !query
+select find_in_set(utf8_binary, 'aaAaaAaA,i̇o ' collate utf8_lcase_rtrim), find_in_set(utf8_lcase, 'aaAaaAaA,i̇o' collate utf8_binary) from t5
+-- !query analysis
+Project [find_in_set(cast(utf8_binary#x as string collate UTF8_LCASE_RTRIM), collate(aaAaaAaA,i̇o , utf8_lcase_rtrim)) AS find_in_set(utf8_binary, collate(aaAaaAaA,i̇o , utf8_lcase_rtrim))#x, find_in_set(cast(utf8_lcase#x as string), collate(aaAaaAaA,i̇o, utf8_binary)) AS find_in_set(utf8_lcase, collate(aaAaaAaA,i̇o, utf8_binary))#x]
++- SubqueryAlias spark_catalog.default.t5
+ +- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+
+
-- !query
select startswith(utf8_binary, utf8_lcase) from t5
-- !query analysis
@@ -1447,6 +1495,14 @@ Project [StartsWith(cast(utf8_binary#x as string collate UTF8_LCASE), collate(aa
+- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+-- !query
+select startswith(utf8_binary, 'aaAaaAaA ' collate utf8_lcase_rtrim), startswith(utf8_lcase, 'aaAaaAaA' collate utf8_binary) from t5
+-- !query analysis
+Project [StartsWith(cast(utf8_binary#x as string collate UTF8_LCASE_RTRIM), collate(aaAaaAaA , utf8_lcase_rtrim)) AS startswith(utf8_binary, collate(aaAaaAaA , utf8_lcase_rtrim))#x, StartsWith(cast(utf8_lcase#x as string), collate(aaAaaAaA, utf8_binary)) AS startswith(utf8_lcase, collate(aaAaaAaA, utf8_binary))#x]
++- SubqueryAlias spark_catalog.default.t5
+ +- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+
+
-- !query
select translate(utf8_lcase, utf8_lcase, '12345') from t5
-- !query analysis
@@ -1529,6 +1585,14 @@ Project [translate(cast(utf8_lcase#x as string), collate(aBc, utf8_binary), 1234
+- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+-- !query
+select translate(utf8_lcase, 'aBc ' collate utf8_binary_rtrim, '12345'), translate(utf8_binary, 'aBc' collate utf8_lcase, '12345') from t5
+-- !query analysis
+Project [translate(cast(utf8_lcase#x as string collate UTF8_BINARY_RTRIM), collate(aBc , utf8_binary_rtrim), cast(12345 as string collate UTF8_BINARY_RTRIM)) AS translate(utf8_lcase, collate(aBc , utf8_binary_rtrim), 12345)#x, translate(cast(utf8_binary#x as string collate UTF8_LCASE), collate(aBc, utf8_lcase), cast(12345 as string collate UTF8_LCASE)) AS translate(utf8_binary, collate(aBc, utf8_lcase), 12345)#x]
++- SubqueryAlias spark_catalog.default.t5
+ +- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+
+
-- !query
select replace(utf8_binary, utf8_lcase, 'abc') from t5
-- !query analysis
@@ -1619,6 +1683,14 @@ Project [replace(cast(utf8_binary#x as string collate UTF8_LCASE), collate(aaAaa
+- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+-- !query
+select replace(utf8_binary, 'aaAaaAaA ' collate utf8_lcase_rtrim, 'abc'), replace(utf8_lcase, 'aaAaaAaA' collate utf8_binary, 'abc') from t5
+-- !query analysis
+Project [replace(cast(utf8_binary#x as string collate UTF8_LCASE_RTRIM), collate(aaAaaAaA , utf8_lcase_rtrim), cast(abc as string collate UTF8_LCASE_RTRIM)) AS replace(utf8_binary, collate(aaAaaAaA , utf8_lcase_rtrim), abc)#x, replace(cast(utf8_lcase#x as string), collate(aaAaaAaA, utf8_binary), abc) AS replace(utf8_lcase, collate(aaAaaAaA, utf8_binary), abc)#x]
++- SubqueryAlias spark_catalog.default.t5
+ +- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+
+
-- !query
select endswith(utf8_binary, utf8_lcase) from t5
-- !query analysis
@@ -1709,6 +1781,14 @@ Project [EndsWith(cast(utf8_binary#x as string collate UTF8_LCASE), collate(aaAa
+- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+-- !query
+select endswith(utf8_binary, 'aaAaaAaA ' collate utf8_lcase_rtrim), endswith(utf8_lcase, 'aaAaaAaA' collate utf8_binary) from t5
+-- !query analysis
+Project [EndsWith(cast(utf8_binary#x as string collate UTF8_LCASE_RTRIM), collate(aaAaaAaA , utf8_lcase_rtrim)) AS endswith(utf8_binary, collate(aaAaaAaA , utf8_lcase_rtrim))#x, EndsWith(cast(utf8_lcase#x as string), collate(aaAaaAaA, utf8_binary)) AS endswith(utf8_lcase, collate(aaAaaAaA, utf8_binary))#x]
++- SubqueryAlias spark_catalog.default.t5
+ +- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+
+
-- !query
select repeat(utf8_binary, 3), repeat(utf8_lcase, 2) from t5
-- !query analysis
@@ -2299,6 +2379,14 @@ Project [rpad(collate(utf8_binary#x, utf8_lcase), 8, collate(utf8_lcase#x, utf8_
+- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+-- !query
+select lpad(utf8_binary collate utf8_binary_rtrim, 8, utf8_lcase collate utf8_binary_rtrim) from t5
+-- !query analysis
+Project [lpad(collate(utf8_binary#x, utf8_binary_rtrim), 8, collate(utf8_lcase#x, utf8_binary_rtrim)) AS lpad(collate(utf8_binary, utf8_binary_rtrim), 8, collate(utf8_lcase, utf8_binary_rtrim))#x]
++- SubqueryAlias spark_catalog.default.t5
+ +- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+
+
-- !query
select rpad(utf8_binary, 8, 'a'), rpad(utf8_lcase, 8, 'a') from t5
-- !query analysis
@@ -2365,6 +2453,14 @@ Project [lpad(collate(utf8_binary#x, utf8_lcase), 8, collate(utf8_lcase#x, utf8_
+- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+-- !query
+select lpad(utf8_binary collate utf8_binary_rtrim, 8, utf8_lcase collate utf8_binary_rtrim) from t5
+-- !query analysis
+Project [lpad(collate(utf8_binary#x, utf8_binary_rtrim), 8, collate(utf8_lcase#x, utf8_binary_rtrim)) AS lpad(collate(utf8_binary, utf8_binary_rtrim), 8, collate(utf8_lcase, utf8_binary_rtrim))#x]
++- SubqueryAlias spark_catalog.default.t5
+ +- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+
+
-- !query
select lpad(utf8_binary, 8, 'a'), lpad(utf8_lcase, 8, 'a') from t5
-- !query analysis
@@ -2471,6 +2567,14 @@ Project [locate(cast(utf8_binary#x as string collate UTF8_LCASE), collate(AaAA,
+- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+-- !query
+select locate(utf8_binary, 'AaAA ' collate utf8_binary_rtrim, 4), locate(utf8_lcase, 'AAa ' collate utf8_binary, 4) from t5
+-- !query analysis
+Project [locate(cast(utf8_binary#x as string collate UTF8_BINARY_RTRIM), collate(AaAA , utf8_binary_rtrim), 4) AS locate(utf8_binary, collate(AaAA , utf8_binary_rtrim), 4)#x, locate(cast(utf8_lcase#x as string), collate(AAa , utf8_binary), 4) AS locate(utf8_lcase, collate(AAa , utf8_binary), 4)#x]
++- SubqueryAlias spark_catalog.default.t5
+ +- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+
+
-- !query
select TRIM(utf8_binary, utf8_lcase) from t5
-- !query analysis
@@ -2545,6 +2649,14 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
}
+-- !query
+select TRIM(utf8_binary collate utf8_binary_rtrim, utf8_lcase collate utf8_binary_rtrim) from t5
+-- !query analysis
+Project [trim(collate(utf8_lcase#x, utf8_binary_rtrim), Some(collate(utf8_binary#x, utf8_binary_rtrim))) AS TRIM(BOTH collate(utf8_binary, utf8_binary_rtrim) FROM collate(utf8_lcase, utf8_binary_rtrim))#x]
++- SubqueryAlias spark_catalog.default.t5
+ +- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+
+
-- !query
select TRIM('ABc', utf8_binary), TRIM('ABc', utf8_lcase) from t5
-- !query analysis
@@ -2635,6 +2747,14 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
}
+-- !query
+select BTRIM(utf8_binary collate utf8_binary_rtrim, utf8_lcase collate utf8_binary_rtrim) from t5
+-- !query analysis
+Project [btrim(collate(utf8_binary#x, utf8_binary_rtrim), collate(utf8_lcase#x, utf8_binary_rtrim)) AS btrim(collate(utf8_binary, utf8_binary_rtrim), collate(utf8_lcase, utf8_binary_rtrim))#x]
++- SubqueryAlias spark_catalog.default.t5
+ +- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+
+
-- !query
select BTRIM('ABc', utf8_binary), BTRIM('ABc', utf8_lcase) from t5
-- !query analysis
@@ -2725,6 +2845,14 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
}
+-- !query
+select LTRIM(utf8_binary collate utf8_binary_rtrim, utf8_lcase collate utf8_binary_rtrim) from t5
+-- !query analysis
+Project [ltrim(collate(utf8_lcase#x, utf8_binary_rtrim), Some(collate(utf8_binary#x, utf8_binary_rtrim))) AS TRIM(LEADING collate(utf8_binary, utf8_binary_rtrim) FROM collate(utf8_lcase, utf8_binary_rtrim))#x]
++- SubqueryAlias spark_catalog.default.t5
+ +- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+
+
-- !query
select LTRIM('ABc', utf8_binary), LTRIM('ABc', utf8_lcase) from t5
-- !query analysis
@@ -2815,6 +2943,14 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
}
+-- !query
+select RTRIM(utf8_binary collate utf8_binary_rtrim, utf8_lcase collate utf8_binary_rtrim) from t5
+-- !query analysis
+Project [rtrim(collate(utf8_lcase#x, utf8_binary_rtrim), Some(collate(utf8_binary#x, utf8_binary_rtrim))) AS TRIM(TRAILING collate(utf8_binary, utf8_binary_rtrim) FROM collate(utf8_lcase, utf8_binary_rtrim))#x]
++- SubqueryAlias spark_catalog.default.t5
+ +- Relation spark_catalog.default.t5[s#x,utf8_binary#x,utf8_lcase#x] parquet
+
+
-- !query
select RTRIM('ABc', utf8_binary), RTRIM('ABc', utf8_lcase) from t5
-- !query analysis
diff --git a/sql/core/src/test/resources/sql-tests/inputs/collations.sql b/sql/core/src/test/resources/sql-tests/inputs/collations.sql
index bbbb229ad1cd..b4d33bb0196c 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/collations.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/collations.sql
@@ -117,6 +117,8 @@ insert into t5 values ('aaAaAAaA', 'aaAaAAaA', 'aaAaaAaA');
insert into t5 values ('aaAaAAaA', 'aaAaAAaA', 'aaAaaAaAaaAaaAaAaaAaaAaA');
insert into t5 values ('bbAbaAbA', 'bbAbAAbA', 'a');
insert into t5 values ('İo', 'İo', 'İo');
+insert into t5 values ('İo', 'İo', 'İo ');
+insert into t5 values ('İo', 'İo ', 'İo');
insert into t5 values ('İo', 'İo', 'i̇o');
insert into t5 values ('efd2', 'efd2', 'efd2');
insert into t5 values ('Hello, world! Nice day.', 'Hello, world! Nice day.', 'Hello, world! Nice day.');
@@ -170,6 +172,7 @@ select split_part(utf8_binary collate utf8_lcase, utf8_lcase collate utf8_lcase,
select split_part(utf8_binary collate unicode_ai, utf8_lcase collate unicode_ai, 2) from t5;
select split_part(utf8_binary, 'a', 3), split_part(utf8_lcase, 'a', 3) from t5;
select split_part(utf8_binary, 'a' collate utf8_lcase, 3), split_part(utf8_lcase, 'a' collate utf8_binary, 3) from t5;
+select split_part(utf8_binary, 'a ' collate utf8_lcase_rtrim, 3), split_part(utf8_lcase, 'a' collate utf8_binary, 3) from t5;
-- Contains
select contains(utf8_binary, utf8_lcase) from t5;
@@ -180,6 +183,7 @@ select contains(utf8_binary collate utf8_lcase, utf8_lcase collate utf8_lcase) f
select contains(utf8_binary collate unicode_ai, utf8_lcase collate unicode_ai) from t5;
select contains(utf8_binary, 'a'), contains(utf8_lcase, 'a') from t5;
select contains(utf8_binary, 'AaAA' collate utf8_lcase), contains(utf8_lcase, 'AAa' collate utf8_binary) from t5;
+select contains(utf8_binary, 'AaAA ' collate utf8_lcase_rtrim), contains(utf8_lcase, 'AAa ' collate utf8_binary_rtrim) from t5;
-- SubstringIndex
select substring_index(utf8_binary, utf8_lcase, 2) from t5;
@@ -190,6 +194,7 @@ select substring_index(utf8_binary collate utf8_lcase, utf8_lcase collate utf8_l
select substring_index(utf8_binary collate unicode_ai, utf8_lcase collate unicode_ai, 2) from t5;
select substring_index(utf8_binary, 'a', 2), substring_index(utf8_lcase, 'a', 2) from t5;
select substring_index(utf8_binary, 'AaAA' collate utf8_lcase, 2), substring_index(utf8_lcase, 'AAa' collate utf8_binary, 2) from t5;
+select substring_index(utf8_binary, 'AaAA ' collate utf8_lcase_rtrim, 2), substring_index(utf8_lcase, 'AAa' collate utf8_binary, 2) from t5;
-- StringInStr
select instr(utf8_binary, utf8_lcase) from t5;
@@ -209,7 +214,7 @@ select find_in_set(utf8_binary, utf8_lcase collate utf8_binary) from t5;
select find_in_set(utf8_binary collate utf8_lcase, utf8_lcase collate utf8_lcase) from t5;
select find_in_set(utf8_binary, 'aaAaaAaA,i̇o'), find_in_set(utf8_lcase, 'aaAaaAaA,i̇o') from t5;
select find_in_set(utf8_binary, 'aaAaaAaA,i̇o' collate utf8_lcase), find_in_set(utf8_lcase, 'aaAaaAaA,i̇o' collate utf8_binary) from t5;
-
+select find_in_set(utf8_binary, 'aaAaaAaA,i̇o ' collate utf8_lcase_rtrim), find_in_set(utf8_lcase, 'aaAaaAaA,i̇o' collate utf8_binary) from t5;
-- StartsWith
select startswith(utf8_binary, utf8_lcase) from t5;
select startswith(s, utf8_binary) from t5;
@@ -219,6 +224,7 @@ select startswith(utf8_binary collate utf8_lcase, utf8_lcase collate utf8_lcase)
select startswith(utf8_binary collate unicode_ai, utf8_lcase collate unicode_ai) from t5;
select startswith(utf8_binary, 'aaAaaAaA'), startswith(utf8_lcase, 'aaAaaAaA') from t5;
select startswith(utf8_binary, 'aaAaaAaA' collate utf8_lcase), startswith(utf8_lcase, 'aaAaaAaA' collate utf8_binary) from t5;
+select startswith(utf8_binary, 'aaAaaAaA ' collate utf8_lcase_rtrim), startswith(utf8_lcase, 'aaAaaAaA' collate utf8_binary) from t5;
-- StringTranslate
select translate(utf8_lcase, utf8_lcase, '12345') from t5;
@@ -228,6 +234,7 @@ select translate(utf8_binary, 'SQL' collate utf8_lcase, '12345' collate utf8_lca
select translate(utf8_binary, 'SQL' collate unicode_ai, '12345' collate unicode_ai) from t5;
select translate(utf8_lcase, 'aaAaaAaA', '12345'), translate(utf8_binary, 'aaAaaAaA', '12345') from t5;
select translate(utf8_lcase, 'aBc' collate utf8_binary, '12345'), translate(utf8_binary, 'aBc' collate utf8_lcase, '12345') from t5;
+select translate(utf8_lcase, 'aBc ' collate utf8_binary_rtrim, '12345'), translate(utf8_binary, 'aBc' collate utf8_lcase, '12345') from t5;
-- Replace
select replace(utf8_binary, utf8_lcase, 'abc') from t5;
@@ -238,6 +245,7 @@ select replace(utf8_binary collate utf8_lcase, utf8_lcase collate utf8_lcase, 'a
select replace(utf8_binary collate unicode_ai, utf8_lcase collate unicode_ai, 'abc') from t5;
select replace(utf8_binary, 'aaAaaAaA', 'abc'), replace(utf8_lcase, 'aaAaaAaA', 'abc') from t5;
select replace(utf8_binary, 'aaAaaAaA' collate utf8_lcase, 'abc'), replace(utf8_lcase, 'aaAaaAaA' collate utf8_binary, 'abc') from t5;
+select replace(utf8_binary, 'aaAaaAaA ' collate utf8_lcase_rtrim, 'abc'), replace(utf8_lcase, 'aaAaaAaA' collate utf8_binary, 'abc') from t5;
-- EndsWith
select endswith(utf8_binary, utf8_lcase) from t5;
@@ -248,6 +256,7 @@ select endswith(utf8_binary collate utf8_lcase, utf8_lcase collate utf8_lcase) f
select endswith(utf8_binary collate unicode_ai, utf8_lcase collate unicode_ai) from t5;
select endswith(utf8_binary, 'aaAaaAaA'), endswith(utf8_lcase, 'aaAaaAaA') from t5;
select endswith(utf8_binary, 'aaAaaAaA' collate utf8_lcase), endswith(utf8_lcase, 'aaAaaAaA' collate utf8_binary) from t5;
+select endswith(utf8_binary, 'aaAaaAaA ' collate utf8_lcase_rtrim), endswith(utf8_lcase, 'aaAaaAaA' collate utf8_binary) from t5;
-- StringRepeat
select repeat(utf8_binary, 3), repeat(utf8_lcase, 2) from t5;
@@ -362,6 +371,7 @@ select rpad(s, 8, utf8_binary) from t5;
select rpad(utf8_binary collate utf8_binary, 8, s collate utf8_lcase) from t5;
select rpad(utf8_binary, 8, utf8_lcase collate utf8_binary) from t5;
select rpad(utf8_binary collate utf8_lcase, 8, utf8_lcase collate utf8_lcase) from t5;
+select lpad(utf8_binary collate utf8_binary_rtrim, 8, utf8_lcase collate utf8_binary_rtrim) from t5;
select rpad(utf8_binary, 8, 'a'), rpad(utf8_lcase, 8, 'a') from t5;
select rpad(utf8_binary, 8, 'AaAA' collate utf8_lcase), rpad(utf8_lcase, 8, 'AAa' collate utf8_binary) from t5;
@@ -371,6 +381,7 @@ select lpad(s, 8, utf8_binary) from t5;
select lpad(utf8_binary collate utf8_binary, 8, s collate utf8_lcase) from t5;
select lpad(utf8_binary, 8, utf8_lcase collate utf8_binary) from t5;
select lpad(utf8_binary collate utf8_lcase, 8, utf8_lcase collate utf8_lcase) from t5;
+select lpad(utf8_binary collate utf8_binary_rtrim, 8, utf8_lcase collate utf8_binary_rtrim) from t5;
select lpad(utf8_binary, 8, 'a'), lpad(utf8_lcase, 8, 'a') from t5;
select lpad(utf8_binary, 8, 'AaAA' collate utf8_lcase), lpad(utf8_lcase, 8, 'AAa' collate utf8_binary) from t5;
@@ -383,6 +394,7 @@ select locate(utf8_binary collate utf8_lcase, utf8_lcase collate utf8_lcase, 3)
select locate(utf8_binary collate unicode_ai, utf8_lcase collate unicode_ai, 3) from t5;
select locate(utf8_binary, 'a'), locate(utf8_lcase, 'a') from t5;
select locate(utf8_binary, 'AaAA' collate utf8_lcase, 4), locate(utf8_lcase, 'AAa' collate utf8_binary, 4) from t5;
+select locate(utf8_binary, 'AaAA ' collate utf8_binary_rtrim, 4), locate(utf8_lcase, 'AAa ' collate utf8_binary, 4) from t5;
-- StringTrim
select TRIM(utf8_binary, utf8_lcase) from t5;
@@ -391,6 +403,7 @@ select TRIM(utf8_binary collate utf8_binary, s collate utf8_lcase) from t5;
select TRIM(utf8_binary, utf8_lcase collate utf8_binary) from t5;
select TRIM(utf8_binary collate utf8_lcase, utf8_lcase collate utf8_lcase) from t5;
select TRIM(utf8_binary collate unicode_ai, utf8_lcase collate unicode_ai) from t5;
+select TRIM(utf8_binary collate utf8_binary_rtrim, utf8_lcase collate utf8_binary_rtrim) from t5;
select TRIM('ABc', utf8_binary), TRIM('ABc', utf8_lcase) from t5;
select TRIM('ABc' collate utf8_lcase, utf8_binary), TRIM('AAa' collate utf8_binary, utf8_lcase) from t5;
-- StringTrimBoth
@@ -400,6 +413,7 @@ select BTRIM(utf8_binary collate utf8_binary, s collate utf8_lcase) from t5;
select BTRIM(utf8_binary, utf8_lcase collate utf8_binary) from t5;
select BTRIM(utf8_binary collate utf8_lcase, utf8_lcase collate utf8_lcase) from t5;
select BTRIM(utf8_binary collate unicode_ai, utf8_lcase collate unicode_ai) from t5;
+select BTRIM(utf8_binary collate utf8_binary_rtrim, utf8_lcase collate utf8_binary_rtrim) from t5;
select BTRIM('ABc', utf8_binary), BTRIM('ABc', utf8_lcase) from t5;
select BTRIM('ABc' collate utf8_lcase, utf8_binary), BTRIM('AAa' collate utf8_binary, utf8_lcase) from t5;
-- StringTrimLeft
@@ -409,6 +423,7 @@ select LTRIM(utf8_binary collate utf8_binary, s collate utf8_lcase) from t5;
select LTRIM(utf8_binary, utf8_lcase collate utf8_binary) from t5;
select LTRIM(utf8_binary collate utf8_lcase, utf8_lcase collate utf8_lcase) from t5;
select LTRIM(utf8_binary collate unicode_ai, utf8_lcase collate unicode_ai) from t5;
+select LTRIM(utf8_binary collate utf8_binary_rtrim, utf8_lcase collate utf8_binary_rtrim) from t5;
select LTRIM('ABc', utf8_binary), LTRIM('ABc', utf8_lcase) from t5;
select LTRIM('ABc' collate utf8_lcase, utf8_binary), LTRIM('AAa' collate utf8_binary, utf8_lcase) from t5;
-- StringTrimRight
@@ -418,6 +433,7 @@ select RTRIM(utf8_binary collate utf8_binary, s collate utf8_lcase) from t5;
select RTRIM(utf8_binary, utf8_lcase collate utf8_binary) from t5;
select RTRIM(utf8_binary collate utf8_lcase, utf8_lcase collate utf8_lcase) from t5;
select RTRIM(utf8_binary collate unicode_ai, utf8_lcase collate unicode_ai) from t5;
+select RTRIM(utf8_binary collate utf8_binary_rtrim, utf8_lcase collate utf8_binary_rtrim) from t5;
select RTRIM('ABc', utf8_binary), RTRIM('ABc', utf8_lcase) from t5;
select RTRIM('ABc' collate utf8_lcase, utf8_binary), RTRIM('AAa' collate utf8_binary, utf8_lcase) from t5;
diff --git a/sql/core/src/test/resources/sql-tests/results/collations.sql.out b/sql/core/src/test/resources/sql-tests/results/collations.sql.out
index d64b8869905d..f92fc5de8c3f 100644
--- a/sql/core/src/test/resources/sql-tests/results/collations.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/collations.sql.out
@@ -708,6 +708,22 @@ struct<>
+-- !query
+insert into t5 values ('İo', 'İo', 'İo ')
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+insert into t5 values ('İo', 'İo ', 'İo')
+-- !query schema
+struct<>
+-- !query output
+
+
+
-- !query
insert into t5 values ('İo', 'İo', 'i̇o')
-- !query schema
@@ -893,6 +909,8 @@ abc abc
efd2 efd2
i̇o i̇o
sitTing sitTing
+İo İo
+İo İo
İo İo
@@ -942,6 +960,8 @@ abcdcba SQL
bbAbAAbA SQL
efd2 SQL
kitten SQL
+İo SQL
+İo SQL
İo SQL
İo SQL
@@ -963,6 +983,8 @@ abc,word abc,word
efd2,word efd2,word
i̇o,word İo,word
sitTing,word kitten,word
+İo ,word İo,word
+İo,word İo ,word
İo,word İo,word
@@ -983,6 +1005,8 @@ abc,word abc,word
efd2,word efd2,word
i̇o,word İo,word
sitTing,word kitten,word
+İo ,word İo,word
+İo,word İo ,word
İo,word İo,word
@@ -1004,6 +1028,8 @@ efd2
kitten
İo
İo
+İo
+İo
-- !query
@@ -1054,6 +1080,8 @@ efd2
kitten
İo
İo
+İo
+İo
-- !query
@@ -1074,6 +1102,8 @@ efd2
kitten
İo
İo
+İo
+İo
-- !query
@@ -1094,6 +1124,8 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
-- !query
@@ -1114,6 +1146,8 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
-- !query
@@ -1148,7 +1182,9 @@ struct<split_part(s, utf8_binary, 1):string>
+
bbAbaAbA
+İo
-- !query
@@ -1191,6 +1227,8 @@ struct<split_part(collate(utf8_binary, utf8_lcase), collate(utf8_lcase, utf8_lca
+
+
b
@@ -1235,6 +1273,8 @@ struct<split_part(utf8_binary, a, 3):string,split_part(utf8_lcase, a, 3):string
+
+
A
A
A
@@ -1255,6 +1295,30 @@ struct<split_part(utf8_binary, collate(a, utf8_lcase), 3):string collate UTF8_LC
+
+
+ A
+ A
+ A
+
+
+-- !query
+select split_part(utf8_binary, 'a ' collate utf8_lcase_rtrim, 3), split_part(utf8_lcase, 'a' collate utf8_binary, 3) from t5
+-- !query schema
+struct<split_part(utf8_binary, collate(a , utf8_lcase_rtrim), 3):string collate UTF8_LCASE_RTRIM,split_part(utf8_lcase, collate(a, utf8_binary), 3):string>
+-- !query output
+
+
+
+
+
+
+
+
+
+
+
+
A
A
A
@@ -1281,6 +1345,8 @@ select contains(s, utf8_binary) from t5
struct<contains(s, utf8_binary):boolean>
-- !query output
false
+false
+true
true
true
true
@@ -1322,6 +1388,8 @@ false
false
false
false
+false
+true
true
true
true
@@ -1338,6 +1406,8 @@ struct<contains(collate(utf8_binary, utf8_lcase), collate(utf8_lcase, utf8_lcase
false
false
false
+false
+true
true
true
true
@@ -1386,6 +1456,8 @@ false false
false false
false false
false false
+false false
+false false
false true
true false
true true
@@ -1411,6 +1483,30 @@ false false
false false
false false
false false
+false false
+false false
+true false
+true false
+true true
+
+
+-- !query
+select contains(utf8_binary, 'AaAA ' collate utf8_lcase_rtrim), contains(utf8_lcase, 'AAa ' collate utf8_binary_rtrim) from t5
+-- !query schema
+struct<contains(utf8_binary, collate(AaAA , utf8_lcase_rtrim)):boolean,contains(utf8_lcase, collate(AAa , utf8_binary_rtrim)):boolean>
+-- !query output
+false false
+false false
+false false
+false false
+false false
+false false
+false false
+false false
+false false
+false false
+false false
+false false
true false
true false
true true
@@ -1448,7 +1544,9 @@ struct<substring_index(s, utf8_binary, 1):string>
+
bbAbaAbA
+İo
-- !query
@@ -1484,6 +1582,8 @@ efd2
kitten
İo
İo
+İo
+İo
-- !query
@@ -1504,6 +1604,8 @@ efd2
kitten
İo
İo
+İo
+İo
-- !query
@@ -1550,6 +1652,8 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
-- !query
@@ -1570,6 +1674,30 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
+
+
+-- !query
+select substring_index(utf8_binary, 'AaAA ' collate utf8_lcase_rtrim, 2), substring_index(utf8_lcase, 'AAa' collate utf8_binary, 2) from t5
+-- !query schema
+struct<substring_index(utf8_binary, collate(AaAA , utf8_lcase_rtrim), 2):string collate UTF8_LCASE_RTRIM,substring_index(utf8_lcase, collate(AAa, utf8_binary), 2):string>
+-- !query output
+Hello, world! Nice day. Hello, world! Nice day.
+Something else. Nothing here. Something else. Nothing here.
+Spark SQL
+a aaAaAAaA
+a aaAaaAaA
+a aaAaaAaAaaAaaAaAaaAaaAaA
+abc abc
+abcdcba aBcDCbA
+bbAbAAbA a
+efd2 efd2
+kitten sitTing
+İo i̇o
+İo İo
+İo İo
+İo İo
-- !query
@@ -1593,6 +1721,8 @@ select instr(s, utf8_binary) from t5
struct<instr(s, utf8_binary):int>
-- !query output
0
+0
+1
1
1
1
@@ -1634,6 +1764,8 @@ struct<instr(utf8_binary, collate(utf8_lcase, utf8_binary)):int>
0
0
0
+0
+1
1
1
1
@@ -1650,6 +1782,8 @@ struct<instr(collate(utf8_binary, utf8_lcase), collate(utf8_lcase, utf8_lcase)):
0
0
0
+0
+1
1
1
1
@@ -1698,6 +1832,8 @@ struct<instr(utf8_binary, a):int,instr(utf8_lcase, a):int>
0 0
0 0
0 0
+0 0
+0 0
0 1
1 1
1 1
@@ -1723,6 +1859,8 @@ struct<instr(utf8_binary, collate(AaAA, utf8_lcase)):int,instr(utf8_lcase, colla
0 0
0 0
0 0
+0 0
+0 0
1 0
1 0
1 5
@@ -1750,6 +1888,8 @@ struct<find_in_set(s, utf8_binary):int>
-- !query output
0
0
+0
+1
1
1
1
@@ -1791,6 +1931,8 @@ struct<find_in_set(utf8_binary, collate(utf8_lcase, utf8_binary)):int>
0
0
0
+0
+0
1
1
1
@@ -1808,6 +1950,8 @@ struct<find_in_set(collate(utf8_binary, utf8_lcase), collate(utf8_lcase, utf8_lc
0
0
0
+0
+0
1
1
1
@@ -1832,10 +1976,12 @@ struct<find_in_set(utf8_binary, aaAaaAaA,i̇o):int,find_in_set(utf8_lcase, aaAaa
0 0
0 0
0 0
+0 0
0 1
0 1
0 2
0 2
+0 2
-- !query
@@ -1851,10 +1997,34 @@ struct<find_in_set(utf8_binary, collate(aaAaaAaA,i̇o, utf8_lcase)):int,find_in_
0 0
0 0
0 0
+0 0
1 0
1 0
1 1
2 0
+2 0
+2 2
+
+
+-- !query
+select find_in_set(utf8_binary, 'aaAaaAaA,i̇o ' collate utf8_lcase_rtrim), find_in_set(utf8_lcase, 'aaAaaAaA,i̇o' collate utf8_binary) from t5
+-- !query schema
+struct<find_in_set(utf8_binary, collate(aaAaaAaA,i̇o , utf8_lcase_rtrim)):int,find_in_set(utf8_lcase, collate(aaAaaAaA,i̇o, utf8_binary)):int>
+-- !query output
+0 0
+0 0
+0 0
+0 0
+0 0
+0 0
+0 0
+0 0
+1 0
+1 0
+1 1
+2 0
+2 0
+2 0
2 2
@@ -1879,6 +2049,8 @@ select startswith(s, utf8_binary) from t5
struct<startswith(s, utf8_binary):boolean>
-- !query output
false
+false
+true
true
true
true
@@ -1920,6 +2092,8 @@ false
false
false
false
+false
+true
true
true
true
@@ -1937,6 +2111,8 @@ false
false
false
false
+false
+true
true
true
true
@@ -1989,6 +2165,8 @@ false false
false false
false false
false false
+false false
+false false
false true
false true
false true
@@ -2009,6 +2187,30 @@ false false
false false
false false
false false
+false false
+false false
+true false
+true true
+true true
+
+
+-- !query
+select startswith(utf8_binary, 'aaAaaAaA ' collate utf8_lcase_rtrim), startswith(utf8_lcase, 'aaAaaAaA' collate utf8_binary) from t5
+-- !query schema
+struct<startswith(utf8_binary, collate(aaAaaAaA , utf8_lcase_rtrim)):boolean,startswith(utf8_lcase, collate(aaAaaAaA, utf8_binary)):boolean>
+-- !query output
+false false
+false false
+false false
+false false
+false false
+false false
+false false
+false false
+false false
+false false
+false false
+false false
true false
true true
true true
@@ -2024,6 +2226,8 @@ struct<translate(utf8_lcase, utf8_lcase, 12345):string collate UTF8_LCASE>
11111111
111111111111111111111111
12
+12
+123
123
123
123
@@ -2082,6 +2286,8 @@ efd2
kitten
İo
İo
+İo
+İo
-- !query
@@ -2128,6 +2334,8 @@ efd2 efd2
i̇o İo
sitTing kitten
İo İo
+İo İo
+İo İo
-- !query
@@ -2148,6 +2356,30 @@ efd2 efd2
i̇o İo
sitTing kitten
İo İo
+İo İo
+İo İo
+
+
+-- !query
+select translate(utf8_lcase, 'aBc ' collate utf8_binary_rtrim, '12345'), translate(utf8_binary, 'aBc' collate utf8_lcase, '12345') from t5
+-- !query schema
+struct<translate(utf8_lcase, collate(aBc , utf8_binary_rtrim), 12345):string collate UTF8_BINARY_RTRIM,translate(utf8_binary, collate(aBc, utf8_lcase), 12345):string collate UTF8_LCASE>
+-- !query output
+1 22121121
+11A11A1A 11111111
+11A11A1A11A11A1A11A11A1A 11111111
+11A1AA1A 11111111
+123DCbA 123d321
+1b3 123
+Hello,4world!4Ni3e4d1y. Hello, world! Ni3e d1y.
+SQL Sp1rk
+Something4else.4Nothing4here. Something else. Nothing here.
+efd2 efd2
+i̇o İo
+sitTing kitten
+İo İo
+İo İo
+İo4 İo
-- !query
@@ -2182,7 +2414,9 @@ abc
abc
abc
abc
+abc
bbAbaAbA
+İo
-- !query
@@ -2214,10 +2448,12 @@ abc
abc
abc
abc
+abc
abcdcba
bbAbAAbA
kitten
İo
+İo
-- !query
@@ -2236,8 +2472,10 @@ abc
abc
abc
abc
+abc
bbabcbabcabcbabc
kitten
+İo
-- !query
@@ -2284,6 +2522,8 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
-- !query
@@ -2304,6 +2544,30 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
+
+
+-- !query
+select replace(utf8_binary, 'aaAaaAaA ' collate utf8_lcase_rtrim, 'abc'), replace(utf8_lcase, 'aaAaaAaA' collate utf8_binary, 'abc') from t5
+-- !query schema
+struct<replace(utf8_binary, collate(aaAaaAaA , utf8_lcase_rtrim), abc):string collate UTF8_LCASE_RTRIM,replace(utf8_lcase, collate(aaAaaAaA, utf8_binary), abc):string>
+-- !query output
+Hello, world! Nice day. Hello, world! Nice day.
+Something else. Nothing here. Something else. Nothing here.
+Spark SQL
+aaAaAAaA aaAaAAaA
+aaAaAAaA abc
+aaAaAAaA abcabcabc
+abc abc
+abcdcba aBcDCbA
+bbAbAAbA a
+efd2 efd2
+kitten sitTing
+İo i̇o
+İo İo
+İo İo
+İo İo
-- !query
@@ -2327,6 +2591,8 @@ select endswith(s, utf8_binary) from t5
struct<endswith(s, utf8_binary):boolean>
-- !query output
false
+false
+true
true
true
true
@@ -2368,6 +2634,8 @@ false
false
false
false
+false
+false
true
true
true
@@ -2384,6 +2652,8 @@ struct<endswith(collate(utf8_binary, utf8_lcase), collate(utf8_lcase, utf8_lcase
false
false
false
+false
+false
true
true
true
@@ -2437,6 +2707,8 @@ false false
false false
false false
false false
+false false
+false false
false true
false true
false true
@@ -2457,6 +2729,30 @@ false false
false false
false false
false false
+false false
+false false
+true false
+true true
+true true
+
+
+-- !query
+select endswith(utf8_binary, 'aaAaaAaA ' collate utf8_lcase_rtrim), endswith(utf8_lcase, 'aaAaaAaA' collate utf8_binary) from t5
+-- !query schema
+struct<endswith(utf8_binary, collate(aaAaaAaA , utf8_lcase_rtrim)):boolean,endswith(utf8_lcase, collate(aaAaaAaA, utf8_binary)):boolean>
+-- !query output
+false false
+false false
+false false
+false false
+false false
+false false
+false false
+false false
+false false
+false false
+false false
+false false
true false
true true
true true
@@ -2478,7 +2774,9 @@ abcdcbaabcdcbaabcdcba aBcDCbAaBcDCbA
bbAbAAbAbbAbAAbAbbAbAAbA aa
efd2efd2efd2 efd2efd2
kittenkittenkitten sitTingsitTing
+İo İo İo İoİo
İoİoİo i̇oi̇o
+İoİoİo İo İo
İoİoİo İoİo
@@ -2498,7 +2796,9 @@ abcdcbaabcdcbaabcdcba aBcDCbAaBcDCbA
bbAbAAbAbbAbAAbAbbAbAAbA aa
efd2efd2efd2 efd2efd2
kittenkittenkitten sitTingsitTing
+İo İo İo İoİo
İoİoİo i̇oi̇o
+İoİoİo İo İo
İoİoİo İoİo
@@ -2511,6 +2811,8 @@ struct<ascii(utf8_binary):int,ascii(utf8_lcase):int>
107 115
304 105
304 304
+304 304
+304 304
72 72
83 83
83 83
@@ -2531,6 +2833,8 @@ struct<ascii(collate(utf8_binary, utf8_lcase)):int,ascii(collate(utf8_lcase, utf
107 115
304 105
304 304
+304 304
+304 304
72 72
83 83
83 83
@@ -2587,6 +2891,8 @@ ZWZkMg== ZWZkMg==
a2l0dGVu c2l0VGluZw==
xLBv acyHbw==
xLBv xLBv
+xLBv xLBvIA==
+xLBvIA== xLBv
-- !query
@@ -2607,6 +2913,8 @@ ZWZkMg== ZWZkMg==
a2l0dGVu c2l0VGluZw==
xLBv acyHbw==
xLBv xLBv
+xLBv xLBvIA==
+xLBvIA== xLBv
-- !query
@@ -2627,6 +2935,8 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
-- !query
@@ -2647,6 +2957,8 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
-- !query
@@ -2685,6 +2997,8 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
-- !query
@@ -2705,6 +3019,8 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
-- !query
@@ -2725,6 +3041,8 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
-- !query
@@ -2745,6 +3063,8 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
-- !query
@@ -2765,6 +3085,8 @@ struct<sentences(utf8_binary, , ):array<array<string>>,sentences(utf8_lcase, , )
[["kitten"]] [["sitTing"]]
[["İo"]] [["i̇o"]]
[["İo"]] [["İo"]]
+[["İo"]] [["İo"]]
+[["İo"]] [["İo"]]
-- !query
@@ -2785,6 +3107,8 @@ struct<sentences(collate(utf8_binary, utf8_lcase), , ):array<array<string collat
[["kitten"]] [["sitTing"]]
[["İo"]] [["i̇o"]]
[["İo"]] [["İo"]]
+[["İo"]] [["İo"]]
+[["İo"]] [["İo"]]
-- !query
@@ -2805,6 +3129,8 @@ SOMETHING ELSE. NOTHING HERE. SOMETHING ELSE. NOTHING HERE.
SPARK SQL
İO İO
İO İO
+İO İO
+İO İO
-- !query
@@ -2825,6 +3151,8 @@ SOMETHING ELSE. NOTHING HERE. SOMETHING ELSE. NOTHING HERE.
SPARK SQL
İO İO
İO İO
+İO İO
+İO İO
-- !query
@@ -2842,6 +3170,8 @@ efd2 efd2
hello, world! nice day. hello, world! nice day.
i̇o i̇o
i̇o i̇o
+i̇o i̇o
+i̇o i̇o
kitten sitting
something else. nothing here. something else. nothing here.
spark sql
@@ -2862,6 +3192,8 @@ efd2 efd2
hello, world! nice day. hello, world! nice day.
i̇o i̇o
i̇o i̇o
+i̇o i̇o
+i̇o i̇o
kitten sitting
something else. nothing here. something else. nothing here.
spark sql
@@ -2885,6 +3217,8 @@ Something Else. Nothing Here. Something Else. Nothing Here.
Spark Sql
İo İo
İo İo
+İo İo
+İo İo
-- !query
@@ -2905,6 +3239,8 @@ Something Else. Nothing Here. Something Else. Nothing Here.
Spark Sql
İo İo
İo İo
+İo İo
+İo İo
-- !query
@@ -2940,6 +3276,8 @@ efd2
kitten
İo
İo
+İo
+İo
-- !query
@@ -2975,6 +3313,8 @@ eefd2
ksitTing
İi̇o
İİo
+İİo
+İİo
-- !query
@@ -2995,6 +3335,8 @@ eefd2
ksitTing
İi̇o
İİo
+İİo
+İİo
-- !query
@@ -3015,6 +3357,8 @@ ead2 ead2
katten satTing
İa iao
İa İa
+İa İa
+İa İa
-- !query
@@ -3035,6 +3379,8 @@ eAaAA eAAa
kAaAAn sAAaing
İAaAA iAAa
İAaAA İAAa
+İAaAA İAAa
+İAaAA İAAa
-- !query
@@ -3079,6 +3425,8 @@ S162 S400
S535 S535
İo I000
İo İo
+İo İo
+İo İo
-- !query
@@ -3099,6 +3447,8 @@ S162 S400
S535 S535
İo I000
İo İo
+İo İo
+İo İo
-- !query
@@ -3108,8 +3458,10 @@ struct<length(utf8_binary):int,length(utf8_lcase):int>
-- !query output
2 2
2 3
+2 3
23 23
29 29
+3 2
3 3
4 4
5 3
@@ -3128,8 +3480,10 @@ struct<length(collate(utf8_binary, utf8_lcase)):int,length(collate(utf8_lcase, u
-- !query output
2 2
2 3
+2 3
23 23
29 29
+3 2
3 3
4 4
5 3
@@ -3151,6 +3505,8 @@ struct<bit_length(utf8_binary):int,bit_length(utf8_lcase):int>
24 24
24 24
24 32
+24 32
+32 24
32 32
40 24
48 56
@@ -3171,6 +3527,8 @@ struct<bit_length(collate(utf8_binary, utf8_lcase)):int,bit_length(collate(utf8_
24 24
24 24
24 32
+24 32
+32 24
32 32
40 24
48 56
@@ -3191,6 +3549,8 @@ struct<octet_length(utf8_binary):int,octet_length(utf8_lcase):int>
3 3
3 3
3 4
+3 4
+4 3
4 4
5 3
6 7
@@ -3211,6 +3571,8 @@ struct<octet_length(collate(utf8_binary, utf8_lcase)):int,octet_length(collate(u
3 3
3 3
3 4
+3 4
+4 3
4 4
5 3
6 7
@@ -3262,6 +3624,8 @@ struct<levenshtein(s, utf8_binary):int>
0
0
0
+0
+1
1
@@ -3292,6 +3656,8 @@ struct<levenshtein(utf8_binary, collate(utf8_lcase, utf8_binary)):int>
0
0
1
+1
+1
16
2
4
@@ -3312,6 +3678,8 @@ struct<levenshtein(collate(utf8_binary, utf8_lcase), collate(utf8_lcase, utf8_lc
0
0
1
+1
+1
16
2
4
@@ -3328,8 +3696,10 @@ struct<levenshtein(utf8_binary, a):int,levenshtein(utf8_lcase, a):int>
2 2
2 2
2 3
+2 3
22 22
29 29
+3 2
4 3
4 4
6 6
@@ -3356,6 +3726,8 @@ struct<levenshtein(utf8_binary, collate(AaAA, utf8_lcase), 3):int,levenshtein(ut
-1 3
-1 3
-1 3
+-1 3
+-1 3
-1 4
3 3
@@ -3378,6 +3750,8 @@ true true
true true
true true
true true
+true true
+true true
-- !query
@@ -3398,6 +3772,8 @@ true true
true true
true true
true true
+true true
+true true
-- !query
@@ -3418,6 +3794,8 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
-- !query
@@ -3438,6 +3816,8 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
-- !query
@@ -3458,6 +3838,8 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
-- !query
@@ -3478,6 +3860,8 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
-- !query
@@ -3498,6 +3882,8 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
-- !query
@@ -3518,6 +3904,8 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
-- !query
@@ -3535,7 +3923,9 @@ el el
fd fd
it it
o o
+o o
o ̇o
+o o
om om
pa QL
@@ -3555,7 +3945,9 @@ el el
fd fd
it it
o o
+o o
o ̇o
+o o
om om
pa QL
@@ -3574,8 +3966,10 @@ bc bc
d2 d2
e. e.
en ng
+o İo
rk QL
y. y.
+İo o
İo İo
İo ̇o
@@ -3594,8 +3988,10 @@ bc bc
d2 d2
e. e.
en ng
+o İo
rk QL
y. y.
+İo o
İo İo
İo ̇o
@@ -3618,6 +4014,8 @@ ef ef
ki si
İo i̇
İo İo
+İo İo
+İo İo
-- !query
@@ -3638,6 +4036,8 @@ ef ef
ki si
İo i̇
İo İo
+İo İo
+İo İo
-- !query
@@ -3671,6 +4071,8 @@ abcdcbaa
bbAbaAbA
efd2efd2
kittenki
+İoİo İo
+İoİoİoİo
İoİoİoİo
İoİoİoİo
@@ -3706,7 +4108,9 @@ abcdcbaa
bbAbAAbA
efd2efd2
kittensi
+İo İoİoİ
İoi̇oi̇o
+İoİo İo
İoİoİoİo
@@ -3726,10 +4130,34 @@ abcdcbaa
bbAbAAbA
efd2efd2
kittensi
+İo İoİoİ
İoi̇oi̇o
+İoİo İo
İoİoİoİo
+-- !query
+select lpad(utf8_binary collate utf8_binary_rtrim, 8, utf8_lcase collate utf8_binary_rtrim) from t5
+-- !query schema
+struct<lpad(collate(utf8_binary, utf8_binary_rtrim), 8, collate(utf8_lcase, utf8_binary_rtrim)):string collate UTF8_BINARY_RTRIM>
+-- !query output
+Hello, w
+SQLSpark
+Somethin
+aaAaAAaA
+aaAaAAaA
+aaAaAAaA
+aabcdcba
+abcababc
+bbAbAAbA
+efd2efd2
+i̇oi̇oİo
+sikitten
+İo İo İo
+İoİoİoİo
+İoİoİİo
+
+
-- !query
select rpad(utf8_binary, 8, 'a'), rpad(utf8_lcase, 8, 'a') from t5
-- !query schema
@@ -3746,7 +4174,9 @@ abcdcbaa aBcDCbAa
bbAbAAbA aaaaaaaa
efd2aaaa efd2aaaa
kittenaa sitTinga
+İo aaaaa İoaaaaaa
İoaaaaaa i̇oaaaaa
+İoaaaaaa İo aaaaa
İoaaaaaa İoaaaaaa
@@ -3766,7 +4196,9 @@ abcdcbaA aBcDCbAA
bbAbAAbA aAAaAAaA
efd2AaAA efd2AAaA
kittenAa sitTingA
+İo AaAAA İoAAaAAa
İoAaAAAa i̇oAAaAA
+İoAaAAAa İo AAaAA
İoAaAAAa İoAAaAAa
@@ -3801,6 +4233,8 @@ abcababc
bbAbaAbA
efd2efd2
kikitten
+İo İo İo
+İoİoİoİo
İoİoİoİo
İoİoİoİo
@@ -3837,7 +4271,9 @@ bbAbAAbA
efd2efd2
i̇oi̇oİo
sikitten
+İo İo İo
İoİoİoİo
+İoİoİİo
-- !query
@@ -3857,7 +4293,31 @@ bbAbAAbA
efd2efd2
i̇oi̇oİo
sikitten
+İo İo İo
İoİoİoİo
+İoİoİİo
+
+
+-- !query
+select lpad(utf8_binary collate utf8_binary_rtrim, 8, utf8_lcase collate utf8_binary_rtrim) from t5
+-- !query schema
+struct<lpad(collate(utf8_binary, utf8_binary_rtrim), 8, collate(utf8_lcase, utf8_binary_rtrim)):string collate UTF8_BINARY_RTRIM>
+-- !query output
+Hello, w
+SQLSpark
+Somethin
+aaAaAAaA
+aaAaAAaA
+aaAaAAaA
+aabcdcba
+abcababc
+bbAbAAbA
+efd2efd2
+i̇oi̇oİo
+sikitten
+İo İo İo
+İoİoİoİo
+İoİoİİo
-- !query
@@ -3874,6 +4334,8 @@ aaaSpark aaaaaSQL
aaaaaabc aaaaaabc
aaaaaaİo aaaaaaİo
aaaaaaİo aaaaai̇o
+aaaaaaİo aaaaaİo
+aaaaaİo aaaaaaİo
aaaaefd2 aaaaefd2
aabcdcba aaBcDCbA
aakitten asitTing
@@ -3888,6 +4350,8 @@ struct<lpad(utf8_binary, 8, collate(AaAA, utf8_lcase)):string collate UTF8_LCASE
AaAAAabc AAaAAabc
AaAAAaİo AAaAAaİo
AaAAAaİo AAaAAi̇o
+AaAAAaİo AAaAAİo
+AaAAAİo AAaAAaİo
AaAAefd2 AAaAefd2
AaASpark AAaAASQL
Aabcdcba AaBcDCbA
@@ -3933,6 +4397,8 @@ struct<locate(s, utf8_binary, 1):int>
1
1
1
+1
+1
-- !query
@@ -3962,6 +4428,8 @@ struct<locate(utf8_binary, collate(utf8_lcase, utf8_binary), 1):int>
0
0
0
+0
+1
1
1
1
@@ -3987,6 +4455,8 @@ struct<locate(collate(utf8_binary, utf8_lcase), collate(utf8_lcase, utf8_lcase),
0
0
0
+0
+0
3
@@ -4033,6 +4503,8 @@ struct<locate(utf8_binary, a, 1):int,locate(utf8_lcase, a, 1):int>
0 0
0 0
0 0
+0 0
+0 0
0 1
@@ -4054,6 +4526,30 @@ struct<locate(utf8_binary, collate(AaAA, utf8_lcase), 4):int,locate(utf8_lcase,
0 0
0 0
0 0
+0 0
+0 0
+
+
+-- !query
+select locate(utf8_binary, 'AaAA ' collate utf8_binary_rtrim, 4), locate(utf8_lcase, 'AAa ' collate utf8_binary, 4) from t5
+-- !query schema
+struct<locate(utf8_binary, collate(AaAA , utf8_binary_rtrim), 4):int,locate(utf8_lcase, collate(AAa , utf8_binary), 4):int>
+-- !query output
+0 0
+0 0
+0 0
+0 0
+0 0
+0 0
+0 0
+0 0
+0 0
+0 0
+0 0
+0 0
+0 0
+0 0
+0 0
-- !query
@@ -4107,6 +4603,8 @@ struct<TRIM(BOTH utf8_binary FROM collate(utf8_lcase, utf8_binary)):string>
+
+
BcDCbA
QL
a
@@ -4130,6 +4628,8 @@ struct<TRIM(BOTH collate(utf8_binary, utf8_lcase) FROM collate(utf8_lcase, utf8_
+
+
QL
sitTing
@@ -4160,6 +4660,28 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
}
+-- !query
+select TRIM(utf8_binary collate utf8_binary_rtrim, utf8_lcase collate utf8_binary_rtrim) from t5
+-- !query schema
+struct<TRIM(BOTH collate(utf8_binary, utf8_binary_rtrim) FROM collate(utf8_lcase, utf8_binary_rtrim)):string collate UTF8_BINARY_RTRIM>
+-- !query output
+
+
+
+
+
+
+
+
+
+
+BcDCbA
+QL
+a
+i̇
+sitTing
+
+
-- !query
select TRIM('ABc', utf8_binary), TRIM('ABc', utf8_lcase) from t5
-- !query schema
@@ -4178,6 +4700,8 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
-- !query
@@ -4198,6 +4722,8 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
-- !query
@@ -4232,6 +4758,8 @@ struct<btrim(s, utf8_binary):string>
+
+
a
@@ -4263,6 +4791,8 @@ struct<btrim(utf8_binary, collate(utf8_lcase, utf8_binary)):string>
+
+
bbAbAAbA
d
kitte
@@ -4284,6 +4814,8 @@ struct<btrim(collate(utf8_binary, utf8_lcase), collate(utf8_lcase, utf8_lcase)):
+
+
bbAbAAb
kitte
park
@@ -4316,6 +4848,28 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
}
+-- !query
+select BTRIM(utf8_binary collate utf8_binary_rtrim, utf8_lcase collate utf8_binary_rtrim) from t5
+-- !query schema
+struct<btrim(collate(utf8_binary, utf8_binary_rtrim), collate(utf8_lcase, utf8_binary_rtrim)):string collate UTF8_BINARY_RTRIM>
+-- !query output
+
+
+
+
+
+
+
+
+
+
+bbAbAAbA
+d
+kitte
+park
+İ
+
+
-- !query
select BTRIM('ABc', utf8_binary), BTRIM('ABc', utf8_lcase) from t5
-- !query schema
@@ -4330,6 +4884,8 @@ ABc ABc
ABc ABc
ABc ABc
ABc ABc
+ABc ABc
+ABc ABc
Bc Bc
Bc Bc
Bc Bc
@@ -4348,6 +4904,8 @@ ABc AAa
ABc AAa
ABc AAa
ABc AAa
+ABc AAa
+ABc AAa
B AA
Bc
Bc
@@ -4407,6 +4965,8 @@ struct<TRIM(LEADING utf8_binary FROM collate(utf8_lcase, utf8_binary)):string>
+
+
BcDCbA
QL
a
@@ -4430,6 +4990,8 @@ struct<TRIM(LEADING collate(utf8_binary, utf8_lcase) FROM collate(utf8_lcase, ut
+
+
QL
sitTing
@@ -4460,6 +5022,28 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
}
+-- !query
+select LTRIM(utf8_binary collate utf8_binary_rtrim, utf8_lcase collate utf8_binary_rtrim) from t5
+-- !query schema
+struct<TRIM(LEADING collate(utf8_binary, utf8_binary_rtrim) FROM collate(utf8_lcase, utf8_binary_rtrim)):string collate UTF8_BINARY_RTRIM>
+-- !query output
+
+
+
+
+
+
+
+
+
+
+BcDCbA
+QL
+a
+i̇o
+sitTing
+
+
-- !query
select LTRIM('ABc', utf8_binary), LTRIM('ABc', utf8_lcase) from t5
-- !query schema
@@ -4478,6 +5062,8 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
-- !query
@@ -4498,6 +5084,8 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
-- !query
@@ -4523,6 +5111,20 @@ struct<TRIM(TRAILING s FROM utf8_binary):string>
+
+
+
+
+
+
+
+
+
+
+
+İo
+
+
-- !query
select RTRIM(utf8_binary collate utf8_binary, s collate utf8_lcase) from t5
-- !query schema
@@ -4551,11 +5153,13 @@ struct<TRIM(TRAILING utf8_binary FROM collate(utf8_lcase, utf8_binary)):string>
+
SQL
a
aBcDCbA
i̇
sitTing
+İo
-- !query
@@ -4574,8 +5178,10 @@ struct<TRIM(TRAILING collate(utf8_binary, utf8_lcase) FROM collate(utf8_lcase, u
+
SQL
sitTing
+İo
-- !query
@@ -4604,6 +5210,28 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
}
+-- !query
+select RTRIM(utf8_binary collate utf8_binary_rtrim, utf8_lcase collate utf8_binary_rtrim) from t5
+-- !query schema
+struct<TRIM(TRAILING collate(utf8_binary, utf8_binary_rtrim) FROM collate(utf8_lcase, utf8_binary_rtrim)):string collate UTF8_BINARY_RTRIM>
+-- !query output
+
+
+
+
+
+
+
+
+
+
+SQL
+a
+aBcDCbA
+i̇
+sitTing
+
+
-- !query
select RTRIM('ABc', utf8_binary), RTRIM('ABc', utf8_lcase) from t5
-- !query schema
@@ -4622,6 +5250,8 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
-- !query
@@ -4642,6 +5272,8 @@ efd2 efd2
kitten sitTing
İo i̇o
İo İo
+İo İo
+İo İo
-- !query
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CollationStringExpressionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CollationStringExpressionsSuite.scala
index 2e942ef442ef..9ee2cfb964fe 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CollationStringExpressionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CollationStringExpressionsSuite.scala
@@ -38,8 +38,13 @@ class CollationStringExpressionsSuite
result: R)
val testCases = Seq(
ConcatWsTestCase(" ", Array("Spark", "SQL"), "UTF8_BINARY", "Spark SQL"),
+ ConcatWsTestCase(" ", Array("Spark", "SQL"), "UTF8_BINARY_RTRIM", "Spark SQL"),
ConcatWsTestCase(" ", Array("Spark", "SQL"), "UTF8_LCASE", "Spark SQL"),
+ ConcatWsTestCase(" ", Array("Spark", "SQL"), "UTF8_LCASE_RTRIM", "Spark SQL"),
ConcatWsTestCase(" ", Array("Spark", "SQL"), "UNICODE", "Spark SQL"),
+ ConcatWsTestCase(" ", Array("Spark", "SQL"), "UNICODE_RTRIM", "Spark SQL"),
+ ConcatWsTestCase(" ", Array("Spark", "SQL"), "UNICODE_CI", "Spark SQL"),
+ ConcatWsTestCase(" ", Array("Spark", "SQL"), "UNICODE_CI_RTRIM", "Spark SQL"),
ConcatWsTestCase(" ", Array("Spark", "SQL"), "UNICODE_CI", "Spark SQL"),
ConcatWsTestCase(" ", Array("Spark", "Unterstützung"), "DE_CI_AI", "Spark Unterstützung")
)
@@ -65,8 +70,13 @@ class CollationStringExpressionsSuite
case class EltTestCase[R](index: Integer, inputs: Array[String], collation: String, result: R)
val testCases = Seq(
EltTestCase(1, Array("Spark", "SQL"), "UTF8_BINARY", "Spark"),
+ EltTestCase(1, Array("Spark", "SQL"), "UTF8_BINARY_RTRIM", "Spark"),
EltTestCase(1, Array("Spark", "SQL"), "UTF8_LCASE", "Spark"),
+ EltTestCase(1, Array("Spark", "SQL"), "UTF8_LCASE_RTRIM", "Spark"),
EltTestCase(2, Array("Spark", "SQL"), "UNICODE", "SQL"),
+ EltTestCase(2, Array("Spark", "SQL"), "UNICODE_RTRIM", "SQL"),
+ EltTestCase(2, Array("Spark", "SQL"), "UNICODE_CI", "SQL"),
+ EltTestCase(2, Array("Spark", "SQL"), "UNICODE_CI_RTRIM", "SQL"),
EltTestCase(2, Array("Spark", "SQL"), "UNICODE_CI", "SQL"),
EltTestCase(2, Array("Spark", "Unterstützung"), "DE_CI", "Unterstützung")
)
@@ -96,8 +106,13 @@ class CollationStringExpressionsSuite
result: R)
val testCases = Seq(
SplitPartTestCase("1a2", "a", 2, "UTF8_BINARY", "2"),
+ SplitPartTestCase("1a2", "a ", 1, "UTF8_BINARY_RTRIM", "1"),
SplitPartTestCase("1a2", "a", 2, "UNICODE", "2"),
+ SplitPartTestCase("1a 2", "a ", 2, "UNICODE_RTRIM", " 2"),
SplitPartTestCase("1a2", "A", 2, "UTF8_LCASE", "2"),
+ SplitPartTestCase("1 a2", "A ", 2, "UTF8_LCASE_RTRIM", "2"),
+ SplitPartTestCase("1a2", "A", 2, "UNICODE_CI", "2"),
+ SplitPartTestCase("1 a2 ", "A ", 2, "UNICODE_CI_RTRIM", "2 "),
SplitPartTestCase("1a2", "A", 2, "UNICODE_CI", "2"),
SplitPartTestCase("1ö2", "O", 2, "DE_CI_AI", "2")
)
@@ -145,8 +160,13 @@ class CollationStringExpressionsSuite
result: R)
val testCases = Seq(
StringSplitSQLTestCase("1a2", "a", "UTF8_BINARY", Array("1", "2")),
+ StringSplitSQLTestCase("1a2", "a ", "UTF8_BINARY_RTRIM", Array("1", "2")),
StringSplitSQLTestCase("1a2", "a", "UNICODE", Array("1", "2")),
+ StringSplitSQLTestCase("1a 2", "a ", "UNICODE_RTRIM", Array("1", " 2")),
StringSplitSQLTestCase("1a2", "A", "UTF8_LCASE", Array("1", "2")),
+ StringSplitSQLTestCase("1 a2", "A ", "UTF8_LCASE_RTRIM", Array("1 ", "2")),
+ StringSplitSQLTestCase("1a2", "A", "UNICODE_CI", Array("1", "2")),
+ StringSplitSQLTestCase("1 a2 ", "A ", "UNICODE_CI_RTRIM", Array("1 ", "2 ")),
StringSplitSQLTestCase("1a2", "A", "UNICODE_CI", Array("1", "2")),
StringSplitSQLTestCase("1ä2", "Ä", "DE_CI", Array("1", "2")),
StringSplitSQLTestCase("1ä2", "A", "DE_CI_AI", Array("1", "2"))
@@ -192,8 +212,13 @@ class CollationStringExpressionsSuite
case class ContainsTestCase[R](left: String, right: String, collation: String, result: R)
val testCases = Seq(
ContainsTestCase("", "", "UTF8_BINARY", true),
+ ContainsTestCase("", " ", "UTF8_BINARY_RTRIM", true),
ContainsTestCase("abcde", "C", "UNICODE", false),
+ ContainsTestCase("abcde", " C ", "UNICODE_RTRIM", false),
ContainsTestCase("abcde", "FGH", "UTF8_LCASE", false),
+ ContainsTestCase("abcde", "ABC ", "UTF8_LCASE_RTRIM", true),
+ ContainsTestCase("abcde", "BCD", "UNICODE_CI", true),
+ ContainsTestCase("ab c de ", "B C D ", "UNICODE_CI_RTRIM", true),
ContainsTestCase("abcde", "BCD", "UNICODE_CI", true),
ContainsTestCase("Priča o Maču u kamenu", "MAC", "SR_CI_AI", true),
ContainsTestCase("Priča o Maču u kamenu", "MAC", "SR_CI", false),
@@ -246,8 +271,14 @@ class CollationStringExpressionsSuite
val testCases = Seq(
SubstringIndexTestCase("wwwgapachegorg", "g", -3, "UTF8_BINARY", "apachegorg"),
SubstringIndexTestCase("www||apache||org", "||", 2, "UTF8_BINARY", "www||apache"),
+ SubstringIndexTestCase("wwwgapachegorg", "g ", -3, "UTF8_BINARY_RTRIM", "apachegorg"),
+ SubstringIndexTestCase("www ||apache||org", "|| ", 2, "UTF8_BINARY_RTRIM", "www ||apache"),
SubstringIndexTestCase("wwwXapacheXorg", "x", 2, "UTF8_LCASE", "wwwXapache"),
+ SubstringIndexTestCase("AAA ", "a ", -2, "UTF8_LCASE_RTRIM", "A "),
SubstringIndexTestCase("aaaaaaaaaa", "aa", 2, "UNICODE", "a"),
+ SubstringIndexTestCase("aaaaaaaaaa ", "aa ", 2, "UNICODE_RTRIM", "a"),
+ SubstringIndexTestCase("wwwmapacheMorg", "M", -2, "UNICODE_CI", "apacheMorg"),
+ SubstringIndexTestCase("AA A ", "a ", -2, "UNICODE_CI_RTRIM", " A "),
SubstringIndexTestCase("wwwmapacheMorg", "M", -2, "UNICODE_CI", "apacheMorg"),
SubstringIndexTestCase("wwwüapacheüorg", "U", 2, "DE_CI_AI", "wwwüapache")
)
@@ -294,11 +325,16 @@ class CollationStringExpressionsSuite
case class StringInStrTestCase[R](str: String, substr: String, collation: String, result: R)
val testCases = Seq(
StringInStrTestCase("test大千世界X大千世界", "大千", "UTF8_BINARY", 5),
+ StringInStrTestCase("test大千世界X大千世界", "大千 ", "UTF8_BINARY_RTRIM", 5),
StringInStrTestCase("test大千世界X大千世界", "界x", "UTF8_LCASE", 8),
+ StringInStrTestCase(" test大千世界X大千世界 ", "界x ", "UTF8_LCASE_RTRIM", 9),
StringInStrTestCase("test大千世界X大千世界", "界x", "UNICODE", 0),
+ StringInStrTestCase("test大千世界X大千世界", "界x", "UNICODE_RTRIM", 0),
StringInStrTestCase("test大千世界X大千世界", "界y", "UNICODE_CI", 0),
StringInStrTestCase("test大千世界X大千世界", "界x", "UNICODE_CI", 8),
- StringInStrTestCase("abİo12", "i̇o", "UNICODE_CI", 3)
+ StringInStrTestCase("abİo12", "i̇o", "UNICODE_CI", 3),
+ StringInStrTestCase("test大千世界X大千世界", "大 ", "UNICODE_CI_RTRIM", 5),
+ StringInStrTestCase("test大千世界X大千世界", " 大 ", "UNICODE_CI_RTRIM", 0)
)
val unsupportedTestCase = StringInStrTestCase("a", "abcde", "UNICODE_AI", 0)
testCases.foreach(t => {
@@ -338,10 +374,17 @@ class CollationStringExpressionsSuite
case class FindInSetTestCase[R](left: String, right: String, collation: String, result: R)
val testCases = Seq(
FindInSetTestCase("AB", "abc,b,ab,c,def", "UTF8_BINARY", 0),
+ FindInSetTestCase("b ", "abc,b,ab,c,def", "UTF8_BINARY_RTRIM", 2),
+ FindInSetTestCase("def", "abc,b,ab,c,def ", "UTF8_BINARY_RTRIM", 5),
FindInSetTestCase("C", "abc,b,ab,c,def", "UTF8_LCASE", 4),
+ FindInSetTestCase("C ", "abc,b,ab,c ,def", "UTF8_LCASE_RTRIM", 4),
FindInSetTestCase("d,ef", "abc,b,ab,c,def", "UNICODE", 0),
+ FindInSetTestCase(" def", "abc,b,ab,c,def", "UNICODE_RTRIM", 0),
FindInSetTestCase("i̇o", "ab,İo,12", "UNICODE_CI", 2),
- FindInSetTestCase("İo", "ab,i̇o,12", "UNICODE_CI", 2)
+ FindInSetTestCase("İo", "ab,i̇o,12", "UNICODE_CI", 2),
+ FindInSetTestCase("İo", "ab,i̇o,12", "UNICODE_CI", 2),
+ FindInSetTestCase("a", "A ,B ,C", "UNICODE_CI_RTRIM", 1),
+ FindInSetTestCase(" a", "A ,B ,C", "UNICODE_CI_RTRIM", 0)
)
testCases.foreach(t => {
// Unit test.
@@ -361,8 +404,13 @@ class CollationStringExpressionsSuite
case class StartsWithTestCase[R](left: String, right: String, collation: String, result: R)
val testCases = Seq(
StartsWithTestCase("", "", "UTF8_BINARY", true),
+ StartsWithTestCase("", " ", "UTF8_BINARY_RTRIM", true),
StartsWithTestCase("abcde", "A", "UNICODE", false),
+ StartsWithTestCase("abcde", "a ", "UNICODE_RTRIM", true),
StartsWithTestCase("abcde", "FGH", "UTF8_LCASE", false),
+ StartsWithTestCase("abcde ", "FGH ", "UTF8_LCASE_RTRIM", false),
+ StartsWithTestCase("abcde", "ABC", "UNICODE_CI", true),
+ StartsWithTestCase("a b c de ", "A B C ", "UNICODE_CI_RTRIM", true),
StartsWithTestCase("abcde", "ABC", "UNICODE_CI", true),
StartsWithTestCase("Šuma", "šum", "SR_CI_AI", true),
StartsWithTestCase("Šuma", "šum", "SR", false)
@@ -410,8 +458,13 @@ class CollationStringExpressionsSuite
result: R)
val testCases = Seq(
StringTranslateTestCase("Translate", "Rnlt", "12", "UTF8_BINARY", "Tra2sae"),
+ StringTranslateTestCase(" abc ", "abc", "123", "UTF8_BINARY_RTRIM", " 123 "),
StringTranslateTestCase("Translate", "Rnlt", "1234", "UTF8_LCASE", "41a2s3a4e"),
+ StringTranslateTestCase(" abc ", " AB", "123", "UTF8_LCASE_RTRIM", "123c1"),
StringTranslateTestCase("Translate", "Rn", "\u0000\u0000", "UNICODE", "Traslate"),
+ StringTranslateTestCase(" a b c ", "abc ", "1234", "UNICODE_RTRIM", "4142434"),
+ StringTranslateTestCase("Translate", "Rn", "1234", "UNICODE_CI", "T1a2slate"),
+ StringTranslateTestCase(" abc ", "AB ", "123", "UNICODE_CI_RTRIM", "312c3"),
StringTranslateTestCase("Translate", "Rn", "1234", "UNICODE_CI", "T1a2slate"),
StringTranslateTestCase("Êtèréêë", "te", "12", "AF_CI_AI", "212r222")
)
@@ -461,11 +514,15 @@ class CollationStringExpressionsSuite
result: R)
val testCases = Seq(
StringReplaceTestCase("r世eplace", "pl", "123", "UTF8_BINARY", "r世e123ace"),
+ StringReplaceTestCase(" abc ", "b ", "x", "UTF8_BINARY_RTRIM", " abc "),
StringReplaceTestCase("repl世ace", "PL", "AB", "UTF8_LCASE", "reAB世ace"),
+ StringReplaceTestCase(" abc ", " AB", "123", "UTF8_LCASE_RTRIM", "123c "),
StringReplaceTestCase("abcdabcd", "bc", "", "UNICODE", "adad"),
+ StringReplaceTestCase(" abc ", "b ", "x", "UNICODE_RTRIM", " abc "),
StringReplaceTestCase("aBc世abc", "b", "12", "UNICODE_CI", "a12c世a12c"),
StringReplaceTestCase("abi̇o12i̇o", "İo", "yy", "UNICODE_CI", "abyy12yy"),
StringReplaceTestCase("abİo12i̇o", "i̇o", "xx", "UNICODE_CI", "abxx12xx"),
+ StringReplaceTestCase(" ABC ", "bc ", "123", "UNICODE_CI_RTRIM", " A123"),
StringReplaceTestCase("češalj", "eSal", "A", "SR_CI_AI", "čAj")
)
val unsupportedTestCase = StringReplaceTestCase("abcde", "A", "B", "UNICODE_AI", "abcde")
@@ -509,8 +566,13 @@ class CollationStringExpressionsSuite
case class EndsWithTestCase[R](left: String, right: String, collation: String, result: R)
val testCases = Seq(
EndsWithTestCase("", "", "UTF8_BINARY", true),
+ EndsWithTestCase("", " ", "UTF8_BINARY_RTRIM", true),
EndsWithTestCase("abcde", "E", "UNICODE", false),
+ EndsWithTestCase("abcde ", "E ", "UNICODE_RTRIM", false),
EndsWithTestCase("abcde", "FGH", "UTF8_LCASE", false),
+ EndsWithTestCase("abcde ", "FGH ", "UTF8_LCASE_RTRIM", false),
+ EndsWithTestCase("abcde", "CDE", "UNICODE_CI", true),
+ EndsWithTestCase("abc d e ", "C D E", "UNICODE_CI_RTRIM", true),
EndsWithTestCase("abcde", "CDE", "UNICODE_CI", true),
EndsWithTestCase("xnigħat", "għat", "MT", true),
// The following two test cases showcase different behavior based on collation.
@@ -557,9 +619,13 @@ class CollationStringExpressionsSuite
case class StringRepeatTestCase[R](str: String, times: Integer, collation: String, result: R)
val testCases = Seq(
StringRepeatTestCase("", 1, "UTF8_BINARY", ""),
+ StringRepeatTestCase(" ", 1, "UTF8_BINARY_RTRIM", " "),
StringRepeatTestCase("a", 0, "UNICODE", ""),
+ StringRepeatTestCase("a", 0, "UNICODE_RTRIM", ""),
StringRepeatTestCase("XY", 3, "UTF8_LCASE", "XYXYXY"),
- StringRepeatTestCase("123", 2, "UNICODE_CI", "123123")
+ StringRepeatTestCase("XY ", 3, "UTF8_LCASE_RTRIM", "XY XY XY "),
+ StringRepeatTestCase("123", 2, "UNICODE_CI", "123123"),
+ StringRepeatTestCase("123 ", 2, "UNICODE_CI_RTRIM", "123 123 ")
)
testCases.foreach(t => {
// Unit test.
@@ -579,9 +645,13 @@ class CollationStringExpressionsSuite
case class AsciiTestCase[R](input: String, collation: String, result: R)
val testCases = Seq(
AsciiTestCase("a", "UTF8_BINARY", 97),
+ AsciiTestCase("a ", "UTF8_BINARY_RTRIM", 97),
AsciiTestCase("B", "UTF8_LCASE", 66),
+ AsciiTestCase("B ", "UTF8_LCASE_RTRIM", 66),
AsciiTestCase("#", "UNICODE", 35),
- AsciiTestCase("!", "UNICODE_CI", 33)
+ AsciiTestCase("# ", "UNICODE_RTRIM", 35),
+ AsciiTestCase("!", "UNICODE_CI", 33),
+ AsciiTestCase("! ", "UNICODE_CI_RTRIM", 33)
)
testCases.foreach(t => {
// Unit test.
@@ -599,9 +669,13 @@ class CollationStringExpressionsSuite
case class ChrTestCase[R](input: Long, collation: String, result: R)
val testCases = Seq(
ChrTestCase(65, "UTF8_BINARY", "A"),
+ ChrTestCase(65, "UTF8_BINARY_RTRIM", "A"),
ChrTestCase(66, "UTF8_LCASE", "B"),
+ ChrTestCase(66, "UTF8_LCASE_RTRIM", "B"),
ChrTestCase(97, "UNICODE", "a"),
- ChrTestCase(98, "UNICODE_CI", "b")
+ ChrTestCase(97, "UNICODE_RTRIM", "a"),
+ ChrTestCase(98, "UNICODE_CI", "b"),
+ ChrTestCase(98, "UNICODE_CI_RTRIM", "b")
)
testCases.foreach(t => {
// Unit test.
@@ -619,9 +693,13 @@ class CollationStringExpressionsSuite
case class UnBase64TestCase[R](input: String, collation: String, result: R)
val testCases = Seq(
UnBase64TestCase("QUJD", "UTF8_BINARY", Array(65, 66, 67)),
+ UnBase64TestCase("QUJD", "UTF8_BINARY_RTRIM", Array(65, 66, 67)),
UnBase64TestCase("eHl6", "UTF8_LCASE", Array(120, 121, 122)),
+ UnBase64TestCase("eHl6", "UTF8_LCASE_RTRIM", Array(120, 121, 122)),
UnBase64TestCase("IyMj", "UNICODE", Array(35, 35, 35)),
- UnBase64TestCase("IQ==", "UNICODE_CI", Array(33))
+ UnBase64TestCase("IyMj", "UNICODE_RTRIM", Array(35, 35, 35)),
+ UnBase64TestCase("IQ==", "UNICODE_CI", Array(33)),
+ UnBase64TestCase("IQ==", "UNICODE_CI_RTRIM", Array(33))
)
testCases.foreach(t => {
// Unit test.
@@ -639,9 +717,13 @@ class CollationStringExpressionsSuite
case class Base64TestCase[R](input: Array[Byte], collation: String, result: R)
val testCases = Seq(
Base64TestCase(Array(65, 66, 67), "UTF8_BINARY", "QUJD"),
+ Base64TestCase(Array(65, 66, 67), "UTF8_BINARY_RTRIM", "QUJD"),
Base64TestCase(Array(120, 121, 122), "UTF8_LCASE", "eHl6"),
+ Base64TestCase(Array(120, 121, 122), "UTF8_LCASE_RTRIM", "eHl6"),
Base64TestCase(Array(35, 35, 35), "UNICODE", "IyMj"),
- Base64TestCase(Array(33), "UNICODE_CI", "IQ==")
+ Base64TestCase(Array(35, 35, 35), "UNICODE_RTRIM", "IyMj"),
+ Base64TestCase(Array(33), "UNICODE_CI", "IQ=="),
+ Base64TestCase(Array(33), "UNICODE_CI_RTRIM", "IQ==")
)
testCases.foreach(t => {
// Unit test.
@@ -660,8 +742,13 @@ class CollationStringExpressionsSuite
case class FormatNumberTestCase[R](x: Double, d: String, collation: String, r: R)
val testCases = Seq(
FormatNumberTestCase(123.123, "###.###", "UTF8_BINARY", "123.123"),
+ FormatNumberTestCase(123.123, "###.###", "UTF8_BINARY_RTRIM", "123.123"),
FormatNumberTestCase(99.99, "##.##", "UTF8_LCASE", "99.99"),
+ FormatNumberTestCase(99.99, "##.##", "UTF8_LCASE_RTRIM", "99.99"),
FormatNumberTestCase(123.123, "###.###", "UNICODE", "123.123"),
+ FormatNumberTestCase(123.123, "###.###", "UNICODE_RTRIM", "123.123"),
+ FormatNumberTestCase(99.99, "##.##", "UNICODE_CI", "99.99"),
+ FormatNumberTestCase(99.99, "##.##", "UNICODE_CI_RTRIM", "99.99"),
FormatNumberTestCase(99.99, "##.##", "UNICODE_CI", "99.99"),
FormatNumberTestCase(99.999, "##.###", "AF_CI_AI", "99.999")
)
@@ -683,9 +770,13 @@ class CollationStringExpressionsSuite
case class DecodeTestCase[R](input: String, collation: String, result: R)
val testCases = Seq(
DecodeTestCase("a", "UTF8_BINARY", "a"),
+ DecodeTestCase("a", "UTF8_BINARY_RTRIM", "a"),
DecodeTestCase("A", "UTF8_LCASE", "A"),
+ DecodeTestCase("A", "UTF8_LCASE_RTRIM", "A"),
DecodeTestCase("b", "UNICODE", "b"),
- DecodeTestCase("B", "UNICODE_CI", "B")
+ DecodeTestCase("b", "UNICODE_RTRIM", "b"),
+ DecodeTestCase("B", "UNICODE_CI", "B"),
+ DecodeTestCase("B", "UNICODE_CI_RTRIM", "B")
)
testCases.foreach(t => {
// Unit test.
@@ -712,9 +803,13 @@ class CollationStringExpressionsSuite
case class EncodeTestCase[R](input: String, collation: String, result: R)
val testCases = Seq(
EncodeTestCase("a", "UTF8_BINARY", Array(97)),
+ EncodeTestCase("a ", "UTF8_BINARY_RTRIM", Array(97, 32)),
EncodeTestCase("A", "UTF8_LCASE", Array(65)),
+ EncodeTestCase("A ", "UTF8_LCASE_RTRIM", Array(65, 32)),
EncodeTestCase("b", "UNICODE", Array(98)),
- EncodeTestCase("B", "UNICODE_CI", Array(66))
+ EncodeTestCase("b ", "UNICODE_RTRIM", Array(98, 32)),
+ EncodeTestCase("B", "UNICODE_CI", Array(66)),
+ EncodeTestCase("B ", "UNICODE_CI_RTRIM", Array(66, 32))
)
testCases.foreach(t => {
// Unit test.
@@ -734,9 +829,13 @@ class CollationStringExpressionsSuite
case class ToBinaryTestCase[R](expr: String, format: String, collation: String, result: R)
val testCases = Seq(
ToBinaryTestCase("a", "utf-8", "UTF8_BINARY", Array(97)),
+ ToBinaryTestCase("a ", "utf-8", "UTF8_BINARY_RTRIM", Array(97, 32)),
ToBinaryTestCase("A", "utf-8", "UTF8_LCASE", Array(65)),
+ ToBinaryTestCase("A ", "utf-8", "UTF8_LCASE_RTRIM", Array(65, 32)),
ToBinaryTestCase("b", "utf-8", "UNICODE", Array(98)),
- ToBinaryTestCase("B", "utf-8", "UNICODE_CI", Array(66))
+ ToBinaryTestCase("b ", "utf-8", "UNICODE_RTRIM", Array(98, 32)),
+ ToBinaryTestCase("B", "utf-8", "UNICODE_CI", Array(66)),
+ ToBinaryTestCase("B ", "utf-8", "UNICODE_CI_RTRIM", Array(66, 32))
)
testCases.foreach(t => {
// Unit test.
@@ -798,8 +897,13 @@ class CollationStringExpressionsSuite
case class UpperTestCase[R](input: String, collation: String, result: R)
val testCases = Seq(
UpperTestCase("aBc", "UTF8_BINARY", "ABC"),
+ UpperTestCase("aBc ", "UTF8_BINARY_RTRIM", "ABC "),
UpperTestCase("aBc", "UTF8_LCASE", "ABC"),
+ UpperTestCase("aBc ", "UTF8_LCASE_RTRIM", "ABC "),
UpperTestCase("aBc", "UNICODE", "ABC"),
+ UpperTestCase("aBc ", "UNICODE_RTRIM", "ABC "),
+ UpperTestCase("aBc", "UNICODE_CI", "ABC"),
+ UpperTestCase("aBc ", "UNICODE_CI_RTRIM", "ABC "),
UpperTestCase("aBc", "UNICODE_CI", "ABC"),
UpperTestCase("xnìgħat", "MT_CI_AI", "XNÌGĦAT")
)
@@ -819,8 +923,13 @@ class CollationStringExpressionsSuite
case class LowerTestCase[R](input: String, collation: String, result: R)
val testCases = Seq(
LowerTestCase("aBc", "UTF8_BINARY", "abc"),
+ LowerTestCase("aBc ", "UTF8_BINARY_RTRIM", "abc "),
LowerTestCase("aBc", "UTF8_LCASE", "abc"),
+ LowerTestCase("aBc ", "UTF8_LCASE_RTRIM", "abc "),
LowerTestCase("aBc", "UNICODE", "abc"),
+ LowerTestCase("aBc ", "UNICODE_RTRIM", "abc "),
+ LowerTestCase("aBc", "UNICODE_CI", "abc"),
+ LowerTestCase("aBc ", "UNICODE_CI_RTRIM", "abc "),
LowerTestCase("aBc", "UNICODE_CI", "abc"),
LowerTestCase("VeRGrÖßeRn", "DE_CI", "vergrößern")
)
@@ -840,8 +949,13 @@ class CollationStringExpressionsSuite
case class InitCapTestCase[R](input: String, collation: String, result: R)
val testCases = Seq(
InitCapTestCase("aBc ABc", "UTF8_BINARY", "Abc Abc"),
+ InitCapTestCase(" aBc ABc ", "UTF8_BINARY_RTRIM", " Abc Abc "),
InitCapTestCase("aBc ABc", "UTF8_LCASE", "Abc Abc"),
+ InitCapTestCase(" aBc ABc ", "UTF8_LCASE_RTRIM", " Abc Abc "),
InitCapTestCase("aBc ABc", "UNICODE", "Abc Abc"),
+ InitCapTestCase(" aBc ABc ", "UNICODE_RTRIM", " Abc Abc "),
+ InitCapTestCase("aBc ABc", "UNICODE_CI", "Abc Abc"),
+ InitCapTestCase(" aBc ABc ", "UNICODE_CI_RTRIM", " Abc Abc "),
InitCapTestCase("aBc ABc", "UNICODE_CI", "Abc Abc"),
InitCapTestCase("æØÅ ÆøÅ", "NO", "Æøå Æøå")
)
@@ -867,9 +981,13 @@ class CollationStringExpressionsSuite
result: R)
val testCases = Seq(
OverlayTestCase("hello", " world", 6, -1, "UTF8_BINARY", "hello world"),
+ OverlayTestCase("hello ", " world ", 7, -1, "UTF8_BINARY_RTRIM", "hello world "),
OverlayTestCase("nice", " day", 5, -1, "UTF8_LCASE", "nice day"),
+ OverlayTestCase(" nice ", " day ", 7, -1, "UTF8_LCASE_RTRIM", " nice day "),
OverlayTestCase("A", "B", 1, -1, "UNICODE", "B"),
- OverlayTestCase("!", "!!!", 1, -1, "UNICODE_CI", "!!!")
+ OverlayTestCase("A", " B ", 1, -1, "UNICODE_RTRIM", " B "),
+ OverlayTestCase("!", "!!!", 1, -1, "UNICODE_CI", "!!!"),
+ OverlayTestCase("!", " !!! ", 1, -1, "UNICODE_CI_RTRIM", " !!! ")
)
testCases.foreach(t => {
// Unit test.
@@ -895,8 +1013,13 @@ class CollationStringExpressionsSuite
result: R)
val testCases = Seq(
FormatStringTestCase("%s%s", Seq("a", "b"), "UTF8_BINARY", "ab"),
+ FormatStringTestCase("%s%s", Seq("a", "b "), "UTF8_BINARY_RTRIM", "ab "),
FormatStringTestCase("%d", Seq(123), "UTF8_LCASE", "123"),
+ FormatStringTestCase("%d", Seq(123), "UTF8_LCASE_RTRIM", "123"),
FormatStringTestCase("%s%d", Seq("A", 0), "UNICODE", "A0"),
+ FormatStringTestCase("%s%d", Seq(" A ", 0), "UNICODE_RTRIM", " A 0"),
+ FormatStringTestCase("%s%s", Seq("Hello", "!!!"), "UNICODE_CI", "Hello!!!"),
+ FormatStringTestCase("%s%s", Seq(" Hello ", " !!! "), "UNICODE_CI_RTRIM", " Hello !!! "),
FormatStringTestCase("%s%s", Seq("Hello", "!!!"), "UNICODE_CI", "Hello!!!"),
FormatStringTestCase("%s%s", Seq("Storslått", ".?!"), "NN_AI", "Storslått.?!")
)
@@ -927,8 +1050,13 @@ class CollationStringExpressionsSuite
case class SoundExTestCase[R](input: String, collation: String, result: R)
val testCases = Seq(
SoundExTestCase("A", "UTF8_BINARY", "A000"),
+ SoundExTestCase("A", "UTF8_BINARY_RTRIM", "A000"),
SoundExTestCase("!", "UTF8_LCASE", "!"),
+ SoundExTestCase("!", "UTF8_LCASE_RTRIM", "!"),
SoundExTestCase("$", "UNICODE", "$"),
+ SoundExTestCase("$", "UNICODE_RTRIM", "$"),
+ SoundExTestCase("X", "UNICODE_CI", "X000"),
+ SoundExTestCase("X", "UNICODE_CI_RTRIM", "X000"),
SoundExTestCase("X", "UNICODE_CI", "X000"),
SoundExTestCase("ß", "DE", "ß")
)
@@ -948,8 +1076,13 @@ class CollationStringExpressionsSuite
case class LengthTestCase[R](input: String, collation: String, result: R)
val testCases = Seq(
LengthTestCase("", "UTF8_BINARY", 0),
+ LengthTestCase(" ", "UTF8_BINARY_RTRIM", 1),
LengthTestCase("abc", "UTF8_LCASE", 3),
+ LengthTestCase("abc ", "UTF8_LCASE_RTRIM", 4),
LengthTestCase("hello", "UNICODE", 5),
+ LengthTestCase("hello ", "UNICODE_RTRIM", 6),
+ LengthTestCase("ff", "UNICODE_CI", 1),
+ LengthTestCase("ff ", "UNICODE_CI_RTRIM", 2),
LengthTestCase("ff", "UNICODE_CI", 1),
LengthTestCase("groß", "DE_CI_AI", 4),
LengthTestCase("gross", "DE_AI", 5)
@@ -970,8 +1103,13 @@ class CollationStringExpressionsSuite
case class BitLengthTestCase[R](input: String, collation: String, result: R)
val testCases = Seq(
BitLengthTestCase("", "UTF8_BINARY", 0),
+ BitLengthTestCase(" ", "UTF8_BINARY_RTRIM", 8),
BitLengthTestCase("abc", "UTF8_LCASE", 24),
+ BitLengthTestCase("abc ", "UTF8_LCASE_RTRIM", 32),
BitLengthTestCase("hello", "UNICODE", 40),
+ BitLengthTestCase("hello ", "UNICODE_RTRIM", 48),
+ BitLengthTestCase("ff", "UNICODE_CI", 24),
+ BitLengthTestCase("ff ", "UNICODE_CI_RTRIM", 32),
BitLengthTestCase("ff", "UNICODE_CI", 24),
BitLengthTestCase("GROß", "DE", 40)
)
@@ -991,9 +1129,13 @@ class CollationStringExpressionsSuite
case class OctetLengthTestCase[R](input: String, collation: String, result: R)
val testCases = Seq(
OctetLengthTestCase("", "UTF8_BINARY", 0),
+ OctetLengthTestCase(" ", "UTF8_BINARY_RTRIM", 1),
OctetLengthTestCase("abc", "UTF8_LCASE", 3),
+ OctetLengthTestCase("abc ", "UTF8_LCASE_RTRIM", 4),
OctetLengthTestCase("hello", "UNICODE", 5),
- OctetLengthTestCase("ff", "UNICODE_CI", 3)
+ OctetLengthTestCase("hello ", "UNICODE_RTRIM", 6),
+ OctetLengthTestCase("ff", "UNICODE_CI", 3),
+ OctetLengthTestCase("ff ", "UNICODE_CI_RTRIM", 4)
)
testCases.foreach(t => {
// Unit test.
@@ -1011,9 +1153,13 @@ class CollationStringExpressionsSuite
case class LuhncheckTestCase[R](input: String, collation: String, result: R)
val testCases = Seq(
LuhncheckTestCase("123", "UTF8_BINARY", false),
+ LuhncheckTestCase("123", "UTF8_BINARY_RTRIM", false),
LuhncheckTestCase("000", "UTF8_LCASE", true),
+ LuhncheckTestCase("000", "UTF8_LCASE_RTRIM", true),
LuhncheckTestCase("111", "UNICODE", false),
- LuhncheckTestCase("222", "UNICODE_CI", false)
+ LuhncheckTestCase("111", "UNICODE_RTRIM", false),
+ LuhncheckTestCase("222", "UNICODE_CI", false),
+ LuhncheckTestCase("222", "UNICODE_CI_RTRIM", false)
)
testCases.foreach(t => {
// Unit test.
@@ -1036,8 +1182,13 @@ class CollationStringExpressionsSuite
result: R)
val testCases = Seq(
LevenshteinTestCase("kitten", "sitTing", "UTF8_BINARY", None, 4),
+ LevenshteinTestCase("kitten", "sitTing ", "UTF8_BINARY_RTRIM", None, 6),
LevenshteinTestCase("kitten", "sitTing", "UTF8_LCASE", None, 4),
+ LevenshteinTestCase("kitten", "sitTing ", "UTF8_LCASE", None, 6),
LevenshteinTestCase("kitten", "sitTing", "UNICODE", Some(3), -1),
+ LevenshteinTestCase("kitten", "sitTing ", "UNICODE_RTRIM", Some(3), -1),
+ LevenshteinTestCase("kitten", "sitTing", "UNICODE_CI", Some(3), -1),
+ LevenshteinTestCase("kitten ", "sitTing ", "UNICODE_CI_RTRIM", Some(3), -1),
LevenshteinTestCase("kitten", "sitTing", "UNICODE_CI", Some(3), -1),
// Levenshtein function is currently not collation-aware (not considering case or accent).
LevenshteinTestCase("gr", "GR", "UNICODE_CI_AI", None, 2),
@@ -1063,8 +1214,13 @@ class CollationStringExpressionsSuite
case class IsValidUTF8TestCase[R](input: Any, collation: String, result: R)
val testCases = Seq(
IsValidUTF8TestCase(null, "UTF8_BINARY", null),
+ IsValidUTF8TestCase(null, "UTF8_BINARY_RTRIM", null),
IsValidUTF8TestCase("", "UTF8_LCASE", true),
+ IsValidUTF8TestCase("", "UTF8_LCASE_RTRIM", true),
IsValidUTF8TestCase("abc", "UNICODE", true),
+ IsValidUTF8TestCase("abc", "UNICODE_RTRIM", true),
+ IsValidUTF8TestCase("hello", "UNICODE_CI", true),
+ IsValidUTF8TestCase("hello", "UNICODE_CI_RTRIM", true),
IsValidUTF8TestCase("hello", "UNICODE_CI", true),
IsValidUTF8TestCase("ćao", "SR_CI_AI", true)
)
@@ -1085,9 +1241,13 @@ class CollationStringExpressionsSuite
case class MakeValidUTF8TestCase[R](input: String, collation: String, result: R)
val testCases = Seq(
MakeValidUTF8TestCase(null, "UTF8_BINARY", null),
+ MakeValidUTF8TestCase(null, "UTF8_BINARY_RTRIM", null),
MakeValidUTF8TestCase("", "UTF8_LCASE", ""),
+ MakeValidUTF8TestCase("", "UTF8_LCASE_RTRIM", ""),
MakeValidUTF8TestCase("abc", "UNICODE", "abc"),
- MakeValidUTF8TestCase("hello", "UNICODE_CI", "hello")
+ MakeValidUTF8TestCase("abc", "UNICODE_RTRIM", "abc"),
+ MakeValidUTF8TestCase("hello", "UNICODE_CI", "hello"),
+ MakeValidUTF8TestCase("hello", "UNICODE_CI_RTRIM", "hello")
)
testCases.foreach(t => {
// Unit test.
@@ -1106,9 +1266,13 @@ class CollationStringExpressionsSuite
case class ValidateUTF8TestCase[R](input: String, collation: String, result: R)
val testCases = Seq(
ValidateUTF8TestCase(null, "UTF8_BINARY", null),
+ ValidateUTF8TestCase(null, "UTF8_BINARY_RTRIM", null),
ValidateUTF8TestCase("", "UTF8_LCASE", ""),
+ ValidateUTF8TestCase("", "UTF8_LCASE_RTRIM", ""),
ValidateUTF8TestCase("abc", "UNICODE", "abc"),
- ValidateUTF8TestCase("hello", "UNICODE_CI", "hello")
+ ValidateUTF8TestCase("abc", "UNICODE_RTRIM", "abc"),
+ ValidateUTF8TestCase("hello", "UNICODE_CI", "hello"),
+ ValidateUTF8TestCase("hello", "UNICODE_CI_RTRIM", "hello")
)
testCases.foreach(t => {
// Unit test.
@@ -1127,9 +1291,13 @@ class CollationStringExpressionsSuite
case class ValidateUTF8TestCase(input: String, collation: String, result: Any)
val testCases = Seq(
ValidateUTF8TestCase(null, "UTF8_BINARY", null),
+ ValidateUTF8TestCase(null, "UTF8_BINARY_RTRIM", null),
ValidateUTF8TestCase("", "UTF8_LCASE", ""),
+ ValidateUTF8TestCase("", "UTF8_LCASE_RTRIM", ""),
ValidateUTF8TestCase("abc", "UNICODE", "abc"),
- ValidateUTF8TestCase("hello", "UNICODE_CI", "hello")
+ ValidateUTF8TestCase("abc", "UNICODE_RTRIM", "abc"),
+ ValidateUTF8TestCase("hello", "UNICODE_CI", "hello"),
+ ValidateUTF8TestCase("hello", "UNICODE_CI_RTRIM", "hello")
)
testCases.foreach(t => {
// Unit test.
@@ -1153,9 +1321,12 @@ class CollationStringExpressionsSuite
result: R)
val testCases = Seq(
SubstringTestCase("example", 1, Some(100), "UTF8_LCASE", "example"),
+ SubstringTestCase("example ", 1, Some(100), "UTF8_LCASE_RTRIM", "example "),
SubstringTestCase("example", 2, Some(2), "UTF8_BINARY", "xa"),
SubstringTestCase("example", 0, Some(0), "UNICODE", ""),
+ SubstringTestCase("example", 0, Some(0), "UNICODE_RTRIM", ""),
SubstringTestCase("example", -3, Some(2), "UNICODE_CI", "pl"),
+ SubstringTestCase("example ", -3, Some(2), "UNICODE_CI_RTRIM", "le"),
SubstringTestCase(" a世a ", 2, Some(3), "UTF8_LCASE", "a世a"),
SubstringTestCase("", 1, Some(1), "UTF8_LCASE", ""),
SubstringTestCase("", 1, Some(1), "UNICODE", ""),
@@ -1164,6 +1335,8 @@ class CollationStringExpressionsSuite
SubstringTestCase(null, null, Some(null), "UTF8_BINARY", null),
SubstringTestCase(null, null, None, "UNICODE_CI", null),
SubstringTestCase("ÀÃÂĀĂȦÄäåäáâãȻȻȻȻȻǢǼÆ", null, None, "UTF8_BINARY", null),
+ SubstringTestCase("ÀÃÂĀĂȦÄäåäáâãȻȻȻȻȻǢǼÆ", null, None, "UTF8_BINARY_RTRIM", null),
+ SubstringTestCase("", null, None, "UNICODE_CI", null),
SubstringTestCase("", null, None, "UNICODE_CI", null),
SubstringTestCase("xnigħat", 4, Some(2), "MT_CI_AI", "għ")
)
@@ -1188,8 +1361,13 @@ class CollationStringExpressionsSuite
case class LeftTestCase[R](str: String, len: Integer, collation: String, result: R)
val testCases = Seq(
LeftTestCase(null, null, "UTF8_BINARY", null),
+ LeftTestCase(null, null, "UTF8_BINARY_RTRIM", null),
LeftTestCase(" a世a ", 3, "UTF8_LCASE", " a世"),
+ LeftTestCase(" a世a ", 3, "UTF8_LCASE_RTRIM", " a世"),
LeftTestCase("", 1, "UNICODE", ""),
+ LeftTestCase("", 1, "UNICODE_RTRIM", ""),
+ LeftTestCase("ÀÃÂĀĂȦÄäåäáâãȻȻȻȻȻǢǼÆ", 3, "UNICODE", "ÀÃÂ"),
+ LeftTestCase("ÀÃÂĀĂȦÄäåäáâãȻȻȻȻȻǢǼÆ", 3, "UNICODE_RTRIM", "ÀÃÂ"),
LeftTestCase("ÀÃÂĀĂȦÄäåäáâãȻȻȻȻȻǢǼÆ", 3, "UNICODE", "ÀÃÂ"),
LeftTestCase("ÀÃÂĀĂȦÄäåäáâãȻȻȻȻȻǢǼÆ", 7, "NO_AI", "ÀÃÂĀĂȦÄ")
)
@@ -1212,8 +1390,13 @@ class CollationStringExpressionsSuite
case class RightTestCase[R](str: String, len: Integer, collation: String, result: R)
val testCases = Seq(
RightTestCase(null, null, "UTF8_BINARY", null),
+ RightTestCase(null, null, "UTF8_BINARY_RTRIM", null),
RightTestCase(" a世a ", 3, "UTF8_LCASE", "世a "),
+ RightTestCase(" a世a ", 3, "UTF8_LCASE_RTRIM", "世a "),
RightTestCase("", 1, "UNICODE", ""),
+ RightTestCase("", 1, "UNICODE_RTRIM", ""),
+ RightTestCase("ÀÃÂĀĂȦÄäåäáâãȻȻȻȻȻǢǼÆ", 3, "UNICODE", "ǢǼÆ"),
+ RightTestCase("ÀÃÂĀĂȦÄäåäáâãȻȻȻȻȻǢǼÆ", 3, "UNICODE_RTRIM", "ǢǼÆ"),
RightTestCase("ÀÃÂĀĂȦÄäåäáâãȻȻȻȻȻǢǼÆ", 3, "UNICODE", "ǢǼÆ"),
RightTestCase("ÀÃÂĀĂȦÄäåäáâãȻȻȻȻȻǢǼÆ", 5, "NO_CI_AI", "ȻȻǢǼÆ")
)
@@ -1241,13 +1424,18 @@ class CollationStringExpressionsSuite
result: R)
val testCases = Seq(
StringRPadTestCase("", 5, " ", "UTF8_BINARY", " "),
+ StringRPadTestCase("", 5, " ", "UTF8_BINARY_RTRIM", " "),
StringRPadTestCase("abc", 5, " ", "UNICODE", "abc "),
+ StringRPadTestCase("ab c ", 5, " ", "UNICODE_RTRIM", "ab c "),
StringRPadTestCase("Hello", 7, "Wörld", "UTF8_LCASE", "HelloWö"),
StringRPadTestCase("1234567890", 5, "aaaAAa", "UNICODE_CI", "12345"),
+ StringRPadTestCase("12 34567890", 5, "aaaAAa", "UNICODE_CI_RTRIM", "12 34"),
StringRPadTestCase("aaAA", 2, " ", "UTF8_BINARY", "aa"),
StringRPadTestCase("ÀÃÂĀĂȦÄäåäáâãȻȻȻȻȻǢǼÆ℀℃", 2, "1", "UTF8_LCASE", "ÀÃ"),
StringRPadTestCase("ĂȦÄäåäá", 20, "ÀÃÂĀĂȦÄäåäáâãȻȻȻȻȻǢǼÆ", "UNICODE", "ĂȦÄäåäáÀÃÂĀĂȦÄäåäáâã"),
- StringRPadTestCase("aȦÄä", 8, "a1", "UNICODE_CI", "aȦÄäa1a1")
+ StringRPadTestCase("ĂȦÄäåäá", 20, "ÀÃÂĀĂȦÄäåäáâãȻȻ", "UNICODE_RTRIM", "ĂȦÄäåäáÀÃÂĀĂȦÄäåäáâã"),
+ StringRPadTestCase("aȦÄä", 8, "a1", "UNICODE_CI", "aȦÄäa1a1"),
+ StringRPadTestCase("aȦÄä", 8, "a1", "UNICODE_CI_RTRIM", "aȦÄäa1a1")
)
testCases.foreach(t => {
// Unit test.
@@ -1273,13 +1461,17 @@ class CollationStringExpressionsSuite
result: R)
val testCases = Seq(
StringLPadTestCase("", 5, " ", "UTF8_BINARY", " "),
+ StringLPadTestCase("", 5, " ", "UTF8_BINARY_RTRIM", " "),
StringLPadTestCase("abc", 5, " ", "UNICODE", " abc"),
StringLPadTestCase("Hello", 7, "Wörld", "UTF8_LCASE", "WöHello"),
+ StringLPadTestCase("Hello", 7, "W örld", "UTF8_LCASE_RTRIM", "W Hello"),
StringLPadTestCase("1234567890", 5, "aaaAAa", "UNICODE_CI", "12345"),
StringLPadTestCase("aaAA", 2, " ", "UTF8_BINARY", "aa"),
StringLPadTestCase("ÀÃÂĀĂȦÄäåäáâãȻȻȻȻȻǢǼÆ℀℃", 2, "1", "UTF8_LCASE", "ÀÃ"),
StringLPadTestCase("ĂȦÄäåäá", 20, "ÀÃÂĀĂȦÄäåäáâãȻȻȻȻȻǢǼÆ", "UNICODE", "ÀÃÂĀĂȦÄäåäáâãĂȦÄäåäá"),
- StringLPadTestCase("aȦÄä", 8, "a1", "UNICODE_CI", "a1a1aȦÄä")
+ StringLPadTestCase("ĂȦÄäåäá", 20, "ÀÃÂĀĂȦÄäåäáâãȻȻ", "UNICODE_RTRIM", "ÀÃÂĀĂȦÄäåäáâãĂȦÄäåäá"),
+ StringLPadTestCase("aȦÄä", 8, "a1", "UNICODE_CI", "a1a1aȦÄä"),
+ StringLPadTestCase("aȦÄ ", 8, "a1", "UNICODE_CI_RTRIM", "a1a1aȦÄ ")
)
testCases.foreach(t => {
// Unit test.
@@ -1305,13 +1497,22 @@ class CollationStringExpressionsSuite
result: R)
val testCases = Seq(
StringLocateTestCase("aa", "aaads", 0, "UTF8_BINARY", 0),
+ StringLocateTestCase(" ", "", 1, "UTF8_BINARY_RTRIM", 1),
+ StringLocateTestCase(" abc ", " cdfg abc ", 1, "UTF8_BINARY_RTRIM", 12),
StringLocateTestCase("aa", "Aaads", 0, "UTF8_LCASE", 0),
StringLocateTestCase("界x", "test大千世界X大千世界", 1, "UTF8_LCASE", 8),
StringLocateTestCase("aBc", "abcabc", 4, "UTF8_LCASE", 4),
+ StringLocateTestCase("aa", "Aaads", 0, "UTF8_LCASE_RTRIM", 0),
+ StringLocateTestCase("界 ", "test大千世界X大千世界", 1, "UTF8_LCASE_RTRIM", 8),
+ StringLocateTestCase("aBc", "a bc abc ", 4, "UTF8_LCASE_RTRIM", 6),
StringLocateTestCase("aa", "Aaads", 0, "UNICODE", 0),
StringLocateTestCase("abC", "abCabC", 2, "UNICODE", 4),
+ StringLocateTestCase("aa", "Aaads", 0, "UNICODE_RTRIM", 0),
+ StringLocateTestCase("abC ", "ab C abC ", 2, "UNICODE_RTRIM", 6),
StringLocateTestCase("aa", "Aaads", 0, "UNICODE_CI", 0),
StringLocateTestCase("界x", "test大千世界X大千世界", 1, "UNICODE_CI", 8),
+ StringLocateTestCase("aa", "Aaads", 0, "UNICODE_CI_RTRIM", 0),
+ StringLocateTestCase(" 界", "test大千世界X大千世界", 1, "UNICODE_CI_RTRIM", 0),
StringLocateTestCase("oa", "TÖäöäoAoa", 1, "DE", 8),
StringLocateTestCase("oa", "TÖäöäoAoa", 1, "DE_CI", 6),
StringLocateTestCase("oa", "TÖäöäoAoa", 1, "DE_CI_AI", 2)
@@ -1360,8 +1561,18 @@ class CollationStringExpressionsSuite
result: R)
val testCases = Seq(
StringTrimLeftTestCase("xxasdxx", Some("x"), "UTF8_BINARY", "asdxx"),
+ StringTrimLeftTestCase(" xxasdxx", Some("x"), "UTF8_BINARY_RTRIM", " xxasdxx"),
+ StringTrimLeftTestCase(" xxasdxx", Some("x "), "UTF8_BINARY_RTRIM", "asdxx"),
+ StringTrimLeftTestCase(" xxasdxx ", Some("x "), "UTF8_BINARY_RTRIM", "asdxx "),
StringTrimLeftTestCase("xxasdxx", Some("X"), "UTF8_LCASE", "asdxx"),
+ StringTrimLeftTestCase("xxasdxx ", Some("X "), "UTF8_LCASE_RTRIM", "asdxx "),
+ StringTrimLeftTestCase("xxasdxx ", Some("X"), "UTF8_LCASE_RTRIM", "asdxx "),
+ StringTrimLeftTestCase(" xxasdxx ", Some("X "), "UTF8_LCASE_RTRIM", "asdxx "),
StringTrimLeftTestCase("xxasdxx", Some("y"), "UNICODE", "xxasdxx"),
+ StringTrimLeftTestCase("xxasdxx", Some("y"), "UNICODE_RTRIM", "xxasdxx"),
+ StringTrimLeftTestCase(" asd ", None, "UNICODE_RTRIM", "asd "),
+ StringTrimLeftTestCase(" asd ", None, "UNICODE_CI", "asd "),
+ StringTrimLeftTestCase(" asd ", Some("A"), "UNICODE_CI_RTRIM", " asd "),
StringTrimLeftTestCase(" asd ", None, "UNICODE_CI", "asd "),
StringTrimLeftTestCase("ćĆčČcCabCcČčĆć", Some("ć"), "SR", "ĆčČcCabCcČčĆć"),
StringTrimLeftTestCase("ćĆčČcCabCcČčĆć", Some("Ć"), "SR_CI", "čČcCabCcČčĆć"),
@@ -1409,9 +1620,21 @@ class CollationStringExpressionsSuite
collation: String,
result: R)
val testCases = Seq(
- StringTrimRightTestCase("xxasdxx", Some("x"), "UTF8_BINARY", "xxasd"),
+ StringTrimRightTestCase(" xxasdxx", Some("x "), "UTF8_BINARY", " xxasd"),
+ StringTrimRightTestCase("xxasdxx ", Some("x "), "UTF8_BINARY_RTRIM", "xxasd"),
+ StringTrimRightTestCase("xxasdxx ", Some("x"), "UTF8_BINARY_RTRIM", "xxasd "),
+ StringTrimRightTestCase(" xxasdxx ", Some("x "), "UTF8_BINARY_RTRIM", " xxasd"),
+ StringTrimRightTestCase(" xxasdxx", Some("x"), "UTF8_BINARY_RTRIM", " xxasd"),
StringTrimRightTestCase("xxasdxx", Some("X"), "UTF8_LCASE", "xxasd"),
+ StringTrimRightTestCase("xxasdxx ", Some("X "), "UTF8_LCASE_RTRIM", "xxasd"),
+ StringTrimRightTestCase("xxasdxx ", Some("X"), "UTF8_LCASE_RTRIM", "xxasd "),
+ StringTrimRightTestCase(" xxasdxx ", Some("X "), "UTF8_LCASE_RTRIM", " xxasd"),
+ StringTrimRightTestCase(" xxasdxx", Some("x"), "UTF8_LCASE_RTRIM", " xxasd"),
StringTrimRightTestCase("xxasdxx", Some("y"), "UNICODE", "xxasdxx"),
+ StringTrimRightTestCase("xxasdxx", Some("y"), "UNICODE_RTRIM", "xxasdxx"),
+ StringTrimRightTestCase(" asd ", None, "UNICODE_RTRIM", " asd"),
+ StringTrimRightTestCase(" asd ", None, "UNICODE_CI", " asd"),
+ StringTrimRightTestCase(" asd ", Some("D"), "UNICODE_CI_RTRIM", " as "),
StringTrimRightTestCase(" asd ", None, "UNICODE_CI", " asd"),
StringTrimRightTestCase("ćĆčČcCabCcČčĆć", Some("ć"), "SR", "ćĆčČcCabCcČčĆ"),
StringTrimRightTestCase("ćĆčČcCabCcČčĆć", Some("Ć"), "SR_CI", "ćĆčČcCabCcČč"),
@@ -1461,8 +1684,20 @@ class CollationStringExpressionsSuite
result: R)
val testCases = Seq(
StringTrimTestCase("xxasdxx", Some("x"), "UTF8_BINARY", "asd"),
+ StringTrimTestCase("xxasdxx ", Some("x "), "UTF8_BINARY_RTRIM", "asd"),
+ StringTrimTestCase("xxasdxx ", Some("x"), "UTF8_BINARY_RTRIM", "asd "),
+ StringTrimTestCase(" xxasdxx ", Some("x "), "UTF8_BINARY_RTRIM", "asd"),
+ StringTrimTestCase(" xxasdxx", Some("x"), "UTF8_BINARY_RTRIM", " xxasd"),
StringTrimTestCase("xxasdxx", Some("X"), "UTF8_LCASE", "asd"),
+ StringTrimTestCase("xxasdxx ", Some("X "), "UTF8_LCASE_RTRIM", "asd"),
+ StringTrimTestCase("xxasdxx ", Some("X"), "UTF8_LCASE_RTRIM", "asd "),
+ StringTrimTestCase(" xxasdxx ", Some("X "), "UTF8_LCASE_RTRIM", "asd"),
+ StringTrimTestCase(" xxasdxx", Some("x"), "UTF8_LCASE_RTRIM", " xxasd"),
StringTrimTestCase("xxasdxx", Some("y"), "UNICODE", "xxasdxx"),
+ StringTrimTestCase("xxasdxx", Some("y"), "UNICODE_RTRIM", "xxasdxx"),
+ StringTrimTestCase(" asd ", None, "UNICODE_RTRIM", "asd"),
+ StringTrimTestCase(" asd ", None, "UNICODE_CI", "asd"),
+ StringTrimTestCase(" asd ", Some("D"), "UNICODE_CI_RTRIM", " as "),
StringTrimTestCase(" asd ", None, "UNICODE_CI", "asd"),
StringTrimTestCase("ćĆčČcCabCcČčĆć", Some("ć"), "SR", "ĆčČcCabCcČčĆ"),
StringTrimTestCase("ćĆčČcCabCcČčĆć", Some("Ć"), "SR_CI", "čČcCabCcČč"),
@@ -1512,8 +1747,20 @@ class CollationStringExpressionsSuite
result: R)
val testCases = Seq(
StringTrimBothTestCase("xxasdxx", Some("x"), "UTF8_BINARY", "asd"),
+ StringTrimBothTestCase("xxasdxx ", Some("x "), "UTF8_BINARY_RTRIM", "asd"),
+ StringTrimBothTestCase("xxasdxx ", Some("x"), "UTF8_BINARY_RTRIM", "asd "),
+ StringTrimBothTestCase(" xxasdxx ", Some("x "), "UTF8_BINARY_RTRIM", "asd"),
+ StringTrimBothTestCase(" xxasdxx", Some("x"), "UTF8_BINARY_RTRIM", " xxasd"),
StringTrimBothTestCase("xxasdxx", Some("X"), "UTF8_LCASE", "asd"),
+ StringTrimBothTestCase("xxasdxx ", Some("X "), "UTF8_LCASE_RTRIM", "asd"),
+ StringTrimBothTestCase("xxasdxx ", Some("X"), "UTF8_LCASE_RTRIM", "asd "),
+ StringTrimBothTestCase(" xxasdxx ", Some("X "), "UTF8_LCASE_RTRIM", "asd"),
+ StringTrimBothTestCase(" xxasdxx", Some("x"), "UTF8_LCASE_RTRIM", " xxasd"),
StringTrimBothTestCase("xxasdxx", Some("y"), "UNICODE", "xxasdxx"),
+ StringTrimBothTestCase("xxasdxx", Some("y"), "UNICODE_RTRIM", "xxasdxx"),
+ StringTrimBothTestCase(" asd ", None, "UNICODE_RTRIM", "asd"),
+ StringTrimBothTestCase(" asd ", None, "UNICODE_CI", "asd"),
+ StringTrimBothTestCase(" asd ", Some("D"), "UNICODE_CI_RTRIM", " as "),
StringTrimBothTestCase(" asd ", None, "UNICODE_CI", "asd"),
StringTrimBothTestCase("ćĆčČcCabCcČčĆć", Some("ć"), "SR", "ĆčČcCabCcČčĆ"),
StringTrimBothTestCase("ćĆčČcCabCcČčĆć", Some("Ć"), "SR_CI", "čČcCabCcČč"),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org