You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2024/07/31 05:14:03 UTC
(spark) branch master updated: [SPARK-48725][SQL] Integrate CollationAwareUTF8String.lowerCaseCodePoints into string expressions
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 98c365f6e652 [SPARK-48725][SQL] Integrate CollationAwareUTF8String.lowerCaseCodePoints into string expressions
98c365f6e652 is described below
commit 98c365f6e652f740e51c3f185ebd252918ae824a
Author: Uros Bojanic <15...@users.noreply.github.com>
AuthorDate: Wed Jul 31 13:13:49 2024 +0800
[SPARK-48725][SQL] Integrate CollationAwareUTF8String.lowerCaseCodePoints into string expressions
### What changes were proposed in this pull request?
Use `CollationAwareUTF8String.lowerCaseCodePoints` logic to properly lowercase strings according to UTF8_LCASE collation, instead of relying on `UTF8String.toLowerCase()` method calls.
### Why are the changes needed?
Avoid correctness issues with respect to code-point logic in UTF8_LCASE (arising when Java to performs string lowercasing) and ensure consistent results.
### Does this PR introduce _any_ user-facing change?
Yes, collation aware string function implementations will now rely on `CollationAwareUTF8String` string lowercasing for UTF8_LCASE collation, instead of `UTF8String` logic (which resorts to Java's implementation).
### How was this patch tested?
Existing tests, with some new cases in `CollationSupportSuite`.
6 expressions are affected by this change: `contains`, `instr`, `find_in_set`, `replace`, `locate`, `substr_index`.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #47132 from uros-db/lcase-cp.
Authored-by: Uros Bojanic <15...@users.noreply.github.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../catalyst/util/CollationAwareUTF8String.java | 27 ++-
.../spark/sql/catalyst/util/CollationFactory.java | 35 ++--
.../spark/unsafe/types/CollationSupportSuite.java | 224 +++++++++++++++++++++
3 files changed, 256 insertions(+), 30 deletions(-)
diff --git a/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationAwareUTF8String.java b/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationAwareUTF8String.java
index b9868ca665a6..430d1fb89832 100644
--- a/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationAwareUTF8String.java
+++ b/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationAwareUTF8String.java
@@ -51,9 +51,8 @@ public class CollationAwareUTF8String {
/**
* Returns whether the target string starts with the specified prefix, starting from the
* specified position (0-based index referring to character position in UTF8String), with respect
- * to the UTF8_LCASE collation. The method assumes that the prefix is already lowercased
- * prior to method call to avoid the overhead of calling .toLowerCase() multiple times on the
- * same prefix string.
+ * to the UTF8_LCASE collation. The method assumes that the prefix is already lowercased prior
+ * to method call to avoid the overhead of lowercasing the same prefix string multiple times.
*
* @param target the string to be searched in
* @param lowercasePattern the string to be searched for
@@ -87,7 +86,8 @@ public class CollationAwareUTF8String {
int startPos) {
assert startPos >= 0;
for (int len = 0; len <= target.numChars() - startPos; ++len) {
- if (target.substring(startPos, startPos + len).toLowerCase().equals(lowercasePattern)) {
+ if (lowerCaseCodePoints(target.substring(startPos, startPos + len))
+ .equals(lowercasePattern)) {
return len;
}
}
@@ -123,8 +123,7 @@ public class CollationAwareUTF8String {
* Returns whether the target string ends with the specified suffix, ending at the specified
* position (0-based index referring to character position in UTF8String), with respect to the
* UTF8_LCASE collation. The method assumes that the suffix is already lowercased prior
- * to method call to avoid the overhead of calling .toLowerCase() multiple times on the same
- * suffix string.
+ * to method call to avoid the overhead of lowercasing the same suffix string multiple times.
*
* @param target the string to be searched in
* @param lowercasePattern the string to be searched for
@@ -158,7 +157,7 @@ public class CollationAwareUTF8String {
int endPos) {
assert endPos <= target.numChars();
for (int len = 0; len <= endPos; ++len) {
- if (target.substring(endPos - len, endPos).toLowerCase().equals(lowercasePattern)) {
+ if (lowerCaseCodePoints(target.substring(endPos - len, endPos)).equals(lowercasePattern)) {
return len;
}
}
@@ -191,10 +190,9 @@ public class CollationAwareUTF8String {
}
/**
- * Lowercase UTF8String comparison used for UTF8_LCASE collation. While the default
- * UTF8String comparison is equivalent to a.toLowerCase().binaryCompare(b.toLowerCase()), this
- * method uses code points to compare the strings in a case-insensitive manner using ICU rules,
- * as well as handling special rules for one-to-many case mappings (see: lowerCaseCodePoints).
+ * Lowercase UTF8String comparison used for UTF8_LCASE collation. This method uses lowercased
+ * code points to compare the strings in a case-insensitive manner using ICU rules, taking into
+ * account special rules for one-to-many case mappings (see: lowerCaseCodePoints).
*
* @param left The first UTF8String to compare.
* @param right The second UTF8String to compare.
@@ -325,8 +323,7 @@ public class CollationAwareUTF8String {
return src;
}
- // TODO(SPARK-48725): Use lowerCaseCodePoints instead of UTF8String.toLowerCase.
- UTF8String lowercaseSearch = search.toLowerCase();
+ UTF8String lowercaseSearch = lowerCaseCodePoints(search);
int start = 0;
int end = lowercaseFind(src, lowercaseSearch, start);
@@ -573,7 +570,7 @@ public class CollationAwareUTF8String {
public static int lowercaseIndexOf(final UTF8String target, final UTF8String pattern,
final int start) {
if (pattern.numChars() == 0) return target.indexOfEmpty(start);
- return lowercaseFind(target, pattern.toLowerCase(), start);
+ return lowercaseFind(target, lowerCaseCodePoints(pattern), start);
}
public static int indexOf(final UTF8String target, final UTF8String pattern,
@@ -670,7 +667,7 @@ public class CollationAwareUTF8String {
return UTF8String.EMPTY_UTF8;
}
- UTF8String lowercaseDelimiter = delimiter.toLowerCase();
+ UTF8String lowercaseDelimiter = lowerCaseCodePoints(delimiter);
if (count > 0) {
// Search left to right (note: the start code point is inclusive).
diff --git a/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationFactory.java b/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationFactory.java
index f13f66e384e0..5640a2468d02 100644
--- a/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationFactory.java
+++ b/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationFactory.java
@@ -133,8 +133,8 @@ public final class CollationFactory {
public final boolean supportsBinaryOrdering;
/**
- * Support for Lowercase Equality implies that it is possible to check equality on
- * byte by byte level, but only after calling "UTF8String.toLowerCase" on both arguments.
+ * Support for Lowercase Equality implies that it is possible to check equality on byte by
+ * byte level, but only after calling "UTF8String.lowerCaseCodePoints" on both arguments.
* This allows custom collation support for UTF8_LCASE collation in various Spark
* expressions, as this particular collation is not supported by the external ICU library.
*/
@@ -676,9 +676,9 @@ public final class CollationFactory {
collationName(),
PROVIDER_ICU,
collator,
- (s1, s2) -> collator.compare(s1.toString(), s2.toString()),
+ (s1, s2) -> collator.compare(s1.toValidString(), s2.toValidString()),
ICU_COLLATOR_VERSION,
- s -> (long) collator.getCollationKey(s.toString()).hashCode(),
+ s -> (long) collator.getCollationKey(s.toValidString()).hashCode(),
/* supportsBinaryEquality = */ false,
/* supportsBinaryOrdering = */ false,
/* supportsLowercaseEquality = */ false);
@@ -749,12 +749,15 @@ public final class CollationFactory {
* Returns a StringSearch object for the given pattern and target strings, under collation
* rules corresponding to the given collationId. The external ICU library StringSearch object can
* be used to find occurrences of the pattern in the target string, while respecting collation.
+ * When given invalid UTF8Strings, the method will first convert them to valid strings, and then
+ * instantiate the StringSearch object. However, original UTF8Strings will remain unchanged.
*/
public static StringSearch getStringSearch(
final UTF8String targetUTF8String,
final UTF8String patternUTF8String,
final int collationId) {
- return getStringSearch(targetUTF8String.toString(), patternUTF8String.toString(), collationId);
+ return getStringSearch(targetUTF8String.toValidString(), patternUTF8String.toValidString(),
+ collationId);
}
/**
@@ -763,9 +766,9 @@ public final class CollationFactory {
* be used to find occurrences of the pattern in the target string, while respecting collation.
*/
public static StringSearch getStringSearch(
- final String targetString,
- final String patternString,
- final int collationId) {
+ final String targetString,
+ final String patternString,
+ final int collationId) {
CharacterIterator target = new StringCharacterIterator(targetString);
Collator collator = CollationFactory.fetchCollation(collationId).collator;
return new StringSearch(patternString, target, (RuleBasedCollator) collator);
@@ -775,11 +778,13 @@ public final class CollationFactory {
* Returns a collation-unaware StringSearch object for the given pattern and target strings.
* While this object does not respect collation, it can be used to find occurrences of the pattern
* in the target string for UTF8_BINARY or UTF8_LCASE (if arguments are lowercased).
+ * When given invalid UTF8Strings, the method will first convert them to valid strings, and then
+ * instantiate the StringSearch object. However, original UTF8Strings will remain unchanged.
*/
public static StringSearch getStringSearch(
- final UTF8String targetUTF8String,
- final UTF8String patternUTF8String) {
- return new StringSearch(patternUTF8String.toString(), targetUTF8String.toString());
+ final UTF8String targetUTF8String,
+ final UTF8String patternUTF8String) {
+ return new StringSearch(patternUTF8String.toValidString(), targetUTF8String.toValidString());
}
/**
@@ -818,9 +823,9 @@ public final class CollationFactory {
if (collation.supportsBinaryEquality) {
return input;
} else if (collation.supportsLowercaseEquality) {
- return input.toLowerCase();
+ return CollationAwareUTF8String.lowerCaseCodePoints(input);
} else {
- CollationKey collationKey = collation.collator.getCollationKey(input.toString());
+ CollationKey collationKey = collation.collator.getCollationKey(input.toValidString());
return UTF8String.fromBytes(collationKey.toByteArray());
}
}
@@ -830,9 +835,9 @@ public final class CollationFactory {
if (collation.supportsBinaryEquality) {
return input.getBytes();
} else if (collation.supportsLowercaseEquality) {
- return input.toLowerCase().getBytes();
+ return CollationAwareUTF8String.lowerCaseCodePoints(input).getBytes();
} else {
- return collation.collator.getCollationKey(input.toString()).toByteArray();
+ return collation.collator.getCollationKey(input.toValidString()).toByteArray();
}
}
diff --git a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CollationSupportSuite.java b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CollationSupportSuite.java
index b082ab21944f..4301bf56b6d5 100644
--- a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CollationSupportSuite.java
+++ b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CollationSupportSuite.java
@@ -302,6 +302,43 @@ public class CollationSupportSuite {
assertContains("adİos", "Io", "UTF8_LCASE", false);
assertContains("adİos", "i̇o", "UTF8_LCASE", true);
assertContains("adİos", "İo", "UTF8_LCASE", true);
+ // Greek sigmas.
+ assertContains("σ", "σ", "UTF8_BINARY", true);
+ assertContains("σ", "ς", "UTF8_BINARY", false);
+ assertContains("σ", "Σ", "UTF8_BINARY", false);
+ assertContains("ς", "σ", "UTF8_BINARY", false);
+ assertContains("ς", "ς", "UTF8_BINARY", true);
+ assertContains("ς", "Σ", "UTF8_BINARY", false);
+ assertContains("Σ", "σ", "UTF8_BINARY", false);
+ assertContains("Σ", "ς", "UTF8_BINARY", false);
+ assertContains("Σ", "Σ", "UTF8_BINARY", true);
+ assertContains("σ", "σ", "UTF8_LCASE", true);
+ assertContains("σ", "ς", "UTF8_LCASE", true);
+ assertContains("σ", "Σ", "UTF8_LCASE", true);
+ assertContains("ς", "σ", "UTF8_LCASE", true);
+ assertContains("ς", "ς", "UTF8_LCASE", true);
+ assertContains("ς", "Σ", "UTF8_LCASE", true);
+ assertContains("Σ", "σ", "UTF8_LCASE", true);
+ assertContains("Σ", "ς", "UTF8_LCASE", true);
+ assertContains("Σ", "Σ", "UTF8_LCASE", true);
+ assertContains("σ", "σ", "UNICODE", true);
+ assertContains("σ", "ς", "UNICODE", false);
+ assertContains("σ", "Σ", "UNICODE", false);
+ assertContains("ς", "σ", "UNICODE", false);
+ assertContains("ς", "ς", "UNICODE", true);
+ assertContains("ς", "Σ", "UNICODE", false);
+ assertContains("Σ", "σ", "UNICODE", false);
+ assertContains("Σ", "ς", "UNICODE", false);
+ assertContains("Σ", "Σ", "UNICODE", true);
+ assertContains("σ", "σ", "UNICODE_CI", true);
+ assertContains("σ", "ς", "UNICODE_CI", true);
+ assertContains("σ", "Σ", "UNICODE_CI", true);
+ assertContains("ς", "σ", "UNICODE_CI", true);
+ assertContains("ς", "ς", "UNICODE_CI", true);
+ assertContains("ς", "Σ", "UNICODE_CI", true);
+ assertContains("Σ", "σ", "UNICODE_CI", true);
+ assertContains("Σ", "ς", "UNICODE_CI", true);
+ assertContains("Σ", "Σ", "UNICODE_CI", true);
}
private void assertStartsWith(
@@ -880,6 +917,43 @@ public class CollationSupportSuite {
assertStringInstr("ai̇oxXİo", "Xx", "UTF8_LCASE", 5);
assertStringInstr("abİoi̇o", "\u0307o", "UTF8_LCASE", 6);
assertStringInstr("aİoi̇oxx", "XX", "UTF8_LCASE", 7);
+ // Greek sigmas.
+ assertStringInstr("σ", "σ", "UTF8_BINARY", 1);
+ assertStringInstr("σ", "ς", "UTF8_BINARY", 0);
+ assertStringInstr("σ", "Σ", "UTF8_BINARY", 0);
+ assertStringInstr("ς", "σ", "UTF8_BINARY", 0);
+ assertStringInstr("ς", "ς", "UTF8_BINARY", 1);
+ assertStringInstr("ς", "Σ", "UTF8_BINARY", 0);
+ assertStringInstr("Σ", "σ", "UTF8_BINARY", 0);
+ assertStringInstr("Σ", "ς", "UTF8_BINARY", 0);
+ assertStringInstr("Σ", "Σ", "UTF8_BINARY", 1);
+ assertStringInstr("σ", "σ", "UTF8_LCASE", 1);
+ assertStringInstr("σ", "ς", "UTF8_LCASE", 1);
+ assertStringInstr("σ", "Σ", "UTF8_LCASE", 1);
+ assertStringInstr("ς", "σ", "UTF8_LCASE", 1);
+ assertStringInstr("ς", "ς", "UTF8_LCASE", 1);
+ assertStringInstr("ς", "Σ", "UTF8_LCASE", 1);
+ assertStringInstr("Σ", "σ", "UTF8_LCASE", 1);
+ assertStringInstr("Σ", "ς", "UTF8_LCASE", 1);
+ assertStringInstr("Σ", "Σ", "UTF8_LCASE", 1);
+ assertStringInstr("σ", "σ", "UNICODE", 1);
+ assertStringInstr("σ", "ς", "UNICODE", 0);
+ assertStringInstr("σ", "Σ", "UNICODE", 0);
+ assertStringInstr("ς", "σ", "UNICODE", 0);
+ assertStringInstr("ς", "ς", "UNICODE", 1);
+ assertStringInstr("ς", "Σ", "UNICODE", 0);
+ assertStringInstr("Σ", "σ", "UNICODE", 0);
+ assertStringInstr("Σ", "ς", "UNICODE", 0);
+ assertStringInstr("Σ", "Σ", "UNICODE", 1);
+ assertStringInstr("σ", "σ", "UNICODE_CI", 1);
+ assertStringInstr("σ", "ς", "UNICODE_CI", 1);
+ assertStringInstr("σ", "Σ", "UNICODE_CI", 1);
+ assertStringInstr("ς", "σ", "UNICODE_CI", 1);
+ assertStringInstr("ς", "ς", "UNICODE_CI", 1);
+ assertStringInstr("ς", "Σ", "UNICODE_CI", 1);
+ assertStringInstr("Σ", "σ", "UNICODE_CI", 1);
+ assertStringInstr("Σ", "ς", "UNICODE_CI", 1);
+ assertStringInstr("Σ", "Σ", "UNICODE_CI", 1);
}
private void assertFindInSet(String word, UTF8String set, String collationName,
@@ -981,6 +1055,43 @@ public class CollationSupportSuite {
assertFindInSet("c", UTF8String.fromBytes(
new byte[] { 0x41, (byte) 0xC2, 0x2C, 0x42, 0x2C, 0x43, 0x2C, 0x43, 0x2C, 0x56 }),
"UNICODE_CI", 2);
+ // Greek sigmas.
+ assertFindInSet("σ", UTF8String.fromString("σ"), "UTF8_BINARY", 1);
+ assertFindInSet("σ", UTF8String.fromString("ς"), "UTF8_BINARY", 0);
+ assertFindInSet("σ", UTF8String.fromString("Σ"), "UTF8_BINARY", 0);
+ assertFindInSet("ς", UTF8String.fromString("σ"), "UTF8_BINARY", 0);
+ assertFindInSet("ς", UTF8String.fromString("ς"), "UTF8_BINARY", 1);
+ assertFindInSet("ς", UTF8String.fromString("Σ"), "UTF8_BINARY", 0);
+ assertFindInSet("Σ", UTF8String.fromString("σ"), "UTF8_BINARY", 0);
+ assertFindInSet("Σ", UTF8String.fromString("ς"), "UTF8_BINARY", 0);
+ assertFindInSet("Σ", UTF8String.fromString("Σ"), "UTF8_BINARY", 1);
+ assertFindInSet("σ", UTF8String.fromString("σ"), "UTF8_LCASE", 1);
+ assertFindInSet("σ", UTF8String.fromString("ς"), "UTF8_LCASE", 1);
+ assertFindInSet("σ", UTF8String.fromString("Σ"), "UTF8_LCASE", 1);
+ assertFindInSet("ς", UTF8String.fromString("σ"), "UTF8_LCASE", 1);
+ assertFindInSet("ς", UTF8String.fromString("ς"), "UTF8_LCASE", 1);
+ assertFindInSet("ς", UTF8String.fromString("Σ"), "UTF8_LCASE", 1);
+ assertFindInSet("Σ", UTF8String.fromString("σ"), "UTF8_LCASE", 1);
+ assertFindInSet("Σ", UTF8String.fromString("ς"), "UTF8_LCASE", 1);
+ assertFindInSet("Σ", UTF8String.fromString("Σ"), "UTF8_LCASE", 1);
+ assertFindInSet("σ", UTF8String.fromString("σ"), "UNICODE", 1);
+ assertFindInSet("σ", UTF8String.fromString("ς"), "UNICODE", 0);
+ assertFindInSet("σ", UTF8String.fromString("Σ"), "UNICODE", 0);
+ assertFindInSet("ς", UTF8String.fromString("σ"), "UNICODE", 0);
+ assertFindInSet("ς", UTF8String.fromString("ς"), "UNICODE", 1);
+ assertFindInSet("ς", UTF8String.fromString("Σ"), "UNICODE", 0);
+ assertFindInSet("Σ", UTF8String.fromString("σ"), "UNICODE", 0);
+ assertFindInSet("Σ", UTF8String.fromString("ς"), "UNICODE", 0);
+ assertFindInSet("Σ", UTF8String.fromString("Σ"), "UNICODE", 1);
+ assertFindInSet("σ", UTF8String.fromString("σ"), "UNICODE_CI", 1);
+ assertFindInSet("σ", UTF8String.fromString("ς"), "UNICODE_CI", 1);
+ assertFindInSet("σ", UTF8String.fromString("Σ"), "UNICODE_CI", 1);
+ assertFindInSet("ς", UTF8String.fromString("σ"), "UNICODE_CI", 1);
+ assertFindInSet("ς", UTF8String.fromString("ς"), "UNICODE_CI", 1);
+ assertFindInSet("ς", UTF8String.fromString("Σ"), "UNICODE_CI", 1);
+ assertFindInSet("Σ", UTF8String.fromString("σ"), "UNICODE_CI", 1);
+ assertFindInSet("Σ", UTF8String.fromString("ς"), "UNICODE_CI", 1);
+ assertFindInSet("Σ", UTF8String.fromString("Σ"), "UNICODE_CI", 1);
}
private void assertReplace(String source, String search, String replace, String collationName,
@@ -1034,6 +1145,44 @@ public class CollationSupportSuite {
"İx\u0307İx\u0307İx\u0307"); // != UNICODE_CI
assertReplace("abİo12i̇o", "i̇o", "xx", "UTF8_LCASE", "abxx12xx");
assertReplace("abi̇o12i̇o", "İo", "yy", "UTF8_LCASE", "abyy12yy");
+ // Greek sigmas.
+ assertReplace("σ", "σ", "x", "UTF8_BINARY", "x");
+ assertReplace("σ", "ς", "x", "UTF8_BINARY", "σ");
+ assertReplace("σ", "Σ", "x", "UTF8_BINARY", "σ");
+ assertReplace("ς", "σ", "x", "UTF8_BINARY", "ς");
+ assertReplace("ς", "ς", "x", "UTF8_BINARY", "x");
+ assertReplace("ς", "Σ", "x", "UTF8_BINARY", "ς");
+ assertReplace("Σ", "σ", "x", "UTF8_BINARY", "Σ");
+ assertReplace("Σ", "ς", "x", "UTF8_BINARY", "Σ");
+ assertReplace("Σ", "Σ", "x", "UTF8_BINARY", "x");
+ assertReplace("σ", "σ", "x", "UTF8_LCASE", "x");
+ assertReplace("σ", "ς", "x", "UTF8_LCASE", "x");
+ assertReplace("σ", "Σ", "x", "UTF8_LCASE", "x");
+ assertReplace("ς", "σ", "x", "UTF8_LCASE", "x");
+ assertReplace("ς", "ς", "x", "UTF8_LCASE", "x");
+ assertReplace("ς", "Σ", "x", "UTF8_LCASE", "x");
+ assertReplace("Σ", "σ", "x", "UTF8_LCASE", "x");
+ assertReplace("Σ", "ς", "x", "UTF8_LCASE", "x");
+ assertReplace("Σ", "Σ", "x", "UTF8_LCASE", "x");
+ assertReplace("σ", "σ", "x", "UNICODE", "x");
+ assertReplace("σ", "ς", "x", "UNICODE", "σ");
+ assertReplace("σ", "Σ", "x", "UNICODE", "σ");
+ assertReplace("ς", "σ", "x", "UNICODE", "ς");
+ assertReplace("ς", "ς", "x", "UNICODE", "x");
+ assertReplace("ς", "Σ", "x", "UNICODE", "ς");
+ assertReplace("Σ", "σ", "x", "UNICODE", "Σ");
+ assertReplace("Σ", "ς", "x", "UNICODE", "Σ");
+ assertReplace("Σ", "Σ", "x", "UNICODE", "x");
+ assertReplace("σ", "σ", "x", "UNICODE_CI", "x");
+ assertReplace("σ", "ς", "x", "UNICODE_CI", "x");
+ assertReplace("σ", "Σ", "x", "UNICODE_CI", "x");
+ assertReplace("ς", "σ", "x", "UNICODE_CI", "x");
+ assertReplace("ς", "ς", "x", "UNICODE_CI", "x");
+ assertReplace("ς", "Σ", "x", "UNICODE_CI", "x");
+ assertReplace("Σ", "σ", "x", "UNICODE_CI", "x");
+ assertReplace("Σ", "ς", "x", "UNICODE_CI", "x");
+ assertReplace("Σ", "Σ", "x", "UNICODE_CI", "x");
+
}
private void assertLocate(String substring, String string, Integer start, String collationName,
@@ -1111,6 +1260,43 @@ public class CollationSupportSuite {
assertLocate("İo", "i̇o世界大千世界", 1, "UNICODE_CI", 1);
assertLocate("İo", "大千i̇o世界大千世界", 1, "UNICODE_CI", 3);
assertLocate("İo", "世界i̇o大千世界大千i̇o", 4, "UNICODE_CI", 12);
+ // Greek sigmas.
+ assertLocate("σ", "σ", 1, "UTF8_BINARY", 1);
+ assertLocate("σ", "ς", 1, "UTF8_BINARY", 0);
+ assertLocate("σ", "Σ", 1, "UTF8_BINARY", 0);
+ assertLocate("ς", "σ", 1, "UTF8_BINARY", 0);
+ assertLocate("ς", "ς", 1, "UTF8_BINARY", 1);
+ assertLocate("ς", "Σ", 1, "UTF8_BINARY", 0);
+ assertLocate("Σ", "σ", 1, "UTF8_BINARY", 0);
+ assertLocate("Σ", "ς", 1, "UTF8_BINARY", 0);
+ assertLocate("Σ", "Σ", 1, "UTF8_BINARY", 1);
+ assertLocate("σ", "σ", 1, "UTF8_LCASE", 1);
+ assertLocate("σ", "ς", 1, "UTF8_LCASE", 1);
+ assertLocate("σ", "Σ", 1, "UTF8_LCASE", 1);
+ assertLocate("ς", "σ", 1, "UTF8_LCASE", 1);
+ assertLocate("ς", "ς", 1, "UTF8_LCASE", 1);
+ assertLocate("ς", "Σ", 1, "UTF8_LCASE", 1);
+ assertLocate("Σ", "σ", 1, "UTF8_LCASE", 1);
+ assertLocate("Σ", "ς", 1, "UTF8_LCASE", 1);
+ assertLocate("Σ", "Σ", 1, "UTF8_LCASE", 1);
+ assertLocate("σ", "σ", 1, "UNICODE", 1);
+ assertLocate("σ", "ς", 1, "UNICODE", 0);
+ assertLocate("σ", "Σ", 1, "UNICODE", 0);
+ assertLocate("ς", "σ", 1, "UNICODE", 0);
+ assertLocate("ς", "ς", 1, "UNICODE", 1);
+ assertLocate("ς", "Σ", 1, "UNICODE", 0);
+ assertLocate("Σ", "σ", 1, "UNICODE", 0);
+ assertLocate("Σ", "ς", 1, "UNICODE", 0);
+ assertLocate("Σ", "Σ", 1, "UNICODE", 1);
+ assertLocate("σ", "σ", 1, "UNICODE_CI", 1);
+ assertLocate("σ", "ς", 1, "UNICODE_CI", 1);
+ assertLocate("σ", "Σ", 1, "UNICODE_CI", 1);
+ assertLocate("ς", "σ", 1, "UNICODE_CI", 1);
+ assertLocate("ς", "ς", 1, "UNICODE_CI", 1);
+ assertLocate("ς", "Σ", 1, "UNICODE_CI", 1);
+ assertLocate("Σ", "σ", 1, "UNICODE_CI", 1);
+ assertLocate("Σ", "ς", 1, "UNICODE_CI", 1);
+ assertLocate("Σ", "Σ", 1, "UNICODE_CI", 1);
}
private void assertSubstringIndex(String string, String delimiter, Integer count,
@@ -1220,6 +1406,44 @@ public class CollationSupportSuite {
assertSubstringIndex("ai̇bİoi̇o12i̇oİo", "İo", 3, "UTF8_LCASE", "ai̇bİoi̇o12");
assertSubstringIndex("ai̇bİoi̇o12i̇oİo", "i̇o", 3, "UTF8_LCASE", "ai̇bİoi̇o12");
assertSubstringIndex("bİoi̇o12i̇o", "\u0307oi", 1, "UTF8_LCASE", "bİoi̇o12i̇o");
+ // Greek sigmas.
+ assertSubstringIndex("σ", "σ", 1, "UTF8_BINARY", "");
+ assertSubstringIndex("σ", "ς", 1, "UTF8_BINARY", "σ");
+ assertSubstringIndex("σ", "Σ", 1, "UTF8_BINARY", "σ");
+ assertSubstringIndex("ς", "σ", 1, "UTF8_BINARY", "ς");
+ assertSubstringIndex("ς", "ς", 1, "UTF8_BINARY", "");
+ assertSubstringIndex("ς", "Σ", 1, "UTF8_BINARY", "ς");
+ assertSubstringIndex("Σ", "σ", 1, "UTF8_BINARY", "Σ");
+ assertSubstringIndex("Σ", "ς", 1, "UTF8_BINARY", "Σ");
+ assertSubstringIndex("Σ", "Σ", 1, "UTF8_BINARY", "");
+ assertSubstringIndex("σ", "σ", 1, "UTF8_LCASE", "");
+ assertSubstringIndex("σ", "ς", 1, "UTF8_LCASE", "");
+ assertSubstringIndex("σ", "Σ", 1, "UTF8_LCASE", "");
+ assertSubstringIndex("ς", "σ", 1, "UTF8_LCASE", "");
+ assertSubstringIndex("ς", "ς", 1, "UTF8_LCASE", "");
+ assertSubstringIndex("ς", "Σ", 1, "UTF8_LCASE", "");
+ assertSubstringIndex("Σ", "σ", 1, "UTF8_LCASE", "");
+ assertSubstringIndex("Σ", "ς", 1, "UTF8_LCASE", "");
+ assertSubstringIndex("Σ", "Σ", 1, "UTF8_LCASE", "");
+ assertSubstringIndex("σ", "σ", 1, "UNICODE", "");
+ assertSubstringIndex("σ", "ς", 1, "UNICODE", "σ");
+ assertSubstringIndex("σ", "Σ", 1, "UNICODE", "σ");
+ assertSubstringIndex("ς", "σ", 1, "UNICODE", "ς");
+ assertSubstringIndex("ς", "ς", 1, "UNICODE", "");
+ assertSubstringIndex("ς", "Σ", 1, "UNICODE", "ς");
+ assertSubstringIndex("Σ", "σ", 1, "UNICODE", "Σ");
+ assertSubstringIndex("Σ", "ς", 1, "UNICODE", "Σ");
+ assertSubstringIndex("Σ", "Σ", 1, "UNICODE", "");
+ assertSubstringIndex("σ", "σ", 1, "UNICODE_CI", "");
+ assertSubstringIndex("σ", "ς", 1, "UNICODE_CI", "");
+ assertSubstringIndex("σ", "Σ", 1, "UNICODE_CI", "");
+ assertSubstringIndex("ς", "σ", 1, "UNICODE_CI", "");
+ assertSubstringIndex("ς", "ς", 1, "UNICODE_CI", "");
+ assertSubstringIndex("ς", "Σ", 1, "UNICODE_CI", "");
+ assertSubstringIndex("Σ", "σ", 1, "UNICODE_CI", "");
+ assertSubstringIndex("Σ", "ς", 1, "UNICODE_CI", "");
+ assertSubstringIndex("Σ", "Σ", 1, "UNICODE_CI", "");
+
}
private void assertStringTrim(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org