You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2024/07/31 05:14:03 UTC

(spark) branch master updated: [SPARK-48725][SQL] Integrate CollationAwareUTF8String.lowerCaseCodePoints into string expressions

This is an automated email from the ASF dual-hosted git repository.

wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 98c365f6e652 [SPARK-48725][SQL] Integrate CollationAwareUTF8String.lowerCaseCodePoints into string expressions
98c365f6e652 is described below

commit 98c365f6e652f740e51c3f185ebd252918ae824a
Author: Uros Bojanic <15...@users.noreply.github.com>
AuthorDate: Wed Jul 31 13:13:49 2024 +0800

    [SPARK-48725][SQL] Integrate CollationAwareUTF8String.lowerCaseCodePoints into string expressions
    
    ### What changes were proposed in this pull request?
    Use `CollationAwareUTF8String.lowerCaseCodePoints` logic to properly lowercase strings according to UTF8_LCASE collation, instead of relying on `UTF8String.toLowerCase()` method calls.
    
    ### Why are the changes needed?
    Avoid correctness issues with respect to code-point logic in UTF8_LCASE (arising when Java to performs string lowercasing) and ensure consistent results.
    
    ### Does this PR introduce _any_ user-facing change?
    Yes, collation aware string function implementations will now rely on `CollationAwareUTF8String` string lowercasing for UTF8_LCASE collation, instead of `UTF8String` logic (which resorts to Java's implementation).
    
    ### How was this patch tested?
    Existing tests, with some new cases in `CollationSupportSuite`.
    6 expressions are affected by this change: `contains`, `instr`, `find_in_set`, `replace`, `locate`, `substr_index`.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    No.
    
    Closes #47132 from uros-db/lcase-cp.
    
    Authored-by: Uros Bojanic <15...@users.noreply.github.com>
    Signed-off-by: Wenchen Fan <we...@databricks.com>
---
 .../catalyst/util/CollationAwareUTF8String.java    |  27 ++-
 .../spark/sql/catalyst/util/CollationFactory.java  |  35 ++--
 .../spark/unsafe/types/CollationSupportSuite.java  | 224 +++++++++++++++++++++
 3 files changed, 256 insertions(+), 30 deletions(-)

diff --git a/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationAwareUTF8String.java b/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationAwareUTF8String.java
index b9868ca665a6..430d1fb89832 100644
--- a/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationAwareUTF8String.java
+++ b/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationAwareUTF8String.java
@@ -51,9 +51,8 @@ public class CollationAwareUTF8String {
   /**
    * Returns whether the target string starts with the specified prefix, starting from the
    * specified position (0-based index referring to character position in UTF8String), with respect
-   * to the UTF8_LCASE collation. The method assumes that the prefix is already lowercased
-   * prior to method call to avoid the overhead of calling .toLowerCase() multiple times on the
-   * same prefix string.
+   * to the UTF8_LCASE collation. The method assumes that the prefix is already lowercased prior
+   * to method call to avoid the overhead of lowercasing the same prefix string multiple times.
    *
    * @param target the string to be searched in
    * @param lowercasePattern the string to be searched for
@@ -87,7 +86,8 @@ public class CollationAwareUTF8String {
       int startPos) {
     assert startPos >= 0;
     for (int len = 0; len <= target.numChars() - startPos; ++len) {
-      if (target.substring(startPos, startPos + len).toLowerCase().equals(lowercasePattern)) {
+      if (lowerCaseCodePoints(target.substring(startPos, startPos + len))
+          .equals(lowercasePattern)) {
         return len;
       }
     }
@@ -123,8 +123,7 @@ public class CollationAwareUTF8String {
    * Returns whether the target string ends with the specified suffix, ending at the specified
    * position (0-based index referring to character position in UTF8String), with respect to the
    * UTF8_LCASE collation. The method assumes that the suffix is already lowercased prior
-   * to method call to avoid the overhead of calling .toLowerCase() multiple times on the same
-   * suffix string.
+   * to method call to avoid the overhead of lowercasing the same suffix string multiple times.
    *
    * @param target the string to be searched in
    * @param lowercasePattern the string to be searched for
@@ -158,7 +157,7 @@ public class CollationAwareUTF8String {
       int endPos) {
     assert endPos <= target.numChars();
     for (int len = 0; len <= endPos; ++len) {
-      if (target.substring(endPos - len, endPos).toLowerCase().equals(lowercasePattern)) {
+      if (lowerCaseCodePoints(target.substring(endPos - len, endPos)).equals(lowercasePattern)) {
         return len;
       }
     }
@@ -191,10 +190,9 @@ public class CollationAwareUTF8String {
   }
 
   /**
-   * Lowercase UTF8String comparison used for UTF8_LCASE collation. While the default
-   * UTF8String comparison is equivalent to a.toLowerCase().binaryCompare(b.toLowerCase()), this
-   * method uses code points to compare the strings in a case-insensitive manner using ICU rules,
-   * as well as handling special rules for one-to-many case mappings (see: lowerCaseCodePoints).
+   * Lowercase UTF8String comparison used for UTF8_LCASE collation. This method uses lowercased
+   * code points to compare the strings in a case-insensitive manner using ICU rules, taking into
+   * account special rules for one-to-many case mappings (see: lowerCaseCodePoints).
    *
    * @param left The first UTF8String to compare.
    * @param right The second UTF8String to compare.
@@ -325,8 +323,7 @@ public class CollationAwareUTF8String {
       return src;
     }
 
-    // TODO(SPARK-48725): Use lowerCaseCodePoints instead of UTF8String.toLowerCase.
-    UTF8String lowercaseSearch = search.toLowerCase();
+    UTF8String lowercaseSearch = lowerCaseCodePoints(search);
 
     int start = 0;
     int end = lowercaseFind(src, lowercaseSearch, start);
@@ -573,7 +570,7 @@ public class CollationAwareUTF8String {
   public static int lowercaseIndexOf(final UTF8String target, final UTF8String pattern,
       final int start) {
     if (pattern.numChars() == 0) return target.indexOfEmpty(start);
-    return lowercaseFind(target, pattern.toLowerCase(), start);
+    return lowercaseFind(target, lowerCaseCodePoints(pattern), start);
   }
 
   public static int indexOf(final UTF8String target, final UTF8String pattern,
@@ -670,7 +667,7 @@ public class CollationAwareUTF8String {
       return UTF8String.EMPTY_UTF8;
     }
 
-    UTF8String lowercaseDelimiter = delimiter.toLowerCase();
+    UTF8String lowercaseDelimiter = lowerCaseCodePoints(delimiter);
 
     if (count > 0) {
       // Search left to right (note: the start code point is inclusive).
diff --git a/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationFactory.java b/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationFactory.java
index f13f66e384e0..5640a2468d02 100644
--- a/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationFactory.java
+++ b/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationFactory.java
@@ -133,8 +133,8 @@ public final class CollationFactory {
     public final boolean supportsBinaryOrdering;
 
     /**
-     * Support for Lowercase Equality implies that it is possible to check equality on
-     * byte by byte level, but only after calling "UTF8String.toLowerCase" on both arguments.
+     * Support for Lowercase Equality implies that it is possible to check equality on byte by
+     * byte level, but only after calling "UTF8String.lowerCaseCodePoints" on both arguments.
      * This allows custom collation support for UTF8_LCASE collation in various Spark
      * expressions, as this particular collation is not supported by the external ICU library.
      */
@@ -676,9 +676,9 @@ public final class CollationFactory {
           collationName(),
           PROVIDER_ICU,
           collator,
-          (s1, s2) -> collator.compare(s1.toString(), s2.toString()),
+          (s1, s2) -> collator.compare(s1.toValidString(), s2.toValidString()),
           ICU_COLLATOR_VERSION,
-          s -> (long) collator.getCollationKey(s.toString()).hashCode(),
+          s -> (long) collator.getCollationKey(s.toValidString()).hashCode(),
           /* supportsBinaryEquality = */ false,
           /* supportsBinaryOrdering = */ false,
           /* supportsLowercaseEquality = */ false);
@@ -749,12 +749,15 @@ public final class CollationFactory {
    * Returns a StringSearch object for the given pattern and target strings, under collation
    * rules corresponding to the given collationId. The external ICU library StringSearch object can
    * be used to find occurrences of the pattern in the target string, while respecting collation.
+   * When given invalid UTF8Strings, the method will first convert them to valid strings, and then
+   * instantiate the StringSearch object. However, original UTF8Strings will remain unchanged.
    */
   public static StringSearch getStringSearch(
       final UTF8String targetUTF8String,
       final UTF8String patternUTF8String,
       final int collationId) {
-    return getStringSearch(targetUTF8String.toString(), patternUTF8String.toString(), collationId);
+    return getStringSearch(targetUTF8String.toValidString(), patternUTF8String.toValidString(),
+      collationId);
   }
 
   /**
@@ -763,9 +766,9 @@ public final class CollationFactory {
    * be used to find occurrences of the pattern in the target string, while respecting collation.
    */
   public static StringSearch getStringSearch(
-          final String targetString,
-          final String patternString,
-          final int collationId) {
+      final String targetString,
+      final String patternString,
+      final int collationId) {
     CharacterIterator target = new StringCharacterIterator(targetString);
     Collator collator = CollationFactory.fetchCollation(collationId).collator;
     return new StringSearch(patternString, target, (RuleBasedCollator) collator);
@@ -775,11 +778,13 @@ public final class CollationFactory {
    * Returns a collation-unaware StringSearch object for the given pattern and target strings.
    * While this object does not respect collation, it can be used to find occurrences of the pattern
    * in the target string for UTF8_BINARY or UTF8_LCASE (if arguments are lowercased).
+   * When given invalid UTF8Strings, the method will first convert them to valid strings, and then
+   * instantiate the StringSearch object. However, original UTF8Strings will remain unchanged.
    */
   public static StringSearch getStringSearch(
-          final UTF8String targetUTF8String,
-          final UTF8String patternUTF8String) {
-    return new StringSearch(patternUTF8String.toString(), targetUTF8String.toString());
+      final UTF8String targetUTF8String,
+      final UTF8String patternUTF8String) {
+    return new StringSearch(patternUTF8String.toValidString(), targetUTF8String.toValidString());
   }
 
   /**
@@ -818,9 +823,9 @@ public final class CollationFactory {
     if (collation.supportsBinaryEquality) {
       return input;
     } else if (collation.supportsLowercaseEquality) {
-      return input.toLowerCase();
+      return CollationAwareUTF8String.lowerCaseCodePoints(input);
     } else {
-      CollationKey collationKey = collation.collator.getCollationKey(input.toString());
+      CollationKey collationKey = collation.collator.getCollationKey(input.toValidString());
       return UTF8String.fromBytes(collationKey.toByteArray());
     }
   }
@@ -830,9 +835,9 @@ public final class CollationFactory {
     if (collation.supportsBinaryEquality) {
       return input.getBytes();
     } else if (collation.supportsLowercaseEquality) {
-      return input.toLowerCase().getBytes();
+      return CollationAwareUTF8String.lowerCaseCodePoints(input).getBytes();
     } else {
-      return collation.collator.getCollationKey(input.toString()).toByteArray();
+      return collation.collator.getCollationKey(input.toValidString()).toByteArray();
     }
   }
 
diff --git a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CollationSupportSuite.java b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CollationSupportSuite.java
index b082ab21944f..4301bf56b6d5 100644
--- a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CollationSupportSuite.java
+++ b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CollationSupportSuite.java
@@ -302,6 +302,43 @@ public class CollationSupportSuite {
     assertContains("adİos", "Io", "UTF8_LCASE", false);
     assertContains("adİos", "i̇o", "UTF8_LCASE", true);
     assertContains("adİos", "İo", "UTF8_LCASE", true);
+    // Greek sigmas.
+    assertContains("σ", "σ", "UTF8_BINARY", true);
+    assertContains("σ", "ς", "UTF8_BINARY", false);
+    assertContains("σ", "Σ", "UTF8_BINARY", false);
+    assertContains("ς", "σ", "UTF8_BINARY", false);
+    assertContains("ς", "ς", "UTF8_BINARY", true);
+    assertContains("ς", "Σ", "UTF8_BINARY", false);
+    assertContains("Σ", "σ", "UTF8_BINARY", false);
+    assertContains("Σ", "ς", "UTF8_BINARY", false);
+    assertContains("Σ", "Σ", "UTF8_BINARY", true);
+    assertContains("σ", "σ", "UTF8_LCASE", true);
+    assertContains("σ", "ς", "UTF8_LCASE", true);
+    assertContains("σ", "Σ", "UTF8_LCASE", true);
+    assertContains("ς", "σ", "UTF8_LCASE", true);
+    assertContains("ς", "ς", "UTF8_LCASE", true);
+    assertContains("ς", "Σ", "UTF8_LCASE", true);
+    assertContains("Σ", "σ", "UTF8_LCASE", true);
+    assertContains("Σ", "ς", "UTF8_LCASE", true);
+    assertContains("Σ", "Σ", "UTF8_LCASE", true);
+    assertContains("σ", "σ", "UNICODE", true);
+    assertContains("σ", "ς", "UNICODE", false);
+    assertContains("σ", "Σ", "UNICODE", false);
+    assertContains("ς", "σ", "UNICODE", false);
+    assertContains("ς", "ς", "UNICODE", true);
+    assertContains("ς", "Σ", "UNICODE", false);
+    assertContains("Σ", "σ", "UNICODE", false);
+    assertContains("Σ", "ς", "UNICODE", false);
+    assertContains("Σ", "Σ", "UNICODE", true);
+    assertContains("σ", "σ", "UNICODE_CI", true);
+    assertContains("σ", "ς", "UNICODE_CI", true);
+    assertContains("σ", "Σ", "UNICODE_CI", true);
+    assertContains("ς", "σ", "UNICODE_CI", true);
+    assertContains("ς", "ς", "UNICODE_CI", true);
+    assertContains("ς", "Σ", "UNICODE_CI", true);
+    assertContains("Σ", "σ", "UNICODE_CI", true);
+    assertContains("Σ", "ς", "UNICODE_CI", true);
+    assertContains("Σ", "Σ", "UNICODE_CI", true);
   }
 
   private void assertStartsWith(
@@ -880,6 +917,43 @@ public class CollationSupportSuite {
     assertStringInstr("ai̇oxXİo", "Xx", "UTF8_LCASE", 5);
     assertStringInstr("abİoi̇o", "\u0307o", "UTF8_LCASE", 6);
     assertStringInstr("aİoi̇oxx", "XX", "UTF8_LCASE", 7);
+    // Greek sigmas.
+    assertStringInstr("σ", "σ", "UTF8_BINARY", 1);
+    assertStringInstr("σ", "ς", "UTF8_BINARY", 0);
+    assertStringInstr("σ", "Σ", "UTF8_BINARY", 0);
+    assertStringInstr("ς", "σ", "UTF8_BINARY", 0);
+    assertStringInstr("ς", "ς", "UTF8_BINARY", 1);
+    assertStringInstr("ς", "Σ", "UTF8_BINARY", 0);
+    assertStringInstr("Σ", "σ", "UTF8_BINARY", 0);
+    assertStringInstr("Σ", "ς", "UTF8_BINARY", 0);
+    assertStringInstr("Σ", "Σ", "UTF8_BINARY", 1);
+    assertStringInstr("σ", "σ", "UTF8_LCASE", 1);
+    assertStringInstr("σ", "ς", "UTF8_LCASE", 1);
+    assertStringInstr("σ", "Σ", "UTF8_LCASE", 1);
+    assertStringInstr("ς", "σ", "UTF8_LCASE", 1);
+    assertStringInstr("ς", "ς", "UTF8_LCASE", 1);
+    assertStringInstr("ς", "Σ", "UTF8_LCASE", 1);
+    assertStringInstr("Σ", "σ", "UTF8_LCASE", 1);
+    assertStringInstr("Σ", "ς", "UTF8_LCASE", 1);
+    assertStringInstr("Σ", "Σ", "UTF8_LCASE", 1);
+    assertStringInstr("σ", "σ", "UNICODE", 1);
+    assertStringInstr("σ", "ς", "UNICODE", 0);
+    assertStringInstr("σ", "Σ", "UNICODE", 0);
+    assertStringInstr("ς", "σ", "UNICODE", 0);
+    assertStringInstr("ς", "ς", "UNICODE", 1);
+    assertStringInstr("ς", "Σ", "UNICODE", 0);
+    assertStringInstr("Σ", "σ", "UNICODE", 0);
+    assertStringInstr("Σ", "ς", "UNICODE", 0);
+    assertStringInstr("Σ", "Σ", "UNICODE", 1);
+    assertStringInstr("σ", "σ", "UNICODE_CI", 1);
+    assertStringInstr("σ", "ς", "UNICODE_CI", 1);
+    assertStringInstr("σ", "Σ", "UNICODE_CI", 1);
+    assertStringInstr("ς", "σ", "UNICODE_CI", 1);
+    assertStringInstr("ς", "ς", "UNICODE_CI", 1);
+    assertStringInstr("ς", "Σ", "UNICODE_CI", 1);
+    assertStringInstr("Σ", "σ", "UNICODE_CI", 1);
+    assertStringInstr("Σ", "ς", "UNICODE_CI", 1);
+    assertStringInstr("Σ", "Σ", "UNICODE_CI", 1);
   }
 
   private void assertFindInSet(String word, UTF8String set, String collationName,
@@ -981,6 +1055,43 @@ public class CollationSupportSuite {
     assertFindInSet("c", UTF8String.fromBytes(
       new byte[] { 0x41, (byte) 0xC2, 0x2C, 0x42, 0x2C, 0x43, 0x2C, 0x43, 0x2C, 0x56 }),
       "UNICODE_CI", 2);
+    // Greek sigmas.
+    assertFindInSet("σ", UTF8String.fromString("σ"), "UTF8_BINARY", 1);
+    assertFindInSet("σ", UTF8String.fromString("ς"), "UTF8_BINARY", 0);
+    assertFindInSet("σ", UTF8String.fromString("Σ"), "UTF8_BINARY", 0);
+    assertFindInSet("ς", UTF8String.fromString("σ"), "UTF8_BINARY", 0);
+    assertFindInSet("ς", UTF8String.fromString("ς"), "UTF8_BINARY", 1);
+    assertFindInSet("ς", UTF8String.fromString("Σ"), "UTF8_BINARY", 0);
+    assertFindInSet("Σ", UTF8String.fromString("σ"), "UTF8_BINARY", 0);
+    assertFindInSet("Σ", UTF8String.fromString("ς"), "UTF8_BINARY", 0);
+    assertFindInSet("Σ", UTF8String.fromString("Σ"), "UTF8_BINARY", 1);
+    assertFindInSet("σ", UTF8String.fromString("σ"), "UTF8_LCASE", 1);
+    assertFindInSet("σ", UTF8String.fromString("ς"), "UTF8_LCASE", 1);
+    assertFindInSet("σ", UTF8String.fromString("Σ"), "UTF8_LCASE", 1);
+    assertFindInSet("ς", UTF8String.fromString("σ"), "UTF8_LCASE", 1);
+    assertFindInSet("ς", UTF8String.fromString("ς"), "UTF8_LCASE", 1);
+    assertFindInSet("ς", UTF8String.fromString("Σ"), "UTF8_LCASE", 1);
+    assertFindInSet("Σ", UTF8String.fromString("σ"), "UTF8_LCASE", 1);
+    assertFindInSet("Σ", UTF8String.fromString("ς"), "UTF8_LCASE", 1);
+    assertFindInSet("Σ", UTF8String.fromString("Σ"), "UTF8_LCASE", 1);
+    assertFindInSet("σ", UTF8String.fromString("σ"), "UNICODE", 1);
+    assertFindInSet("σ", UTF8String.fromString("ς"), "UNICODE", 0);
+    assertFindInSet("σ", UTF8String.fromString("Σ"), "UNICODE", 0);
+    assertFindInSet("ς", UTF8String.fromString("σ"), "UNICODE", 0);
+    assertFindInSet("ς", UTF8String.fromString("ς"), "UNICODE", 1);
+    assertFindInSet("ς", UTF8String.fromString("Σ"), "UNICODE", 0);
+    assertFindInSet("Σ", UTF8String.fromString("σ"), "UNICODE", 0);
+    assertFindInSet("Σ", UTF8String.fromString("ς"), "UNICODE", 0);
+    assertFindInSet("Σ", UTF8String.fromString("Σ"), "UNICODE", 1);
+    assertFindInSet("σ", UTF8String.fromString("σ"), "UNICODE_CI", 1);
+    assertFindInSet("σ", UTF8String.fromString("ς"), "UNICODE_CI", 1);
+    assertFindInSet("σ", UTF8String.fromString("Σ"), "UNICODE_CI", 1);
+    assertFindInSet("ς", UTF8String.fromString("σ"), "UNICODE_CI", 1);
+    assertFindInSet("ς", UTF8String.fromString("ς"), "UNICODE_CI", 1);
+    assertFindInSet("ς", UTF8String.fromString("Σ"), "UNICODE_CI", 1);
+    assertFindInSet("Σ", UTF8String.fromString("σ"), "UNICODE_CI", 1);
+    assertFindInSet("Σ", UTF8String.fromString("ς"), "UNICODE_CI", 1);
+    assertFindInSet("Σ", UTF8String.fromString("Σ"), "UNICODE_CI", 1);
   }
 
   private void assertReplace(String source, String search, String replace, String collationName,
@@ -1034,6 +1145,44 @@ public class CollationSupportSuite {
       "İx\u0307İx\u0307İx\u0307"); // != UNICODE_CI
     assertReplace("abİo12i̇o", "i̇o", "xx", "UTF8_LCASE", "abxx12xx");
     assertReplace("abi̇o12i̇o", "İo", "yy", "UTF8_LCASE", "abyy12yy");
+    // Greek sigmas.
+    assertReplace("σ", "σ", "x", "UTF8_BINARY", "x");
+    assertReplace("σ", "ς", "x", "UTF8_BINARY", "σ");
+    assertReplace("σ", "Σ", "x", "UTF8_BINARY", "σ");
+    assertReplace("ς", "σ", "x", "UTF8_BINARY", "ς");
+    assertReplace("ς", "ς", "x", "UTF8_BINARY", "x");
+    assertReplace("ς", "Σ", "x", "UTF8_BINARY", "ς");
+    assertReplace("Σ", "σ", "x", "UTF8_BINARY", "Σ");
+    assertReplace("Σ", "ς", "x", "UTF8_BINARY", "Σ");
+    assertReplace("Σ", "Σ", "x", "UTF8_BINARY", "x");
+    assertReplace("σ", "σ", "x", "UTF8_LCASE", "x");
+    assertReplace("σ", "ς", "x", "UTF8_LCASE", "x");
+    assertReplace("σ", "Σ", "x", "UTF8_LCASE", "x");
+    assertReplace("ς", "σ", "x", "UTF8_LCASE", "x");
+    assertReplace("ς", "ς", "x", "UTF8_LCASE", "x");
+    assertReplace("ς", "Σ", "x", "UTF8_LCASE", "x");
+    assertReplace("Σ", "σ", "x", "UTF8_LCASE", "x");
+    assertReplace("Σ", "ς", "x", "UTF8_LCASE", "x");
+    assertReplace("Σ", "Σ", "x", "UTF8_LCASE", "x");
+    assertReplace("σ", "σ", "x", "UNICODE", "x");
+    assertReplace("σ", "ς", "x", "UNICODE", "σ");
+    assertReplace("σ", "Σ", "x", "UNICODE", "σ");
+    assertReplace("ς", "σ", "x", "UNICODE", "ς");
+    assertReplace("ς", "ς", "x", "UNICODE", "x");
+    assertReplace("ς", "Σ", "x", "UNICODE", "ς");
+    assertReplace("Σ", "σ", "x", "UNICODE", "Σ");
+    assertReplace("Σ", "ς", "x", "UNICODE", "Σ");
+    assertReplace("Σ", "Σ", "x", "UNICODE", "x");
+    assertReplace("σ", "σ", "x", "UNICODE_CI", "x");
+    assertReplace("σ", "ς", "x", "UNICODE_CI", "x");
+    assertReplace("σ", "Σ", "x", "UNICODE_CI", "x");
+    assertReplace("ς", "σ", "x", "UNICODE_CI", "x");
+    assertReplace("ς", "ς", "x", "UNICODE_CI", "x");
+    assertReplace("ς", "Σ", "x", "UNICODE_CI", "x");
+    assertReplace("Σ", "σ", "x", "UNICODE_CI", "x");
+    assertReplace("Σ", "ς", "x", "UNICODE_CI", "x");
+    assertReplace("Σ", "Σ", "x", "UNICODE_CI", "x");
+
   }
 
   private void assertLocate(String substring, String string, Integer start, String collationName,
@@ -1111,6 +1260,43 @@ public class CollationSupportSuite {
     assertLocate("İo", "i̇o世界大千世界", 1, "UNICODE_CI", 1);
     assertLocate("İo", "大千i̇o世界大千世界", 1, "UNICODE_CI", 3);
     assertLocate("İo", "世界i̇o大千世界大千i̇o", 4, "UNICODE_CI", 12);
+    // Greek sigmas.
+    assertLocate("σ", "σ", 1, "UTF8_BINARY", 1);
+    assertLocate("σ", "ς", 1, "UTF8_BINARY", 0);
+    assertLocate("σ", "Σ", 1, "UTF8_BINARY", 0);
+    assertLocate("ς", "σ", 1, "UTF8_BINARY", 0);
+    assertLocate("ς", "ς", 1, "UTF8_BINARY", 1);
+    assertLocate("ς", "Σ", 1, "UTF8_BINARY", 0);
+    assertLocate("Σ", "σ", 1, "UTF8_BINARY", 0);
+    assertLocate("Σ", "ς", 1, "UTF8_BINARY", 0);
+    assertLocate("Σ", "Σ", 1, "UTF8_BINARY", 1);
+    assertLocate("σ", "σ", 1, "UTF8_LCASE", 1);
+    assertLocate("σ", "ς", 1, "UTF8_LCASE", 1);
+    assertLocate("σ", "Σ", 1, "UTF8_LCASE", 1);
+    assertLocate("ς", "σ", 1, "UTF8_LCASE", 1);
+    assertLocate("ς", "ς", 1, "UTF8_LCASE", 1);
+    assertLocate("ς", "Σ", 1, "UTF8_LCASE", 1);
+    assertLocate("Σ", "σ", 1, "UTF8_LCASE", 1);
+    assertLocate("Σ", "ς", 1, "UTF8_LCASE", 1);
+    assertLocate("Σ", "Σ", 1, "UTF8_LCASE", 1);
+    assertLocate("σ", "σ", 1, "UNICODE", 1);
+    assertLocate("σ", "ς", 1, "UNICODE", 0);
+    assertLocate("σ", "Σ", 1, "UNICODE", 0);
+    assertLocate("ς", "σ", 1, "UNICODE", 0);
+    assertLocate("ς", "ς", 1, "UNICODE", 1);
+    assertLocate("ς", "Σ", 1, "UNICODE", 0);
+    assertLocate("Σ", "σ", 1, "UNICODE", 0);
+    assertLocate("Σ", "ς", 1, "UNICODE", 0);
+    assertLocate("Σ", "Σ", 1, "UNICODE", 1);
+    assertLocate("σ", "σ", 1, "UNICODE_CI", 1);
+    assertLocate("σ", "ς", 1, "UNICODE_CI", 1);
+    assertLocate("σ", "Σ", 1, "UNICODE_CI", 1);
+    assertLocate("ς", "σ", 1, "UNICODE_CI", 1);
+    assertLocate("ς", "ς", 1, "UNICODE_CI", 1);
+    assertLocate("ς", "Σ", 1, "UNICODE_CI", 1);
+    assertLocate("Σ", "σ", 1, "UNICODE_CI", 1);
+    assertLocate("Σ", "ς", 1, "UNICODE_CI", 1);
+    assertLocate("Σ", "Σ", 1, "UNICODE_CI", 1);
   }
 
   private void assertSubstringIndex(String string, String delimiter, Integer count,
@@ -1220,6 +1406,44 @@ public class CollationSupportSuite {
     assertSubstringIndex("ai̇bİoi̇o12i̇oİo", "İo", 3, "UTF8_LCASE", "ai̇bİoi̇o12");
     assertSubstringIndex("ai̇bİoi̇o12i̇oİo", "i̇o", 3, "UTF8_LCASE", "ai̇bİoi̇o12");
     assertSubstringIndex("bİoi̇o12i̇o", "\u0307oi", 1, "UTF8_LCASE", "bİoi̇o12i̇o");
+    // Greek sigmas.
+    assertSubstringIndex("σ", "σ", 1, "UTF8_BINARY", "");
+    assertSubstringIndex("σ", "ς", 1, "UTF8_BINARY", "σ");
+    assertSubstringIndex("σ", "Σ", 1, "UTF8_BINARY", "σ");
+    assertSubstringIndex("ς", "σ", 1, "UTF8_BINARY", "ς");
+    assertSubstringIndex("ς", "ς", 1, "UTF8_BINARY", "");
+    assertSubstringIndex("ς", "Σ", 1, "UTF8_BINARY", "ς");
+    assertSubstringIndex("Σ", "σ", 1, "UTF8_BINARY", "Σ");
+    assertSubstringIndex("Σ", "ς", 1, "UTF8_BINARY", "Σ");
+    assertSubstringIndex("Σ", "Σ", 1, "UTF8_BINARY", "");
+    assertSubstringIndex("σ", "σ", 1, "UTF8_LCASE", "");
+    assertSubstringIndex("σ", "ς", 1, "UTF8_LCASE", "");
+    assertSubstringIndex("σ", "Σ", 1, "UTF8_LCASE", "");
+    assertSubstringIndex("ς", "σ", 1, "UTF8_LCASE", "");
+    assertSubstringIndex("ς", "ς", 1, "UTF8_LCASE", "");
+    assertSubstringIndex("ς", "Σ", 1, "UTF8_LCASE", "");
+    assertSubstringIndex("Σ", "σ", 1, "UTF8_LCASE", "");
+    assertSubstringIndex("Σ", "ς", 1, "UTF8_LCASE", "");
+    assertSubstringIndex("Σ", "Σ", 1, "UTF8_LCASE", "");
+    assertSubstringIndex("σ", "σ", 1, "UNICODE", "");
+    assertSubstringIndex("σ", "ς", 1, "UNICODE", "σ");
+    assertSubstringIndex("σ", "Σ", 1, "UNICODE", "σ");
+    assertSubstringIndex("ς", "σ", 1, "UNICODE", "ς");
+    assertSubstringIndex("ς", "ς", 1, "UNICODE", "");
+    assertSubstringIndex("ς", "Σ", 1, "UNICODE", "ς");
+    assertSubstringIndex("Σ", "σ", 1, "UNICODE", "Σ");
+    assertSubstringIndex("Σ", "ς", 1, "UNICODE", "Σ");
+    assertSubstringIndex("Σ", "Σ", 1, "UNICODE", "");
+    assertSubstringIndex("σ", "σ", 1, "UNICODE_CI", "");
+    assertSubstringIndex("σ", "ς", 1, "UNICODE_CI", "");
+    assertSubstringIndex("σ", "Σ", 1, "UNICODE_CI", "");
+    assertSubstringIndex("ς", "σ", 1, "UNICODE_CI", "");
+    assertSubstringIndex("ς", "ς", 1, "UNICODE_CI", "");
+    assertSubstringIndex("ς", "Σ", 1, "UNICODE_CI", "");
+    assertSubstringIndex("Σ", "σ", 1, "UNICODE_CI", "");
+    assertSubstringIndex("Σ", "ς", 1, "UNICODE_CI", "");
+    assertSubstringIndex("Σ", "Σ", 1, "UNICODE_CI", "");
+
   }
 
   private void assertStringTrim(


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org