(opennlp) branch main updated: Spellcheck correction loses the capital of a supplementary-plane initial (#1385)

[rz...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

rzo1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git


The following commit(s) were added to refs/heads/main by this push:
     new 1fd51808e Spellcheck correction loses the capital of a supplementary-plane initial (#1385)
1fd51808e is described below

commit 1fd51808e531c838f37628d5f99a7aa0b2b89e68
Author: Kristian Rickert <kr...@apache.org>
AuthorDate: Tue Oct 6 14:12:03 2026 -0500

    Spellcheck correction loses the capital of a supplementary-plane initial (#1385)
    
    SpellCheckingCharSequenceNormalizer checked and recapitalized the
    first char of a token, not the first code point. A token starting with
    a supplementary-plane capital such as U+10400 lost its capital in the
    correction, and a token made only of supplementary letters was treated
    as having no letters. Both now work on code points.
    
    Assisted-by: Claude Code 2.1.288
    
    
    Reviewed-by: Kristian Rickert <kr...@apache.org>
    
    Signed-off-by: Kristian Rickert <kr...@apache.org>
    Co-authored-by: Claude <no...@anthropic.com>
---
 opennlp-docs/src/docbkx/spellcheck.xml             |  14 +++
 .../SpellCheckingCharSequenceNormalizer.java       |  59 +++++++----
 .../SpellCheckingCharSequenceNormalizerTest.java   |  10 ++
 .../normalizer/SpellCheckingUnicodeCaseTest.java   | 116 +++++++++++++++++++++
 4 files changed, 179 insertions(+), 20 deletions(-)

diff --git a/opennlp-docs/src/docbkx/spellcheck.xml b/opennlp-docs/src/docbkx/spellcheck.xml
index 7a4f047d5..97417360b 100644
--- a/opennlp-docs/src/docbkx/spellcheck.xml
+++ b/opennlp-docs/src/docbkx/spellcheck.xml
@@ -422,6 +422,20 @@ CharSequenceNormalizer normalizer = SpellCheckingCharSequenceNormalizer.builder(
 CharSequence clean = normalizer.normalize("the qick brwn fox"); // -> "the quick brown fox"]]>
 			</programlisting>
 		</para>
+		<para>
+			In per-token mode, letter and capitalization checks include supplementary Unicode code points.
+			A lowercase letter prevents all-uppercase correction. An uppercase token of one code point
+			uses initial-capital correction. Known dictionary words keep their original casing.
+			This example uses U+10428, a lowercase Deseret letter:
+			<programlisting language="java">
+<![CDATA[SymSpell engine = new SymSpell();
+engine.add("\uD801\uDC28word", 1000);
+CharSequenceNormalizer normalizer = new SpellCheckingCharSequenceNormalizer(engine);
+
+String corrected = normalizer.normalize("\uD801\uDC28WROLD").toString();
+// corrected is "\uD801\uDC28word"]]>
+			</programlisting>
+		</para>
 		<para>
 			For stream-based preprocessing two <code>FilterObjectStream&lt;String, String&gt;</code>
 			implementations wrap an existing <code>ObjectStream&lt;String&gt;</code> and correct each element as
diff --git a/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/normalizer/SpellCheckingCharSequenceNormalizer.java b/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/normalizer/SpellCheckingCharSequenceNormalizer.java
index a65007708..31056971f 100644
--- a/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/normalizer/SpellCheckingCharSequenceNormalizer.java
+++ b/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/normalizer/SpellCheckingCharSequenceNormalizer.java
@@ -310,7 +310,12 @@ public class SpellCheckingCharSequenceNormalizer implements CharSequenceNormaliz
     return prefix + corrected + suffix;
   }
 
-  /** @return {@code true} if {@code core} passes the configured length/number/URL guards. */
+  /**
+   * Checks the configured token guards and requires at least one Unicode letter.
+   *
+   * @param core The token to check.
+   * @return Whether the token can be corrected.
+   */
   private boolean isCorrectable(String core) {
     if (core.length() < minTokenLength) {
       return false;
@@ -322,47 +327,61 @@ public class SpellCheckingCharSequenceNormalizer implements CharSequenceNormaliz
       return false;
     }
     // A token with no letters at all (pure symbols) cannot be a spelling error.
-    boolean hasLetter = false;
-    for (int k = 0; k < core.length(); k++) {
-      if (Character.isLetter(core.charAt(k))) {
-        hasLetter = true;
-        break;
+    for (int k = 0; k < core.length();) {
+      final int codePoint = core.codePointAt(k);
+      if (Character.isLetter(codePoint)) {
+        return true;
       }
+      k += Character.charCount(codePoint);
     }
-    return hasLetter;
+    return false;
   }
 
   /**
-   * Re-applies the casing pattern of {@code original} to {@code corrected}: all-upper
-   * stays all-upper, leading-capital stays leading-capital, otherwise the suggestion's
-   * own casing (typically lower-case) is used.
+   * Applies uppercase or initial-capital casing to a correction. Letter checks and
+   * initial-capital conversion use Unicode code points. Other tokens use the
+   * suggestion's casing.
+   *
+   * @param original The source token.
+   * @param corrected The spelling suggestion.
+   * @return The correction with the source token's capitalization pattern.
    */
-  private static String applyCasing(String original, String corrected) {
+  private String applyCasing(String original, String corrected) {
     if (corrected.isEmpty()) {
       return corrected;
     }
     if (isAllUpper(original)) {
       return corrected.toUpperCase(Locale.ROOT);
     }
-    if (Character.isUpperCase(original.charAt(0))) {
-      return Character.toUpperCase(corrected.charAt(0)) + corrected.substring(1);
+    if (Character.isUpperCase(original.codePointAt(0))) {
+      final int initial = corrected.codePointAt(0);
+      return new StringBuilder(corrected.length())
+          .appendCodePoint(Character.toUpperCase(initial))
+          .append(corrected, Character.charCount(initial), corrected.length())
+          .toString();
     }
     return corrected;
   }
 
-  /** {@return whether {@code s} has more than one character and every letter in it is upper-case} */
-  private static boolean isAllUpper(String s) {
+  /**
+   * Checks for at least 2 code points, with all letters uppercase.
+   *
+   * @param s The token to check. Non-letter code points do not affect the case check.
+   * @return Whether the token contains a letter and has an all-uppercase pattern.
+   */
+  private boolean isAllUpper(String s) {
     boolean sawLetter = false;
-    for (int k = 0; k < s.length(); k++) {
-      final char c = s.charAt(k);
-      if (Character.isLetter(c)) {
+    for (int k = 0; k < s.length();) {
+      final int codePoint = s.codePointAt(k);
+      if (Character.isLetter(codePoint)) {
         sawLetter = true;
-        if (!Character.isUpperCase(c)) {
+        if (!Character.isUpperCase(codePoint)) {
           return false;
         }
       }
+      k += Character.charCount(codePoint);
     }
-    return sawLetter && s.length() > 1;
+    return sawLetter && s.codePointCount(0, s.length()) > 1;
   }
 
   /**
diff --git a/opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/normalizer/SpellCheckingCharSequenceNormalizerTest.java b/opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/normalizer/SpellCheckingCharSequenceNormalizerTest.java
index 3e41f00ff..eabfc4a89 100644
--- a/opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/normalizer/SpellCheckingCharSequenceNormalizerTest.java
+++ b/opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/normalizer/SpellCheckingCharSequenceNormalizerTest.java
@@ -106,6 +106,16 @@ public class SpellCheckingCharSequenceNormalizerTest {
     assertEquals("world", norm(normalizer, "wrold"));
   }
 
+  /** Checks a capitalized Deseret initial using a dictionary lookup. */
+  @Test
+  void perTokenPreservesCasingOfSupplementaryPlaneInitial() {
+    final SymSpell engine = new SymSpell();
+    engine.add("𐐨word", 1000);
+    final var normalizer = new SpellCheckingCharSequenceNormalizer(engine);
+    assertEquals("𐐨word", norm(normalizer, "𐐨wrod"));
+    assertEquals("𐐀word", norm(normalizer, "𐐀wrod"));
+  }
+
   @Test
   void minTokenLengthSkipsShortTokens() {
     final var lenient = SpellCheckingCharSequenceNormalizer.builder(symSpell)
diff --git a/opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/normalizer/SpellCheckingUnicodeCaseTest.java b/opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/normalizer/SpellCheckingUnicodeCaseTest.java
new file mode 100644
index 000000000..128eba6b2
--- /dev/null
+++ b/opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/normalizer/SpellCheckingUnicodeCaseTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.spellcheck.normalizer;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+import opennlp.spellcheck.symspell.SymSpell;
+import opennlp.tools.util.StringUtil;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+/** Checks spelling corrections for Unicode letters, including supplementary code points. */
+class SpellCheckingUnicodeCaseTest {
+
+  private static final int FREQUENCY = 1000;
+  private static final String PUNCTUATION_PREFIX = "\"";
+  private static final String PUNCTUATION_SUFFIX = "!\"";
+  private static final String WHITESPACE_PREFIX = "\t";
+  private static final String WHITESPACE_SUFFIX = "\u00a0";
+
+  /**
+   * Corrects a known misspelling using the source token's capitalization.
+   *
+   * @param dictionary The dictionary entry.
+   * @param input The misspelled token.
+   * @param expected The corrected token.
+   */
+  @ParameterizedTest
+  @CsvSource({
+      "world, wrold, world",
+      "world, Wrold, World",
+      "world, WROLD, WORLD",
+      "𐐨𐐩𐐪𐐫, 𐐨𐐩𐐪𐐭, 𐐨𐐩𐐪𐐫",
+      "𐐨𐐩𐐪𐐫, 𐐀𐐩𐐪𐐭, 𐐀𐐩𐐪𐐫",
+      "𐐨𐐩𐐪𐐫, 𐐀𐐁𐐂𐐅, 𐐀𐐁𐐂𐐃",
+      "𐐨world, 𐐨WROLD, 𐐨world",
+      "𐐨world, 𐐀WROLD, 𐐀WORLD",
+      "a𐐩cda, A𐐩CDE, A𐐩cda",
+      "a𐐩cda, A𐐁CDE, A𐐁CDA",
+      "word𐐨, WRLD𐐨, Word𐐨",
+      "word𐐨, WRLD𐐀, WORD𐐀",
+      "𠮷𠮸𠮹𠮻, 𠮷𠮸𠮹𠮺, 𠮷𠮸𠮹𠮻",
+      "ab, A, Ab",
+      "ab, 𐐀, Ab",
+      "ab, 𐐨, ab"
+  })
+  void testCasing(String dictionary, String input, String expected) {
+    final SymSpell engine = new SymSpell();
+    engine.add(dictionary, FREQUENCY);
+    final var suggestions = engine.lookup(StringUtil.toLowerCase(input));
+    assertFalse(suggestions.isEmpty(),
+        "The dictionary must provide a correction for this input");
+    assertEquals(dictionary, suggestions.getFirst().term());
+    final var normalizer = SpellCheckingCharSequenceNormalizer.builder(engine)
+        .minTokenLength(1).build();
+    assertEquals(expected, normalizer.normalize(input).toString());
+    assertEquals(PUNCTUATION_PREFIX + expected + PUNCTUATION_SUFFIX,
+        normalizer.normalize(PUNCTUATION_PREFIX + input + PUNCTUATION_SUFFIX).toString());
+    assertEquals(WHITESPACE_PREFIX + expected + WHITESPACE_SUFFIX,
+        normalizer.normalize(WHITESPACE_PREFIX + input + WHITESPACE_SUFFIX).toString());
+    final StringBuilder mutable = new StringBuilder(input);
+    assertEquals(expected, normalizer.normalize(mutable).toString());
+    assertEquals(input, mutable.toString());
+  }
+
+  /**
+   * Preserves known words and tokens excluded by the normalizer's guards.
+   *
+   * @param dictionary The dictionary entry.
+   * @param input The token to preserve.
+   */
+  @ParameterizedTest
+  @CsvSource({
+      "𐐨𐐩𐐪𐐫, 𐐨𐐩𐐪𐐫",
+      "𐐨𐐩𐐪𐐫, 𐐀𐐁𐐂𐐃",
+      "𐐨𐐩𐐪𐐫, 𐐀𐐩𐐂𐐫",
+      "word𐐨, WORD𐐨",
+      "12346, 12345",
+      "😀😀😀😁, 😀😀😀😀",
+      "www.world.com, www.wrold.com"
+  })
+  void testUnchangedTokens(String dictionary, String input) {
+    final SymSpell engine = new SymSpell();
+    engine.add(dictionary, FREQUENCY);
+    final var normalizer = SpellCheckingCharSequenceNormalizer.builder(engine)
+        .minTokenLength(1).build();
+    assertEquals(input, normalizer.normalize(input).toString());
+  }
+
+  /** Runs the manual example with the default normalizer settings. */
+  @Test
+  void testManualExample() {
+    final SymSpell engine = new SymSpell();
+    engine.add("\uD801\uDC28word", FREQUENCY);
+    final var normalizer = new SpellCheckingCharSequenceNormalizer(engine);
+    assertEquals("\uD801\uDC28word", normalizer.normalize("\uD801\uDC28WROLD").toString());
+  }
+}