(opennlp) 02/06: Fast-path clean text in Confusables.skeleton with a key BitSet

[kr...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

krickert pushed a commit to branch perf-1850-followup
in repository https://gitbox.apache.org/repos/asf/opennlp.git

commit 4919d5705cb3b9a499dfead95a2fa4e22f546b0e
Author: Kristian Rickert <kr...@gmail.com>
AuthorDate: Mon Jul 6 21:57:38 2026 -0400

    Fast-path clean text in Confusables.skeleton with a key BitSet
    
    The skeleton always ran two java.text.Normalizer NFD passes and a
    boxed HashMap lookup per code point, even for text with no confusable
    mapping at all. The prototype map now carries a BitSet over its keys:
    when no code point of the input is a prototype key and the input is
    already in NFD form (trivially true for pure ASCII), the skeleton is
    the input itself and both NFD passes and all allocation are skipped.
    Text containing a key or in un-normalized form takes the unchanged
    full UTS #39 path, whose two NFD passes remain required; there the
    BitSet also pre-filters the per-code-point map lookup so only actual
    keys pay for Integer boxing.
    
    New tests pin the fast-path identity, the ASCII prototype keys that
    must defeat it (m, I, 1, 0), canonical equivalence of precomposed
    input, and a non-ASCII key mapping.
---
 .../opennlp/tools/util/normalizer/Confusables.java | 82 +++++++++++++++++-----
 .../tools/util/normalizer/ConfusablesTest.java     | 41 +++++++++++
 2 files changed, 106 insertions(+), 17 deletions(-)

diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/Confusables.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/Confusables.java
index 6fda61c54..faa47da29 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/Confusables.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/Confusables.java
@@ -23,6 +23,7 @@ import java.io.InputStreamReader;
 import java.io.UncheckedIOException;
 import java.nio.charset.StandardCharsets;
 import java.text.Normalizer;
+import java.util.BitSet;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -49,32 +50,41 @@ public final class Confusables {
 
   private static final String RESOURCE = "confusables.txt";
 
-  // Maps a single confusable code point to its prototype sequence (one or more code points).
-  private static volatile Map<Integer, String> prototypes;
+  // The prototype map plus a BitSet over its keys, so the hot no-mapping test is one O(1) bit read
+  // with no Integer boxing. Immutable after construction; published through the volatile field.
+  private record Data(Map<Integer, String> prototypes, BitSet keys) {
+  }
+
+  private static volatile Data data;
 
   private Confusables() {
   }
 
-  private static Map<Integer, String> prototypes() {
-    Map<Integer, String> map = prototypes;
-    if (map == null) {
+  private static Data data() {
+    Data d = data;
+    if (d == null) {
       synchronized (Confusables.class) {
-        map = prototypes;
-        if (map == null) {
-          map = load();
-          prototypes = map;
+        d = data;
+        if (d == null) {
+          d = load();
+          data = d;
         }
       }
     }
-    return map;
+    return d;
   }
 
-  private static Map<Integer, String> load() {
+  private static Data load() {
     try (InputStream in = Confusables.class.getResourceAsStream(RESOURCE)) {
       if (in == null) {
         throw new IllegalStateException("Missing confusables data resource: " + RESOURCE);
       }
-      return parse(in);
+      final Map<Integer, String> prototypes = parse(in);
+      final BitSet keys = new BitSet();
+      for (final int codePoint : prototypes.keySet()) {
+        keys.set(codePoint);
+      }
+      return new Data(prototypes, keys);
     } catch (IOException e) {
       throw new UncheckedIOException("Unable to read confusables data resource " + RESOURCE, e);
     }
@@ -142,13 +152,51 @@ public final class Confusables {
    * @return The skeleton.
    */
   public static String skeleton(CharSequence text) {
-    final Map<Integer, String> map = prototypes();
+    final Data data = data();
+    final BitSet keys = data.keys();
+
+    // Fast path: when no code point of the text is a prototype key and the text is already in
+    // NFD form (trivially true for pure ASCII, which never decomposes), the skeleton is the text
+    // itself: NFD(text) == text, the map pass changes nothing because no key occurs, and the
+    // second NFD of an unchanged NFD string is again the identity. This skips both Normalizer
+    // passes and all allocation for the common clean-token case. Text that fails either test
+    // (a key, or a non-ASCII code point in un-normalized form) takes the full UTS #39 path below,
+    // whose two NFD passes remain required.
+    final int length = text.length();
+    boolean asciiOnly = true;
+    boolean anyKey = false;
+    int i = 0;
+    while (i < length) {
+      final char c = text.charAt(i);
+      final int codePoint;
+      if (!Character.isHighSurrogate(c)) {
+        codePoint = c;
+        i++;
+      } else {
+        codePoint = Character.codePointAt(text, i);
+        i += Character.charCount(codePoint);
+      }
+      if (codePoint >= 0x80) {
+        asciiOnly = false;
+      }
+      if (keys.get(codePoint)) {
+        anyKey = true;
+        break;
+      }
+    }
+    if (!anyKey && (asciiOnly || Normalizer.isNormalized(text, Normalizer.Form.NFD))) {
+      return text.toString();
+    }
+
+    final Map<Integer, String> map = data.prototypes();
     final String decomposed = Normalizer.normalize(text, Normalizer.Form.NFD);
     final StringBuilder mapped = new StringBuilder(decomposed.length());
-    for (int i = 0; i < decomposed.length(); ) {
-      final int codePoint = decomposed.codePointAt(i);
-      i += Character.charCount(codePoint);
-      final String prototype = map.get(codePoint);
+    for (int j = 0; j < decomposed.length(); ) {
+      final int codePoint = decomposed.codePointAt(j);
+      j += Character.charCount(codePoint);
+      // The BitSet pre-filter keeps the common miss free of Integer boxing; only an actual key
+      // pays for the boxed map lookup.
+      final String prototype = keys.get(codePoint) ? map.get(codePoint) : null;
       if (prototype != null) {
         mapped.append(prototype);
       } else {
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ConfusablesTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ConfusablesTest.java
index 262fe5aa9..fe6bfe54f 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ConfusablesTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ConfusablesTest.java
@@ -20,6 +20,7 @@ import org.junit.jupiter.api.Test;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class ConfusablesTest {
@@ -72,6 +73,46 @@ public class ConfusablesTest {
     assertEquals(Confusables.skeleton("data"), Confusables.skeleton(spoof));
   }
 
+  @Test
+  void testSkeletonReturnsCleanAsciiUnchanged() {
+    // No code point of the text is a prototype key, so the fast path returns the input as-is.
+    final String clean = "hello";
+    assertSame(clean, Confusables.skeleton(clean));
+  }
+
+  @Test
+  void testSkeletonStillMapsAsciiPrototypeKeys() {
+    // ASCII code points that ARE prototype keys must defeat the fast path and map as before.
+    assertEquals("rn", Confusables.skeleton("m"));
+    assertEquals("l", Confusables.skeleton("I"));
+    assertEquals("l", Confusables.skeleton("1"));
+    assertEquals("O", Confusables.skeleton("0"));
+    assertEquals("corn", Confusables.skeleton("com"));
+  }
+
+  @Test
+  void testSkeletonStillDecomposesPrecomposedText() {
+    // A precomposed character is not in NFD form, so it must take the full path and decompose,
+    // and the result must agree with the already-decomposed spelling (canonical equivalence).
+    final String precomposed = cp(0x00E9);          // e with acute, single code point
+    final String decomposed = "e" + cp(0x0301);     // e + combining acute
+    assertEquals(decomposed, Confusables.skeleton(precomposed));
+    assertEquals(Confusables.skeleton(precomposed), Confusables.skeleton(decomposed));
+  }
+
+  @Test
+  void testSkeletonReturnsNfdStableNonKeyTextUnchanged() {
+    // Non-ASCII, already in NFD form, and no prototype key: the fast path applies.
+    final String hiragana = cp(0x3042);
+    assertSame(hiragana, Confusables.skeleton(hiragana));
+  }
+
+  @Test
+  void testSkeletonStillMapsNonAsciiPrototypeKeys() {
+    // U+4E00 is a prototype key (maps to the prolonged sound mark), so it must still map.
+    assertEquals(cp(0x30FC), Confusables.skeleton(cp(0x4E00)));
+  }
+
   @Test
   void testTermConfusableFoldDimension() {
     final String spoof = "p" + cp(0x0430) + "yp" + cp(0x0430) + "l";