You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by rz...@apache.org on 2026/10/08 08:17:45 UTC

(opennlp) branch gh-api-unicode-charclass-cleanup created (now 927fd2484)

This is an automated email from the ASF dual-hosted git repository.

rzo1 pushed a change to branch gh-api-unicode-charclass-cleanup
in repository https://gitbox.apache.org/repos/asf/opennlp.git


      at 927fd2484 GH-1438: Share code point validation and trim scans in CharClass and CodePointSet

This branch includes the following new commits:

     new 927fd2484 GH-1438: Share code point validation and trim scans in CharClass and CodePointSet

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(opennlp) 01/01: GH-1438: Share code point validation and trim scans in CharClass and CodePointSet

Posted by rz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

rzo1 pushed a commit to branch gh-api-unicode-charclass-cleanup
in repository https://gitbox.apache.org/repos/asf/opennlp.git

commit 927fd24848d8c40c158a44b2f310da45054bd122
Author: Richard Zowalla <rz...@apache.org>
AuthorDate: Thu Oct 8 10:17:37 2026 +0200

    GH-1438: Share code point validation and trim scans in CharClass and CodePointSet
    
    CharClass validates replacement code points through CodePointSet and trims
    with shared forward and backward scan helpers. CodePointSet.parse rejects a
    null section with IllegalArgumentException, and the CodePointSet.ofRange
    message for descending bounds no longer ends with a period. The
    UnicodeDash.Category and UnicodeWhitespace.Category Javadoc explains that
    the constant names are Unicode general category abbreviations.
---
 .../opennlp/tools/util/normalizer/CharClass.java   | 83 ++++++++--------------
 .../tools/util/normalizer/CodePointSet.java        | 14 ++--
 .../opennlp/tools/util/normalizer/UnicodeDash.java |  8 ++-
 .../tools/util/normalizer/UnicodeWhitespace.java   |  8 ++-
 .../tools/util/normalizer/CharClassTest.java       | 33 +++++++--
 .../tools/util/normalizer/CodePointSetTest.java    | 12 ++++
 6 files changed, 94 insertions(+), 64 deletions(-)

diff --git a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CharClass.java b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CharClass.java
index 8b2006fd8..b89f38d23 100644
--- a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CharClass.java
+++ b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CharClass.java
@@ -68,7 +68,7 @@ public final class CharClass {
    */
   public static CharClass of(CodePointSet members, int replacement) {
     ParamChecks.requireNonNullArg(members, "members");
-    requireValidCodePoint(replacement);
+    CodePointSet.requireValid(replacement);
     return new CharClass(members, replacement);
   }
 
@@ -247,7 +247,7 @@ public final class CharClass {
   public String collapsePreserving(CharSequence text, CodePointSet keep, int keepReplacement) {
     ParamChecks.requireNonNullArg(text, "text");
     ParamChecks.requireNonNullArg(keep, "keep");
-    requireValidCodePoint(keepReplacement);
+    CodePointSet.requireValid(keepReplacement);
     return collapseRuns(text, preservingReplacement(text, keep, keepReplacement), null);
   }
 
@@ -260,23 +260,8 @@ public final class CharClass {
    */
   public String trim(CharSequence text) {
     ParamChecks.requireNonNullArg(text, "text");
-    final int length = text.length();
-    int start = 0;
-    while (start < length) {
-      final At cp = CodePoints.at(text, start);
-      if (!members.contains(cp.codePoint())) {
-        break;
-      }
-      start = cp.nextIndex(start);
-    }
-    int end = length;
-    while (end > start) {
-      final At cp = CodePoints.before(text, end);
-      if (!members.contains(cp.codePoint())) {
-        break;
-      }
-      end = cp.previousIndex(end);
-    }
+    final int start = skipRun(text, 0);
+    final int end = skipRunBackward(text, start, text.length());
     return text.subSequence(start, end).toString();
   }
 
@@ -365,7 +350,7 @@ public final class CharClass {
                                                int keepReplacement) {
     ParamChecks.requireNonNullArg(text, "text");
     ParamChecks.requireNonNullArg(keep, "keep");
-    requireValidCodePoint(keepReplacement);
+    CodePointSet.requireValid(keepReplacement);
     return collapseRunsAligned(text, preservingReplacement(text, keep, keepReplacement));
   }
 
@@ -388,7 +373,7 @@ public final class CharClass {
                                             int paragraphReplacement) {
     ParamChecks.requireNonNullArg(text, "text");
     ParamChecks.requireNonNullArg(lineBreaks, "lineBreaks");
-    requireValidCodePoint(paragraphReplacement);
+    CodePointSet.requireValid(paragraphReplacement);
     return collapseRuns(text, paragraphReplacement(text, lineBreaks, paragraphReplacement), null);
   }
 
@@ -407,7 +392,7 @@ public final class CharClass {
                                                         int paragraphReplacement) {
     ParamChecks.requireNonNullArg(text, "text");
     ParamChecks.requireNonNullArg(lineBreaks, "lineBreaks");
-    requireValidCodePoint(paragraphReplacement);
+    CodePointSet.requireValid(paragraphReplacement);
     return collapseRunsAligned(text, paragraphReplacement(text, lineBreaks, paragraphReplacement));
   }
 
@@ -423,23 +408,9 @@ public final class CharClass {
   public AlignedText trimAligned(CharSequence text) {
     ParamChecks.requireNonNullArg(text, "text");
     final int length = text.length();
-    int start = 0;
-    while (start < length) {
-      final At cp = CodePoints.at(text, start);
-      if (!members.contains(cp.codePoint())) {
-        break;
-      }
-      start = cp.nextIndex(start);
-    }
-    int end = length;
-    while (end > start) {
-      final At cp = CodePoints.before(text, end);
-      if (!members.contains(cp.codePoint())) {
-        break;
-      }
-      end = cp.previousIndex(end);
-    }
-    final Alignment.Builder alignment = new Alignment.Builder(text.length());
+    final int start = skipRun(text, 0);
+    final int end = skipRunBackward(text, start, length);
+    final Alignment.Builder alignment = new Alignment.Builder(length);
     if (start > 0) {
       alignment.replace(start, 0);
     }
@@ -705,6 +676,27 @@ public final class CharClass {
     return i;
   }
 
+  /**
+   * Moves back past a run of member code points that ends at {@code runEnd}.
+   *
+   * @param text The text to scan.
+   * @param floor The lowest index the scan may reach.
+   * @param runEnd The index after the last code point of the run.
+   * @return The index after the last non-member code point before {@code runEnd}, or
+   *     {@code floor} when every code point in {@code [floor, runEnd)} is a member.
+   */
+  private int skipRunBackward(CharSequence text, int floor, int runEnd) {
+    int i = runEnd;
+    while (i > floor) {
+      final At cp = CodePoints.before(text, i);
+      if (!members.contains(cp.codePoint())) {
+        break;
+      }
+      i = cp.previousIndex(i);
+    }
+    return i;
+  }
+
   /**
    * Counts logical line breaks in a whitespace run. A carriage return immediately followed by a
    * line feed counts as one break.
@@ -739,17 +731,4 @@ public final class CharClass {
     }
     return count;
   }
-
-  /**
-   * Validates that {@code codePoint} is a Unicode code point.
-   *
-   * @param codePoint The value to validate.
-   * @throws IllegalArgumentException Thrown if {@code codePoint} is negative or greater than
-   *     {@link Character#MAX_CODE_POINT}.
-   */
-  private static void requireValidCodePoint(int codePoint) {
-    if (codePoint < 0 || codePoint > Character.MAX_CODE_POINT) {
-      throw new IllegalArgumentException("Not a Unicode code point: " + codePoint);
-    }
-  }
 }
diff --git a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CodePointSet.java b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CodePointSet.java
index 0ea99230e..9a358f917 100644
--- a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CodePointSet.java
+++ b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CodePointSet.java
@@ -23,7 +23,6 @@ import java.nio.file.Path;
 import java.util.BitSet;
 import java.util.List;
 import java.util.Locale;
-import java.util.Objects;
 
 import opennlp.tools.util.ParamChecks;
 import opennlp.tools.util.StringUtil;
@@ -82,7 +81,7 @@ public final class CodePointSet {
     requireValid(lastInclusive);
     if (firstInclusive > lastInclusive) {
       throw new IllegalArgumentException("Range start " + firstInclusive
-          + " must not exceed range end " + lastInclusive + ".");
+          + " must not exceed range end " + lastInclusive);
     }
     final BitSet members = new BitSet();
     members.set(firstInclusive, lastInclusive + 1);
@@ -115,7 +114,7 @@ public final class CodePointSet {
 
   // Package visible so the parser can be exercised directly, without a temporary file.
   static CodePointSet parse(List<String> lines, String section) {
-    Objects.requireNonNull(section, "section");
+    ParamChecks.requireNonNullArg(section, "section");
     final String wanted = StringUtil.trimUnicodeWhitespace(section).toLowerCase(Locale.ROOT);
     final BitSet members = new BitSet();
     String current = null;
@@ -207,7 +206,14 @@ public final class CodePointSet {
     return new IllegalArgumentException("Malformed " + what + " on line " + lineNumber + ": " + raw);
   }
 
-  private static void requireValid(int codePoint) {
+  /**
+   * Validates that {@code codePoint} is a Unicode code point.
+   *
+   * @param codePoint The value to validate.
+   * @throws IllegalArgumentException Thrown if {@code codePoint} is negative or greater than
+   *     {@link Character#MAX_CODE_POINT}.
+   */
+  static void requireValid(int codePoint) {
     if (codePoint < 0 || codePoint > Character.MAX_CODE_POINT) {
       throw new IllegalArgumentException("Not a Unicode code point: " + codePoint);
     }
diff --git a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/UnicodeDash.java b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/UnicodeDash.java
index cd49409a1..8f9b6c705 100644
--- a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/UnicodeDash.java
+++ b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/UnicodeDash.java
@@ -50,7 +50,13 @@ public final class UnicodeDash {
   /** The canonical ASCII dash that dashes are normalized to: {@code U+002D} HYPHEN-MINUS. */
   public static final int HYPHEN_MINUS = 0x002D;
 
-  /** The Unicode general category of a dash code point. */
+  /**
+   * The Unicode general category of a dash code point.
+   *
+   * <p>The constant names are the two-letter general category abbreviations of the Unicode
+   * Character Database, kept as is so they match the Unicode tables and {@code \p{Pd}}-style
+   * property names.</p>
+   */
   public enum Category {
     /** {@code Pd} - dash punctuation. */
     Pd,
diff --git a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/UnicodeWhitespace.java b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/UnicodeWhitespace.java
index c5fba3914..462d81103 100644
--- a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/UnicodeWhitespace.java
+++ b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/UnicodeWhitespace.java
@@ -52,7 +52,13 @@ public final class UnicodeWhitespace {
   /** {@code U+000A} LINE FEED, a line break on its own or the second half of {@code CRLF}. */
   public static final int LINE_FEED = 0x000A;
 
-  /** Unicode general category for a whitespace or related code point. */
+  /**
+   * Unicode general category for a whitespace or related code point.
+   *
+   * <p>The constant names are the two-letter general category abbreviations of the Unicode
+   * Character Database, kept as is so they match the Unicode tables and {@code \p{Zs}}-style
+   * property names.</p>
+   */
   public enum Category {
     /** {@code Cc} - control. */
     Cc,
diff --git a/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CharClassTest.java b/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CharClassTest.java
index e64ed1dce..c3902030a 100644
--- a/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CharClassTest.java
+++ b/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CharClassTest.java
@@ -25,6 +25,7 @@ import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.function.Executable;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
 
 import opennlp.tools.util.Span;
 
@@ -323,12 +324,32 @@ public class CharClassTest {
     assertFalse(WS.contains('_'), "the preset must be unchanged");
   }
 
-  @Test
-  void testOfRejectsInvalidReplacement() {
-    assertThrows(IllegalArgumentException.class,
-        () -> CharClass.of(CodePointSet.of(0x20), -1));
-    assertThrows(IllegalArgumentException.class,
-        () -> CharClass.of(CodePointSet.of(0x20), Character.MAX_CODE_POINT + 1));
+  /**
+   * Lists every call that takes a replacement code point, each invoked with {@code replacement}.
+   *
+   * @param replacement The replacement code point to pass.
+   * @return The named calls.
+   */
+  private static List<Named<Executable>> replacementCalls(int replacement) {
+    return List.of(
+        Named.of("of", () -> CharClass.of(CodePointSet.of(0x20), replacement)),
+        Named.of("collapsePreserving", () -> WS.collapsePreserving("x", NEWLINE, replacement)),
+        Named.of("collapsePreservingAligned",
+            () -> WS.collapsePreservingAligned("x", NEWLINE, replacement)),
+        Named.of("collapseParagraphPreserving",
+            () -> WS.collapseParagraphPreserving("x", NEWLINE, replacement)),
+        Named.of("collapseParagraphPreservingAligned",
+            () -> WS.collapseParagraphPreservingAligned("x", NEWLINE, replacement)));
+  }
+
+  @ParameterizedTest
+  @ValueSource(ints = {-1, Character.MAX_CODE_POINT + 1})
+  void testRejectsInvalidReplacement(int replacement) {
+    for (Named<Executable> call : replacementCalls(replacement)) {
+      assertEquals("Not a Unicode code point: " + replacement,
+          assertThrows(IllegalArgumentException.class, call.getPayload(), call.getName()).getMessage(),
+          call.getName());
+    }
   }
 
   // --- aligned variants (Alignment / AlignedText) ------------------------------------------
diff --git a/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CodePointSetTest.java b/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CodePointSetTest.java
index 6a81ecfc9..0a0671ca4 100644
--- a/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CodePointSetTest.java
+++ b/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CodePointSetTest.java
@@ -130,6 +130,18 @@ public class CodePointSetTest {
         () -> CodePointSet.fromFile(file, null)).getMessage());
   }
 
+  @Test
+  void testParseRejectsNullSection() {
+    assertEquals("section must not be null", assertThrows(IllegalArgumentException.class,
+        () -> CodePointSet.parse(List.of("[s]", "9"), null)).getMessage());
+  }
+
+  @Test
+  void testOfRangeRejectsDescendingBounds() {
+    assertEquals("Range start 66 must not exceed range end 65",
+        assertThrows(IllegalArgumentException.class, () -> CodePointSet.ofRange(66, 65)).getMessage());
+  }
+
   @Test
   void testParseAcceptsSingleHexDigit() {
     assertTrue(CodePointSet.parse(List.of("[s]", "9"), "s").contains(0x9));