You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by rz...@apache.org on 2026/10/08 10:00:52 UTC
(opennlp) branch main updated: GH-1438: Share code point validation and trim scans in CharClass and CodePointSet (#1439)
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/main by this push:
new cf81387cc GH-1438: Share code point validation and trim scans in CharClass and CodePointSet (#1439)
cf81387cc is described below
commit cf81387cc1483590b8adf4228ca5519c529260ea
Author: Richard Zowalla <13...@users.noreply.github.com>
AuthorDate: Thu Oct 8 12:00:45 2026 +0200
GH-1438: Share code point validation and trim scans in CharClass and CodePointSet (#1439)
CharClass validates replacement code points through CodePointSet and trims
with shared forward and backward scan helpers. CodePointSet.parse rejects a
null section with IllegalArgumentException, and the CodePointSet.ofRange
message for descending bounds no longer ends with a period. The
UnicodeDash.Category and UnicodeWhitespace.Category Javadoc explains that
the constant names are Unicode general category abbreviations.
---
.../opennlp/tools/util/normalizer/CharClass.java | 83 ++++++++--------------
.../tools/util/normalizer/CodePointSet.java | 14 ++--
.../opennlp/tools/util/normalizer/UnicodeDash.java | 8 ++-
.../tools/util/normalizer/UnicodeWhitespace.java | 8 ++-
.../tools/util/normalizer/CharClassTest.java | 33 +++++++--
.../tools/util/normalizer/CodePointSetTest.java | 12 ++++
6 files changed, 94 insertions(+), 64 deletions(-)
diff --git a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CharClass.java b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CharClass.java
index 82625c0e8..76dcd8c2f 100644
--- a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CharClass.java
+++ b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CharClass.java
@@ -69,7 +69,7 @@ public final class CharClass {
*/
public static CharClass of(CodePointSet members, int replacement) {
ParamChecks.requireNonNullArg(members, "members");
- requireValidCodePoint(replacement);
+ CodePointSet.requireValid(replacement);
return new CharClass(members, replacement);
}
@@ -249,7 +249,7 @@ public final class CharClass {
public String collapsePreserving(CharSequence text, CodePointSet keep, int keepReplacement) {
ParamChecks.requireNonNullArg(text, "text");
ParamChecks.requireNonNullArg(keep, "keep");
- requireValidCodePoint(keepReplacement);
+ CodePointSet.requireValid(keepReplacement);
return collapseRuns(text, preservingReplacement(text, keep, keepReplacement), null);
}
@@ -262,23 +262,8 @@ public final class CharClass {
*/
public String trim(CharSequence text) {
ParamChecks.requireNonNullArg(text, "text");
- final int length = text.length();
- int start = 0;
- while (start < length) {
- final At cp = CodePoints.at(text, start);
- if (!members.contains(cp.codePoint())) {
- break;
- }
- start = cp.nextIndex(start);
- }
- int end = length;
- while (end > start) {
- final At cp = CodePoints.before(text, end);
- if (!members.contains(cp.codePoint())) {
- break;
- }
- end = cp.previousIndex(end);
- }
+ final int start = skipRun(text, 0);
+ final int end = skipRunBackward(text, start, text.length());
return text.subSequence(start, end).toString();
}
@@ -368,7 +353,7 @@ public final class CharClass {
int keepReplacement) {
ParamChecks.requireNonNullArg(text, "text");
ParamChecks.requireNonNullArg(keep, "keep");
- requireValidCodePoint(keepReplacement);
+ CodePointSet.requireValid(keepReplacement);
return collapseRunsAligned(text, preservingReplacement(text, keep, keepReplacement));
}
@@ -389,7 +374,7 @@ public final class CharClass {
int paragraphReplacement) {
ParamChecks.requireNonNullArg(text, "text");
ParamChecks.requireNonNullArg(lineBreaks, "lineBreaks");
- requireValidCodePoint(paragraphReplacement);
+ CodePointSet.requireValid(paragraphReplacement);
return collapseRuns(text, paragraphReplacement(text, lineBreaks, paragraphReplacement), null);
}
@@ -408,7 +393,7 @@ public final class CharClass {
int paragraphReplacement) {
ParamChecks.requireNonNullArg(text, "text");
ParamChecks.requireNonNullArg(lineBreaks, "lineBreaks");
- requireValidCodePoint(paragraphReplacement);
+ CodePointSet.requireValid(paragraphReplacement);
return collapseRunsAligned(text, paragraphReplacement(text, lineBreaks, paragraphReplacement));
}
@@ -424,23 +409,9 @@ public final class CharClass {
public AlignedText trimAligned(CharSequence text) {
ParamChecks.requireNonNullArg(text, "text");
final int length = text.length();
- int start = 0;
- while (start < length) {
- final At cp = CodePoints.at(text, start);
- if (!members.contains(cp.codePoint())) {
- break;
- }
- start = cp.nextIndex(start);
- }
- int end = length;
- while (end > start) {
- final At cp = CodePoints.before(text, end);
- if (!members.contains(cp.codePoint())) {
- break;
- }
- end = cp.previousIndex(end);
- }
- final Alignment.Builder alignment = new Alignment.Builder(text.length());
+ final int start = skipRun(text, 0);
+ final int end = skipRunBackward(text, start, length);
+ final Alignment.Builder alignment = new Alignment.Builder(length);
if (start > 0) {
alignment.replace(start, 0);
}
@@ -705,6 +676,27 @@ public final class CharClass {
return i;
}
+ /**
+ * Moves back past a run of member code points that ends at {@code runEnd}.
+ *
+ * @param text The text to scan.
+ * @param floor The lowest index the scan may reach.
+ * @param runEnd The index after the last code point of the run.
+ * @return The index after the last non-member code point before {@code runEnd}, or
+ * {@code floor} when every code point in {@code [floor, runEnd)} is a member.
+ */
+ private int skipRunBackward(CharSequence text, int floor, int runEnd) {
+ int i = runEnd;
+ while (i > floor) {
+ final At cp = CodePoints.before(text, i);
+ if (!members.contains(cp.codePoint())) {
+ break;
+ }
+ i = cp.previousIndex(i);
+ }
+ return i;
+ }
+
/**
* Counts logical line breaks in a whitespace run. A carriage return immediately followed by a
* line feed counts as one break.
@@ -739,17 +731,4 @@ public final class CharClass {
}
return count;
}
-
- /**
- * Validates that {@code codePoint} is a Unicode code point.
- *
- * @param codePoint The value to validate.
- * @throws IllegalArgumentException Thrown if {@code codePoint} is negative or greater than
- * {@link Character#MAX_CODE_POINT}.
- */
- private static void requireValidCodePoint(int codePoint) {
- if (codePoint < 0 || codePoint > Character.MAX_CODE_POINT) {
- throw new IllegalArgumentException("Not a Unicode code point: " + codePoint);
- }
- }
}
diff --git a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CodePointSet.java b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CodePointSet.java
index 12f05a297..bf3decca7 100644
--- a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CodePointSet.java
+++ b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CodePointSet.java
@@ -23,7 +23,6 @@ import java.nio.file.Path;
import java.util.BitSet;
import java.util.List;
import java.util.Locale;
-import java.util.Objects;
import opennlp.tools.util.ParamChecks;
import opennlp.tools.util.StringUtil;
@@ -84,7 +83,7 @@ public final class CodePointSet {
requireValid(lastInclusive);
if (firstInclusive > lastInclusive) {
throw new IllegalArgumentException("Range start " + firstInclusive
- + " must not exceed range end " + lastInclusive + ".");
+ + " must not exceed range end " + lastInclusive);
}
final BitSet members = new BitSet();
members.set(firstInclusive, lastInclusive + 1);
@@ -117,7 +116,7 @@ public final class CodePointSet {
// Package visible so the parser can be exercised directly, without a temporary file.
static CodePointSet parse(List<String> lines, String section) {
- Objects.requireNonNull(section, "section");
+ ParamChecks.requireNonNullArg(section, "section");
final String wanted = StringUtil.trimUnicodeWhitespace(section).toLowerCase(Locale.ROOT);
final BitSet members = new BitSet();
String current = null;
@@ -209,7 +208,14 @@ public final class CodePointSet {
return new IllegalArgumentException("Malformed " + what + " on line " + lineNumber + ": " + raw);
}
- private static void requireValid(int codePoint) {
+ /**
+ * Validates that {@code codePoint} is a Unicode code point.
+ *
+ * @param codePoint The value to validate.
+ * @throws IllegalArgumentException Thrown if {@code codePoint} is negative or greater than
+ * {@link Character#MAX_CODE_POINT}.
+ */
+ static void requireValid(int codePoint) {
if (codePoint < 0 || codePoint > Character.MAX_CODE_POINT) {
throw new IllegalArgumentException("Not a Unicode code point: " + codePoint);
}
diff --git a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/UnicodeDash.java b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/UnicodeDash.java
index 438c61d12..7a17ee130 100644
--- a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/UnicodeDash.java
+++ b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/UnicodeDash.java
@@ -53,7 +53,13 @@ public final class UnicodeDash {
/** The canonical ASCII dash that dashes are normalized to: {@code U+002D} HYPHEN-MINUS. */
public static final int HYPHEN_MINUS = 0x002D;
- /** The Unicode general category of a dash code point. */
+ /**
+ * The Unicode general category of a dash code point.
+ *
+ * <p>The constant names are the two-letter general category abbreviations of the Unicode
+ * Character Database, kept as is so they match the Unicode tables and {@code \p{Pd}}-style
+ * property names.</p>
+ */
public enum Category {
/** {@code Pd} - dash punctuation. */
Pd,
diff --git a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/UnicodeWhitespace.java b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/UnicodeWhitespace.java
index 637c4bcf2..b57dad49e 100644
--- a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/UnicodeWhitespace.java
+++ b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/UnicodeWhitespace.java
@@ -56,7 +56,13 @@ public final class UnicodeWhitespace {
/** {@code U+000A} LINE FEED, a line break on its own or the second half of {@code CRLF}. */
public static final int LINE_FEED = 0x000A;
- /** Unicode general category for a whitespace or related code point. */
+ /**
+ * Unicode general category for a whitespace or related code point.
+ *
+ * <p>The constant names are the two-letter general category abbreviations of the Unicode
+ * Character Database, kept as is so they match the Unicode tables and {@code \p{Zs}}-style
+ * property names.</p>
+ */
public enum Category {
/** {@code Cc} - control. */
Cc,
diff --git a/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CharClassTest.java b/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CharClassTest.java
index e64ed1dce..c3902030a 100644
--- a/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CharClassTest.java
+++ b/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CharClassTest.java
@@ -25,6 +25,7 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.function.Executable;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
import opennlp.tools.util.Span;
@@ -323,12 +324,32 @@ public class CharClassTest {
assertFalse(WS.contains('_'), "the preset must be unchanged");
}
- @Test
- void testOfRejectsInvalidReplacement() {
- assertThrows(IllegalArgumentException.class,
- () -> CharClass.of(CodePointSet.of(0x20), -1));
- assertThrows(IllegalArgumentException.class,
- () -> CharClass.of(CodePointSet.of(0x20), Character.MAX_CODE_POINT + 1));
+ /**
+ * Lists every call that takes a replacement code point, each invoked with {@code replacement}.
+ *
+ * @param replacement The replacement code point to pass.
+ * @return The named calls.
+ */
+ private static List<Named<Executable>> replacementCalls(int replacement) {
+ return List.of(
+ Named.of("of", () -> CharClass.of(CodePointSet.of(0x20), replacement)),
+ Named.of("collapsePreserving", () -> WS.collapsePreserving("x", NEWLINE, replacement)),
+ Named.of("collapsePreservingAligned",
+ () -> WS.collapsePreservingAligned("x", NEWLINE, replacement)),
+ Named.of("collapseParagraphPreserving",
+ () -> WS.collapseParagraphPreserving("x", NEWLINE, replacement)),
+ Named.of("collapseParagraphPreservingAligned",
+ () -> WS.collapseParagraphPreservingAligned("x", NEWLINE, replacement)));
+ }
+
+ @ParameterizedTest
+ @ValueSource(ints = {-1, Character.MAX_CODE_POINT + 1})
+ void testRejectsInvalidReplacement(int replacement) {
+ for (Named<Executable> call : replacementCalls(replacement)) {
+ assertEquals("Not a Unicode code point: " + replacement,
+ assertThrows(IllegalArgumentException.class, call.getPayload(), call.getName()).getMessage(),
+ call.getName());
+ }
}
// --- aligned variants (Alignment / AlignedText) ------------------------------------------
diff --git a/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CodePointSetTest.java b/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CodePointSetTest.java
index 6a81ecfc9..0a0671ca4 100644
--- a/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CodePointSetTest.java
+++ b/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CodePointSetTest.java
@@ -130,6 +130,18 @@ public class CodePointSetTest {
() -> CodePointSet.fromFile(file, null)).getMessage());
}
+ @Test
+ void testParseRejectsNullSection() {
+ assertEquals("section must not be null", assertThrows(IllegalArgumentException.class,
+ () -> CodePointSet.parse(List.of("[s]", "9"), null)).getMessage());
+ }
+
+ @Test
+ void testOfRangeRejectsDescendingBounds() {
+ assertEquals("Range start 66 must not exceed range end 65",
+ assertThrows(IllegalArgumentException.class, () -> CodePointSet.ofRange(66, 65)).getMessage());
+ }
+
@Test
void testParseAcceptsSingleHexDigit() {
assertTrue(CodePointSet.parse(List.of("[s]", "9"), "s").contains(0x9));