You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by rz...@apache.org on 2026/10/06 19:01:11 UTC
(opennlp) branch gh-1422-stringutil-test-cleanup created (now 27a590908)
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a change to branch gh-1422-stringutil-test-cleanup
in repository https://gitbox.apache.org/repos/asf/opennlp.git
at 27a590908 GH-1422: Remove duplication in StringUtilTest and the chunker listener test
This branch includes the following new commits:
new 27a590908 GH-1422: Remove duplication in StringUtilTest and the chunker listener test
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(opennlp) 01/01: GH-1422: Remove duplication in StringUtilTest and the chunker listener test
Posted by rz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch gh-1422-stringutil-test-cleanup
in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 27a59090886f59b19aa64c4940184179fc332a5a
Author: Richard Zowalla <rz...@apache.org>
AuthorDate: Tue Oct 6 20:46:45 2026 +0200
GH-1422: Remove duplication in StringUtilTest and the chunker listener test
Merge the per-mode isWhitespace boundary tests, share the info separator source,
turn sample loops into parameterized tests and drop history comments.
Reuse the evaluate helper in ChunkerDetailedFMeasureListenerTest.testEvaluator.
---
.../java/opennlp/tools/util/StringUtilTest.java | 187 +++++++++------------
.../ChunkerDetailedFMeasureListenerTest.java | 21 +--
2 files changed, 82 insertions(+), 126 deletions(-)
diff --git a/opennlp-api/src/test/java/opennlp/tools/util/StringUtilTest.java b/opennlp-api/src/test/java/opennlp/tools/util/StringUtilTest.java
index 8c7732da1..e71ab8675 100644
--- a/opennlp-api/src/test/java/opennlp/tools/util/StringUtilTest.java
+++ b/opennlp-api/src/test/java/opennlp/tools/util/StringUtilTest.java
@@ -28,6 +28,7 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.EnumSource;
import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.ValueSource;
@@ -44,6 +45,9 @@ public class StringUtilTest {
private static final int[] INFO_SEPARATORS = {0x001C, 0x001D, 0x001E, 0x001F};
+ private static final int[] SPACE_SEPARATORS = {0x1680, 0x2000, 0x2001, 0x2002, 0x2003,
+ 0x2004, 0x2005, 0x2006, 0x2008, 0x2009, 0x200A, 0x2028, 0x2029, 0x205F, 0x3000};
+
private static final int DESERET_CAPITAL_BEE = 0x10412; // supplementary-plane letter
private static final int DESERET_SMALL_BEE = 0x1043A;
private static final int GRINNING_FACE = 0x1F600; // emoji, two chars
@@ -56,6 +60,10 @@ public class StringUtilTest {
return UnicodeWhitespace.lookalikes();
}
+ private static IntStream infoSeparators() {
+ return Arrays.stream(INFO_SEPARATORS);
+ }
+
private static Stream<CharSequence> nonStringCharSequences() {
return Stream.of(
new StringBuilder("a\u00A0b"),
@@ -72,7 +80,7 @@ public class StringUtilTest {
}
// -------------------------------------------------------------------------
- // isWhitespace / isUnicodeWhitespace (existing pins, kept)
+ // isWhitespace / isUnicodeWhitespace
// -------------------------------------------------------------------------
@Test
@@ -87,47 +95,20 @@ public class StringUtilTest {
}
/**
- * Pins the exact semantics of {@link StringUtil#isWhitespace(int)} under the default
- * {@link WhitespaceMode#UNICODE}, at the code points where the JVM predicates and the
- * Unicode {@code White_Space} property disagree, so the predicate cannot drift silently:
- * it excludes the {@code U+001C..U+001F} information separators and includes the next
- * line control {@code U+0085}, agreeing with {@link StringUtil#isUnicodeWhitespace(int)}.
- */
- @Test
- void testIsWhitespaceBoundaryCodePointsDefaultIsUnicode() {
- for (int cp = 0x0009; cp <= 0x000D; cp++) {
- Assertions.assertTrue(StringUtil.isWhitespace(cp), "U+" + Integer.toHexString(cp));
- }
- Assertions.assertTrue(StringUtil.isWhitespace(0x0020));
-
- for (int cp : INFO_SEPARATORS) {
- Assertions.assertFalse(StringUtil.isWhitespace(cp), "U+" + Integer.toHexString(cp));
- Assertions.assertFalse(StringUtil.isWhitespace((char) cp), "U+" + Integer.toHexString(cp));
- }
-
- Assertions.assertTrue(StringUtil.isWhitespace(0x0085));
- Assertions.assertTrue(StringUtil.isWhitespace((char) 0x0085));
-
- int[] separators = {0x1680, 0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006,
- 0x2008, 0x2009, 0x200A, 0x2028, 0x2029, 0x205F, 0x3000};
- for (int cp : separators) {
- Assertions.assertTrue(StringUtil.isWhitespace(cp), "U+" + Integer.toHexString(cp));
- }
-
- Assertions.assertFalse(StringUtil.isWhitespace(0x200B));
- Assertions.assertFalse(StringUtil.isWhitespace(0xFEFF));
- }
-
- /**
- * Pins the exact semantics of {@link StringUtil#isWhitespace(int)} under
- * {@link WhitespaceMode#LEGACY}: the union of {@link Character#isWhitespace(int)} and the
- * {@code Zs} category, the opposite of the Unicode {@code White_Space} set at
- * {@code U+001C..U+001F} and {@code U+0085}. Trained sentence-detector and tokenizer
- * models built under this definition depend on it.
+ * Pins the exact semantics of {@link StringUtil#isWhitespace(int)} in each
+ * {@link WhitespaceMode}, at the code points where the JVM predicates and the Unicode
+ * {@code White_Space} property disagree, so the predicate cannot drift silently.
+ * {@link WhitespaceMode#UNICODE} excludes the {@code U+001C..U+001F} information separators
+ * and includes the next line control {@code U+0085}, agreeing with
+ * {@link StringUtil#isUnicodeWhitespace(int)}. {@link WhitespaceMode#LEGACY} is the union of
+ * {@link Character#isWhitespace(int)} and the {@code Zs} category, the opposite at those
+ * code points; trained sentence-detector and tokenizer models built under it depend on it.
*/
- @Test
- void testIsWhitespaceBoundaryCodePointsUnderLegacyMode() {
- WhitespaceMode.setActive(WhitespaceMode.LEGACY);
+ @ParameterizedTest
+ @EnumSource(WhitespaceMode.class)
+ void testIsWhitespaceBoundaryCodePoints(WhitespaceMode mode) {
+ WhitespaceMode.setActive(mode);
+ final boolean unicode = mode == WhitespaceMode.UNICODE;
for (int cp = 0x0009; cp <= 0x000D; cp++) {
Assertions.assertTrue(StringUtil.isWhitespace(cp), "U+" + Integer.toHexString(cp));
@@ -135,16 +116,16 @@ public class StringUtilTest {
Assertions.assertTrue(StringUtil.isWhitespace(0x0020));
for (int cp : INFO_SEPARATORS) {
- Assertions.assertTrue(StringUtil.isWhitespace(cp), "U+" + Integer.toHexString(cp));
- Assertions.assertTrue(StringUtil.isWhitespace((char) cp), "U+" + Integer.toHexString(cp));
+ Assertions.assertEquals(!unicode, StringUtil.isWhitespace(cp),
+ "U+" + Integer.toHexString(cp));
+ Assertions.assertEquals(!unicode, StringUtil.isWhitespace((char) cp),
+ "U+" + Integer.toHexString(cp));
}
- Assertions.assertFalse(StringUtil.isWhitespace(0x0085));
- Assertions.assertFalse(StringUtil.isWhitespace((char) 0x0085));
+ Assertions.assertEquals(unicode, StringUtil.isWhitespace(0x0085));
+ Assertions.assertEquals(unicode, StringUtil.isWhitespace((char) 0x0085));
- int[] separators = {0x1680, 0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006,
- 0x2008, 0x2009, 0x200A, 0x2028, 0x2029, 0x205F, 0x3000};
- for (int cp : separators) {
+ for (int cp : SPACE_SEPARATORS) {
Assertions.assertTrue(StringUtil.isWhitespace(cp), "U+" + Integer.toHexString(cp));
}
@@ -414,7 +395,7 @@ public class StringUtilTest {
@Test
void testSplitOnUnicodeWhitespaceCollapsesMixedRuns() {
- // Tab, space, NBSP, NEL, ideographic space between two tokens — one boundary.
+ // Tab, space, NBSP, NEL, ideographic space between two tokens form one boundary.
final String input = "left" + "\t \u00A0\u0085\u3000" + "right";
Assertions.assertArrayEquals(new String[] {"left", "right"},
StringUtil.splitOnUnicodeWhitespace(input));
@@ -442,7 +423,7 @@ public class StringUtilTest {
@ParameterizedTest
@MethodSource("lookalikes")
void testSplitOnUnicodeWhitespaceLookalikesDoNotSeparate(RelatedCharacter related) {
- // ZWSP/BOM/etc. are not White_Space — they stay glued inside the token.
+ // ZWSP/BOM/etc. are not White_Space; they stay glued inside the token.
final String glue = cp(related.codePoint());
Assertions.assertArrayEquals(new String[] {"a" + glue + "b"},
StringUtil.splitOnUnicodeWhitespace("a" + glue + "b"),
@@ -450,7 +431,7 @@ public class StringUtilTest {
}
@ParameterizedTest
- @ValueSource(ints = {0x001C, 0x001D, 0x001E, 0x001F})
+ @MethodSource("infoSeparators")
void testSplitOnUnicodeWhitespaceInfoSeparatorsDoNotSeparate(int infoSep) {
// Character.isWhitespace includes these; Unicode White_Space does not.
Assertions.assertArrayEquals(new String[] {"a" + cp(infoSep) + "b"},
@@ -517,14 +498,14 @@ public class StringUtilTest {
StringUtil.splitOnUnicodeWhitespace(run + "x" + run + "y" + run));
}
- @Test
- void testSplitOnUnicodeWhitespaceSingleWhitespaceBetweenEmptyYieldsEmpty() {
- // Only whitespace → empty array, never [""].
- for (WhitespaceCharacter ws : UnicodeWhitespace.all()) {
- Assertions.assertArrayEquals(new String[0],
- StringUtil.splitOnUnicodeWhitespace(cp(ws.codePoint())),
- () -> ws.toUnicodeNotation());
- }
+ @ParameterizedTest
+ @MethodSource("whitespace")
+ void testSplitOnUnicodeWhitespaceSingleWhitespaceBetweenEmptyYieldsEmpty(
+ WhitespaceCharacter ws) {
+ // Only whitespace gives an empty array, never [""].
+ Assertions.assertArrayEquals(new String[0],
+ StringUtil.splitOnUnicodeWhitespace(cp(ws.codePoint())),
+ () -> ws.toUnicodeNotation());
}
@ParameterizedTest
@@ -552,8 +533,8 @@ public class StringUtilTest {
}
@Test
- void testSplitOnUnicodeWhitespaceAllTwentyFiveAsMixedSeparators() {
- // Build "t0 <ws0> t1 <ws1> ... t24 <ws24> t25" using every White_Space code point.
+ void testSplitOnUnicodeWhitespaceAllWhiteSpaceCodePointsAsMixedSeparators() {
+ // Build "t0 <ws0> t1 <ws1> ... tN" using every White_Space code point.
final List<WhitespaceCharacter> all = UnicodeWhitespace.all();
final StringBuilder sb = new StringBuilder();
for (int i = 0; i < all.size(); i++) {
@@ -608,7 +589,7 @@ public class StringUtilTest {
@MethodSource("lookalikes")
void testTrimUnicodeWhitespaceLookalikesAreNotTrimmed(RelatedCharacter related) {
final String glue = cp(related.codePoint());
- // Leading/trailing lookalikes stay — they are not White_Space.
+ // Leading/trailing lookalikes stay; they are not White_Space.
Assertions.assertEquals(glue + "x" + glue,
StringUtil.trimUnicodeWhitespace(glue + "x" + glue),
() -> related.toUnicodeNotation());
@@ -619,7 +600,7 @@ public class StringUtilTest {
}
@ParameterizedTest
- @ValueSource(ints = {0x001C, 0x001D, 0x001E, 0x001F})
+ @MethodSource("infoSeparators")
void testTrimUnicodeWhitespaceInfoSeparatorsAreNotTrimmed(int infoSep) {
final String sep = cp(infoSep);
Assertions.assertEquals(sep + "x" + sep, StringUtil.trimUnicodeWhitespace(sep + "x" + sep));
@@ -663,7 +644,7 @@ public class StringUtilTest {
WhitespaceMode.setActive(WhitespaceMode.LEGACY);
Assertions.assertEquals("a\u0085b", StringUtil.trimUnicodeWhitespace(" a\u0085b "));
- // Info separators still not trimmed under LEGACY — Unicode helper is unconditional.
+ // Info separators still not trimmed under LEGACY; the Unicode helper is unconditional.
Assertions.assertEquals("\u001Cx\u001C",
StringUtil.trimUnicodeWhitespace("\u001Cx\u001C"));
// But NEL is still trimmed (Unicode), even though LEGACY isWhitespace(NEL) is false.
@@ -714,7 +695,7 @@ public class StringUtilTest {
}
@Test
- void testIsUnicodeBlankAllTwentyFiveTogether() {
+ void testIsUnicodeBlankAllWhiteSpaceCodePointsTogether() {
final String allWs = UnicodeWhitespace.all().stream()
.map(ws -> cp(ws.codePoint()))
.collect(Collectors.joining());
@@ -732,13 +713,12 @@ public class StringUtilTest {
}
@ParameterizedTest
- @ValueSource(ints = {0x001C, 0x001D, 0x001E, 0x001F})
+ @MethodSource("infoSeparators")
void testIsUnicodeBlankInfoSeparatorsAreNotBlank(int infoSep) {
Assertions.assertFalse(StringUtil.isUnicodeBlank(cp(infoSep)));
// JVM String.isBlank uses Character.isWhitespace, which *does* treat info separators
- // as blank — pin that we disagree.
+ // as blank; pin that we disagree.
Assertions.assertTrue(cp(infoSep).isBlank());
- Assertions.assertFalse(StringUtil.isUnicodeBlank(cp(infoSep)));
}
@Test
@@ -777,9 +757,8 @@ public class StringUtilTest {
// Cross-helper invariants
// -------------------------------------------------------------------------
- @Test
- void testBlankIffSplitEmptyAndTrimEmpty() {
- final String[] samples = {
+ private static Stream<String> blankSamples() {
+ return Stream.of(
"",
" ",
"\t\n\u00A0\u0085\u3000",
@@ -791,34 +770,29 @@ public class StringUtilTest {
cp(DESERET_CAPITAL_BEE),
join(0x00A0, DESERET_CAPITAL_BEE, 0x3000),
UnicodeWhitespace.all().stream().map(ws -> cp(ws.codePoint()))
- .collect(Collectors.joining())
- };
- for (String sample : samples) {
- final boolean blank = StringUtil.isUnicodeBlank(sample);
- final String[] split = StringUtil.splitOnUnicodeWhitespace(sample);
- final String trimmed = StringUtil.trimUnicodeWhitespace(sample);
- Assertions.assertEquals(blank, split.length == 0,
- () -> "blank iff split empty for: " + Arrays.toString(sample.codePoints().toArray()));
- Assertions.assertEquals(blank, trimmed.isEmpty(),
- () -> "blank iff trim empty for: " + Arrays.toString(sample.codePoints().toArray()));
- }
+ .collect(Collectors.joining()));
}
- @Test
- void testTrimThenSplitEqualsSplit() {
- // Leading/trailing whitespace is ignored by split, so trim-then-split is a no-op.
- final String[] samples = {
- " a b ",
- "\u00A0hello\u3000world\u0085",
- "\t\t alone \n",
- "a\u2007b\u2009c"
- };
- for (String sample : samples) {
- Assertions.assertArrayEquals(
- StringUtil.splitOnUnicodeWhitespace(sample),
- StringUtil.splitOnUnicodeWhitespace(StringUtil.trimUnicodeWhitespace(sample)),
- () -> sample);
- }
+ @ParameterizedTest
+ @MethodSource("blankSamples")
+ void testBlankIffSplitEmptyAndTrimEmpty(String sample) {
+ final boolean blank = StringUtil.isUnicodeBlank(sample);
+ final String[] split = StringUtil.splitOnUnicodeWhitespace(sample);
+ final String trimmed = StringUtil.trimUnicodeWhitespace(sample);
+ Assertions.assertEquals(blank, split.length == 0,
+ () -> "blank iff split empty for: " + Arrays.toString(sample.codePoints().toArray()));
+ Assertions.assertEquals(blank, trimmed.isEmpty(),
+ () -> "blank iff trim empty for: " + Arrays.toString(sample.codePoints().toArray()));
+ }
+
+ // Leading/trailing whitespace is ignored by split, so trim-then-split is a no-op.
+ @ParameterizedTest
+ @ValueSource(strings = {" a b ", "\u00A0hello\u3000world\u0085", "\t\t alone \n",
+ "a\u2007b\u2009c"})
+ void testTrimThenSplitEqualsSplit(String sample) {
+ Assertions.assertArrayEquals(
+ StringUtil.splitOnUnicodeWhitespace(sample),
+ StringUtil.splitOnUnicodeWhitespace(StringUtil.trimUnicodeWhitespace(sample)));
}
@Test
@@ -830,21 +804,18 @@ public class StringUtilTest {
StringUtil.splitOnUnicodeWhitespace(String.join(" ", tokens)));
}
- @Test
- void testSplitDoesNotEmitEmptyTokensAroundSeparators() {
- // Regex split("\\s+") on a leading-space string yields a leading "" with limit -1;
- // our scanner must never emit empty strings.
- for (String sample : List.of(" a", "a ", " a ", " a b ", "\u00A0a\u00A0")) {
- for (String token : StringUtil.splitOnUnicodeWhitespace(sample)) {
- Assertions.assertFalse(token.isEmpty(), () -> "empty token from: '" + sample + "'");
- Assertions.assertFalse(StringUtil.isUnicodeBlank(token),
- () -> "whitespace-only token from: '" + sample + "'");
- }
+ @ParameterizedTest
+ @ValueSource(strings = {" a", "a ", " a ", " a b ", "\u00A0a\u00A0"})
+ void testSplitDoesNotEmitEmptyTokensAroundSeparators(String sample) {
+ for (String token : StringUtil.splitOnUnicodeWhitespace(sample)) {
+ Assertions.assertFalse(token.isEmpty(), () -> "empty token from: '" + sample + "'");
+ Assertions.assertFalse(StringUtil.isUnicodeBlank(token),
+ () -> "whitespace-only token from: '" + sample + "'");
}
}
// -------------------------------------------------------------------------
- // Existing non-whitespace StringUtil coverage
+ // toLowerCase, toUpperCase, isEmpty, isBlank
// -------------------------------------------------------------------------
@Test
diff --git a/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/chunker/ChunkerDetailedFMeasureListenerTest.java b/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/chunker/ChunkerDetailedFMeasureListenerTest.java
index f71d5fafa..b1cbf0ad8 100644
--- a/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/chunker/ChunkerDetailedFMeasureListenerTest.java
+++ b/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/chunker/ChunkerDetailedFMeasureListenerTest.java
@@ -27,7 +27,6 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import opennlp.tools.RestoreDefaultLocale;
-import opennlp.tools.chunker.Chunker;
import opennlp.tools.chunker.ChunkerEvaluator;
import opennlp.tools.formats.ResourceAsStreamFactory;
import opennlp.tools.util.PlainTextByLineStream;
@@ -38,25 +37,11 @@ public class ChunkerDetailedFMeasureListenerTest {
@Test
void testEvaluator() throws IOException {
- ResourceAsStreamFactory inPredicted = new ResourceAsStreamFactory(
- getClass(), "/opennlp/tools/cmdline/chunker/output.txt");
- ResourceAsStreamFactory inExpected = new ResourceAsStreamFactory(getClass(),
- "/opennlp/tools/cmdline/chunker/output.txt");
- ResourceAsStreamFactory detailedOutputStream = new ResourceAsStreamFactory(
- getClass(), "/opennlp/tools/cmdline/chunker/detailedOutput.txt");
-
- DummyChunkSampleStream predictedSample = new DummyChunkSampleStream(
- new PlainTextByLineStream(inPredicted, StandardCharsets.UTF_8), true);
-
- DummyChunkSampleStream expectedSample = new DummyChunkSampleStream(
- new PlainTextByLineStream(inExpected, StandardCharsets.UTF_8), false);
-
- Chunker dummyChunker = new DummyChunker(predictedSample);
-
ChunkerDetailedFMeasureListener listener = new ChunkerDetailedFMeasureListener();
- ChunkerEvaluator evaluator = new ChunkerEvaluator(dummyChunker, listener);
+ evaluate(listener);
- evaluator.evaluate(expectedSample);
+ ResourceAsStreamFactory detailedOutputStream = new ResourceAsStreamFactory(
+ getClass(), "/opennlp/tools/cmdline/chunker/detailedOutput.txt");
StringBuilder expected = new StringBuilder();
BufferedReader reader = new BufferedReader(