You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by ma...@apache.org on 2026/10/09 08:13:25 UTC
(opennlp) branch main updated: GH-1420: Remove duplication and add readResolve in opennlp-runtime (#1427)
This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/main by this push:
new eef3909dc GH-1420: Remove duplication and add readResolve in opennlp-runtime (#1427)
eef3909dc is described below
commit eef3909dc72b557041ef616fcd82ff78f490ea0b
Author: Richard Zowalla <13...@users.noreply.github.com>
AuthorDate: Fri Oct 9 10:13:19 2026 +0200
GH-1420: Remove duplication and add readResolve in opennlp-runtime (#1427)
* GH-1420: Remove duplication and add readResolve in opennlp-runtime
Adds readResolve to the normalizer singletons, reuses shared helpers in place of
local copies in main and test code, and rejects POS dictionary entries without tags.
---
.../tools/lemmatizer/LemmatizerAnnotator.java | 3 ++
.../tools/namefind/NameFinderAnnotator.java | 3 ++
.../java/opennlp/tools/postag/POSDictionary.java | 6 ++-
.../opennlp/tools/postag/POSTaggerAnnotator.java | 3 ++
.../sentdetect/SentenceDetectorAnnotator.java | 3 ++
.../opennlp/tools/stemmer/StemmerAnnotator.java | 3 ++
.../tools/stemmer/hunspell/AffixCondition.java | 20 +------
.../tools/stemmer/hunspell/HunspellDictionary.java | 18 ++-----
.../tools/termvector/TermVectorAnnotator.java | 29 +++++-----
.../opennlp/tools/tokenize/TokenizerAnnotator.java | 5 +-
.../opennlp/tools/tokenize/uax29/WordType.java | 11 ++--
.../opennlp/tools/util/normalizer/AsciiChars.java | 19 +++++++
.../normalizer/BulletCharSequenceNormalizer.java | 5 ++
.../ConfusableSkeletonCharSequenceNormalizer.java | 5 ++
.../normalizer/DashCharSequenceNormalizer.java | 5 ++
.../normalizer/DigitCharSequenceNormalizer.java | 5 ++
.../tools/util/normalizer/EmojiAnnotations.java | 4 +-
.../tools/util/normalizer/EmojiEmoticons.java | 27 ++--------
.../opennlp/tools/util/normalizer/EmojiFlags.java | 2 +-
.../EmojiToEmoticonCharSequenceNormalizer.java | 5 ++
.../EmoticonToEmojiCharSequenceNormalizer.java | 5 ++
.../FullCaseFoldCharSequenceNormalizer.java | 5 ++
.../GermanUmlautCharSequenceNormalizer.java | 5 ++
.../util/normalizer/NfcCharSequenceNormalizer.java | 5 ++
.../normalizer/NfkcCharSequenceNormalizer.java | 5 ++
.../normalizer/QuoteCharSequenceNormalizer.java | 5 ++
.../SocialMediaCharSequenceNormalizer.java | 5 +-
.../tools/util/normalizer/TermAnalyzer.java | 5 +-
.../util/normalizer/UnicodeEmojiSequences.java | 4 +-
.../util/normalizer/UrlCharSequenceNormalizer.java | 25 ++-------
.../tools/chunker/ChunkerAnnotatorTest.java | 18 ++-----
.../tools/document/DocumentAnalyzerTest.java | 49 +++++++++++++----
.../document/DocumentPipelineExampleTest.java | 10 ++--
...{TestComponents.java => DocumentTestStubs.java} | 25 +++++++--
.../opennlp/tools/parser/ParserAnnotatorTest.java | 15 +-----
.../opennlp/tools/postag/POSDictionaryTest.java | 10 ++++
.../opennlp/tools/stemmer/StemmerFactoryTest.java | 4 ++
.../tools/termvector/SingleSpaceTokens.java | 46 ++--------------
.../tools/termvector/TermVectorAnnotatorTest.java | 5 ++
.../tools/termvector/TermVectorPipelineTest.java | 5 +-
.../tokenize/uax29/WordBreakPropertyTest.java | 2 +-
.../tools/tokenize/uax29/WordSegmenterTest.java | 5 +-
.../tools/tokenize/uax29/WordTokenizerTest.java | 5 +-
.../opennlp/tools/tokenize/uax29/WordTypeTest.java | 5 +-
.../opennlp/tools/util/InstallerTestSupport.java | 42 ---------------
.../tools/util/ResourceInstallerHttpTest.java | 16 +++---
.../opennlp/tools/util/ResourceInstallerTest.java | 61 ++++++++++------------
.../AccentFoldCharSequenceNormalizerTest.java | 5 +-
.../normalizer/AlignedNormalizerPipelineTest.java | 9 ++--
.../CaseFoldCharSequenceNormalizerTest.java | 5 +-
.../CharSequenceNormalizerContractTest.java | 33 ++++++++++++
.../util/normalizer/ConfusableSkeletonTest.java | 5 +-
.../tools/util/normalizer/ConfusablesTest.java | 5 +-
...CharSequenceNormalizerCharacterizationTest.java | 5 +-
.../EmojiCharSequenceNormalizerTest.java | 6 +--
.../EmojiToEmoticonCharSequenceNormalizerTest.java | 5 +-
.../EmoticonToEmojiCharSequenceNormalizerTest.java | 5 +-
.../FullCaseFoldCharSequenceNormalizerTest.java | 5 +-
.../GermanUmlautCharSequenceNormalizerTest.java | 5 +-
...ervingWhitespaceCharSequenceNormalizerTest.java | 5 +-
.../util/normalizer/NormalizationProfilesTest.java | 13 ++---
...ervingWhitespaceCharSequenceNormalizerTest.java | 5 +-
.../util/normalizer/SetBasedNormalizerTest.java | 5 +-
.../normalizer/TermAnalyzerMultilingualTest.java | 5 +-
.../tools/util/normalizer/TermAnalyzerTest.java | 9 +---
.../tools/util/normalizer/TextNormalizerTest.java | 5 +-
.../UnicodeCharSequenceNormalizerTest.java | 5 +-
.../normalizer/UnicodeEmojiSequencesParseTest.java | 5 +-
68 files changed, 338 insertions(+), 385 deletions(-)
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java
index 820a41335..b78f7e4fc 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java
@@ -40,6 +40,9 @@ import opennlp.tools.util.ParamChecks;
* lemmatizer are sliced per sentence; the produced lemma layer stays aligned with
* {@link Layers#TOKENS} by position.</p>
*
+ * <p>The adapter holds no per-call state; it is as thread-safe as the lemmatizer it
+ * wraps.</p>
+ *
* @since 3.0.0
*/
public final class LemmatizerAnnotator implements DocumentAnnotator {
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderAnnotator.java
index 655be794f..eb8012b00 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderAnnotator.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderAnnotator.java
@@ -46,6 +46,9 @@ import opennlp.tools.util.Span;
* <p>Spans the finder returns without a type are recorded with the {@link #UNTYPED}
* entity type.</p>
*
+ * <p>The adapter holds no per-call state; it is as thread-safe as the name finder it
+ * wraps.</p>
+ *
* @since 3.0.0
*/
public final class NameFinderAnnotator implements DocumentAnnotator {
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/POSDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/POSDictionary.java
index 6c18869ba..e0b642d5c 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/POSDictionary.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/POSDictionary.java
@@ -205,7 +205,8 @@ public class POSDictionary implements Iterable<String>, MutableTagDictionary, Se
* @return A valid {@link POSDictionary} instance.
*
* @throws IOException Thrown if IO errors occurred during creation.
- * @throws InvalidFormatException Thrown if the entries don't have exactly one token.
+ * @throws InvalidFormatException Thrown if an entry does not have exactly one token or has
+ * no tags attribute.
*/
public static POSDictionary create(InputStream in) throws IOException {
@@ -214,6 +215,9 @@ public class POSDictionary implements Iterable<String>, MutableTagDictionary, Se
boolean isCaseSensitive = DictionaryEntryPersistor.create(in, entry -> {
String tagString = entry.attributes().getValue("tags");
+ if (tagString == null) {
+ throw new InvalidFormatException("Each entry must have a tags attribute! " + entry.tokens());
+ }
String[] tags = StringUtil.split(tagString, ' ');
StringList word = entry.tokens();
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/POSTaggerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/POSTaggerAnnotator.java
index a450eba5a..20d88f1f1 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/POSTaggerAnnotator.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/POSTaggerAnnotator.java
@@ -40,6 +40,9 @@ import opennlp.tools.util.ParamChecks;
* sentence; the produced tag layer stays aligned with {@link Layers#TOKENS} by
* position.</p>
*
+ * <p>The adapter holds no per-call state; it is as thread-safe as the tagger it
+ * wraps.</p>
+ *
* @since 3.0.0
*/
public final class POSTaggerAnnotator implements DocumentAnnotator {
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentdetect/SentenceDetectorAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentdetect/SentenceDetectorAnnotator.java
index 4595eecee..c5bc47a26 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentdetect/SentenceDetectorAnnotator.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentdetect/SentenceDetectorAnnotator.java
@@ -36,6 +36,9 @@ import opennlp.tools.util.Span;
* <p>The wrapped detector stays the primary API for single-task use; this adapter calls
* it like any other caller would.</p>
*
+ * <p>The adapter holds no per-call state; it is as thread-safe as the detector it
+ * wraps.</p>
+ *
* @since 3.0.0
*/
public final class SentenceDetectorAnnotator implements DocumentAnnotator {
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java
index 75ff04aad..41d480b61 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java
@@ -36,6 +36,9 @@ import opennlp.tools.util.ParamChecks;
* <p>Stemming operates on the token surface alone, so this annotator requires only the
* token layer; no part-of-speech tags are involved.</p>
*
+ * <p>The adapter holds no per-call state; it is as thread-safe as the stemmer it
+ * wraps.</p>
+ *
* @since 3.0.0
*/
public final class StemmerAnnotator implements DocumentAnnotator {
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java
index 19ce657cc..9c82a020f 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java
@@ -85,7 +85,7 @@ final class AffixCondition {
negate = true;
members = members.substring(1);
}
- positions.add(toCodePoints(members));
+ positions.add(members.codePoints().toArray());
negations.add(negate);
i = end + 1;
} else if (codePoint == '.') {
@@ -106,24 +106,6 @@ final class AffixCondition {
return new AffixCondition(accepted, negated, suffix);
}
- /**
- * Collects the code points of a character-class body.
- *
- * @param members The class body text.
- * @return The code points in order. Never {@code null}.
- */
- private static int[] toCodePoints(String members) {
- final int[] codePoints = new int[members.codePointCount(0, members.length())];
- int i = 0;
- int out = 0;
- while (i < members.length()) {
- final int codePoint = members.codePointAt(i);
- codePoints[out++] = codePoint;
- i += Character.charCount(codePoint);
- }
- return codePoints;
- }
-
/**
* Tests a candidate stem against the condition at its anchored side: the last
* positions of the stem for a suffix condition, the first positions for a prefix
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java
index 7cc05bdb2..294103598 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java
@@ -1145,21 +1145,11 @@ public final class HunspellDictionary {
*/
static byte[] readBounded(InputStream in, int maxBytes, String label)
throws IOException {
- final byte[] chunk = new byte[8192];
- byte[] buffer = new byte[Math.min(8192, maxBytes)];
- int size = 0;
- int n;
- while ((n = in.read(chunk)) >= 0) {
- if (size + n > maxBytes) {
- throw new IOException(label + " size exceeds safe limit of " + maxBytes);
- }
- if (size + n > buffer.length) {
- buffer = Arrays.copyOf(buffer, Math.min(maxBytes, Math.max(buffer.length * 2, size + n)));
- }
- System.arraycopy(chunk, 0, buffer, size, n);
- size += n;
+ final byte[] bytes = in.readNBytes(maxBytes + 1);
+ if (bytes.length > maxBytes) {
+ throw new IOException(label + " size exceeds safe limit of " + maxBytes);
}
- return size == buffer.length ? buffer : Arrays.copyOf(buffer, size);
+ return bytes;
}
/**
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java
index 1e7bcc0e3..6bed8d62e 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java
@@ -40,21 +40,11 @@ import opennlp.tools.util.normalizer.OffsetAwareNormalizer;
* {@link TermVector} per distinct term, carrying the term string, its occurrence count,
* and (in {@link Mode#FULL full mode}) the occurrence offsets.
*
- * <p>Term identity comes from the annotator's inputs, not from logic of its own. Without
- * a normalizer, the term is the token layer's value as-is, that is, the token's covered
- * text in the original document. With a plain {@link CharSequenceNormalizer}, the general
- * path, each token's covered text is normalized on its own to produce the term, so any
- * normalizer works: case folding, NFC, accent folding, a stemmer-backed normalizer. With
- * an {@link OffsetAwareNormalizer}, the whole document text is normalized once with its
- * alignment recorded, each token span is mapped forward to the normalized form, and the
- * covered normalized text is the term; this path can see across token boundaries but is
- * limited to alignment-reporting normalizers. On every path, tokens that differ only by
- * a normalization fold (case, an eszett expansion, collapsed whitespace) group together,
- * and the occurrence spans emitted in {@link Mode#FULL full mode} are the token layer's
- * own spans and therefore always point into the original text. A token whose normalized
- * form is empty, for example one the normalizer deleted entirely, is omitted from the
- * layer; an empty string is no term, and the token layer still accounts for the
- * token.</p>
+ * <p>Term identity comes from the configured normalizer: without one, the term is the
+ * token's covered text; with a {@link CharSequenceNormalizer}, each token is normalized
+ * on its own; with an {@link OffsetAwareNormalizer}, the document is normalized once and
+ * each token span is mapped into the normalized text. Occurrence spans always point into
+ * the original text. A token whose term is empty is omitted.</p>
*
* <p>The layer is {@link LayerKey.Scope#DOCUMENT document-scoped}: each {@link TermVector}
* is a whole-document statistic, so the annotations carry no span of their own and the
@@ -278,6 +268,15 @@ public final class TermVectorAnnotator implements DocumentAnnotator {
return Set.of(TERM_VECTORS);
}
+ /**
+ * {@return the adapter's simple class name, which names it in pipeline validation
+ * messages}
+ */
+ @Override
+ public String toString() {
+ return getClass().getSimpleName();
+ }
+
/**
* Determines the term one token groups under: its covered text as-is, its covered text
* normalized on its own when a plain per-token normalizer is present, or the covered
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerAnnotator.java
index 366e54430..01184b9f1 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerAnnotator.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerAnnotator.java
@@ -38,6 +38,9 @@ import opennlp.tools.util.Span;
* is absent is the whole text tokenized at once. Either way, every token span refers to
* the original document text.</p>
*
+ * <p>The adapter holds no per-call state; it is as thread-safe as the tokenizer it
+ * wraps.</p>
+ *
* @since 3.0.0
*/
public final class TokenizerAnnotator implements DocumentAnnotator {
@@ -91,7 +94,7 @@ public final class TokenizerAnnotator implements DocumentAnnotator {
*/
private void addTokens(List<Annotation<String>> tokens, String text, int offset) {
for (final Span span : tokenizer.tokenizePos(text)) {
- final Span shifted = new Span(span.getStart() + offset, span.getEnd() + offset);
+ final Span shifted = new Span(span, offset);
tokens.add(new Annotation<>(shifted, span.getCoveredText(text).toString()));
}
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/uax29/WordType.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/uax29/WordType.java
index c44d7ef85..74d6dfc9e 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/uax29/WordType.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/uax29/WordType.java
@@ -18,6 +18,8 @@ package opennlp.tools.tokenize.uax29;
import java.util.BitSet;
+import opennlp.tools.util.normalizer.EmojiFlags;
+
/**
* The category of a {@linkplain WordTokenizer word token}. {@link #ALPHANUMERIC} and
* {@link #NUMERIC} cover letter and digit words; the remaining categories identify scripts and
@@ -50,9 +52,6 @@ public enum WordType {
/** An emoji, emoji sequence, or regional-indicator flag. */
EMOJI;
- private static final int REGIONAL_INDICATOR_FIRST = 0x1F1E6;
- private static final int REGIONAL_INDICATOR_LAST = 0x1F1FF;
-
// No code point below this can belong to a script-specific category (the lowest is Thai, U+0E00),
// so Latin, Greek, Cyrillic, and ASCII text skips the relatively costly script lookup entirely.
private static final int LOWEST_SCRIPT_CODE_POINT = 0x0E00;
@@ -112,7 +111,7 @@ public enum WordType {
}
continue;
}
- if (ExtendedPictographic.is(pictographs, codePoint) || isRegionalIndicator(codePoint)) {
+ if (ExtendedPictographic.is(pictographs, codePoint) || EmojiFlags.isRegionalIndicator(codePoint)) {
return EMOJI;
}
if (codePoint >= LOWEST_SCRIPT_CODE_POINT && script == null) {
@@ -136,10 +135,6 @@ public enum WordType {
return null;
}
- private static boolean isRegionalIndicator(int codePoint) {
- return codePoint >= REGIONAL_INDICATOR_FIRST && codePoint <= REGIONAL_INDICATOR_LAST;
- }
-
// Maps a code point to a script-specific token type, or null for scripts (Latin, Greek, ...) that
// fall through to the generic alphanumeric category.
private static WordType scriptType(int codePoint) {
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AsciiChars.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AsciiChars.java
index fcc24373d..98a01cd6b 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AsciiChars.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AsciiChars.java
@@ -79,4 +79,23 @@ final class AsciiChars {
static boolean caseInsensitiveEquals(int a, int b) {
return toLower(a) == toLower(b);
}
+
+ /**
+ * {@return whether the chars of {@code text} at {@code at} equal {@code literal} exactly}
+ *
+ * @param text The text to look into.
+ * @param at The index the comparison starts at.
+ * @param literal The chars to compare against.
+ */
+ static boolean regionMatches(CharSequence text, int at, String literal) {
+ if (at + literal.length() > text.length()) {
+ return false;
+ }
+ for (int k = 0; k < literal.length(); k++) {
+ if (text.charAt(at + k) != literal.charAt(k)) {
+ return false;
+ }
+ }
+ return true;
+ }
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/BulletCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/BulletCharSequenceNormalizer.java
index 81ad02b42..895167830 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/BulletCharSequenceNormalizer.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/BulletCharSequenceNormalizer.java
@@ -54,4 +54,9 @@ public class BulletCharSequenceNormalizer implements OffsetAwareNormalizer {
public AlignedText normalizeAligned(CharSequence text) {
return BULLETS.normalizeAligned(text);
}
+
+ /** {@return the shared instance, so deserialization keeps the singleton} */
+ private Object readResolve() {
+ return INSTANCE;
+ }
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ConfusableSkeletonCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ConfusableSkeletonCharSequenceNormalizer.java
index eaed23f77..57a4d5136 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ConfusableSkeletonCharSequenceNormalizer.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ConfusableSkeletonCharSequenceNormalizer.java
@@ -48,4 +48,9 @@ public class ConfusableSkeletonCharSequenceNormalizer implements CharSequenceNor
ParamChecks.requireNonNullArg(text, "text");
return Confusables.skeleton(text);
}
+
+ /** {@return the shared instance, so deserialization keeps the singleton} */
+ private Object readResolve() {
+ return INSTANCE;
+ }
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/DashCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/DashCharSequenceNormalizer.java
index 69431bae2..85ca067fe 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/DashCharSequenceNormalizer.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/DashCharSequenceNormalizer.java
@@ -49,4 +49,9 @@ public class DashCharSequenceNormalizer implements OffsetAwareNormalizer {
public AlignedText normalizeAligned(CharSequence text) {
return DASHES.normalizeAligned(text);
}
+
+ /** {@return the shared instance, so deserialization keeps the singleton} */
+ private Object readResolve() {
+ return INSTANCE;
+ }
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/DigitCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/DigitCharSequenceNormalizer.java
index 13dbeb7cd..4d25095ca 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/DigitCharSequenceNormalizer.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/DigitCharSequenceNormalizer.java
@@ -62,4 +62,9 @@ public class DigitCharSequenceNormalizer implements OffsetAwareNormalizer {
public AlignedText normalizeAligned(CharSequence text) {
return CharClass.substituteAligned(text, DigitCharSequenceNormalizer::toAscii);
}
+
+ /** {@return the shared instance, so deserialization keeps the singleton} */
+ private Object readResolve() {
+ return INSTANCE;
+ }
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiAnnotations.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiAnnotations.java
index 32f9ab7dc..3c1462b07 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiAnnotations.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiAnnotations.java
@@ -103,7 +103,7 @@ public final class EmojiAnnotations {
static String stripPresentationSelector(CharSequence symbol) {
final int length = symbol.length();
int i = 0;
- while (i < length && symbol.charAt(i) != 0xFE0F) {
+ while (i < length && symbol.charAt(i) != UnicodeEmojiSequences.VARIATION_SELECTOR_EMOJI) {
i++;
}
if (i == length) {
@@ -113,7 +113,7 @@ public final class EmojiAnnotations {
stripped.append(symbol, 0, i);
for (int k = i + 1; k < length; k++) {
final char c = symbol.charAt(k);
- if (c != 0xFE0F) {
+ if (c != UnicodeEmojiSequences.VARIATION_SELECTOR_EMOJI) {
stripped.append(c);
}
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiEmoticons.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiEmoticons.java
index 3ce7d0716..131c138fb 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiEmoticons.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiEmoticons.java
@@ -58,8 +58,6 @@ final class EmojiEmoticons {
private static final char MAPPING_CODE_POINT_SEPARATOR = ' ';
private static final int ZERO_WIDTH_JOINER = 0x200D;
- private static final int VARIATION_SELECTOR_TEXT = 0xFE0E;
- private static final int VARIATION_SELECTOR_EMOJI = 0xFE0F;
private static final BundledUnicodeData.Lazy<EmojiEmoticons> INSTANCE =
new BundledUnicodeData.Lazy<>(() -> new EmojiEmoticons(loadBundled()));
@@ -259,7 +257,7 @@ final class EmojiEmoticons {
for (int index = 0; index < candidates.size(); index++) {
final Mapping candidate = candidates.get(index);
int end = i + candidate.source().length();
- if (end > text.length() || !regionMatches(text, i, candidate.source())) {
+ if (!AsciiChars.regionMatches(text, i, candidate.source())) {
continue;
}
if (delimited) {
@@ -269,12 +267,14 @@ final class EmojiEmoticons {
continue;
}
// Absorb one trailing emoji-presentation selector into the fold.
- if (end < text.length() && Character.codePointAt(text, end) == VARIATION_SELECTOR_EMOJI) {
+ if (end < text.length()
+ && Character.codePointAt(text, end) == UnicodeEmojiSequences.VARIATION_SELECTOR_EMOJI) {
end++;
}
if (end < text.length()) {
final int following = Character.codePointAt(text, end);
- if (following == ZERO_WIDTH_JOINER || following == VARIATION_SELECTOR_TEXT) {
+ if (following == ZERO_WIDTH_JOINER
+ || following == UnicodeEmojiSequences.VARIATION_SELECTOR_TEXT) {
continue;
}
}
@@ -303,23 +303,6 @@ final class EmojiEmoticons {
return end == text.length() || CharClass.whitespace().contains(Character.codePointAt(text, end));
}
- /**
- * {@return whether {@code text} contains exactly {@code source} starting at {@code start}}
- *
- * @param text the text being scanned.
- * @param start the position to compare from. The caller guarantees
- * {@code start + source.length() <= text.length()}.
- * @param source the sequence to compare against.
- */
- private static boolean regionMatches(CharSequence text, int start, String source) {
- for (int k = 0; k < source.length(); k++) {
- if (text.charAt(start + k) != source.charAt(k)) {
- return false;
- }
- }
- return true;
- }
-
/**
* {@return the tables parsed from the bundled {@code emoji-emoticons.txt} resource}
*
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiFlags.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiFlags.java
index dfa4316a0..40dc56348 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiFlags.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiFlags.java
@@ -57,7 +57,7 @@ public final class EmojiFlags {
*
* @param codePoint The code point to test.
*/
- private static boolean isRegionalIndicator(int codePoint) {
+ public static boolean isRegionalIndicator(int codePoint) {
return codePoint >= FIRST_REGIONAL_INDICATOR && codePoint <= LAST_REGIONAL_INDICATOR;
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiToEmoticonCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiToEmoticonCharSequenceNormalizer.java
index 212c1a848..fb2cac5d6 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiToEmoticonCharSequenceNormalizer.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiToEmoticonCharSequenceNormalizer.java
@@ -58,4 +58,9 @@ public final class EmojiToEmoticonCharSequenceNormalizer implements OffsetAwareN
ParamChecks.requireNonNullArg(text, "text");
return EmojiEmoticons.getInstance().emojiToEmoticonAligned(text);
}
+
+ /** {@return the shared instance, so deserialization keeps the singleton} */
+ private Object readResolve() {
+ return INSTANCE;
+ }
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmoticonToEmojiCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmoticonToEmojiCharSequenceNormalizer.java
index cba7c237a..400557f68 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmoticonToEmojiCharSequenceNormalizer.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmoticonToEmojiCharSequenceNormalizer.java
@@ -59,4 +59,9 @@ public final class EmoticonToEmojiCharSequenceNormalizer implements OffsetAwareN
ParamChecks.requireNonNullArg(text, "text");
return EmojiEmoticons.getInstance().emoticonToEmojiAligned(text);
}
+
+ /** {@return the shared instance, so deserialization keeps the singleton} */
+ private Object readResolve() {
+ return INSTANCE;
+ }
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/FullCaseFoldCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/FullCaseFoldCharSequenceNormalizer.java
index 4c79ebb52..4552153c2 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/FullCaseFoldCharSequenceNormalizer.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/FullCaseFoldCharSequenceNormalizer.java
@@ -173,4 +173,9 @@ public final class FullCaseFoldCharSequenceNormalizer implements OffsetAwareNorm
});
return map;
}
+
+ /** {@return the shared instance, so deserialization keeps the singleton} */
+ private Object readResolve() {
+ return INSTANCE;
+ }
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/GermanUmlautCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/GermanUmlautCharSequenceNormalizer.java
index dc2f994d8..2794fed7e 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/GermanUmlautCharSequenceNormalizer.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/GermanUmlautCharSequenceNormalizer.java
@@ -83,4 +83,9 @@ public class GermanUmlautCharSequenceNormalizer implements OffsetAwareNormalizer
default -> null;
};
}
+
+ /** {@return the shared instance, so deserialization keeps the singleton} */
+ private Object readResolve() {
+ return INSTANCE;
+ }
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NfcCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NfcCharSequenceNormalizer.java
index 16c30240f..7d5807c36 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NfcCharSequenceNormalizer.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NfcCharSequenceNormalizer.java
@@ -46,4 +46,9 @@ public class NfcCharSequenceNormalizer implements CharSequenceNormalizer {
ParamChecks.requireNonNullArg(text, "text");
return Normalizer.normalize(text, Normalizer.Form.NFC);
}
+
+ /** {@return the shared instance, so deserialization keeps the singleton} */
+ private Object readResolve() {
+ return INSTANCE;
+ }
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NfkcCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NfkcCharSequenceNormalizer.java
index 3aec2bf3f..fdd6ccc3a 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NfkcCharSequenceNormalizer.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NfkcCharSequenceNormalizer.java
@@ -47,4 +47,9 @@ public class NfkcCharSequenceNormalizer implements CharSequenceNormalizer {
ParamChecks.requireNonNullArg(text, "text");
return Normalizer.normalize(text, Normalizer.Form.NFKC);
}
+
+ /** {@return the shared instance, so deserialization keeps the singleton} */
+ private Object readResolve() {
+ return INSTANCE;
+ }
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/QuoteCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/QuoteCharSequenceNormalizer.java
index 45ca730be..330205ead 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/QuoteCharSequenceNormalizer.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/QuoteCharSequenceNormalizer.java
@@ -73,4 +73,9 @@ public class QuoteCharSequenceNormalizer implements OffsetAwareNormalizer {
final AlignedText single = SINGLE.normalizeAligned(text);
return single.andThen(DOUBLE.normalizeAligned(single.normalized()));
}
+
+ /** {@return the shared instance, so deserialization keeps the singleton} */
+ private Object readResolve() {
+ return INSTANCE;
+ }
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/SocialMediaCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/SocialMediaCharSequenceNormalizer.java
index 124222034..3c3cb26f9 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/SocialMediaCharSequenceNormalizer.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/SocialMediaCharSequenceNormalizer.java
@@ -170,10 +170,7 @@ public class SocialMediaCharSequenceNormalizer implements CharSequenceNormalizer
/** {@return whether {@code codePoint} is an ASCII letter, digit, or underscore} */
private boolean isAsciiWord(int codePoint) {
- return codePoint >= 'a' && codePoint <= 'z'
- || codePoint >= 'A' && codePoint <= 'Z'
- || codePoint >= '0' && codePoint <= '9'
- || codePoint == '_';
+ return codePoint == '_' || AsciiChars.ALPHANUMERIC.contains(codePoint);
}
/**
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TermAnalyzer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TermAnalyzer.java
index c2e31078c..02fb1a0ae 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TermAnalyzer.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TermAnalyzer.java
@@ -487,7 +487,10 @@ public final class TermAnalyzer {
}
/**
- * Sets the tokenizer used by {@link TermAnalyzer#analyze(CharSequence)}.
+ * Sets the tokenizer used by {@link TermAnalyzer#analyze(CharSequence)}. The parameter is
+ * the UAX #29 {@link WordTokenizer} on purpose: the analyzer segments any
+ * {@link CharSequence} without copying it to a {@link String}, and its terms are defined
+ * by UAX #29 word boundaries rather than by an arbitrary tokenizer.
*
* @param value The tokenizer. Must not be {@code null}.
* @return this builder
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UnicodeEmojiSequences.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UnicodeEmojiSequences.java
index 86837083f..0106bca28 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UnicodeEmojiSequences.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UnicodeEmojiSequences.java
@@ -60,10 +60,10 @@ final class UnicodeEmojiSequences {
* U+FE0E asks for text presentation. It has no {@code Emoji_Component} property but it binds
* to the symbol before it the same way U+FE0F does, so it counts as a structural component.
*/
- private static final int VARIATION_SELECTOR_TEXT = 0xFE0E;
+ static final int VARIATION_SELECTOR_TEXT = 0xFE0E;
/** U+FE0F asks for emoji presentation. */
- private static final int VARIATION_SELECTOR_EMOJI = 0xFE0F;
+ static final int VARIATION_SELECTOR_EMOJI = 0xFE0F;
/**
* The bases of the keycap sequences: {@code #}, {@code *} and the ASCII digits. They have the
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizer.java
index 1b2348252..2a16a1f23 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizer.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizer.java
@@ -313,14 +313,14 @@ public class UrlCharSequenceNormalizer implements CharSequenceNormalizer {
*/
private int matchLegacyUrlEnd(CharSequence text, int start) {
final int length = text.length();
- if (!regionEquals(text, start, HTTP)) {
+ if (!AsciiChars.regionMatches(text, start, HTTP)) {
return -1;
}
int bodyStart = start + HTTP.length();
if (bodyStart < length && text.charAt(bodyStart) == 's'
- && regionEquals(text, bodyStart + 1, SCHEME_SEPARATOR)) {
+ && AsciiChars.regionMatches(text, bodyStart + 1, SCHEME_SEPARATOR)) {
bodyStart += 1 + SCHEME_SEPARATOR.length();
- } else if (regionEquals(text, bodyStart, SCHEME_SEPARATOR)) {
+ } else if (AsciiChars.regionMatches(text, bodyStart, SCHEME_SEPARATOR)) {
bodyStart += SCHEME_SEPARATOR.length();
} else {
return -1;
@@ -334,23 +334,4 @@ public class UrlCharSequenceNormalizer implements CharSequenceNormalizer {
}
return end;
}
-
- /**
- * {@return whether the chars at {@code at} equal {@code literal} exactly}
- *
- * @param text The text to look into; never null.
- * @param at The index the comparison starts at.
- * @param literal The chars to compare against; never null.
- */
- private boolean regionEquals(CharSequence text, int at, String literal) {
- if (at + literal.length() > text.length()) {
- return false;
- }
- for (int k = 0; k < literal.length(); k++) {
- if (text.charAt(at + k) != literal.charAt(k)) {
- return false;
- }
- }
- return true;
- }
}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/chunker/ChunkerAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/chunker/ChunkerAnnotatorTest.java
index 3214c1389..bdc7e416d 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/chunker/ChunkerAnnotatorTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/chunker/ChunkerAnnotatorTest.java
@@ -26,6 +26,7 @@ import org.junit.jupiter.api.Test;
import opennlp.tools.document.Annotation;
import opennlp.tools.document.Document;
+import opennlp.tools.document.DocumentTestStubs;
import opennlp.tools.document.Layers;
import opennlp.tools.util.Sequence;
import opennlp.tools.util.Span;
@@ -78,17 +79,6 @@ public class ChunkerAnnotatorTest {
}
}
- private static List<Annotation<String>> tokens(String text, String... forms) {
- final List<Annotation<String>> annotations = new ArrayList<>(forms.length);
- int cursor = 0;
- for (final String form : forms) {
- final int start = text.indexOf(form, cursor);
- annotations.add(new Annotation<>(new Span(start, start + form.length()), form));
- cursor = start + form.length();
- }
- return annotations;
- }
-
private static List<Annotation<String>> values(List<Annotation<String>> tokens,
String... tags) {
final List<Annotation<String>> annotations = new ArrayList<>(tags.length);
@@ -101,7 +91,7 @@ public class ChunkerAnnotatorTest {
/** Two sentences whose noun runs straddle neither sentence boundary. */
private static Document twoSentences() {
final String text = "Mary Jones leads Acme. She joined Acme Corp.";
- final List<Annotation<String>> toks = tokens(text,
+ final List<Annotation<String>> toks = DocumentTestStubs.tokens(text,
"Mary", "Jones", "leads", "Acme", ".", "She", "joined", "Acme", "Corp", ".");
return Document.of(text)
.with(Layers.SENTENCES, List.of(
@@ -161,7 +151,7 @@ public class ChunkerAnnotatorTest {
() -> annotator.annotate(null));
final Document untagged = Document.of("Mary.")
.with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 5), "s")))
- .with(Layers.TOKENS, tokens("Mary.", "Mary", "."));
+ .with(Layers.TOKENS, DocumentTestStubs.tokens("Mary.", "Mary", "."));
Assertions.assertThrows(IllegalArgumentException.class,
() -> annotator.annotate(untagged));
}
@@ -169,7 +159,7 @@ public class ChunkerAnnotatorTest {
@Test
void testRejectsMisalignedTagLayer() {
final String text = "Mary.";
- final List<Annotation<String>> toks = tokens(text, "Mary", ".");
+ final List<Annotation<String>> toks = DocumentTestStubs.tokens(text, "Mary", ".");
final Document document = Document.of(text)
.with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 5), "s")))
.with(Layers.TOKENS, toks)
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java
index 7af7a7516..63f6d1cc5 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java
@@ -29,6 +29,7 @@ import opennlp.tools.namefind.TokenNameFinder;
import opennlp.tools.postag.POSTagger;
import opennlp.tools.postag.POSTaggerAnnotator;
import opennlp.tools.sentdetect.SentenceDetectorAnnotator;
+import opennlp.tools.tokenize.Tokenizer;
import opennlp.tools.tokenize.TokenizerAnnotator;
import opennlp.tools.util.Sequence;
import opennlp.tools.util.Span;
@@ -39,7 +40,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Tests the {@link DocumentAnalyzer} pipeline over the adapter annotators, using the
- * deterministic components from {@link TestComponents} and a fixed-vocabulary tagger.
+ * deterministic components from {@link DocumentTestStubs} and a fixed-vocabulary tagger.
* The point under test is the pipeline mechanics and span arithmetic, not model quality.
*/
public class DocumentAnalyzerTest {
@@ -94,9 +95,9 @@ public class DocumentAnalyzerTest {
@Test
void testAdaptersNameThemselvesByClassName() {
assertEquals("SentenceDetectorAnnotator",
- new SentenceDetectorAnnotator(TestComponents.PERIOD_SPLITTER).toString());
+ new SentenceDetectorAnnotator(DocumentTestStubs.PERIOD_SPLITTER).toString());
assertEquals("TokenizerAnnotator",
- new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER).toString());
+ new TokenizerAnnotator(DocumentTestStubs.SPACE_TOKENIZER).toString());
assertEquals("POSTaggerAnnotator", new POSTaggerAnnotator(TAGGER).toString());
assertEquals("NameFinderAnnotator", new NameFinderAnnotator(NO_NAMES).toString());
}
@@ -104,8 +105,8 @@ public class DocumentAnalyzerTest {
@Test
void testPipelineProducesAlignedLayersInOriginalCoordinates() {
final Document document = DocumentAnalyzer.builder()
- .add(new SentenceDetectorAnnotator(TestComponents.PERIOD_SPLITTER))
- .add(new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER))
+ .add(new SentenceDetectorAnnotator(DocumentTestStubs.PERIOD_SPLITTER))
+ .add(new TokenizerAnnotator(DocumentTestStubs.SPACE_TOKENIZER))
.add(new POSTaggerAnnotator(TAGGER))
.build()
.analyze("the dog barks. she eats.");
@@ -135,8 +136,8 @@ public class DocumentAnalyzerTest {
@ValueSource(strings = {"", " "})
void testEmptyAndBlankInputProduceEmptyLayers(String text) {
final DocumentAnalyzer analyzer = DocumentAnalyzer.builder()
- .add(new SentenceDetectorAnnotator(TestComponents.PERIOD_SPLITTER))
- .add(new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER))
+ .add(new SentenceDetectorAnnotator(DocumentTestStubs.PERIOD_SPLITTER))
+ .add(new TokenizerAnnotator(DocumentTestStubs.SPACE_TOKENIZER))
.add(new POSTaggerAnnotator(TAGGER))
.add(new NameFinderAnnotator(NO_NAMES))
.build();
@@ -155,16 +156,42 @@ public class DocumentAnalyzerTest {
*/
@Test
void testTokenizerHonorsPresentButEmptySentenceLayer() {
- final Document document = new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER)
+ final Document document = new TokenizerAnnotator(DocumentTestStubs.SPACE_TOKENIZER)
.annotate(Document.of("the dog").with(Layers.SENTENCES, List.of()));
assertTrue(document.layers().contains(Layers.TOKENS));
assertTrue(document.get(Layers.TOKENS).isEmpty());
}
+ /**
+ * Verifies that shifting a token span into document coordinates keeps the type and
+ * probability the tokenizer reported.
+ */
+ @Test
+ void testTokenizerKeepsSpanTypeAndProbabilityWhenShifting() {
+ final Tokenizer typed = new Tokenizer() {
+
+ @Override
+ public String[] tokenize(String s) {
+ throw new UnsupportedOperationException("the adapter only calls tokenizePos");
+ }
+
+ @Override
+ public Span[] tokenizePos(String s) {
+ return new Span[] {new Span(0, s.length(), "word", 0.5)};
+ }
+ };
+ final Document document = new TokenizerAnnotator(typed)
+ .annotate(Document.of("ab cd").with(Layers.SENTENCES, List.of(
+ new Annotation<>(new Span(0, 2), "ab"), new Annotation<>(new Span(3, 5), "cd"))));
+ final Span shifted = document.get(Layers.TOKENS).get(1).span();
+ assertEquals(new Span(3, 5, "word"), shifted);
+ assertEquals(0.5, shifted.getProb());
+ }
+
@Test
void testTokenizerWorksWithoutSentences() {
final Document document = DocumentAnalyzer.builder()
- .add(new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER))
+ .add(new TokenizerAnnotator(DocumentTestStubs.SPACE_TOKENIZER))
.build()
.analyze("the dog");
assertEquals(2, document.get(Layers.TOKENS).size());
@@ -193,8 +220,8 @@ public class DocumentAnalyzerTest {
@Test
void testAnnotatorAdaptersRejectNullDocuments() {
final List<DocumentAnnotator> adapters = List.of(
- new SentenceDetectorAnnotator(TestComponents.PERIOD_SPLITTER),
- new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER),
+ new SentenceDetectorAnnotator(DocumentTestStubs.PERIOD_SPLITTER),
+ new TokenizerAnnotator(DocumentTestStubs.SPACE_TOKENIZER),
new POSTaggerAnnotator(TAGGER),
new NameFinderAnnotator(NO_NAMES));
for (final DocumentAnnotator adapter : adapters) {
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java
index f38737d66..c9b66bfbc 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java
@@ -41,7 +41,7 @@ import static org.junit.jupiter.api.Assertions.assertNull;
* its spans in original text coordinates.
*
* <p>The wrapped components are the tiny deterministic stand-ins from
- * {@link TestComponents}, so every expected span and value below follows directly from
+ * {@link DocumentTestStubs}, so every expected span and value below follows directly from
* the input text. The point under demonstration is how the layers connect, not the
* quality of any single step.</p>
*/
@@ -154,8 +154,8 @@ public class DocumentPipelineExampleTest {
@Test
void testFullPipelineStory() {
final DocumentAnalyzer analyzer = DocumentAnalyzer.builder()
- .add(new SentenceDetectorAnnotator(TestComponents.PERIOD_SPLITTER))
- .add(new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER))
+ .add(new SentenceDetectorAnnotator(DocumentTestStubs.PERIOD_SPLITTER))
+ .add(new TokenizerAnnotator(DocumentTestStubs.SPACE_TOKENIZER))
.add(new POSTaggerAnnotator(DICTIONARY_TAGGER))
.add(new TokenLengthAnnotator())
.build();
@@ -229,8 +229,8 @@ public class DocumentPipelineExampleTest {
@Test
void testAnalyzerIsReusableAcrossTexts() {
final DocumentAnalyzer analyzer = DocumentAnalyzer.builder()
- .add(new SentenceDetectorAnnotator(TestComponents.PERIOD_SPLITTER))
- .add(new TokenizerAnnotator(TestComponents.SPACE_TOKENIZER))
+ .add(new SentenceDetectorAnnotator(DocumentTestStubs.PERIOD_SPLITTER))
+ .add(new TokenizerAnnotator(DocumentTestStubs.SPACE_TOKENIZER))
.build();
final Document first = analyzer.analyze("The dog barks.");
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/TestComponents.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentTestStubs.java
similarity index 76%
rename from opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/TestComponents.java
rename to opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentTestStubs.java
index 6145e5b79..61d88d1f4 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/TestComponents.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/DocumentTestStubs.java
@@ -28,7 +28,7 @@ import opennlp.tools.util.Span;
* Deterministic stand-in components shared by the document pipeline tests, so every
* expected span in those tests follows directly from the definitions here.
*/
-final class TestComponents {
+public final class DocumentTestStubs {
/**
* A deterministic sentence detector that ends a sentence after every period and
@@ -62,7 +62,7 @@ final class TestComponents {
* other characters, including sentence-final periods, attached to their token. Only
* the span-producing method is implemented because the adapter calls no other method.
*/
- static final Tokenizer SPACE_TOKENIZER = new Tokenizer() {
+ public static final Tokenizer SPACE_TOKENIZER = new Tokenizer() {
@Override
public String[] tokenize(String s) {
@@ -86,7 +86,26 @@ final class TestComponents {
}
};
- private TestComponents() {
+ /**
+ * Builds a token layer by locating each form in {@code text}, searching from the end of
+ * the previous form, so tokens may be separated by any run of other characters.
+ *
+ * @param text The document text.
+ * @param forms The token forms, in text order.
+ * @return One annotation per form, valued with the form.
+ */
+ public static List<Annotation<String>> tokens(String text, String... forms) {
+ final List<Annotation<String>> annotations = new ArrayList<>(forms.length);
+ int cursor = 0;
+ for (final String form : forms) {
+ final int start = text.indexOf(form, cursor);
+ annotations.add(new Annotation<>(new Span(start, start + form.length()), form));
+ cursor = start + form.length();
+ }
+ return annotations;
+ }
+
+ private DocumentTestStubs() {
// Not instantiated; this class provides shared test fixtures only.
}
}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/parser/ParserAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/parser/ParserAnnotatorTest.java
index dee78a2ef..8259937b6 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/parser/ParserAnnotatorTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/parser/ParserAnnotatorTest.java
@@ -17,7 +17,6 @@
package opennlp.tools.parser;
-import java.util.ArrayList;
import java.util.List;
import java.util.Set;
@@ -26,6 +25,7 @@ import org.junit.jupiter.api.Test;
import opennlp.tools.document.Annotation;
import opennlp.tools.document.Document;
+import opennlp.tools.document.DocumentTestStubs;
import opennlp.tools.document.Layers;
import opennlp.tools.parser.ParserAnnotator.Phrase;
import opennlp.tools.util.Span;
@@ -77,23 +77,12 @@ public class ParserAnnotatorTest {
}
}
- private static List<Annotation<String>> tokens(String text, String... forms) {
- final List<Annotation<String>> annotations = new ArrayList<>(forms.length);
- int cursor = 0;
- for (final String form : forms) {
- final int start = text.indexOf(form, cursor);
- annotations.add(new Annotation<>(new Span(start, start + form.length()), form));
- cursor = start + form.length();
- }
- return annotations;
- }
-
/** One six-token sentence whose text carries a double space the parse text lacks. */
private static Document sentence() {
final String text = "The dog of Mary ran.";
return Document.of(text)
.with(Layers.SENTENCES, List.of(new Annotation<>(new Span(0, 21), "s")))
- .with(Layers.TOKENS, tokens(text, "The", "dog", "of", "Mary", "ran", "."));
+ .with(Layers.TOKENS, DocumentTestStubs.tokens(text, "The", "dog", "of", "Mary", "ran", "."));
}
@Test
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/postag/POSDictionaryTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/postag/POSDictionaryTest.java
index 440f00296..b3cf9b764 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/postag/POSDictionaryTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/postag/POSDictionaryTest.java
@@ -26,6 +26,8 @@ import java.nio.charset.StandardCharsets;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import opennlp.tools.util.InvalidFormatException;
+
/**
* Tests for the {@link POSDictionary} class.
*/
@@ -80,6 +82,14 @@ public class POSDictionaryTest {
Assertions.assertArrayEquals(new String[] {"NN", "NNS", "VBP"}, dict.getTags("run"));
}
+ @Test
+ void testLoadingEntryWithoutTagsAttributeThrowsInvalidFormatException() {
+ final String xml = "<dictionary><entry><token>run</token></entry></dictionary>";
+
+ Assertions.assertThrows(InvalidFormatException.class, () ->
+ POSDictionary.create(new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))));
+ }
+
@Test
void testCaseSensitiveDictionary() throws IOException {
POSDictionary dict = loadDictionary("TagDictionaryCaseSensitive.xml");
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerFactoryTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerFactoryTest.java
index 9a19d0110..ccc848f55 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerFactoryTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerFactoryTest.java
@@ -342,6 +342,10 @@ class StemmerFactoryTest {
() -> new CachingStemmer(factory).stem(null));
Assertions.assertThrows(IllegalArgumentException.class,
() -> new SharingStemmer(factory).stem(null));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new CachingStemmer(factory).stemAll(null));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new SharingStemmer(factory).stemAll(null));
Assertions.assertThrows(IllegalArgumentException.class,
() -> new PorterStemmer().stem((CharSequence) null));
}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java
index 4f7beb464..f8d8a328a 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java
@@ -21,7 +21,7 @@ import java.util.ArrayList;
import java.util.List;
import opennlp.tools.document.Annotation;
-import opennlp.tools.tokenize.Tokenizer;
+import opennlp.tools.document.DocumentTestStubs;
import opennlp.tools.util.Span;
/**
@@ -32,57 +32,19 @@ import opennlp.tools.util.Span;
*/
final class SingleSpaceTokens {
- /**
- * A {@link Tokenizer} view of {@link #spans(String)}. Only the span-producing method
- * is implemented because the annotator adapter calls no other method.
- */
- static final Tokenizer TOKENIZER = new Tokenizer() {
-
- @Override
- public String[] tokenize(String s) {
- throw new UnsupportedOperationException("the adapter only calls tokenizePos");
- }
-
- @Override
- public Span[] tokenizePos(String s) {
- return spans(s).toArray(new Span[0]);
- }
- };
-
private SingleSpaceTokens() {
}
/**
- * Computes the token spans of a text split on single space characters.
- *
- * @param text The text to split.
- * @return One span per token, in text order.
- */
- static List<Span> spans(String text) {
- final List<Span> spans = new ArrayList<>();
- int start = -1;
- for (int i = 0; i <= text.length(); i++) {
- final boolean boundary = i == text.length() || text.charAt(i) == ' ';
- if (boundary && start >= 0) {
- spans.add(new Span(start, i));
- start = -1;
- } else if (!boundary && start < 0) {
- start = i;
- }
- }
- return spans;
- }
-
- /**
- * Builds a token layer from {@link #spans(String)}, each token valued with its covered
- * text.
+ * Builds a token layer from {@link DocumentTestStubs#SPACE_TOKENIZER}, each token valued
+ * with its covered text.
*
* @param text The text to split.
* @return One annotation per token, in text order.
*/
static List<Annotation<String>> tokens(String text) {
final List<Annotation<String>> tokens = new ArrayList<>();
- for (final Span span : spans(text)) {
+ for (final Span span : DocumentTestStubs.SPACE_TOKENIZER.tokenizePos(text)) {
tokens.add(new Annotation<>(span, text.substring(span.getStart(), span.getEnd())));
}
return tokens;
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java
index 33559371f..6ae67c80b 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java
@@ -421,4 +421,9 @@ public class TermVectorAnnotatorTest {
assertEquals(Set.of(Layers.TOKENS), annotator.requires());
assertEquals(Set.of(TermVectorAnnotator.TERM_VECTORS), annotator.provides());
}
+
+ @Test
+ void testToStringNamesTheAnnotator() {
+ assertEquals("TermVectorAnnotator", new TermVectorAnnotator().toString());
+ }
}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java
index 8510cfdac..ea6b86055 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java
@@ -25,6 +25,7 @@ import org.junit.jupiter.api.Test;
import opennlp.tools.document.Annotation;
import opennlp.tools.document.Document;
import opennlp.tools.document.DocumentAnalyzer;
+import opennlp.tools.document.DocumentTestStubs;
import opennlp.tools.document.Layers;
import opennlp.tools.tokenize.TokenizerAnnotator;
import opennlp.tools.util.Span;
@@ -47,7 +48,7 @@ public class TermVectorPipelineTest {
@Test
void testTokenizerAndTermVectorPipeline() {
final DocumentAnalyzer analyzer = DocumentAnalyzer.builder()
- .add(new TokenizerAnnotator(SingleSpaceTokens.TOKENIZER))
+ .add(new TokenizerAnnotator(DocumentTestStubs.SPACE_TOKENIZER))
.add(new TermVectorAnnotator())
.build();
@@ -82,7 +83,7 @@ public class TermVectorPipelineTest {
@Test
void testScoringOnlyPipeline() {
final DocumentAnalyzer analyzer = DocumentAnalyzer.builder()
- .add(new TokenizerAnnotator(SingleSpaceTokens.TOKENIZER))
+ .add(new TokenizerAnnotator(DocumentTestStubs.SPACE_TOKENIZER))
.add(new TermVectorAnnotator(TermVectorAnnotator.Mode.SCORING_ONLY))
.build();
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordBreakPropertyTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordBreakPropertyTest.java
index fa2ee00b3..bc84f5c98 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordBreakPropertyTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordBreakPropertyTest.java
@@ -87,7 +87,7 @@ public class WordBreakPropertyTest {
@Test
void testFromPropertyNameRejectsUnknown() {
assertEquals(WordBreak.ALETTER, WordBreak.fromPropertyName("ALetter"));
- org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class,
+ assertThrows(IllegalArgumentException.class,
() -> WordBreak.fromPropertyName("NotAValue"));
}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordSegmenterTest.java
index 6e120b7be..6bdd4121d 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordSegmenterTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordSegmenterTest.java
@@ -23,15 +23,12 @@ import org.junit.jupiter.api.Test;
import opennlp.tools.util.Span;
+import static opennlp.tools.util.normalizer.NormalizerTestUtil.cp;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class WordSegmenterTest {
- private static String cp(int codePoint) {
- return new String(Character.toChars(codePoint));
- }
-
private static List<String> words(String text) {
final List<String> out = new ArrayList<>();
for (final Span span : WordSegmenter.segments(text)) {
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordTokenizerTest.java
index bd26fb8f0..a4866a836 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordTokenizerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordTokenizerTest.java
@@ -23,6 +23,7 @@ import org.junit.jupiter.api.Test;
import opennlp.tools.tokenize.Tokenizer;
import opennlp.tools.util.Span;
+import static opennlp.tools.util.normalizer.NormalizerTestUtil.cp;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -31,10 +32,6 @@ public class WordTokenizerTest {
private static final WordTokenizer TOKENIZER = new WordTokenizer();
- private static String cp(int codePoint) {
- return new String(Character.toChars(codePoint));
- }
-
private static List<String> words(String text) {
return List.of(TOKENIZER.tokenize(text));
}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordTypeTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordTypeTest.java
index 1cd5a1f06..c23b62086 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordTypeTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordTypeTest.java
@@ -18,6 +18,7 @@ package opennlp.tools.tokenize.uax29;
import org.junit.jupiter.api.Test;
+import static opennlp.tools.util.normalizer.NormalizerTestUtil.cp;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
@@ -27,10 +28,6 @@ public class WordTypeTest {
return WordType.of(token, 0, token.length());
}
- private static String cp(int codePoint) {
- return new String(Character.toChars(codePoint));
- }
-
@Test
void lettersClassifyAlphanumeric() {
assertEquals(WordType.ALPHANUMERIC, of("great"));
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/InstallerTestSupport.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/InstallerTestSupport.java
index a341bc9c2..cd22e9e80 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/InstallerTestSupport.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/InstallerTestSupport.java
@@ -58,48 +58,6 @@ final class InstallerTestSupport {
TarArchives.entry(tar, name, content);
}
- /**
- * Builds a gzip-compressed tar archive from name and content pairs.
- *
- * @param entries Pairs of entry name and UTF-8 text content. Must not be {@code null}.
- * @return The archive bytes. Never {@code null}.
- * @throws IOException Thrown if assembling the archive fails.
- */
- static byte[] tarGz(String[][] entries) throws IOException {
- return TarArchives.gzippedTar(entries);
- }
-
- /**
- * Compresses the given bytes with gzip.
- *
- * @param content The bytes to compress. Must not be {@code null}.
- * @return The gzip-compressed bytes. Never {@code null}.
- * @throws IOException Thrown if compressing fails.
- */
- static byte[] gzip(byte[] content) throws IOException {
- return TarArchives.gzip(content);
- }
-
- /**
- * Computes the SHA-256 of the given bytes as a lowercase hex string.
- *
- * @param content The bytes to digest. Must not be {@code null}.
- * @return The 64-character lowercase hex digest. Never {@code null}.
- */
- static String sha256(byte[] content) {
- return DigestTestUtil.sha256(content);
- }
-
- /**
- * Computes the SHA-512 of the given bytes as a lowercase hex string.
- *
- * @param content The bytes to digest. Must not be {@code null}.
- * @return The 128-character lowercase hex digest. Never {@code null}.
- */
- static String sha512(byte[] content) {
- return DigestTestUtil.sha512(content);
- }
-
/**
* Lists every regular file below the given directory as relative paths with forward
* slashes, sorted lexicographically, so tests can assert the exact installed file
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ResourceInstallerHttpTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ResourceInstallerHttpTest.java
index 7e27bb528..578132060 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ResourceInstallerHttpTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ResourceInstallerHttpTest.java
@@ -46,11 +46,11 @@ import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
+import static opennlp.tools.util.DigestTestUtil.sha256;
import static opennlp.tools.util.InstallerTestSupport.KIBIBYTE;
import static opennlp.tools.util.InstallerTestSupport.MEBIBYTE;
import static opennlp.tools.util.InstallerTestSupport.installedFiles;
-import static opennlp.tools.util.InstallerTestSupport.sha256;
-import static opennlp.tools.util.InstallerTestSupport.tarGz;
+import static opennlp.tools.util.archive.TarArchives.gzippedTar;
/**
* Exercises {@link ResourceInstaller} against a local scripted HTTP server: happy
@@ -145,7 +145,7 @@ public class ResourceInstallerHttpTest {
final AtomicBoolean fetched = new AtomicBoolean();
server.route("/corpus.tar.gz", out -> {
fetched.set(true);
- StubServer.ok(out, tarGz(new String[][] {{"corpus/data.txt", "unverified"}}));
+ StubServer.ok(out, gzippedTar(new String[][] {{"corpus/data.txt", "unverified"}}));
});
final URI source = server.uri("/corpus.tar.gz");
@@ -172,7 +172,7 @@ public class ResourceInstallerHttpTest {
@Test
void testHttpDownloadInstallsArchive(@TempDir Path target) throws Exception {
- final byte[] archive = tarGz(new String[][] {{"corpus/data.txt", "over http"}});
+ final byte[] archive = gzippedTar(new String[][] {{"corpus/data.txt", "over http"}});
server.route("/corpus.tar.gz", out -> StubServer.ok(out, archive));
ResourceInstaller.install(server.uri("/corpus.tar.gz"), target, sha256(archive));
@@ -204,7 +204,7 @@ public class ResourceInstallerHttpTest {
@Test
void testAbsoluteRedirectIsFollowed(@TempDir Path target) throws Exception {
- final byte[] archive = tarGz(new String[][] {{"corpus/data.txt", "moved"}});
+ final byte[] archive = gzippedTar(new String[][] {{"corpus/data.txt", "moved"}});
server.route("/old.tar.gz", out -> StubServer.redirect(out,
server.uri("/new.tar.gz").toString()));
server.route("/new.tar.gz", out -> StubServer.ok(out, archive));
@@ -218,7 +218,7 @@ public class ResourceInstallerHttpTest {
@Test
void testRelativeRedirectIsResolvedAgainstTheSource(@TempDir Path target)
throws Exception {
- final byte[] archive = tarGz(new String[][] {{"corpus/data.txt", "relative"}});
+ final byte[] archive = gzippedTar(new String[][] {{"corpus/data.txt", "relative"}});
server.route("/mirror/old.tar.gz",
out -> StubServer.redirect(out, "new.tar.gz"));
server.route("/mirror/new.tar.gz", out -> StubServer.ok(out, archive));
@@ -234,7 +234,7 @@ public class ResourceInstallerHttpTest {
"307 Temporary Redirect", "308 Permanent Redirect"})
void testEveryRedirectStatusIsFollowed(String status, @TempDir Path target)
throws Exception {
- final byte[] archive = tarGz(new String[][] {{"corpus/data.txt", "followed"}});
+ final byte[] archive = gzippedTar(new String[][] {{"corpus/data.txt", "followed"}});
server.route("/old.tar.gz", out -> StubServer.redirect(out, status, "/new.tar.gz"));
server.route("/new.tar.gz", out -> StubServer.ok(out, archive));
@@ -415,7 +415,7 @@ public class ResourceInstallerHttpTest {
@Test
void testTimeoutBeyondTheMillisecondRangeIsCapped(@TempDir Path target)
throws Exception {
- final byte[] archive = tarGz(new String[][] {{"payload/data.txt", "content"}});
+ final byte[] archive = gzippedTar(new String[][] {{"payload/data.txt", "content"}});
server.route("/payload.tar.gz", out -> StubServer.ok(out, archive));
final Duration beyondMillis = Duration.ofSeconds(Long.MAX_VALUE / 1000 + 1);
final ResourceInstaller.Limits limits = ResourceInstaller.Limits.builder()
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ResourceInstallerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ResourceInstallerTest.java
index e7bb12ea1..c2fe6eeef 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ResourceInstallerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ResourceInstallerTest.java
@@ -33,7 +33,6 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.stream.Stream;
-import java.util.zip.GZIPOutputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -50,16 +49,16 @@ import org.junit.jupiter.params.provider.ValueSource;
import opennlp.tools.util.archive.TarArchives;
+import static opennlp.tools.util.DigestTestUtil.sha256;
+import static opennlp.tools.util.DigestTestUtil.sha512;
import static opennlp.tools.util.InstallerTestSupport.BLOCK;
import static opennlp.tools.util.InstallerTestSupport.KIBIBYTE;
import static opennlp.tools.util.InstallerTestSupport.MEBIBYTE;
import static opennlp.tools.util.InstallerTestSupport.TERMINATOR_SIZE;
-import static opennlp.tools.util.InstallerTestSupport.gzip;
import static opennlp.tools.util.InstallerTestSupport.installedFiles;
-import static opennlp.tools.util.InstallerTestSupport.sha256;
-import static opennlp.tools.util.InstallerTestSupport.sha512;
import static opennlp.tools.util.InstallerTestSupport.tarEntry;
-import static opennlp.tools.util.InstallerTestSupport.tarGz;
+import static opennlp.tools.util.archive.TarArchives.gzip;
+import static opennlp.tools.util.archive.TarArchives.gzippedTar;
public class ResourceInstallerTest {
@@ -145,7 +144,7 @@ public class ResourceInstallerTest {
@Test
void testInstallEndToEndUsageExample(@TempDir Path source, @TempDir Path target)
throws Exception {
- final byte[] archive = tarGz(new String[][] {
+ final byte[] archive = gzippedTar(new String[][] {
{"corpus/README", "A tiny example corpus.\n"},
{"corpus/tokens.txt", "the\ncat\n"},
{"corpus/pos/tags.tsv", "the\tDET\ncat\tNOUN\n"}});
@@ -169,7 +168,7 @@ public class ResourceInstallerTest {
@Test
void testTarGzUnpacksWithStructure(@TempDir Path source, @TempDir Path target)
throws Exception {
- final byte[] archive = tarGz(new String[][] {
+ final byte[] archive = gzippedTar(new String[][] {
{"corpus-1.0/train.conllu", "# sent_id = 1\n"},
{"corpus-1.0/sub/readme.txt", "hello"}});
final Path file = source.resolve("corpus.tgz");
@@ -186,7 +185,7 @@ public class ResourceInstallerTest {
@Test
void testChecksumMismatchFailsBeforeUnpacking(@TempDir Path source,
@TempDir Path target) throws Exception {
- final byte[] archive = tarGz(new String[][] {{"a/file.txt", "content"}});
+ final byte[] archive = gzippedTar(new String[][] {{"a/file.txt", "content"}});
final Path file = source.resolve("archive.tar.gz");
Files.write(file, archive);
@@ -201,7 +200,7 @@ public class ResourceInstallerTest {
@Test
void testChecksumComparisonIgnoresHexLetterCase(@TempDir Path source,
@TempDir Path target) throws Exception {
- final byte[] archive = tarGz(new String[][] {{"data/entry.txt", "payload"}});
+ final byte[] archive = gzippedTar(new String[][] {{"data/entry.txt", "payload"}});
final Path file = source.resolve("cased.tar.gz");
Files.write(file, archive);
// The uppercase digest must differ textually from the lowercase one, otherwise
@@ -217,7 +216,7 @@ public class ResourceInstallerTest {
@Test
void testChecksumIgnoresUnicodeWhitespace(@TempDir Path source,
@TempDir Path target) throws Exception {
- final byte[] archive = tarGz(new String[][] {{"corpus/data.txt", "verified"}});
+ final byte[] archive = gzippedTar(new String[][] {{"corpus/data.txt", "verified"}});
final Path file = source.resolve("unicode-space.tar.gz");
Files.write(file, archive);
final String emSpace = Character.toString(0x2003);
@@ -232,7 +231,7 @@ public class ResourceInstallerTest {
@Test
void testEscapingEntriesAreRejected(@TempDir Path source, @TempDir Path target)
throws Exception {
- final byte[] archive = tarGz(new String[][] {{"../escape.txt", "bad"}});
+ final byte[] archive = gzippedTar(new String[][] {{"../escape.txt", "bad"}});
final Path file = source.resolve("evil.tar.gz");
Files.write(file, archive);
@@ -243,7 +242,7 @@ public class ResourceInstallerTest {
@Test
void testAbsoluteTarEntryIsRejected(@TempDir Path source, @TempDir Path target)
throws Exception {
- final byte[] archive = tarGz(new String[][] {
+ final byte[] archive = gzippedTar(new String[][] {
{"/absolute-escape-attempt/evil.txt", "bad"}});
final Path file = source.resolve("absolute.tar.gz");
Files.write(file, archive);
@@ -365,12 +364,8 @@ public class ResourceInstallerTest {
@Test
void testPlainGzipDecompressesToTheSourceName(@TempDir Path source,
@TempDir Path target) throws Exception {
- final ByteArrayOutputStream out = new ByteArrayOutputStream();
- try (GZIPOutputStream gzip = new GZIPOutputStream(out)) {
- gzip.write("word\tlemma\n".getBytes(StandardCharsets.UTF_8));
- }
final Path file = source.resolve("lexicon.tsv.gz");
- Files.write(file, out.toByteArray());
+ Files.write(file, gzip("word\tlemma\n".getBytes(StandardCharsets.UTF_8)));
ResourceInstaller.install(file.toUri(), target);
@@ -458,7 +453,7 @@ public class ResourceInstallerTest {
@Test
void testSha512ChecksumVerifies(@TempDir Path source, @TempDir Path target)
throws Exception {
- final byte[] archive = tarGz(new String[][] {{"data/entry.txt", "payload"}});
+ final byte[] archive = gzippedTar(new String[][] {{"data/entry.txt", "payload"}});
final Path file = source.resolve("checked.tar.gz");
Files.write(file, archive);
@@ -471,7 +466,7 @@ public class ResourceInstallerTest {
@Test
void testSha512ChecksumMismatchFailsBeforeUnpacking(@TempDir Path source,
@TempDir Path target) throws Exception {
- final byte[] archive = tarGz(new String[][] {{"a/file.txt", "content"}});
+ final byte[] archive = gzippedTar(new String[][] {{"a/file.txt", "content"}});
final Path file = source.resolve("archive.tar.gz");
Files.write(file, archive);
@@ -486,7 +481,7 @@ public class ResourceInstallerTest {
@Test
void testFailedTarUnpackLeavesTargetEmpty(@TempDir Path source, @TempDir Path target)
throws Exception {
- final byte[] archive = tarGz(new String[][] {
+ final byte[] archive = gzippedTar(new String[][] {
{"good.txt", "fine"},
{"../escape.txt", "bad"}});
final Path file = source.resolve("partial.tar.gz");
@@ -670,7 +665,7 @@ public class ResourceInstallerTest {
void testFailedInstallKeepsPreexistingTargetContent(@TempDir Path source,
@TempDir Path target) throws Exception {
Files.writeString(target.resolve("existing.txt"), "keep");
- final byte[] archive = tarGz(new String[][] {
+ final byte[] archive = gzippedTar(new String[][] {
{"good.txt", "fine"},
{"../escape.txt", "bad"}});
final Path file = source.resolve("partial.tar.gz");
@@ -685,7 +680,7 @@ public class ResourceInstallerTest {
@Test
void testInstallationLeavesNoStagingResidue(@TempDir Path source, @TempDir Path target)
throws Exception {
- final byte[] archive = tarGz(new String[][] {{"corpus/data.txt", "content"}});
+ final byte[] archive = gzippedTar(new String[][] {{"corpus/data.txt", "content"}});
final Path file = source.resolve("clean.tar.gz");
Files.write(file, archive);
@@ -881,7 +876,7 @@ public class ResourceInstallerTest {
@Test
void testInstallWithinCustomLimitsSucceeds(@TempDir Path source, @TempDir Path target)
throws Exception {
- final byte[] archive = tarGz(new String[][] {{"corpus/data.txt", "small"}});
+ final byte[] archive = gzippedTar(new String[][] {{"corpus/data.txt", "small"}});
final Path file = source.resolve("small.tar.gz");
Files.write(file, archive);
@@ -929,7 +924,7 @@ public class ResourceInstallerTest {
@Test
void testTarEntryCountLimitRejectsArchive(@TempDir Path source, @TempDir Path target)
throws Exception {
- final byte[] archive = tarGz(new String[][] {
+ final byte[] archive = gzippedTar(new String[][] {
{"corpus/one.txt", "1"},
{"corpus/two.txt", "2"},
{"corpus/three.txt", "3"}});
@@ -984,7 +979,7 @@ public class ResourceInstallerTest {
@Test
void testEntryCountExactlyAtLimitSucceeds(@TempDir Path source, @TempDir Path target)
throws Exception {
- final byte[] archive = tarGz(new String[][] {
+ final byte[] archive = gzippedTar(new String[][] {
{"corpus/one.txt", "1"},
{"corpus/two.txt", "2"}});
final Path file = source.resolve("exact.tar.gz");
@@ -999,7 +994,7 @@ public class ResourceInstallerTest {
@Test
void testSha512ChecksumComparisonIgnoresHexLetterCase(@TempDir Path source,
@TempDir Path target) throws Exception {
- final byte[] archive = tarGz(new String[][] {{"data/entry.txt", "payload"}});
+ final byte[] archive = gzippedTar(new String[][] {{"data/entry.txt", "payload"}});
final Path file = source.resolve("cased512.tar.gz");
Files.write(file, archive);
final String upperCase = sha512(archive).toUpperCase(Locale.ROOT);
@@ -1056,8 +1051,8 @@ public class ResourceInstallerTest {
@Test
void testReinstallOverAnExistingFileIsRejected(@TempDir Path source, @TempDir Path target)
throws Exception {
- final byte[] first = tarGz(new String[][] {{"corpus/data.txt", "version one"}});
- final byte[] second = tarGz(new String[][] {{"corpus/data.txt", "version two"}});
+ final byte[] first = gzippedTar(new String[][] {{"corpus/data.txt", "version one"}});
+ final byte[] second = gzippedTar(new String[][] {{"corpus/data.txt", "version two"}});
final Path firstFile = source.resolve("first.tar.gz");
final Path secondFile = source.resolve("second.tar.gz");
Files.write(firstFile, first);
@@ -1084,7 +1079,7 @@ public class ResourceInstallerTest {
throws Exception {
Files.createDirectories(target.resolve("corpus"));
Files.writeString(target.resolve("corpus/data.txt"), "keep");
- final byte[] archive = tarGz(new String[][] {
+ final byte[] archive = gzippedTar(new String[][] {
{"corpus/fresh.txt", "new"},
{"corpus/data.txt", "replacement"}});
final Path file = source.resolve("colliding.tar.gz");
@@ -1107,7 +1102,7 @@ public class ResourceInstallerTest {
@TempDir Path target, @TempDir Path outside) throws Exception {
final Path link = target.resolve("link");
Files.createSymbolicLink(link, outside);
- final byte[] archive = tarGz(new String[][] {{"link/planted.txt", "escaped"}});
+ final byte[] archive = gzippedTar(new String[][] {{"link/planted.txt", "escaped"}});
final Path file = source.resolve("symlink.tar.gz");
Files.write(file, archive);
@@ -1124,7 +1119,7 @@ public class ResourceInstallerTest {
void testNestedDirectoryThatIsNotASymlinkStillInstalls(@TempDir Path source,
@TempDir Path target) throws Exception {
Files.createDirectory(target.resolve("link"));
- final byte[] archive = tarGz(new String[][] {{"link/planted.txt", "fine"}});
+ final byte[] archive = gzippedTar(new String[][] {{"link/planted.txt", "fine"}});
final Path file = source.resolve("nested.tar.gz");
Files.write(file, archive);
@@ -1296,7 +1291,7 @@ public class ResourceInstallerTest {
Files.writeString(staleStaging.resolve("partial.txt"), "half");
Files.writeString(target.resolve(".opennlp-downloadOLD.part"), "half");
final Path file = source.resolve("corpus.tar.gz");
- Files.write(file, tarGz(new String[][] {{"corpus/data.txt", "content"}}));
+ Files.write(file, gzippedTar(new String[][] {{"corpus/data.txt", "content"}}));
ResourceInstaller.install(file.toUri(), target);
@@ -1309,7 +1304,7 @@ public class ResourceInstallerTest {
@Test
void testFailedInstallRemovesTheTargetDirectoryItCreated(@TempDir Path source,
@TempDir Path parent) throws Exception {
- final byte[] archive = tarGz(new String[][] {{"corpus/data.txt", "content"}});
+ final byte[] archive = gzippedTar(new String[][] {{"corpus/data.txt", "content"}});
final Path file = source.resolve("corpus.tar.gz");
Files.write(file, archive);
final Path target = parent.resolve("fresh");
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/AccentFoldCharSequenceNormalizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/AccentFoldCharSequenceNormalizerTest.java
index 5db1a4683..7b1f815a6 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/AccentFoldCharSequenceNormalizerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/AccentFoldCharSequenceNormalizerTest.java
@@ -20,16 +20,13 @@ import java.util.Set;
import org.junit.jupiter.api.Test;
+import static opennlp.tools.util.normalizer.NormalizerTestUtil.cp;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class AccentFoldCharSequenceNormalizerTest {
- private static String cp(int codePoint) {
- return new String(Character.toChars(codePoint));
- }
-
private static String fold(String text) {
return AccentFoldCharSequenceNormalizer.getInstance().normalize(text).toString();
}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/AlignedNormalizerPipelineTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/AlignedNormalizerPipelineTest.java
index a4da8ed56..3d4e60475 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/AlignedNormalizerPipelineTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/AlignedNormalizerPipelineTest.java
@@ -20,6 +20,7 @@ import org.junit.jupiter.api.Test;
import opennlp.tools.util.Span;
+import static opennlp.tools.util.normalizer.NormalizerTestUtil.cp;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertSame;
@@ -39,10 +40,6 @@ public class AlignedNormalizerPipelineTest {
private static final int YEZIDI_HYPHEN = 0x10EAD; // a supplementary (non-BMP) dash
private static final int MATH_BOLD_DIGIT_ZERO = 0x1D7CE; // a supplementary decimal digit
- private static String cp(int codePoint) {
- return new String(Character.toChars(codePoint));
- }
-
private static String covered(AlignedText aligned, int normalizedStart, int normalizedEnd) {
final Span span = aligned.toOriginalSpan(normalizedStart, normalizedEnd);
return aligned.original().subSequence(span.getStart(), span.getEnd()).toString();
@@ -187,8 +184,8 @@ public class AlignedNormalizerPipelineTest {
assertFalse(NfkcCharSequenceNormalizer.getInstance() instanceof OffsetAwareNormalizer);
assertFalse(CaseFoldCharSequenceNormalizer.getInstance() instanceof OffsetAwareNormalizer);
assertFalse(AccentFoldCharSequenceNormalizer.getInstance() instanceof OffsetAwareNormalizer);
- assertFalse(ConfusableSkeletonCharSequenceNormalizer.getInstance()
- instanceof OffsetAwareNormalizer);
+ final CharSequenceNormalizer skeleton = ConfusableSkeletonCharSequenceNormalizer.getInstance();
+ assertFalse(skeleton instanceof OffsetAwareNormalizer);
}
@Test
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/CaseFoldCharSequenceNormalizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/CaseFoldCharSequenceNormalizerTest.java
index 1fb8cdc24..4157c396c 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/CaseFoldCharSequenceNormalizerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/CaseFoldCharSequenceNormalizerTest.java
@@ -20,16 +20,13 @@ import java.util.Locale;
import org.junit.jupiter.api.Test;
+import static opennlp.tools.util.normalizer.NormalizerTestUtil.cp;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
public class CaseFoldCharSequenceNormalizerTest {
- private static String cp(int codePoint) {
- return new String(Character.toChars(codePoint));
- }
-
@Test
void testRootLowerCases() {
assertEquals("hello world",
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/CharSequenceNormalizerContractTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/CharSequenceNormalizerContractTest.java
index 240548336..488855293 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/CharSequenceNormalizerContractTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/CharSequenceNormalizerContractTest.java
@@ -16,6 +16,10 @@
*/
package opennlp.tools.util.normalizer;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
@@ -24,6 +28,7 @@ import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
/**
@@ -76,6 +81,34 @@ class CharSequenceNormalizerContractTest {
name + " must pass empty text through");
}
+ static Stream<Arguments> singletons() {
+ return Stream.of(
+ Arguments.of("bullet", BulletCharSequenceNormalizer.getInstance()),
+ Arguments.of("confusableSkeleton", ConfusableSkeletonCharSequenceNormalizer.getInstance()),
+ Arguments.of("dash", DashCharSequenceNormalizer.getInstance()),
+ Arguments.of("digit", DigitCharSequenceNormalizer.getInstance()),
+ Arguments.of("emojiToEmoticon", EmojiToEmoticonCharSequenceNormalizer.getInstance()),
+ Arguments.of("emoticonToEmoji", EmoticonToEmojiCharSequenceNormalizer.getInstance()),
+ Arguments.of("fullCaseFold", FullCaseFoldCharSequenceNormalizer.getInstance()),
+ Arguments.of("germanUmlaut", GermanUmlautCharSequenceNormalizer.getInstance()),
+ Arguments.of("nfc", NfcCharSequenceNormalizer.getInstance()),
+ Arguments.of("nfkc", NfkcCharSequenceNormalizer.getInstance()),
+ Arguments.of("quote", QuoteCharSequenceNormalizer.getInstance()));
+ }
+
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("singletons")
+ void testDeserializationReturnsSharedInstance(String name, CharSequenceNormalizer normalizer)
+ throws Exception {
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ try (ObjectOutputStream out = new ObjectOutputStream(bytes)) {
+ out.writeObject(normalizer);
+ }
+ try (ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) {
+ assertSame(normalizer, in.readObject(), name + " must deserialize to its shared instance");
+ }
+ }
+
@Test
void testNumberJavadocExample() {
assertEquals("a b ",
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ConfusableSkeletonTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ConfusableSkeletonTest.java
index 61a9e1401..426cc94cd 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ConfusableSkeletonTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ConfusableSkeletonTest.java
@@ -18,16 +18,13 @@ package opennlp.tools.util.normalizer;
import org.junit.jupiter.api.Test;
+import static opennlp.tools.util.normalizer.NormalizerTestUtil.cp;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class ConfusableSkeletonTest {
- private static String cp(int codePoint) {
- return new String(Character.toChars(codePoint));
- }
-
@Test
void testSkeletonFoldsKnownLookalikesToTheirLatinPrototype() {
// Known single-character mappings from the bundled UTS #39 table.
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ConfusablesTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ConfusablesTest.java
index 3a1491495..a3d44b2a2 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ConfusablesTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ConfusablesTest.java
@@ -18,6 +18,7 @@ package opennlp.tools.util.normalizer;
import org.junit.jupiter.api.Test;
+import static opennlp.tools.util.normalizer.NormalizerTestUtil.cp;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertSame;
@@ -26,10 +27,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
public class ConfusablesTest {
- private static String cp(int codePoint) {
- return new String(Character.toChars(codePoint));
- }
-
@Test
void testCyrillicLetterIsConfusableWithLatin() {
final String cyrillicA = cp(0x0430); // CYRILLIC SMALL LETTER A, looks like Latin 'a'
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerCharacterizationTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerCharacterizationTest.java
index ec06f406d..7d5932085 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerCharacterizationTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerCharacterizationTest.java
@@ -31,6 +31,7 @@ import org.junit.jupiter.params.provider.MethodSource;
import opennlp.tools.util.CompatibilityMode;
+import static opennlp.tools.util.normalizer.NormalizerTestUtil.cp;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -73,10 +74,6 @@ public class EmojiCharSequenceNormalizerCharacterizationTest {
CompatibilityMode.reset();
}
- private static String cp(int... codePoints) {
- return new String(codePoints, 0, codePoints.length);
- }
-
static Stream<Arguments> emojiRuns() {
return Stream.of(
Arguments.of("", ""),
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerTest.java
index 981da786e..5017e267c 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerTest.java
@@ -35,6 +35,8 @@ import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.ValueSource;
+import static opennlp.tools.util.normalizer.NormalizerTestUtil.cp;
+
public class EmojiCharSequenceNormalizerTest {
private static final EmojiCharSequenceNormalizer NORMALIZER =
EmojiCharSequenceNormalizer.getInstance();
@@ -45,10 +47,6 @@ public class EmojiCharSequenceNormalizerTest {
*/
private static final int EMOJI_17_SEQUENCE_COUNT = 4973;
- private static String cp(int... codePoints) {
- return new String(codePoints, 0, codePoints.length);
- }
-
private static Stream<Arguments> emojiSequences() {
return Stream.of(
Arguments.of(cp(0x1F600), "supplementary singleton"),
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiToEmoticonCharSequenceNormalizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiToEmoticonCharSequenceNormalizerTest.java
index af409141f..7eac965c5 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiToEmoticonCharSequenceNormalizerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiToEmoticonCharSequenceNormalizerTest.java
@@ -20,6 +20,7 @@ import org.junit.jupiter.api.Test;
import opennlp.tools.util.Span;
+import static opennlp.tools.util.normalizer.NormalizerTestUtil.cp;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -29,10 +30,6 @@ public class EmojiToEmoticonCharSequenceNormalizerTest {
return EmojiToEmoticonCharSequenceNormalizer.getInstance();
}
- private static String cp(int codePoint) {
- return new String(Character.toChars(codePoint));
- }
-
@Test
void nullInputIsRejectedAtThePublicBoundary() {
assertThrows(IllegalArgumentException.class, () -> norm().normalize(null));
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmoticonToEmojiCharSequenceNormalizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmoticonToEmojiCharSequenceNormalizerTest.java
index 1e9badec4..fba5e4e75 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmoticonToEmojiCharSequenceNormalizerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmoticonToEmojiCharSequenceNormalizerTest.java
@@ -25,6 +25,7 @@ import opennlp.tools.tokenize.uax29.WordTokenizer;
import opennlp.tools.tokenize.uax29.WordType;
import opennlp.tools.util.Span;
+import static opennlp.tools.util.normalizer.NormalizerTestUtil.cp;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -35,10 +36,6 @@ public class EmoticonToEmojiCharSequenceNormalizerTest {
return EmoticonToEmojiCharSequenceNormalizer.getInstance();
}
- private static String cp(int codePoint) {
- return new String(Character.toChars(codePoint));
- }
-
@Test
void nullInputIsRejectedAtThePublicBoundary() {
assertThrows(IllegalArgumentException.class, () -> norm().normalize(null));
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/FullCaseFoldCharSequenceNormalizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/FullCaseFoldCharSequenceNormalizerTest.java
index 4deeee84b..c3b0b7b29 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/FullCaseFoldCharSequenceNormalizerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/FullCaseFoldCharSequenceNormalizerTest.java
@@ -27,6 +27,7 @@ import org.junit.jupiter.params.provider.ValueSource;
import opennlp.tools.util.Span;
+import static opennlp.tools.util.normalizer.NormalizerTestUtil.cp;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -37,10 +38,6 @@ public class FullCaseFoldCharSequenceNormalizerTest {
return FullCaseFoldCharSequenceNormalizer.getInstance();
}
- private static String cp(int codePoint) {
- return new String(Character.toChars(codePoint));
- }
-
@Test
void getInstanceReturnsTheSharedSingleton() {
assertSame(norm(), norm());
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/GermanUmlautCharSequenceNormalizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/GermanUmlautCharSequenceNormalizerTest.java
index ac9abdabd..27576c813 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/GermanUmlautCharSequenceNormalizerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/GermanUmlautCharSequenceNormalizerTest.java
@@ -20,6 +20,7 @@ import java.text.Normalizer;
import org.junit.jupiter.api.Test;
+import static opennlp.tools.util.normalizer.NormalizerTestUtil.cp;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class GermanUmlautCharSequenceNormalizerTest {
@@ -27,10 +28,6 @@ public class GermanUmlautCharSequenceNormalizerTest {
private static final GermanUmlautCharSequenceNormalizer FOLD =
GermanUmlautCharSequenceNormalizer.getInstance();
- private static String cp(int codePoint) {
- return new String(Character.toChars(codePoint));
- }
-
private static String fold(String text) {
return FOLD.normalize(text).toString();
}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/LineBreakPreservingWhitespaceCharSequenceNormalizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/LineBreakPreservingWhitespaceCharSequenceNormalizerTest.java
index 7c2ffe7ca..1c2474ac3 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/LineBreakPreservingWhitespaceCharSequenceNormalizerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/LineBreakPreservingWhitespaceCharSequenceNormalizerTest.java
@@ -20,6 +20,7 @@ import org.junit.jupiter.api.Test;
import opennlp.tools.util.Span;
+import static opennlp.tools.util.normalizer.NormalizerTestUtil.cp;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -30,10 +31,6 @@ public class LineBreakPreservingWhitespaceCharSequenceNormalizerTest {
return LineBreakPreservingWhitespaceCharSequenceNormalizer.getInstance();
}
- private static String cp(int codePoint) {
- return new String(Character.toChars(codePoint));
- }
-
@Test
void getInstanceReturnsTheSharedSingleton() {
assertSame(norm(), norm());
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NormalizationProfilesTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NormalizationProfilesTest.java
index f0f1207d9..f344af918 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NormalizationProfilesTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NormalizationProfilesTest.java
@@ -25,6 +25,8 @@ import java.util.concurrent.Future;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import opennlp.tools.langdetect.Language;
import opennlp.tools.langdetect.LanguageDetector;
@@ -95,12 +97,11 @@ public class NormalizationProfilesTest {
profile.matchingAnalyzer().dimensions());
}
- @Test
- void testRomanceLanguagesUseTheGenericFold() {
- for (final String language : List.of("fra", "spa", "por", "ita", "cat")) {
- assertSame(AccentFoldCharSequenceNormalizer.getInstance(),
- NormalizationProfiles.forLanguage(language).orElseThrow().accentFold());
- }
+ @ParameterizedTest
+ @ValueSource(strings = {"fra", "spa", "por", "ita", "cat"})
+ void testRomanceLanguagesUseTheGenericFold(String language) {
+ assertSame(AccentFoldCharSequenceNormalizer.getInstance(),
+ NormalizationProfiles.forLanguage(language).orElseThrow().accentFold());
}
@Test
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ParagraphPreservingWhitespaceCharSequenceNormalizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ParagraphPreservingWhitespaceCharSequenceNormalizerTest.java
index 250ad75c1..e04243e50 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ParagraphPreservingWhitespaceCharSequenceNormalizerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ParagraphPreservingWhitespaceCharSequenceNormalizerTest.java
@@ -29,6 +29,7 @@ import org.junit.jupiter.params.provider.MethodSource;
import opennlp.tools.util.Span;
+import static opennlp.tools.util.normalizer.NormalizerTestUtil.cp;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -39,10 +40,6 @@ public class ParagraphPreservingWhitespaceCharSequenceNormalizerTest {
return ParagraphPreservingWhitespaceCharSequenceNormalizer.getInstance();
}
- private static String cp(int codePoint) {
- return new String(Character.toChars(codePoint));
- }
-
static Stream<Arguments> unicodeLineBreakCodePoints() {
return UnicodeWhitespace.lineBreaks().stream()
.map(ws -> Arguments.of(ws.codePoint(), ws.abbreviation()));
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/SetBasedNormalizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/SetBasedNormalizerTest.java
index 0197370aa..20d5fd2aa 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/SetBasedNormalizerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/SetBasedNormalizerTest.java
@@ -18,15 +18,12 @@ package opennlp.tools.util.normalizer;
import org.junit.jupiter.api.Test;
+import static opennlp.tools.util.normalizer.NormalizerTestUtil.cp;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
public class SetBasedNormalizerTest {
- private static String cp(int codePoint) {
- return new String(Character.toChars(codePoint));
- }
-
private static String quotes(String text) {
return QuoteCharSequenceNormalizer.getInstance().normalize(text).toString();
}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerMultilingualTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerMultilingualTest.java
index cc567001d..2b3f19f20 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerMultilingualTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerMultilingualTest.java
@@ -18,6 +18,7 @@ package opennlp.tools.util.normalizer;
import org.junit.jupiter.api.Test;
+import static opennlp.tools.util.normalizer.NormalizerTestUtil.cp;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
@@ -28,10 +29,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
*/
public class TermAnalyzerMultilingualTest {
- private static String cp(int... codePoints) {
- return new String(codePoints, 0, codePoints.length);
- }
-
private static String normalized(TermAnalyzer analyzer, String text) {
return analyzer.analyze(text).get(0).normalized();
}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerTest.java
index 39232e16e..3daa3a83c 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerTest.java
@@ -36,6 +36,7 @@ import opennlp.tools.stemmer.snowball.SnowballStemmer;
import opennlp.tools.stemmer.snowball.SnowballStemmerFactory;
import opennlp.tools.util.Span;
+import static opennlp.tools.util.normalizer.NormalizerTestUtil.cp;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
@@ -44,10 +45,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
public class TermAnalyzerTest {
- private static String cp(int codePoint) {
- return new String(Character.toChars(codePoint));
- }
-
@Test
void testNoDimensionsLeavesTokenUnchanged() {
final TermAnalyzer analyzer = TermAnalyzer.builder().build();
@@ -169,9 +166,7 @@ public class TermAnalyzerTest {
@Test
void testLemmatizerReturningNullFailsLoudlyInsteadOfOverflowing() {
// A contract-violating Lemmatizer that returns a null lemma must surface as a clear
- // IllegalStateException. Before this guard the null was cached under LEMMA, read as "absent"
- // by Term.at's lazy cache, and recomputed through normalized() forever, surfacing as a
- // StackOverflowError far from the cause.
+ // IllegalStateException.
final Lemmatizer broken = new Lemmatizer() {
@Override
public String[] lemmatize(String[] tokens, String[] tags) {
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TextNormalizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TextNormalizerTest.java
index 7845b1223..e102d3ece 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TextNormalizerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TextNormalizerTest.java
@@ -20,15 +20,12 @@ import java.util.Locale;
import org.junit.jupiter.api.Test;
+import static opennlp.tools.util.normalizer.NormalizerTestUtil.cp;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
public class TextNormalizerTest {
- private static String cp(int codePoint) {
- return new String(Character.toChars(codePoint));
- }
-
@Test
void testNormalizersApplyInOrder() {
final CharSequenceNormalizer n = TextNormalizer.builder().caseFold().accentFold().build();
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/UnicodeCharSequenceNormalizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/UnicodeCharSequenceNormalizerTest.java
index 7a700739f..e2c011733 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/UnicodeCharSequenceNormalizerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/UnicodeCharSequenceNormalizerTest.java
@@ -18,6 +18,7 @@ package opennlp.tools.util.normalizer;
import org.junit.jupiter.api.Test;
+import static opennlp.tools.util.normalizer.NormalizerTestUtil.cp;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
@@ -27,10 +28,6 @@ import static org.junit.jupiter.api.Assertions.assertSame;
*/
public class UnicodeCharSequenceNormalizerTest {
- private static String cp(int codePoint) {
- return new String(Character.toChars(codePoint));
- }
-
@Test
void testWhitespaceCollapsesUnicodeRunsAndTrims() {
final String input = " a" + cp(0x00A0) + cp(0x00A0) + "b" + cp(0x3000) + " ";
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/UnicodeEmojiSequencesParseTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/UnicodeEmojiSequencesParseTest.java
index d7cfd5c25..274430de4 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/UnicodeEmojiSequencesParseTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/UnicodeEmojiSequencesParseTest.java
@@ -22,6 +22,7 @@ import java.nio.charset.StandardCharsets;
import org.junit.jupiter.api.Test;
+import static opennlp.tools.util.normalizer.NormalizerTestUtil.cp;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -43,10 +44,6 @@ class UnicodeEmojiSequencesParseTest {
return new ByteArrayInputStream(data.getBytes(StandardCharsets.US_ASCII));
}
- private String cp(int... codePoints) {
- return new String(codePoints, 0, codePoints.length);
- }
-
@Test
void parseReadsWellFormedRecords() throws Exception {
final UnicodeEmojiSequences sequences = UnicodeEmojiSequences.parse(in(WELL_FORMED));