You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by rz...@apache.org on 2026/10/08 10:15:39 UTC
(opennlp) 01/01: GH-1449: Share ONNX inference between DocumentCategorizerDL and NameFinderDL with one failure message
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch gh-dl-cleanup
in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 03e9d437bc166bd404f613d07eb54e5130ceae5d
Author: Richard Zowalla <rz...@apache.org>
AuthorDate: Thu Oct 8 12:15:28 2026 +0200
GH-1449: Share ONNX inference between DocumentCategorizerDL and NameFinderDL with one failure message
Move tensor building, session execution and error wrapping into
AbstractDL.runModel and copy InferenceOptions once into an
InferenceSettings record. Both components now report a failed model run
as "Unable to run the ONNX model: <cause>". Add the missing Javadoc and
@throws, trim implementation narrative, and keep only current behavior
in the README.
---
opennlp-core/opennlp-ml/opennlp-dl/README.md | 27 +--
.../src/main/java/opennlp/dl/AbstractDL.java | 216 +++++++++++++++++----
.../src/main/java/opennlp/dl/JsonScan.java | 6 +-
.../opennlp/dl/doccat/DocumentCategorizerDL.java | 134 ++++++-------
.../java/opennlp/dl/namefinder/NameFinderDL.java | 150 ++++++--------
.../java/opennlp/dl/AbstractDLChunkingTest.java | 24 +--
.../opennlp/dl/InferenceOptionsValidationTest.java | 22 +++
.../src/test/java/opennlp/dl/LoadVocabTest.java | 19 ++
.../dl/doccat/DocumentCategorizerDLTest.java | 21 +-
.../opennlp/dl/namefinder/NameFinderDLTest.java | 18 +-
10 files changed, 396 insertions(+), 241 deletions(-)
diff --git a/opennlp-core/opennlp-ml/opennlp-dl/README.md b/opennlp-core/opennlp-ml/opennlp-dl/README.md
index 39cc41b56..54ec4faa3 100644
--- a/opennlp-core/opennlp-ml/opennlp-dl/README.md
+++ b/opennlp-core/opennlp-ml/opennlp-dl/README.md
@@ -87,25 +87,18 @@ Export a Huggingface classification (e.g. sentiment) model to ONNX, e.g.:
python -m transformers.onnx --model=nlptown/bert-base-multilingual-uncased-sentiment --feature sequence-classification exported
```
-## Behavior changes in this release
-
-Integrators upgrading from an earlier `opennlp-dl` should note these intentional changes (OPENNLP-1850):
+## Input validation and overlapping spans
- `NameFinderDL.find(...)` reports spans in the coordinates of the joined input it ran inference on,
- which differ from the original text only when length-changing dash folding is enabled. Use the new
- `NameFinderDL.findInOriginal(...)` (from `OffsetMappingNameFinder`) for original-text coordinates.
-- Spans that overlap at chunk boundaries are now merged longest-wins; `find(...)` previously returned
- every decoded span, overlaps included.
-- Chunking splits on the Unicode `White_Space` set rather than `String#split("\\s+")`, and
- whitespace-only input now yields no spans without running the model.
-- `DocumentCategorizerDL.categorize(...)` now rejects `null`/empty input, and a document with no
- non-whitespace token, with `IllegalArgumentException` rather than running the model on empty input.
-- A `null` input array, a `null` token passed to `NameFinderDL`, or a `null` document passed to
- `DocumentCategorizerDL` is rejected with `IllegalArgumentException` instead of surfacing as a
- `NullPointerException` deep in processing or being joined into the text as the literal string
- `"null"`.
-- The example label constants `NameFinderDL.I_PER` and `NameFinderDL.B_PER` were removed; supply your
- own label strings (any `B-<TYPE>`/`I-<TYPE>` pair works, as described above).
+ which differ from the original text only when length-changing dash folding is enabled.
+ `NameFinderDL.findInOriginal(...)` reports original-text coordinates; code that holds a
+ `TokenNameFinder` can check for `OffsetMappingNameFinder` to reach it.
+- Spans that overlap where consecutive chunks overlap are resolved longest-wins, ties going to the
+ higher probability.
+- Whitespace-only input yields no spans without running the model.
+- `NameFinderDL` rejects a `null` input array or a `null` token, and `DocumentCategorizerDL`
+ rejects a `null` or empty input array, a `null` document, and a document with no non-whitespace
+ token, with `IllegalArgumentException`.
## SentenceVectors
diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java
index 44127cee2..9afe7b372 100644
--- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java
+++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java
@@ -19,6 +19,7 @@ package opennlp.dl;
import java.io.File;
import java.io.IOException;
+import java.nio.LongBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -29,7 +30,9 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+import ai.onnxruntime.OnnxTensor;
import ai.onnxruntime.OrtEnvironment;
import ai.onnxruntime.OrtException;
import ai.onnxruntime.OrtSession;
@@ -101,6 +104,40 @@ public abstract class AbstractDL implements AutoCloseable {
protected record TextChunk(String text, int start, int end) {
}
+ /**
+ * The {@link InferenceOptions} values a component reads during inference, copied once at
+ * construction so later changes to the options object do not affect a shared instance.
+ *
+ * @param includeAttentionMask Whether the {@value AbstractDL#ATTENTION_MASK} input is passed
+ * to the model.
+ * @param includeTokenTypeIds Whether the {@value AbstractDL#TOKEN_TYPE_IDS} input is passed
+ * to the model.
+ * @param documentSplitSize The maximum number of whitespace tokens per chunk.
+ * @param splitOverlapSize The number of whitespace tokens shared by consecutive chunks.
+ * @param normalizeWhitespace Whether Unicode whitespace is folded to ASCII spaces.
+ * @param normalizeDashes Whether Unicode dashes are folded to the ASCII hyphen.
+ */
+ @Internal(since = "3.0.0")
+ protected record InferenceSettings(boolean includeAttentionMask, boolean includeTokenTypeIds,
+ int documentSplitSize, int splitOverlapSize,
+ boolean normalizeWhitespace, boolean normalizeDashes) {
+
+ /**
+ * Copies the inference values of {@code options}.
+ *
+ * @param options The options to copy. Must not be {@code null}.
+ * @return The settings.
+ * @throws IllegalArgumentException Thrown if {@code options} is {@code null}.
+ */
+ public static InferenceSettings from(final InferenceOptions options) {
+ ParamChecks.requireNonNullArg(options, "options");
+ return new InferenceSettings(options.isIncludeAttentionMask(),
+ options.isIncludeTokenTypeIds(), options.getDocumentSplitSize(),
+ options.getSplitOverlapSize(), options.isNormalizeWhitespace(),
+ options.isNormalizeDashes());
+ }
+ }
+
/**
* Initializes the shared, immutable inference state: the ONNX environment and session,
* the loaded vocabulary and the configured tokenizer. These fields are {@code final}
@@ -120,6 +157,9 @@ public abstract class AbstractDL implements AutoCloseable {
* @throws InvalidFormatException Thrown if a JSON {@code vocabulary} is malformed, has an
* unsupported layout, or sets {@code normalizer.lowercase} to the other value than
* {@code lowerCase}.
+ * @throws IllegalArgumentException Thrown if {@code model}, {@code vocabulary} or
+ * {@code sessionOptions} is {@code null}, or if the vocabulary lacks the selected
+ * classification, separator or unknown token.
*/
protected AbstractDL(final File model, final File vocabulary,
final OrtSession.SessionOptions sessionOptions, final boolean lowerCase)
@@ -177,6 +217,8 @@ public abstract class AbstractDL implements AutoCloseable {
* @return The configured session options.
*
* @throws OrtException Thrown if the CUDA execution provider cannot be added.
+ * @throws IllegalArgumentException Thrown if {@code inferenceOptions} is {@code null} or its
+ * split sizes are invalid, see {@link #validateSplitOptions(int, int)}.
*/
protected static OrtSession.SessionOptions sessionOptions(final InferenceOptions inferenceOptions)
throws OrtException {
@@ -237,15 +279,9 @@ public abstract class AbstractDL implements AutoCloseable {
Vocabulary readVocabFile(final File vocabFile) throws IOException {
final String read = Files.readString(Path.of(vocabFile.getPath()), StandardCharsets.UTF_8);
final String content = StringUtil.stripByteOrderMark(read);
- final String trimmed = content.trim();
- if (trimmed.startsWith(JSON_OBJECT_START)) {
- try {
- return readJsonVocab(trimmed);
- } catch (IllegalArgumentException e) {
- throw new InvalidFormatException(
- "Vocabulary file " + vocabFile.getName() + ": " + e.getMessage(), e);
- }
+ if (content.trim().startsWith(JSON_OBJECT_START)) {
+ return parseJson(vocabFile, "Vocabulary", read, this::readJsonVocab);
}
final Map<String, Integer> vocab =
@@ -284,6 +320,36 @@ public abstract class AbstractDL implements AutoCloseable {
}
}
+ /**
+ * Parses JSON text read from a file, naming the file when the text is rejected.
+ *
+ * @param file The file the text was read from, named in the message.
+ * @param fileKind The kind of file, such as {@code Vocabulary}, that starts the message.
+ * @param json The JSON text.
+ * @param parser The parser, which throws {@link IllegalArgumentException} for text it rejects.
+ * @param <T> The type of the parsed value.
+ * @return The parsed value.
+ * @throws InvalidFormatException Thrown if {@code parser} rejects the text. The message names
+ * the file and repeats the parser's message.
+ * @throws IllegalArgumentException Thrown if {@code file}, {@code fileKind}, {@code json} or
+ * {@code parser} is {@code null}.
+ */
+ @Internal(since = "3.0.0")
+ protected static <T> T parseJson(final File file, final String fileKind, final String json,
+ final Function<String, T> parser)
+ throws InvalidFormatException {
+ ParamChecks.requireNonNullArg(file, "file");
+ ParamChecks.requireNonNullArg(fileKind, "fileKind");
+ ParamChecks.requireNonNullArg(json, "json");
+ ParamChecks.requireNonNullArg(parser, "parser");
+ try {
+ return parser.apply(json);
+ } catch (IllegalArgumentException e) {
+ throw new InvalidFormatException(
+ fileKind + " file " + file.getName() + ": " + e.getMessage(), e);
+ }
+ }
+
/**
* Creates a {@link WordpieceTokenizer} that uses the
* appropriate special tokens based on the vocabulary.
@@ -293,26 +359,31 @@ public abstract class AbstractDL implements AutoCloseable {
*
* @param vocab The vocabulary map.
* @return A configured {@link WordpieceTokenizer}.
+ * @throws IllegalArgumentException Thrown if the vocabulary has the RoBERTa classification
+ * and separator tokens but no unknown token.
+ * @deprecated Not used by this class; components encode text with
+ * {@link #encodeTokens(CharSequence)}.
*/
+ @Deprecated(since = "3.0.0", forRemoval = true)
protected WordpieceTokenizer createTokenizer(
final Map<String, Integer> vocab) {
return createWordpieceTokenizer(vocab);
}
+ /**
+ * Creates a {@link WordpieceTokenizer} with the special tokens that
+ * {@link #specialTokens(Map)} selects for {@code vocab}.
+ *
+ * @param vocab The vocabulary map.
+ * @return A configured {@link WordpieceTokenizer}.
+ * @throws IllegalArgumentException Thrown if the vocabulary has the RoBERTa classification
+ * and separator tokens but no unknown token.
+ */
static WordpieceTokenizer createWordpieceTokenizer(
final Map<String, Integer> vocab) {
- if (vocab.containsKey(
- WordpieceTokenizer.ROBERTA_CLS_TOKEN)
- && vocab.containsKey(
- WordpieceTokenizer.ROBERTA_SEP_TOKEN)) {
- return new WordpieceTokenizer(
- vocab.keySet(),
- WordpieceTokenizer.ROBERTA_CLS_TOKEN,
- WordpieceTokenizer.ROBERTA_SEP_TOKEN,
- resolveUnknownToken(vocab));
- }
- return new WordpieceTokenizer(vocab.keySet());
+ final SpecialTokens special = specialTokens(vocab);
+ return new WordpieceTokenizer(vocab.keySet(), special.cls(), special.sep(), special.unk());
}
/**
@@ -328,21 +399,85 @@ public abstract class AbstractDL implements AutoCloseable {
static WordpieceEncoder createWordpieceEncoder(
final Map<String, Integer> vocab, final boolean lowerCase) {
ParamChecks.requireNonNullArg(vocab, "vocab");
- if (vocab.containsKey(
- WordpieceTokenizer.ROBERTA_CLS_TOKEN)
- && vocab.containsKey(
- WordpieceTokenizer.ROBERTA_SEP_TOKEN)) {
- return new WordpieceEncoder(
- vocab,
- lowerCase,
- WordpieceTokenizer.ROBERTA_CLS_TOKEN,
- WordpieceTokenizer.ROBERTA_SEP_TOKEN,
- resolveUnknownToken(vocab));
+ final SpecialTokens special = specialTokens(vocab);
+ return new WordpieceEncoder(vocab, lowerCase, special.cls(), special.sep(), special.unk());
+ }
+
+ /**
+ * The classification, separator and unknown tokens a tokenizer adds or emits.
+ *
+ * @param cls The classification token.
+ * @param sep The separator token.
+ * @param unk The unknown token.
+ */
+ private record SpecialTokens(String cls, String sep, String unk) {
+ }
+
+ /**
+ * Selects the special tokens for a vocabulary: the RoBERTa tokens if the vocabulary has the
+ * RoBERTa classification and separator tokens, the BERT tokens otherwise.
+ *
+ * @param vocab The vocabulary map.
+ * @return The special tokens.
+ * @throws IllegalArgumentException Thrown if the vocabulary has the RoBERTa classification
+ * and separator tokens but no unknown token.
+ */
+ private static SpecialTokens specialTokens(final Map<String, Integer> vocab) {
+ if (vocab.containsKey(WordpieceTokenizer.ROBERTA_CLS_TOKEN)
+ && vocab.containsKey(WordpieceTokenizer.ROBERTA_SEP_TOKEN)) {
+ return new SpecialTokens(WordpieceTokenizer.ROBERTA_CLS_TOKEN,
+ WordpieceTokenizer.ROBERTA_SEP_TOKEN, resolveUnknownToken(vocab));
}
- return new WordpieceEncoder(vocab, lowerCase,
- WordpieceTokenizer.BERT_CLS_TOKEN,
- WordpieceTokenizer.BERT_SEP_TOKEN,
- WordpieceTokenizer.BERT_UNK_TOKEN);
+ return new SpecialTokens(WordpieceTokenizer.BERT_CLS_TOKEN,
+ WordpieceTokenizer.BERT_SEP_TOKEN, WordpieceTokenizer.BERT_UNK_TOKEN);
+ }
+
+ /**
+ * Runs the model on one token window and returns its first output. The input tensors are
+ * closed before this method returns.
+ *
+ * @param tokens The encoded token window.
+ * @param settings The settings that decide whether the {@value #ATTENTION_MASK} and
+ * {@value #TOKEN_TYPE_IDS} inputs are passed.
+ * @return The value of the first model output, copied into Java arrays.
+ * @throws IllegalArgumentException Thrown if {@code tokens} or {@code settings} is
+ * {@code null}.
+ * @throws IllegalStateException Thrown if the model cannot be run, with the failure as cause.
+ */
+ @Internal(since = "3.0.0")
+ protected final Object runModel(final Tokens tokens, final InferenceSettings settings) {
+ ParamChecks.requireNonNullArg(tokens, "tokens");
+ ParamChecks.requireNonNullArg(settings, "settings");
+ // At most three inputs (ids, attention mask, token type ids), so size for exactly that.
+ final Map<String, OnnxTensor> inputs = HashMap.newHashMap(3);
+ try {
+ inputs.put(INPUT_IDS, tensor(tokens.ids()));
+ if (settings.includeAttentionMask()) {
+ inputs.put(ATTENTION_MASK, tensor(tokens.mask()));
+ }
+ if (settings.includeTokenTypeIds()) {
+ inputs.put(TOKEN_TYPE_IDS, tensor(tokens.types()));
+ }
+ // getValue() copies the tensor into Java arrays, so the result can be closed safely.
+ try (OrtSession.Result result = session.run(inputs)) {
+ return result.get(0).getValue();
+ }
+ } catch (OrtException | RuntimeException ex) {
+ throw new IllegalStateException("Unable to run the ONNX model: " + ex.getMessage(), ex);
+ } finally {
+ inputs.values().forEach(OnnxTensor::close);
+ }
+ }
+
+ /**
+ * Creates a tensor of shape {@code [1, values.length]} from one model input row.
+ *
+ * @param values The input row.
+ * @return The tensor.
+ * @throws OrtException Thrown if the tensor cannot be created.
+ */
+ private OnnxTensor tensor(final long[] values) throws OrtException {
+ return OnnxTensor.createTensor(env, LongBuffer.wrap(values), new long[] {1, values.length});
}
/**
@@ -399,6 +534,7 @@ public abstract class AbstractDL implements AutoCloseable {
* @param options The {@link InferenceOptions} to consult.
* @param componentDefault The default to apply if the option is not set.
* @return The effective lower casing behavior.
+ * @throws IllegalArgumentException Thrown if {@code options} is {@code null}.
*/
protected static boolean resolveLowerCase(
final InferenceOptions options, final boolean componentDefault) {
@@ -477,8 +613,7 @@ public abstract class AbstractDL implements AutoCloseable {
/**
* Like {@link #normalizeInput(String, boolean, boolean)} but also produces an {@link Alignment}
* from the folded text back to {@code text}, so model output positions map to original character
- * offsets even when a fold changes the string length (a supplementary dash shrinking, or, for
- * folds that may be added later, an expansion such as an ellipsis to three dots).
+ * offsets even when a fold changes the string length, as a supplementary-plane dash does.
*
* @param text The input text.
* @param normalizeWhitespace Whether to fold whitespace to ASCII spaces.
@@ -488,12 +623,7 @@ public abstract class AbstractDL implements AutoCloseable {
@Internal(since = "3.0.0")
protected static AlignedText normalizeInputAligned(final String text,
final boolean normalizeWhitespace, final boolean normalizeDashes) {
- // Compose each enabled fold's alignment with the running alignment so the returned mapping is
- // correct no matter whether a stage changes length. Whitespace folding here is a one-for-one
- // replacement and so is length-preserving today; only dash folding moves offsets (a
- // supplementary-plane dash shrinks from two chars to one). Composing through andThen rather
- // than relying on the whitespace stage staying length-preserving keeps findInOriginal() correct
- // if that ever changes.
+ // Each enabled fold's alignment is composed with the running one.
AlignedText result = identityAligned(text, text);
if (normalizeWhitespace) {
result = result.andThen(WHITESPACE.normalizeAligned(result.normalized()));
@@ -523,6 +653,8 @@ public abstract class AbstractDL implements AutoCloseable {
* @param documentSplitSize The maximum number of whitespace tokens per chunk.
* @param splitOverlapSize The number of tokens shared between consecutive chunks.
* @return The chunk strings, in order.
+ * @throws IllegalArgumentException Thrown if the split settings cannot make progress, see
+ * {@link #validateSplitOptions(int, int)}.
*/
@Internal(since = "3.0.0")
protected static List<String> whitespaceChunks(final String text, final int documentSplitSize,
@@ -544,6 +676,8 @@ public abstract class AbstractDL implements AutoCloseable {
* @param documentSplitSize The maximum number of whitespace tokens per chunk.
* @param splitOverlapSize The number of tokens shared between consecutive chunks.
* @return The chunks, in order, each with its character span in {@code text}.
+ * @throws IllegalArgumentException Thrown if the split settings cannot make progress, see
+ * {@link #validateSplitOptions(int, int)}.
*/
@Internal(since = "3.0.0")
protected static List<TextChunk> whitespaceChunkSpans(final String text,
diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/JsonScan.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/JsonScan.java
index b817c1bef..86c6b8ed6 100644
--- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/JsonScan.java
+++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/JsonScan.java
@@ -67,9 +67,9 @@ public final class JsonScan {
private static final String SIMPLE_ESCAPES = "\"\\/bfnrt";
/** The length of a two-character escape such as {@code \n}. */
private static final int ESCAPE_LENGTH = 2;
- /** The number of hexadecimal digits of a {@code \\u} escape. */
+ /** The number of hexadecimal digits of a <code>\u</code> escape. */
private static final int UNICODE_ESCAPE_DIGITS = 4;
- /** The length of a {@code \\u} escape including the backslash and the {@code u}. */
+ /** The length of a <code>\u</code> escape including the backslash and the {@code u}. */
private static final int UNICODE_ESCAPE_LENGTH = ESCAPE_LENGTH + UNICODE_ESCAPE_DIGITS;
/** The number of characters quoted in the message for malformed text. */
private static final int MESSAGE_CONTEXT_LENGTH = 20;
@@ -362,7 +362,7 @@ public final class JsonScan {
/**
* Finds the closing quote of a string literal, checking each escape on the way: a backslash
- * must be followed by one of the escape characters of RFC 8259, and {@code \\u} by four
+ * must be followed by one of the escape characters of RFC 8259, and <code>\u</code> by four
* hexadecimal digits.
*
* @param text The JSON text.
diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerDL.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerDL.java
index 4082c122e..d2c1c2570 100644
--- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerDL.java
+++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerDL.java
@@ -19,7 +19,6 @@ package opennlp.dl.doccat;
import java.io.File;
import java.io.IOException;
-import java.nio.LongBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
@@ -32,7 +31,6 @@ import java.util.SortedMap;
import java.util.TreeMap;
import java.util.stream.IntStream;
-import ai.onnxruntime.OnnxTensor;
import ai.onnxruntime.OrtEnvironment;
import ai.onnxruntime.OrtException;
import ai.onnxruntime.OrtSession;
@@ -79,14 +77,7 @@ public class DocumentCategorizerDL extends AbstractDL implements DocumentCategor
private final Map<Integer, String> categories;
private final ClassificationScoringStrategy classificationScoringStrategy;
- // Inference options are snapshotted into final fields at construction so a shared
- // instance never reads the caller's mutable InferenceOptions during inference.
- private final boolean includeAttentionMask;
- private final boolean includeTokenTypeIds;
- private final int documentSplitSize;
- private final int splitOverlapSize;
- private final boolean normalizeWhitespace;
- private final boolean normalizeDashes;
+ private final InferenceSettings settings;
/**
* Test-only constructor that injects an already-built {@link OrtSession} (or {@code null}),
@@ -99,12 +90,7 @@ public class DocumentCategorizerDL extends AbstractDL implements DocumentCategor
super(env, session, vocab, resolveLowerCase(inferenceOptions, LOWER_CASE_DEFAULT));
this.categories = Map.copyOf(categories);
this.classificationScoringStrategy = classificationScoringStrategy;
- this.includeAttentionMask = inferenceOptions.isIncludeAttentionMask();
- this.includeTokenTypeIds = inferenceOptions.isIncludeTokenTypeIds();
- this.documentSplitSize = inferenceOptions.getDocumentSplitSize();
- this.splitOverlapSize = inferenceOptions.getSplitOverlapSize();
- this.normalizeWhitespace = inferenceOptions.isNormalizeWhitespace();
- this.normalizeDashes = inferenceOptions.isNormalizeDashes();
+ this.settings = InferenceSettings.from(inferenceOptions);
}
/**
@@ -121,6 +107,10 @@ public class DocumentCategorizerDL extends AbstractDL implements DocumentCategor
* @throws IOException Thrown if errors occurred loading the {@code model} or {@code vocabulary}.
* @throws InvalidFormatException Thrown if a JSON {@code vocabulary} is malformed, has an
* unsupported layout, or sets a lower casing that disagrees with {@code inferenceOptions}.
+ * @throws IllegalArgumentException Thrown if {@code model}, {@code vocabulary},
+ * {@code categories}, {@code classificationScoringStrategy} or {@code inferenceOptions} is
+ * {@code null}, if the split sizes of {@code inferenceOptions} are invalid, or if the
+ * vocabulary lacks the classification, separator or unknown token.
*/
public DocumentCategorizerDL(File model, File vocabulary, Map<Integer, String> categories,
ClassificationScoringStrategy classificationScoringStrategy,
@@ -129,17 +119,12 @@ public class DocumentCategorizerDL extends AbstractDL implements DocumentCategor
super(model, vocabulary,
sessionOptions(validateConstructorArguments(
- inferenceOptions, categories, classificationScoringStrategy)),
+ inferenceOptions, categories, "categories", classificationScoringStrategy)),
resolveLowerCase(inferenceOptions, LOWER_CASE_DEFAULT));
this.categories = Map.copyOf(categories);
this.classificationScoringStrategy = classificationScoringStrategy;
- this.includeAttentionMask = inferenceOptions.isIncludeAttentionMask();
- this.includeTokenTypeIds = inferenceOptions.isIncludeTokenTypeIds();
- this.documentSplitSize = inferenceOptions.getDocumentSplitSize();
- this.splitOverlapSize = inferenceOptions.getSplitOverlapSize();
- this.normalizeWhitespace = inferenceOptions.isNormalizeWhitespace();
- this.normalizeDashes = inferenceOptions.isNormalizeDashes();
+ this.settings = InferenceSettings.from(inferenceOptions);
}
@@ -161,6 +146,10 @@ public class DocumentCategorizerDL extends AbstractDL implements DocumentCategor
* unsupported layout, or sets a lower casing that disagrees with {@code inferenceOptions},
* or if {@code config} is not well-formed JSON, its {@code id2label} member does not map
* keys to strings, or a key is not an integer.
+ * @throws IllegalArgumentException Thrown if {@code model}, {@code vocabulary}, {@code config},
+ * {@code classificationScoringStrategy} or {@code inferenceOptions} is {@code null}, if the
+ * split sizes of {@code inferenceOptions} are invalid, or if the vocabulary lacks the
+ * classification, separator or unknown token.
*/
public DocumentCategorizerDL(File model, File vocabulary, File config,
ClassificationScoringStrategy classificationScoringStrategy,
@@ -169,25 +158,30 @@ public class DocumentCategorizerDL extends AbstractDL implements DocumentCategor
super(model, vocabulary,
sessionOptions(validateConstructorArguments(
- inferenceOptions, config, classificationScoringStrategy)),
+ inferenceOptions, config, "config", classificationScoringStrategy)),
resolveLowerCase(inferenceOptions, LOWER_CASE_DEFAULT));
this.categories = Map.copyOf(readCategories(config));
this.classificationScoringStrategy = classificationScoringStrategy;
- this.includeAttentionMask = inferenceOptions.isIncludeAttentionMask();
- this.includeTokenTypeIds = inferenceOptions.isIncludeTokenTypeIds();
- this.documentSplitSize = inferenceOptions.getDocumentSplitSize();
- this.splitOverlapSize = inferenceOptions.getSplitOverlapSize();
- this.normalizeWhitespace = inferenceOptions.isNormalizeWhitespace();
- this.normalizeDashes = inferenceOptions.isNormalizeDashes();
+ this.settings = InferenceSettings.from(inferenceOptions);
}
+ /**
+ * Checks the arguments of a public constructor that the base class does not check.
+ *
+ * @param inferenceOptions The inference options.
+ * @param labels The categories or the configuration file that supplies them.
+ * @param labelsName The parameter name of {@code labels}, used in the message.
+ * @param classificationScoringStrategy The scoring strategy.
+ * @return {@code inferenceOptions}.
+ * @throws IllegalArgumentException Thrown if an argument is {@code null}.
+ */
private static InferenceOptions validateConstructorArguments(
- final InferenceOptions inferenceOptions, final Object categoriesOrConfig,
+ final InferenceOptions inferenceOptions, final Object labels, final String labelsName,
final ClassificationScoringStrategy classificationScoringStrategy) {
ParamChecks.requireNonNullArg(inferenceOptions, "inferenceOptions");
- ParamChecks.requireNonNullArg(categoriesOrConfig, "categoriesOrConfig");
+ ParamChecks.requireNonNullArg(labels, labelsName);
ParamChecks.requireNonNullArg(classificationScoringStrategy, "classificationScoringStrategy");
return inferenceOptions;
}
@@ -200,8 +194,9 @@ public class DocumentCategorizerDL extends AbstractDL implements DocumentCategor
*
* @param strings The document to categorize; {@code strings[0]} is classified.
* @return The per-category probabilities.
- * @throws IllegalArgumentException If {@code strings} is {@code null} or empty, or if
- * {@code strings[0]} has no tokens to classify (it is empty or only whitespace).
+ * @throws IllegalArgumentException If {@code strings} is {@code null} or empty, if
+ * {@code strings[0]} is {@code null}, or if {@code strings[0]} has no tokens to classify
+ * (it is empty or only whitespace).
* @throws IllegalStateException If inference fails or the model returns an unexpected output.
*/
@Override
@@ -242,40 +237,15 @@ public class DocumentCategorizerDL extends AbstractDL implements DocumentCategor
}
/**
- * Runs the model on one token window and returns its raw per-category logits. A failure executing
- * the model (an {@link OrtException} or any runtime fault) is wrapped as an
- * {@link IllegalStateException}; an unexpected output shape is its own loud failure.
+ * Runs the model on one token window and returns its raw per-category logits.
+ *
+ * @param t The encoded token window.
+ * @return The logits.
+ * @throws IllegalStateException Thrown if the model cannot be run or returns an output that is
+ * neither {@code float[][]} nor {@code float[]}.
*/
private float[] infer(final Tokens t) {
-
- // At most three inputs (ids, attention mask, token type ids), so size for exactly that.
- final Map<String, OnnxTensor> inputs = HashMap.newHashMap(3);
- final Object output;
- try {
- inputs.put(INPUT_IDS, OnnxTensor.createTensor(env,
- LongBuffer.wrap(t.ids()), new long[] {1, t.ids().length}));
-
- if (includeAttentionMask) {
- inputs.put(ATTENTION_MASK, OnnxTensor.createTensor(env,
- LongBuffer.wrap(t.mask()), new long[] {1, t.mask().length}));
- }
-
- if (includeTokenTypeIds) {
- inputs.put(TOKEN_TYPE_IDS, OnnxTensor.createTensor(env,
- LongBuffer.wrap(t.types()), new long[] {1, t.types().length}));
- }
-
- // getValue() copies the tensor into Java arrays, so the result can be closed safely.
- try (OrtSession.Result result = session.run(inputs)) {
- output = result.get(0).getValue();
- }
- } catch (OrtException | RuntimeException ex) {
- throw new IllegalStateException("Unable to perform document classification inference", ex);
- } finally {
- inputs.values().forEach(OnnxTensor::close);
- }
-
- return logitsFromOutput(output);
+ return logitsFromOutput(runModel(t, settings));
}
// Package-visible so the output-shape dispatch, including the null and unexpected-type failures,
@@ -375,12 +345,14 @@ public class DocumentCategorizerDL extends AbstractDL implements DocumentCategor
private List<Tokens> tokenize(final String input) {
- final String text = normalizeInput(input, normalizeWhitespace, normalizeDashes);
+ final String text =
+ normalizeInput(input, settings.normalizeWhitespace(), settings.normalizeDashes());
// Segment long input text into overlapping chunks (split on Unicode whitespace) configured by
// InferenceOptions before feeding each chunk into BERT.
// https://medium.com/analytics-vidhya/text-classification-with-bert-using-transformers-for-long-text-inputs-f54833994dfd
- final List<String> groups = whitespaceChunks(text, documentSplitSize, splitOverlapSize);
+ final List<String> groups =
+ whitespaceChunks(text, settings.documentSplitSize(), settings.splitOverlapSize());
final List<Tokens> t = new ArrayList<>(groups.size());
for (final String group : groups) {
@@ -434,16 +406,24 @@ public class DocumentCategorizerDL extends AbstractDL implements DocumentCategor
*/
static Map<Integer, String> readCategories(File config) throws IOException {
final String json = Files.readString(config.toPath(), StandardCharsets.UTF_8);
- final Map<Integer, String> categories = new HashMap<>();
- try {
- for (Map.Entry<String, String> label : DocumentCategorizerConfig.fromJson(json).id2label().entrySet()) {
- categories.put(parseIndex(label.getKey()), label.getValue());
- }
- } catch (IllegalArgumentException e) {
- throw new InvalidFormatException(
- "Configuration file " + config.getName() + ": " + e.getMessage(), e);
+ return parseJson(config, "Configuration", json, DocumentCategorizerDL::parseCategories);
+ }
+
+ /**
+ * Reads the categories from the {@code id2label} member of a model configuration.
+ *
+ * @param json The JSON text of the configuration.
+ * @return The categories by output index, empty if the configuration has no {@code id2label}.
+ * @throws IllegalArgumentException Thrown if the text is not well-formed JSON, its
+ * {@code id2label} member does not map keys to strings, or a key is not an integer.
+ */
+ private static Map<Integer, String> parseCategories(String json) {
+ final Map<Integer, String> parsed = new HashMap<>();
+ for (Map.Entry<String, String> label
+ : DocumentCategorizerConfig.fromJson(json).id2label().entrySet()) {
+ parsed.put(parseIndex(label.getKey()), label.getValue());
}
- return categories;
+ return parsed;
}
/**
diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/namefinder/NameFinderDL.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/namefinder/NameFinderDL.java
index b9b0baf33..99358f549 100644
--- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/namefinder/NameFinderDL.java
+++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/namefinder/NameFinderDL.java
@@ -19,16 +19,14 @@ package opennlp.dl.namefinder;
import java.io.File;
import java.io.IOException;
-import java.nio.LongBuffer;
import java.util.ArrayList;
import java.util.Comparator;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
-import ai.onnxruntime.OnnxTensor;
+import ai.onnxruntime.OrtEnvironment;
import ai.onnxruntime.OrtException;
import ai.onnxruntime.OrtSession;
import org.slf4j.Logger;
@@ -105,14 +103,27 @@ public class NameFinderDL extends AbstractDL implements OffsetMappingNameFinder
private final SentenceDetector sentenceDetector;
private final Map<Integer, String> ids2Labels;
- // Inference options are snapshotted into final fields at construction so a shared
- // instance never reads the caller's mutable InferenceOptions during inference.
- private final boolean includeAttentionMask;
- private final boolean includeTokenTypeIds;
- private final int documentSplitSize;
- private final int splitOverlapSize;
- private final boolean normalizeWhitespace;
- private final boolean normalizeDashes;
+ private final InferenceSettings settings;
+
+ /**
+ * Test-only constructor that injects an already-built {@link OrtSession} (or {@code null}),
+ * bypassing model loading so inference paths can be exercised in unit tests.
+ *
+ * @param env The ONNX environment, or {@code null}.
+ * @param session The ONNX session, or {@code null}.
+ * @param vocab The vocabulary used by the tokenizer.
+ * @param ids2Labels The mapping of model output indices to BIO labels.
+ * @param inferenceOptions {@link InferenceOptions} to control the inference.
+ * @param sentenceDetector The {@link SentenceDetector} to be used.
+ */
+ NameFinderDL(OrtEnvironment env, OrtSession session, Map<String, Integer> vocab,
+ Map<Integer, String> ids2Labels, InferenceOptions inferenceOptions,
+ SentenceDetector sentenceDetector) {
+ super(env, session, vocab, resolveLowerCase(inferenceOptions, LOWER_CASE_DEFAULT));
+ this.ids2Labels = Map.copyOf(ids2Labels);
+ this.settings = InferenceSettings.from(inferenceOptions);
+ this.sentenceDetector = sentenceDetector;
+ }
/**
* Instantiates a {@link opennlp.tools.namefind.TokenNameFinder name finder} using ONNX models.
@@ -129,8 +140,9 @@ public class NameFinderDL extends AbstractDL implements OffsetMappingNameFinder
* @throws opennlp.tools.util.InvalidFormatException Thrown if a JSON {@code vocabulary}
* is malformed, has an unsupported layout, or sets a lower casing that disagrees with
* the lower casing the component is configured with.
- * @throws IllegalArgumentException Thrown if {@code inferenceOptions}, {@code ids2Labels}, or
- * the sentence detector is {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code model}, {@code vocabulary},
+ * {@code ids2Labels} or {@code sentenceDetector} is {@code null}, or if the vocabulary
+ * lacks the classification, separator or unknown token.
*/
public NameFinderDL(File model, File vocabulary, Map<Integer, String> ids2Labels,
SentenceDetector sentenceDetector) throws IOException, OrtException {
@@ -155,8 +167,10 @@ public class NameFinderDL extends AbstractDL implements OffsetMappingNameFinder
* @throws opennlp.tools.util.InvalidFormatException Thrown if a JSON {@code vocabulary}
* is malformed, has an unsupported layout, or sets a lower casing that disagrees with
* the lower casing the component is configured with.
- * @throws IllegalArgumentException Thrown if {@code inferenceOptions}, {@code ids2Labels}, or
- * the sentence detector is {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code model}, {@code vocabulary},
+ * {@code ids2Labels}, {@code inferenceOptions} or {@code sentenceDetector} is
+ * {@code null}, if the split sizes of {@code inferenceOptions} are invalid, or if the
+ * vocabulary lacks the classification, separator or unknown token.
*/
public NameFinderDL(File model, File vocabulary, Map<Integer, String> ids2Labels,
InferenceOptions inferenceOptions,
@@ -168,16 +182,20 @@ public class NameFinderDL extends AbstractDL implements OffsetMappingNameFinder
resolveLowerCase(inferenceOptions, LOWER_CASE_DEFAULT));
this.ids2Labels = Map.copyOf(ids2Labels);
- this.includeAttentionMask = inferenceOptions.isIncludeAttentionMask();
- this.includeTokenTypeIds = inferenceOptions.isIncludeTokenTypeIds();
- this.documentSplitSize = inferenceOptions.getDocumentSplitSize();
- this.splitOverlapSize = inferenceOptions.getSplitOverlapSize();
- this.normalizeWhitespace = inferenceOptions.isNormalizeWhitespace();
- this.normalizeDashes = inferenceOptions.isNormalizeDashes();
+ this.settings = InferenceSettings.from(inferenceOptions);
this.sentenceDetector = sentenceDetector;
}
+ /**
+ * Checks the arguments of a public constructor that the base class does not check.
+ *
+ * @param inferenceOptions The inference options.
+ * @param ids2Labels The mapping of model output indices to BIO labels.
+ * @param sentenceDetector The sentence detector.
+ * @return {@code inferenceOptions}.
+ * @throws IllegalArgumentException Thrown if an argument is {@code null}.
+ */
private static InferenceOptions validateConstructorArguments(
final InferenceOptions inferenceOptions, final Map<Integer, String> ids2Labels,
final SentenceDetector sentenceDetector) {
@@ -194,13 +212,7 @@ public class NameFinderDL extends AbstractDL implements OffsetMappingNameFinder
* <p>Joins the provided tokens with spaces, sentence-splits the joined text, runs each sentence
* through the ONNX token-classification model, decodes BIO labels into {@link Span spans}, and
* resolves those spans to character offsets in the joined text <em>after</em> any optional input
- * normalization.</p>
- *
- * <p>Note: this returns correct original offsets in every case except one. Whitespace folding is
- * length-preserving, so it never moves offsets. Only dash folding can change the input length, and
- * only for a non-BMP dash; so when {@code normalizeDashes} is enabled and the input contains a
- * supplementary-plane dash, the returned spans are offsets into the normalized text rather than
- * the original. For an exact original mapping in that case, use {@link #findInOriginal(String[])}.</p>
+ * normalization; use {@link #findInOriginal(String[])} for offsets in the original input.</p>
*
* @throws IllegalStateException Thrown if inference fails, if the model output shape is not
* the expected {@code float[batch][token][label]} form, if the model output contains
@@ -219,10 +231,8 @@ public class NameFinderDL extends AbstractDL implements OffsetMappingNameFinder
* Finds names and returns their {@link Span spans} in coordinates of the original joined input
* ({@code String.join(" ", input)}), regardless of any whitespace or dash normalization applied
* before inference. Spans are mapped back through the normalization {@link Alignment}, so a fold
- * that changes the input length (a supplementary dash shrinking, or an expansion) does not shift
- * the reported offsets. This implements {@link OffsetMappingNameFinder}, so an interface-typed
- * caller can reach the offset-correct path with
- * {@code finder instanceof OffsetMappingNameFinder}.
+ * that changes the input length, such as a supplementary-plane dash, does not shift the reported
+ * offsets.
*
* @param input The tokens to search.
* @return The detected spans, in original-input character coordinates.
@@ -266,7 +276,8 @@ public class NameFinderDL extends AbstractDL implements OffsetMappingNameFinder
// Join the tokens here because they will be tokenized using Wordpiece during inference.
final AlignedText normalized =
- normalizeInputAligned(String.join(" ", input), normalizeWhitespace, normalizeDashes);
+ normalizeInputAligned(String.join(" ", input), settings.normalizeWhitespace(),
+ settings.normalizeDashes());
final String text = normalized.normalizedString();
// sentPosDetect (not sentDetect) so each sentence's offset in the full text is known.
@@ -317,17 +328,12 @@ public class NameFinderDL extends AbstractDL implements OffsetMappingNameFinder
*/
static List<Span> mergeOverlappingSpans(final List<Span> spans) {
if (spans.size() < 2) {
- // Return a fresh list so the caller always owns the result, matching the >= 2 path below
- // (which returns a new list); the input is never handed back aliased.
+ // Always return a new list, never the input.
return new ArrayList<>(spans);
}
final List<Span> byDominance = new ArrayList<>(spans);
byDominance.sort(BY_LENGTH_THEN_PROBABILITY);
- // Kept spans never overlap each other, so they form a start-sorted partition. A candidate can
- // only intersect the kept span that starts at or just before it (floor) or the next kept span
- // that starts within it (ceiling); checking those two is O(log n) instead of scanning every kept
- // span, making the whole longest-wins pass O(n log n) rather than O(n^2). Keyed by start, the map
- // also yields the result already in document order.
+ // Kept spans are disjoint, so a candidate can only overlap its floor or ceiling entry by start.
final TreeMap<Integer, Span> kept = new TreeMap<>();
for (final Span candidate : byDominance) {
final Map.Entry<Integer, Span> before = kept.floorEntry(candidate.getStart());
@@ -344,46 +350,16 @@ public class NameFinderDL extends AbstractDL implements OffsetMappingNameFinder
}
/**
- * Runs the model on one token window and returns the per-token label score rows. A failure
- * executing the model (an {@link OrtException} or any runtime fault) is surfaced as an
- * {@link IllegalStateException} (cause preserved); an unexpected output shape is its own loud
- * failure. This mirrors the fail-loud contract of the sibling {@code DocumentCategorizerDL}.
+ * Runs the model on one token window and returns the per-token label score rows.
*
* @param tokens The tokens for one chunk to run inference on.
* @return The {@code [token][label]} score matrix for the chunk.
+ * @throws IllegalStateException Thrown if the model cannot be run, or if its output is not a
+ * non-empty {@code float[batch][token][label]} array.
*/
private float[][] infer(final Tokens tokens) {
- // At most three inputs (ids, attention mask, token type ids), so size for exactly that.
- final Map<String, OnnxTensor> inputs = HashMap.newHashMap(3);
- final Object output;
- try {
- inputs.put(INPUT_IDS, OnnxTensor.createTensor(env, LongBuffer.wrap(tokens.ids()),
- new long[] {1, tokens.ids().length}));
-
- if (includeAttentionMask) {
- inputs.put(ATTENTION_MASK, OnnxTensor.createTensor(env,
- LongBuffer.wrap(tokens.mask()), new long[] {1, tokens.mask().length}));
- }
-
- if (includeTokenTypeIds) {
- inputs.put(TOKEN_TYPE_IDS, OnnxTensor.createTensor(env,
- LongBuffer.wrap(tokens.types()), new long[] {1, tokens.types().length}));
- }
-
- // getValue() copies the tensor into Java arrays, so the result can be closed safely.
- try (OrtSession.Result result = session.run(inputs)) {
- output = result.get(0).getValue();
- }
- } catch (OrtException ex) {
- throw new IllegalStateException(
- "Unable to perform name finder inference: " + ex.getMessage(), ex);
- } catch (RuntimeException ex) {
- throw new IllegalStateException(
- "Unexpected runtime failure during name finder inference: " + ex.getMessage(), ex);
- } finally {
- inputs.values().forEach(OnnxTensor::close);
- }
+ final Object output = runModel(tokens, settings);
// The model returns one score row per token, batched: float[batch][token][label]. Any other
// shape (or an empty batch) is a model-contract violation, surfaced on its own rather than as
@@ -430,9 +406,8 @@ public class NameFinderDL extends AbstractDL implements OffsetMappingNameFinder
* @param tokens The WordPiece tokens produced for the text.
* @param tokenLabelScores The per-token label scores returned by the model.
* @param id2Labels The mapping from model output indexes to BIO labels.
- * @param searchStart The character offset in {@code text} to begin locating spans from. Threading
- * a monotonic cursor across the chunks and sentences of a single {@link #find(String[])} call
- * keeps a repeated entity surface form from being emitted twice at the same first occurrence.
+ * @param searchStart The character offset in {@code text} to begin locating spans from; spans
+ * are located between it and the end of {@code text}.
* @return The decoded {@link Span spans}.
*/
static List<Span> decodeSpans(String text, String[] tokens, float[][] tokenLabelScores,
@@ -448,10 +423,11 @@ public class NameFinderDL extends AbstractDL implements OffsetMappingNameFinder
* @param tokens The WordPiece tokens produced for the text.
* @param tokenLabelScores The per-token label scores returned by the model.
* @param id2Labels The mapping from model output indexes to BIO labels.
- * @param searchStart The first character offset in {@code text} to search.
+ * @param searchStart The first character offset in {@code text} to search. During
+ * {@link #find(String[])}, this is the start of the chunk the tokens were encoded from.
* @param searchEnd The exclusive upper bound for locating reconstructed spans. During
- * {@link #find(String[])}, this is the current sentence end so an entity from one sentence
- * cannot be resolved to an identical surface form in a later sentence.
+ * {@link #find(String[])}, this is the end of that chunk, so an entity is located only
+ * within the text its chunk covers.
* @return The decoded {@link Span spans}.
*/
static List<Span> decodeSpans(String text, String[] tokens, float[][] tokenLabelScores,
@@ -663,13 +639,10 @@ public class NameFinderDL extends AbstractDL implements OffsetMappingNameFinder
/**
* Locates reconstructed span text in a bounded region of the original input text.
*
- * <p>Matching is a single forward cursor scan, not a regular expression. Each space in the
- * reconstructed span matches a run of zero or more Unicode whitespace characters in the source
- * (so an entity whose WordPiece pieces were rejoined with spaces, such as {@code "AT & T"} for
- * {@code "AT&T"}, is still located), and every other code point matches case-insensitively.
- * Using a cursor avoids {@link java.util.regex.Pattern}/{@link java.util.regex.Matcher}
- * allocation and the ReDoS surface of regular expressions, and recognizes Unicode whitespace
- * that Java's {@code \s} does not.</p>
+ * <p>Each space in the reconstructed span matches a run of zero or more Unicode
+ * {@code White_Space} characters in the source (so an entity whose WordPiece pieces were
+ * rejoined with spaces, such as {@code "AT & T"} for {@code "AT&T"}, is still located), and
+ * every other code point matches case-insensitively.</p>
*
* @param text The original text.
* @param span The reconstructed span text, with sub-tokens separated by single ASCII spaces.
@@ -758,7 +731,8 @@ public class NameFinderDL extends AbstractDL implements OffsetMappingNameFinder
// InferenceOptions before feeding each chunk into BERT, keeping each chunk's character span so
// its decoded spans can be bounded to the region the chunk covers.
// https://medium.com/analytics-vidhya/text-classification-with-bert-using-transformers-for-long-text-inputs-f54833994dfd
- final List<TextChunk> chunks = whitespaceChunkSpans(text, documentSplitSize, splitOverlapSize);
+ final List<TextChunk> chunks =
+ whitespaceChunkSpans(text, settings.documentSplitSize(), settings.splitOverlapSize());
final List<ChunkTokens> t = new ArrayList<>(chunks.size());
for (final TextChunk chunk : chunks) {
diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/AbstractDLChunkingTest.java b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/AbstractDLChunkingTest.java
index 13957ea9b..9797c1f0e 100644
--- a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/AbstractDLChunkingTest.java
+++ b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/AbstractDLChunkingTest.java
@@ -17,8 +17,12 @@
package opennlp.dl;
import java.util.List;
+import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
import opennlp.tools.util.Span;
import opennlp.tools.util.normalizer.AlignedText;
@@ -48,19 +52,17 @@ public class AbstractDLChunkingTest {
assertEquals(List.of("a b c"), AbstractDL.whitespaceChunks(" a b\tc ", 100, 0));
}
- @Test
- void testAppliesChunkSizeWithoutOverlap() {
- assertEquals(List.of("a b", "c d"), AbstractDL.whitespaceChunks("a b c d", 2, 0));
+ private static Stream<Arguments> chunkCases() {
+ return Stream.of(
+ Arguments.of("a b c d", 2, 0, List.of("a b", "c d")),
+ Arguments.of("a b c d", 2, 1, List.of("a b", "b c", "c d")),
+ Arguments.of("", 100, 0, List.of()));
}
- @Test
- void testAppliesChunkOverlap() {
- assertEquals(List.of("a b", "b c", "c d"), AbstractDL.whitespaceChunks("a b c d", 2, 1));
- }
-
- @Test
- void testEmptyTextYieldsNoChunks() {
- assertEquals(List.of(), AbstractDL.whitespaceChunks("", 100, 0));
+ @ParameterizedTest
+ @MethodSource("chunkCases")
+ void testAppliesChunkSizeAndOverlap(String text, int size, int overlap, List<String> expected) {
+ assertEquals(expected, AbstractDL.whitespaceChunks(text, size, overlap));
}
@Test
diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/InferenceOptionsValidationTest.java b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/InferenceOptionsValidationTest.java
index d54597793..27f207477 100644
--- a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/InferenceOptionsValidationTest.java
+++ b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/InferenceOptionsValidationTest.java
@@ -20,6 +20,7 @@ package opennlp.dl;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
public class InferenceOptionsValidationTest {
@@ -74,4 +75,25 @@ public class InferenceOptionsValidationTest {
assertThrows(IllegalArgumentException.class, () -> AbstractDL.validateSplitOptions(options));
}
+
+ @Test
+ void testInferenceSettingsCopyTheOptionsOnce() {
+ final InferenceOptions options = new InferenceOptions();
+ options.setIncludeAttentionMask(false);
+ options.setIncludeTokenTypeIds(false);
+ options.setDocumentSplitSize(7);
+ options.setSplitOverlapSize(3);
+ options.setNormalizeWhitespace(true);
+ options.setNormalizeDashes(true);
+
+ final AbstractDL.InferenceSettings settings = AbstractDL.InferenceSettings.from(options);
+ options.setDocumentSplitSize(9);
+
+ assertEquals(new AbstractDL.InferenceSettings(false, false, 7, 3, true, true), settings);
+ }
+
+ @Test
+ void testInferenceSettingsRejectNullOptions() {
+ assertThrows(IllegalArgumentException.class, () -> AbstractDL.InferenceSettings.from(null));
+ }
}
diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/LoadVocabTest.java b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/LoadVocabTest.java
index f28e690e1..4428469ae 100644
--- a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/LoadVocabTest.java
+++ b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/LoadVocabTest.java
@@ -552,6 +552,25 @@ public class LoadVocabTest {
assertEquals(Map.of("a", 0, "b", 1), DL.loadVocab(tempFile));
}
+ @ParameterizedTest
+ @ValueSource(strings = {" \n\t{\"a\": 0, \"b\": 1} \n", "\uFEFF\r\n{\"a\": 0, \"b\": 1}"})
+ void testJsonVocabFileWithLeadingWhitespaceIsReadAsJson(String content) throws IOException {
+ final File tempFile = vocabFile("vocab-space.json", content);
+
+ assertEquals(Map.of("a", 0, "b", 1), DL.loadVocab(tempFile));
+ }
+
+ @Test
+ void testJsonVocabFileErrorNamesTheOffsetInTheFile() throws IOException {
+ final String content = "\n {\"a\": \"x\"}";
+ final File tempFile = vocabFile("vocab-offset.json", content);
+
+ final InvalidFormatException e =
+ assertThrows(InvalidFormatException.class, () -> DL.loadVocab(tempFile));
+ assertTrue(e.getMessage().contains(tempFile.getName()), e.getMessage());
+ assertTrue(e.getMessage().contains("offset " + content.indexOf("\"x\"")), e.getMessage());
+ }
+
@Test
void testPlainTextVocabFileWithAByteOrderMarkKeepsTheFirstToken() throws IOException {
final File tempFile = vocabFile("vocab-bom.txt", "\uFEFF[CLS]\n[SEP]\n");
diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/doccat/DocumentCategorizerDLTest.java b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/doccat/DocumentCategorizerDLTest.java
index 5c31b3b43..9b6b7373c 100644
--- a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/doccat/DocumentCategorizerDLTest.java
+++ b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/doccat/DocumentCategorizerDLTest.java
@@ -64,12 +64,31 @@ public class DocumentCategorizerDLTest {
new AverageClassificationScoringStrategy(), new InferenceOptions());
}
+ @Test
+ void testConstructorNamesTheNullCategoriesParameter() {
+ final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () ->
+ new DocumentCategorizerDL(new File("model.onnx"), new File("vocab.txt"),
+ (Map<Integer, String>) null, new AverageClassificationScoringStrategy(),
+ new InferenceOptions()));
+
+ assertEquals("categories must not be null", e.getMessage());
+ }
+
+ @Test
+ void testConstructorNamesTheNullConfigParameter() {
+ final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () ->
+ new DocumentCategorizerDL(new File("model.onnx"), new File("vocab.txt"),
+ (File) null, new AverageClassificationScoringStrategy(), new InferenceOptions()));
+
+ assertEquals("config must not be null", e.getMessage());
+ }
+
@Test
void testCategorizeFailsLoudlyWhenInferenceFails() {
final IllegalStateException e = assertThrows(IllegalStateException.class, () ->
categorizerWithoutSession().categorize(new String[] {"hello world"}));
- assertTrue(e.getMessage().contains("document classification inference"));
+ assertTrue(e.getMessage().contains("Unable to run the ONNX model"), e.getMessage());
assertTrue(e.getCause() instanceof RuntimeException);
}
diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/namefinder/NameFinderDLTest.java b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/namefinder/NameFinderDLTest.java
index 931b09d03..a07c51a3f 100644
--- a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/namefinder/NameFinderDLTest.java
+++ b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/namefinder/NameFinderDLTest.java
@@ -24,6 +24,8 @@ import java.util.Map;
import org.junit.jupiter.api.Test;
+import opennlp.dl.InferenceOptions;
+import opennlp.tools.sentdetect.NewlineSentenceDetector;
import opennlp.tools.tokenize.WordpieceTokenizer;
import opennlp.tools.util.Span;
@@ -54,6 +56,18 @@ public class NameFinderDLTest {
return vocab;
}
+ @Test
+ void testFindFailsWithModelErrorWhenInferenceFails() {
+ final NameFinderDL finder = new NameFinderDL(null, null, vocab(), ID_TO_LABELS,
+ new InferenceOptions(), new NewlineSentenceDetector());
+
+ final IllegalStateException e = assertThrows(IllegalStateException.class, () ->
+ finder.find(new String[] {"hello", "world"}));
+
+ assertTrue(e.getMessage().startsWith("Unable to run the ONNX model: "), e.getMessage());
+ assertTrue(e.getCause() instanceof NullPointerException, String.valueOf(e.getCause()));
+ }
+
@Test
void testDecodeSpansUsesBioEntityTypesAndBoundedProbabilities() {
final String text = "Alice visited New York City.";
@@ -128,9 +142,7 @@ public class NameFinderDLTest {
@Test
void testDecodeSpansSearchStartLocatesNextOccurrence() {
- // "Paris" appears twice. Threading the cursor past the first occurrence (as find() does
- // across chunks/sentences) locates the second one instead of re-emitting the first, so a
- // repeated entity is not duplicated at the same offset.
+ // searchStart past the first occurrence locates the second one.
final String text = "Paris and Paris";
final String[] tokens = {"[CLS]", "Paris", "[SEP]"};
final float[][] scores = {scoresFor(0), scoresFor(3), scoresFor(0)};