You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by ma...@apache.org on 2026/10/03 09:59:28 UTC
(opennlp) branch main updated: Minor: Share layer alignment and span mapping across document annotators (#1357)
This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/main by this push:
new 932dbd94c Minor: Share layer alignment and span mapping across document annotators (#1357)
932dbd94c is described below
commit 932dbd94c65a21167187d961da3e338c31409670
Author: Richard Zowalla <13...@users.noreply.github.com>
AuthorDate: Sat Oct 3 11:59:23 2026 +0200
Minor: Share layer alignment and span mapping across document annotators (#1357)
* Minor: Share layer alignment and span mapping across document annotators
DocumentAnnotators gains requireAligned, values and toCharacterSpan.
ChunkerAnnotator, LemmatizerAnnotator and NameFinderAnnotator use them
instead of their own alignment check, tag gather and span mapping.
TermVectorAnnotator uses DocumentAnnotators.requireLayers.
* Pin that requireAligned reports an absent layer as absent
The Javadoc of DocumentAnnotators.requireAligned asks for a document that
carries both layers, but the method compares sizes only. Document.get returns
an empty list for an absent layer, so one absent layer is reported as
misaligned and a document without either layer passes.
* Check layer presence in requireAligned and map parser spans through the helper
---------
Signed-off-by: Kristian Rickert <kr...@apache.org>
Co-authored-by: Kristian Rickert <kr...@apache.org>
---
.../opennlp/tools/document/DocumentAnnotators.java | 107 ++++++++++++-
.../tools/document/DocumentAnnotatorsTest.java | 170 ++++++++++++++++++++-
.../opennlp/tools/chunker/ChunkerAnnotator.java | 27 ++--
.../tools/lemmatizer/LemmatizerAnnotator.java | 10 +-
.../tools/namefind/NameFinderAnnotator.java | 19 +--
.../java/opennlp/tools/parser/ParserAnnotator.java | 5 +-
.../tools/termvector/TermVectorAnnotator.java | 9 +-
7 files changed, 291 insertions(+), 56 deletions(-)
diff --git a/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotators.java b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotators.java
index fd5f81b85..f4fca8ed1 100644
--- a/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotators.java
+++ b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotators.java
@@ -20,9 +20,12 @@ package opennlp.tools.document;
import java.util.List;
import java.util.Set;
+import opennlp.tools.util.Span;
+
/**
* Support methods shared by {@link DocumentAnnotator} implementations: the
- * required-layer check and the per-sentence walk over the token layer.
+ * required-layer and layer-alignment checks, the per-sentence walk over the token
+ * layer, and the mapping of sentence-relative token spans onto character spans.
*
* <p>These helpers keep the annotators' shared behavior identical across
* implementations: an absent required layer is always rejected with the same message
@@ -100,11 +103,7 @@ public final class DocumentAnnotators {
if (count == 0) {
continue;
}
- final String[] words = new String[count];
- for (int i = 0; i < count; i++) {
- words[i] = tokens.get(first + i).value();
- }
- consumer.accept(first, words);
+ consumer.accept(first, values(tokens, first, count));
}
if (next != tokens.size()) {
throw new IllegalArgumentException("token at " + tokens.get(next).span()
@@ -112,6 +111,102 @@ public final class DocumentAnnotators {
}
}
+ /**
+ * Verifies that two layers of a document hold the same number of annotations, so the
+ * annotation at each position of one belongs to the annotation at the same position of
+ * the other.
+ *
+ * @param document The document to check. Must not be {@code null}.
+ * @param layer The first layer, for example {@link Layers#TOKENS}.
+ * @param aligned The layer that must be aligned with {@code layer}, for example
+ * {@link Layers#POS_TAGS}.
+ * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, if a
+ * layer is absent, or if the two layers differ in size; the message names the
+ * absent layer, or both layers.
+ */
+ public static void requireAligned(Document document, LayerKey<?> layer,
+ LayerKey<?> aligned) {
+ requireLayers(document, layer, aligned);
+ if (document.get(layer).size() != document.get(aligned).size()) {
+ throw new IllegalArgumentException("document needs aligned "
+ + layer + " and " + aligned + " layers");
+ }
+ }
+
+ /**
+ * Collects the values of a contiguous run of annotations, for example the tags of one
+ * sentence's tokens during {@link #forEachSentence(List, List, SentenceTokenConsumer)}.
+ *
+ * @param layer The layer to read. Must not be {@code null}.
+ * @param first The position of the run's first annotation in {@code layer}.
+ * @param count The number of annotations in the run.
+ * @return The values at the layer positions {@code [first, first + count)}, in layer
+ * order. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code layer} is {@code null}, or if the
+ * run does not lie inside the layer.
+ */
+ public static String[] values(List<Annotation<String>> layer, int first, int count) {
+ if (layer == null) {
+ throw new IllegalArgumentException("layer must not be null");
+ }
+ if (first < 0 || count < 0 || first > layer.size() - count) {
+ throw new IllegalArgumentException("run of " + count + " annotations at " + first
+ + " lies outside the layer's " + layer.size() + " annotations");
+ }
+ final String[] values = new String[count];
+ for (int i = 0; i < count; i++) {
+ values[i] = layer.get(first + i).value();
+ }
+ return values;
+ }
+
+ /**
+ * Maps a span over one sentence's tokens onto the character span those tokens cover.
+ *
+ * <p>A sentence-level component such as a chunker or a name finder indexes the tokens
+ * within the sentence. The span is shifted by the sentence's first token position and
+ * resolved through the token layer, whose spans already refer to the original text, so
+ * the result runs from the start of the span's first token to the end of its last
+ * token. An empty span covers no token and therefore has no character span; it is
+ * rejected together with the out-of-bounds ones.</p>
+ *
+ * @param tokens The token layer. Must not be {@code null}.
+ * @param first The position of the sentence's first token in {@code tokens}.
+ * @param count The number of tokens in the sentence.
+ * @param span The span over the sentence's tokens, with an exclusive end. Must not be
+ * {@code null}.
+ * @param source How the rejection message introduces {@code span}, for example
+ * {@code "chunker returned chunk"}. Must not be {@code null}.
+ * @return The character span covered by the tokens, without a type. Never
+ * {@code null}.
+ * @throws IllegalArgumentException Thrown if an argument is {@code null}, if the
+ * sentence does not lie inside {@code tokens}, or if {@code span} is empty or
+ * reaches outside the sentence's {@code count} tokens; the message for a
+ * rejected span starts with {@code source}.
+ */
+ public static Span toCharacterSpan(List<Annotation<String>> tokens, int first,
+ int count, Span span, String source) {
+ if (tokens == null) {
+ throw new IllegalArgumentException("tokens must not be null");
+ }
+ if (span == null) {
+ throw new IllegalArgumentException("span must not be null");
+ }
+ if (source == null) {
+ throw new IllegalArgumentException("source must not be null");
+ }
+ if (first < 0 || count < 0 || first > tokens.size() - count) {
+ throw new IllegalArgumentException("sentence of " + count + " tokens at " + first
+ + " lies outside the token layer's " + tokens.size() + " tokens");
+ }
+ if (span.getStart() < 0 || span.getEnd() > count || span.getStart() >= span.getEnd()) {
+ throw new IllegalArgumentException(source + " " + span
+ + " outside the sentence's " + count + " tokens");
+ }
+ return new Span(tokens.get(first + span.getStart()).span().getStart(),
+ tokens.get(first + span.getEnd() - 1).span().getEnd());
+ }
+
private DocumentAnnotators() {
// Not instantiated; this class provides static support methods only.
}
diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnnotatorsTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnnotatorsTest.java
index bef5db118..a7c1792ce 100644
--- a/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnnotatorsTest.java
+++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnnotatorsTest.java
@@ -21,17 +21,21 @@ import java.util.ArrayList;
import java.util.List;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
import opennlp.tools.util.Span;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Tests the {@link DocumentAnnotators} support methods directly: the required-layer
- * check's exact rejection messages and the per-sentence walk's slicing, skipping, and
- * loud rejection of a token outside every sentence. The adapter tests exercise the same
+ * and alignment checks' exact rejection messages, the per-sentence walk's slicing,
+ * skipping, and loud rejection of a token outside every sentence, the value gather,
+ * and the token-to-character span mapping. The adapter tests exercise the same
* behavior through the annotators; this class pins the helpers as public API on their
* own.
*/
@@ -118,4 +122,166 @@ public class DocumentAnnotatorsTest {
(first, words) -> runs.add(List.of(words)));
assertTrue(runs.isEmpty());
}
+
+ @Test
+ void testRequireAlignedAcceptsEqualSizes() {
+ final Document document = Document.of("Ana runs")
+ .with(Layers.TOKENS, List.of(
+ new Annotation<>(new Span(0, 3), "Ana"),
+ new Annotation<>(new Span(4, 8), "runs")))
+ .with(Layers.POS_TAGS, List.of(
+ new Annotation<>(new Span(0, 3), "NNP"),
+ new Annotation<>(new Span(4, 8), "VBZ")));
+ DocumentAnnotators.requireAligned(document, Layers.TOKENS, Layers.POS_TAGS);
+ }
+
+ @Test
+ void testRequireAlignedAcceptsEmptyLayers() {
+ final Document document = Document.of("")
+ .with(Layers.TOKENS, List.of())
+ .with(Layers.POS_TAGS, List.of());
+ DocumentAnnotators.requireAligned(document, Layers.TOKENS, Layers.POS_TAGS);
+ }
+
+ @Test
+ void testRequireAlignedRejectsDifferentSizes() {
+ final Document document = Document.of("Ana runs")
+ .with(Layers.TOKENS, List.of(
+ new Annotation<>(new Span(0, 3), "Ana"),
+ new Annotation<>(new Span(4, 8), "runs")))
+ .with(Layers.POS_TAGS, List.of(new Annotation<>(new Span(0, 3), "NNP")));
+ final IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+ () -> DocumentAnnotators.requireAligned(document, Layers.TOKENS, Layers.POS_TAGS));
+ assertEquals("document needs aligned " + Layers.TOKENS + " and " + Layers.POS_TAGS
+ + " layers", e.getMessage());
+ }
+
+ /**
+ * Verifies that an absent layer is reported as absent, not as misaligned, and that two
+ * absent layers do not pass as aligned.
+ */
+ @Test
+ void testRequireAlignedRejectsAbsentLayers() {
+ final Document oneLayer = Document.of("Ana runs")
+ .with(Layers.TOKENS, List.of(
+ new Annotation<>(new Span(0, 3), "Ana"),
+ new Annotation<>(new Span(4, 8), "runs")));
+ final IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+ () -> DocumentAnnotators.requireAligned(oneLayer, Layers.TOKENS, Layers.POS_TAGS));
+ assertEquals("document lacks the required layer " + Layers.POS_TAGS, e.getMessage());
+
+ final Document noLayers = Document.of("Ana runs");
+ assertThrows(IllegalArgumentException.class,
+ () -> DocumentAnnotators.requireAligned(noLayers, Layers.TOKENS, Layers.POS_TAGS));
+ }
+
+ @Test
+ void testRequireAlignedRejectsNullDocument() {
+ final IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+ () -> DocumentAnnotators.requireAligned(null, Layers.TOKENS, Layers.POS_TAGS));
+ assertEquals("document must not be null", e.getMessage());
+ }
+
+ /**
+ * Verifies that the gather returns the run's values in layer order, including values
+ * outside the Basic Multilingual Plane unchanged, and an empty array for an empty run.
+ */
+ @Test
+ void testValuesGathersTheRun() {
+ final List<Annotation<String>> layer = List.of(
+ new Annotation<>(new Span(0, 1), "a"),
+ new Annotation<>(new Span(2, 4), "\uD83D\uDE00"),
+ new Annotation<>(new Span(5, 6), "c"));
+ assertArrayEquals(new String[] {"\uD83D\uDE00", "c"},
+ DocumentAnnotators.values(layer, 1, 2));
+ assertArrayEquals(new String[] {"a", "\uD83D\uDE00", "c"},
+ DocumentAnnotators.values(layer, 0, 3));
+ assertArrayEquals(new String[0], DocumentAnnotators.values(layer, 3, 0));
+ }
+
+ @ParameterizedTest(name = "run of {1} at {0}")
+ @CsvSource({"-1, 1", "0, -1", "2, 2", "4, 0", "1, " + Integer.MAX_VALUE})
+ void testValuesRejectsRunOutsideTheLayer(int first, int count) {
+ final List<Annotation<String>> layer = List.of(
+ new Annotation<>(new Span(0, 1), "a"),
+ new Annotation<>(new Span(2, 3), "b"),
+ new Annotation<>(new Span(4, 5), "c"));
+ final IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+ () -> DocumentAnnotators.values(layer, first, count));
+ assertEquals("run of " + count + " annotations at " + first
+ + " lies outside the layer's 3 annotations", e.getMessage());
+ }
+
+ @Test
+ void testValuesRejectsNullLayer() {
+ final IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+ () -> DocumentAnnotators.values(null, 0, 0));
+ assertEquals("layer must not be null", e.getMessage());
+ }
+
+ /**
+ * Verifies that a sentence-relative span is shifted by the sentence's first token and
+ * resolved to the characters from its first token's start to its last token's end.
+ */
+ @ParameterizedTest(name = "span [{0}..{1}) maps to [{2}..{3})")
+ @CsvSource({"0, 1, 4, 7", "1, 3, 8, 17", "0, 3, 4, 17", "2, 3, 13, 17"})
+ void testToCharacterSpanMapsTokenRun(int start, int end, int charStart, int charEnd) {
+ final Span covered = DocumentAnnotators.toCharacterSpan(sentenceTokens(), 1, 3,
+ new Span(start, end, "type"), "finder returned mention");
+ assertEquals(new Span(charStart, charEnd), covered);
+ }
+
+ @ParameterizedTest(name = "span [{0}..{1})")
+ @CsvSource({"0, 4", "3, 4", "2, 2", "3, 3"})
+ void testToCharacterSpanRejectsSpanOutsideSentence(int start, int end) {
+ final Span span = new Span(start, end, "NP");
+ final IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+ () -> DocumentAnnotators.toCharacterSpan(sentenceTokens(), 1, 3, span,
+ "chunker returned chunk"));
+ assertEquals("chunker returned chunk " + span + " outside the sentence's 3 tokens",
+ e.getMessage());
+ }
+
+ @Test
+ void testToCharacterSpanRejectsEmptySpanWithExactMessage() {
+ final IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+ () -> DocumentAnnotators.toCharacterSpan(sentenceTokens(), 1, 3, new Span(0, 0),
+ "finder returned mention"));
+ assertEquals("finder returned mention [0..0) outside the sentence's 3 tokens",
+ e.getMessage());
+ }
+
+ @ParameterizedTest(name = "sentence of {1} at {0}")
+ @CsvSource({"-1, 1", "2, 3", "0, -1", "1, " + Integer.MAX_VALUE})
+ void testToCharacterSpanRejectsSentenceOutsideTokens(int first, int count) {
+ final IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+ () -> DocumentAnnotators.toCharacterSpan(sentenceTokens(), first, count,
+ new Span(0, 1), "finder returned mention"));
+ assertEquals("sentence of " + count + " tokens at " + first
+ + " lies outside the token layer's 4 tokens", e.getMessage());
+ }
+
+ @Test
+ void testToCharacterSpanRejectsNullArguments() {
+ final List<Annotation<String>> tokens = sentenceTokens();
+ final Span span = new Span(0, 1);
+ assertEquals("tokens must not be null", assertThrows(IllegalArgumentException.class,
+ () -> DocumentAnnotators.toCharacterSpan(null, 0, 1, span, "x")).getMessage());
+ assertEquals("span must not be null", assertThrows(IllegalArgumentException.class,
+ () -> DocumentAnnotators.toCharacterSpan(tokens, 0, 1, null, "x")).getMessage());
+ assertEquals("source must not be null", assertThrows(IllegalArgumentException.class,
+ () -> DocumentAnnotators.toCharacterSpan(tokens, 0, 1, span, null)).getMessage());
+ }
+
+ /**
+ * {@return the token layer of "Hi. Ana sees Bob!": a one-token sentence followed by
+ * a three-token sentence starting at position 1}
+ */
+ private static List<Annotation<String>> sentenceTokens() {
+ return List.of(
+ new Annotation<>(new Span(0, 3), "Hi."),
+ new Annotation<>(new Span(4, 7), "Ana"),
+ new Annotation<>(new Span(8, 12), "sees"),
+ new Annotation<>(new Span(13, 17), "Bob!"));
+ }
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/chunker/ChunkerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/chunker/ChunkerAnnotator.java
index 8bd4a17a4..25f9ffaf8 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/chunker/ChunkerAnnotator.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/chunker/ChunkerAnnotator.java
@@ -54,6 +54,9 @@ public final class ChunkerAnnotator implements DocumentAnnotator {
*/
public static final LayerKey<String> CHUNKS = Layers.key("chunks", String.class);
+ /** Introduces a rejected chunk in exception messages. */
+ private static final String CHUNK_SOURCE = "chunker returned chunk";
+
private final Chunker chunker;
/**
@@ -91,32 +94,20 @@ public final class ChunkerAnnotator implements DocumentAnnotator {
public Document annotate(Document document) {
DocumentAnnotators.requireLayers(document, Layers.SENTENCES, Layers.TOKENS,
Layers.POS_TAGS);
+ DocumentAnnotators.requireAligned(document, Layers.TOKENS, Layers.POS_TAGS);
final List<Annotation<String>> sentences = document.get(Layers.SENTENCES);
final List<Annotation<String>> tokens = document.get(Layers.TOKENS);
final List<Annotation<String>> tags = document.get(Layers.POS_TAGS);
- if (tags.size() != tokens.size()) {
- throw new IllegalArgumentException("document needs aligned "
- + Layers.TOKENS + " and " + Layers.POS_TAGS + " layers");
- }
final List<Annotation<String>> chunks = new ArrayList<>();
DocumentAnnotators.forEachSentence(sentences, tokens, (first, words) -> {
- final String[] sentenceTags = new String[words.length];
- for (int i = 0; i < words.length; i++) {
- sentenceTags[i] = tags.get(first + i).value();
- }
+ final String[] sentenceTags = DocumentAnnotators.values(tags, first, words.length);
for (final Span chunk : chunker.chunkAsSpans(words, sentenceTags)) {
- if (chunk.getStart() < 0 || chunk.getEnd() > words.length
- || chunk.getStart() >= chunk.getEnd()) {
- throw new IllegalArgumentException("chunker returned chunk " + chunk
- + " outside the sentence's " + words.length + " tokens");
- }
+ final Span covered = DocumentAnnotators.toCharacterSpan(tokens, first,
+ words.length, chunk, CHUNK_SOURCE);
if (chunk.getType() == null) {
- throw new IllegalArgumentException(
- "chunker returned chunk " + chunk + " without a type");
+ throw new IllegalArgumentException(CHUNK_SOURCE + " " + chunk + " without a type");
}
- chunks.add(new Annotation<>(new Span(
- tokens.get(first + chunk.getStart()).span().getStart(),
- tokens.get(first + chunk.getEnd() - 1).span().getEnd()), chunk.getType()));
+ chunks.add(new Annotation<>(covered, chunk.getType()));
}
});
return document.with(CHUNKS, chunks);
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java
index 2c633798e..babf0aca2 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java
@@ -89,19 +89,13 @@ public final class LemmatizerAnnotator implements DocumentAnnotator {
public Document annotate(Document document) {
DocumentAnnotators.requireLayers(document,
Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS);
+ DocumentAnnotators.requireAligned(document, Layers.TOKENS, Layers.POS_TAGS);
final List<Annotation<String>> sentences = document.get(Layers.SENTENCES);
final List<Annotation<String>> tokens = document.get(Layers.TOKENS);
final List<Annotation<String>> tags = document.get(Layers.POS_TAGS);
- if (tags.size() != tokens.size()) {
- throw new IllegalArgumentException("document needs aligned "
- + Layers.TOKENS + " and " + Layers.POS_TAGS + " layers");
- }
final List<Annotation<String>> layer = new ArrayList<>(tokens.size());
DocumentAnnotators.forEachSentence(sentences, tokens, (first, words) -> {
- final String[] posTags = new String[words.length];
- for (int i = 0; i < words.length; i++) {
- posTags[i] = tags.get(first + i).value();
- }
+ final String[] posTags = DocumentAnnotators.values(tags, first, words.length);
final String[] lemmas = lemmatizer.lemmatize(words, posTags);
if (lemmas.length != words.length) {
throw new IllegalArgumentException("lemmatizer returned " + lemmas.length
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderAnnotator.java
index 43baa231e..8705393c1 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderAnnotator.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderAnnotator.java
@@ -57,6 +57,9 @@ public final class NameFinderAnnotator implements DocumentAnnotator {
*/
public static final String UNTYPED = NameSample.DEFAULT_TYPE;
+ /** Introduces a rejected mention in exception messages. */
+ private static final String MENTION_SOURCE = "finder returned mention";
+
private final TokenNameFinder finder;
/**
@@ -103,21 +106,11 @@ public final class NameFinderAnnotator implements DocumentAnnotator {
// cannot leak finder state into the next one.
try {
DocumentAnnotators.forEachSentence(sentences, tokens, (first, words) -> {
- // The finder indexes within the sentence; shifting by the sentence's first
- // token position turns every mention boundary into a document-wide token
- // index, whose token spans already refer to the original text. An empty
- // mention is rejected with the out-of-bounds ones: it covers no token, so it
- // has no character span.
for (final Span mention : finder.find(words)) {
- if (mention.getStart() < 0 || mention.getEnd() > words.length
- || mention.getStart() >= mention.getEnd()) {
- throw new IllegalArgumentException("finder returned mention " + mention
- + " outside the sentence's " + words.length + " tokens");
- }
- final int start = tokens.get(first + mention.getStart()).span().getStart();
- final int end = tokens.get(first + mention.getEnd() - 1).span().getEnd();
+ final Span covered = DocumentAnnotators.toCharacterSpan(tokens, first,
+ words.length, mention, MENTION_SOURCE);
final String type = mention.getType() == null ? UNTYPED : mention.getType();
- entities.add(new Annotation<>(new Span(start, end), type));
+ entities.add(new Annotation<>(covered, type));
}
});
} finally {
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/parser/ParserAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/parser/ParserAnnotator.java
index e38b2fdb2..820521c62 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/parser/ParserAnnotator.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/parser/ParserAnnotator.java
@@ -154,8 +154,9 @@ public final class ParserAnnotator implements DocumentAnnotator {
throw new IllegalArgumentException("parser returned node " + node.getType()
+ " with head " + head + " outside the sentence's " + starts.length + " tokens");
}
- phrases.add(new Annotation<>(new Span(tokens.get(first + from).span().getStart(),
- tokens.get(first + to).span().getEnd()),
+ final Span covered = DocumentAnnotators.toCharacterSpan(tokens, first, starts.length,
+ new Span(from, to + 1), "parser returned node " + node.getType() + " at");
+ phrases.add(new Annotation<>(covered,
new Phrase(node.getType(), tokens.get(first + head).span())));
for (final Parse child : node.getChildren()) {
collect(child, first, starts, length, tokens, phrases);
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java
index 4ed12d0d1..285ca691c 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java
@@ -26,6 +26,7 @@ import java.util.Set;
import opennlp.tools.document.Annotation;
import opennlp.tools.document.Document;
import opennlp.tools.document.DocumentAnnotator;
+import opennlp.tools.document.DocumentAnnotators;
import opennlp.tools.document.LayerKey;
import opennlp.tools.document.Layers;
import opennlp.tools.util.Span;
@@ -211,13 +212,7 @@ public final class TermVectorAnnotator implements DocumentAnnotator {
*/
@Override
public Document annotate(Document document) {
- if (document == null) {
- throw new IllegalArgumentException("document must not be null");
- }
- if (!document.layers().contains(Layers.TOKENS)) {
- throw new IllegalArgumentException("document lacks the required layer "
- + Layers.TOKENS);
- }
+ DocumentAnnotators.requireLayers(document, Layers.TOKENS);
final List<Annotation<String>> tokens = document.get(Layers.TOKENS);
final AlignedText aligned =
normalizer != null ? normalizer.normalizeAligned(document.text()) : null;