You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by kr...@apache.org on 2026/08/16 01:12:59 UTC
(opennlp) 06/06: OPENNLP-1897: Review round: reject null span elements, finalize the annotator, share the space-split test fixture, pin supplementary-plane offsets
This is an automated email from the ASF dual-hosted git repository.
krickert pushed a commit to branch OPENNLP-1897-term-vectors
in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 9829831e0ac35940a0ce6b947e5fb2c469ab3906
Author: Kristian Rickert <kr...@gmail.com>
AuthorDate: Sat Aug 15 21:08:37 2026 -0400
OPENNLP-1897: Review round: reject null span elements, finalize the annotator, share the space-split test fixture, pin supplementary-plane offsets
---
.../java/opennlp/tools/termvector/TermVector.java | 15 +++-
.../tools/termvector/TermVectorAnnotator.java | 2 +-
.../tools/termvector/SingleSpaceTokens.java | 90 ++++++++++++++++++++++
.../tools/termvector/TermVectorAnnotatorTest.java | 53 ++++++++-----
.../tools/termvector/TermVectorPipelineTest.java | 39 +---------
.../opennlp/tools/termvector/TermVectorTest.java | 9 +++
6 files changed, 147 insertions(+), 61 deletions(-)
diff --git a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVector.java b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVector.java
index 676bf0131..42c654257 100644
--- a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVector.java
+++ b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVector.java
@@ -43,8 +43,9 @@ import opennlp.tools.util.Span;
* @param term The term string. Must not be {@code null}.
* @param frequency The number of occurrences in the document. Must be at least one.
* @param spans The occurrence spans in original text coordinates, one per occurrence,
- * or an empty list for a scoring-only vector. Must not be {@code null} and,
- * when non-empty, must contain exactly {@code frequency} spans.
+ * or an empty list for a scoring-only vector. Must not be {@code null} or
+ * contain {@code null} and, when non-empty, must hold exactly
+ * {@code frequency} spans.
*
* @since 3.0.0
*/
@@ -55,8 +56,9 @@ public record TermVector(String term, int frequency, List<Span> spans) {
* Validates the term vector and detaches the span list from the caller's input.
*
* @throws IllegalArgumentException Thrown if {@code term} is {@code null},
- * {@code frequency} is below one, {@code spans} is {@code null}, or a
- * non-empty {@code spans} list does not hold exactly {@code frequency} spans.
+ * {@code frequency} is below one, {@code spans} is or contains {@code null},
+ * or a non-empty {@code spans} list does not hold exactly {@code frequency}
+ * spans.
*/
public TermVector {
if (term == null) {
@@ -68,6 +70,11 @@ public record TermVector(String term, int frequency, List<Span> spans) {
if (spans == null) {
throw new IllegalArgumentException("spans must not be null");
}
+ for (final Span span : spans) {
+ if (span == null) {
+ throw new IllegalArgumentException("spans must not contain null");
+ }
+ }
if (!spans.isEmpty() && spans.size() != frequency) {
throw new IllegalArgumentException("a full term vector holds one span per "
+ "occurrence: frequency is " + frequency + " but spans holds " + spans.size());
diff --git a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java
index 27282db16..bec73882f 100644
--- a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java
+++ b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java
@@ -63,7 +63,7 @@ import opennlp.tools.util.normalizer.OffsetAwareNormalizer;
*
* @since 3.0.0
*/
-public class TermVectorAnnotator implements DocumentAnnotator {
+public final class TermVectorAnnotator implements DocumentAnnotator {
/**
* The key of the term vector layer this annotator provides: a document-scoped layer
diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java b/opennlp-api/src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java
new file mode 100644
index 000000000..4f7beb464
--- /dev/null
+++ b/opennlp-api/src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.termvector;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import opennlp.tools.document.Annotation;
+import opennlp.tools.tokenize.Tokenizer;
+import opennlp.tools.util.Span;
+
+/**
+ * Deterministic single-space tokenization shared by the term vector tests: splits on
+ * single space characters and keeps all other characters, including sentence-final
+ * periods, attached to their token. Runs of spaces yield no empty tokens, so every
+ * expected span follows directly from the input text.
+ */
+final class SingleSpaceTokens {
+
+ /**
+ * A {@link Tokenizer} view of {@link #spans(String)}. Only the span-producing method
+ * is implemented because the annotator adapter calls no other method.
+ */
+ static final Tokenizer TOKENIZER = new Tokenizer() {
+
+ @Override
+ public String[] tokenize(String s) {
+ throw new UnsupportedOperationException("the adapter only calls tokenizePos");
+ }
+
+ @Override
+ public Span[] tokenizePos(String s) {
+ return spans(s).toArray(new Span[0]);
+ }
+ };
+
+ private SingleSpaceTokens() {
+ }
+
+ /**
+ * Computes the token spans of a text split on single space characters.
+ *
+ * @param text The text to split.
+ * @return One span per token, in text order.
+ */
+ static List<Span> spans(String text) {
+ final List<Span> spans = new ArrayList<>();
+ int start = -1;
+ for (int i = 0; i <= text.length(); i++) {
+ final boolean boundary = i == text.length() || text.charAt(i) == ' ';
+ if (boundary && start >= 0) {
+ spans.add(new Span(start, i));
+ start = -1;
+ } else if (!boundary && start < 0) {
+ start = i;
+ }
+ }
+ return spans;
+ }
+
+ /**
+ * Builds a token layer from {@link #spans(String)}, each token valued with its covered
+ * text.
+ *
+ * @param text The text to split.
+ * @return One annotation per token, in text order.
+ */
+ static List<Annotation<String>> tokens(String text) {
+ final List<Annotation<String>> tokens = new ArrayList<>();
+ for (final Span span : spans(text)) {
+ tokens.add(new Annotation<>(span, text.substring(span.getStart(), span.getEnd())));
+ }
+ return tokens;
+ }
+}
diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java
index 03eeb130b..e928d5dfc 100644
--- a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java
+++ b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java
@@ -18,7 +18,6 @@
package opennlp.tools.termvector;
import java.io.Serial;
-import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Set;
@@ -148,30 +147,18 @@ public class TermVectorAnnotatorTest {
* A plain normalizer that deletes every digit, so a token made up of digits alone
* normalizes to the empty string on the per-token path.
*/
- private static final CharSequenceNormalizer PLAIN_DROP_DIGITS =
- text -> text.toString().replaceAll("\\d", "");
-
- /**
- * Splits a text on single space characters into a token layer, skipping the empty
- * tokens a collapsed run would produce, so double spaces yield no tokens.
- */
- private static List<Annotation<String>> splitOnSpaces(String text) {
- final List<Annotation<String>> tokens = new ArrayList<>();
- int start = -1;
- for (int i = 0; i <= text.length(); i++) {
- final boolean boundary = i == text.length() || text.charAt(i) == ' ';
- if (boundary && start >= 0) {
- tokens.add(new Annotation<>(new Span(start, i), text.substring(start, i)));
- start = -1;
- } else if (!boundary && start < 0) {
- start = i;
+ private static final CharSequenceNormalizer PLAIN_DROP_DIGITS = text -> {
+ final StringBuilder out = new StringBuilder(text.length());
+ for (int i = 0; i < text.length(); i++) {
+ if (!Character.isDigit(text.charAt(i))) {
+ out.append(text.charAt(i));
}
}
- return tokens;
- }
+ return out.toString();
+ };
private static Document documentWithTokens(String text) {
- return Document.of(text).with(Layers.TOKENS, splitOnSpaces(text));
+ return Document.of(text).with(Layers.TOKENS, SingleSpaceTokens.tokens(text));
}
@Test
@@ -364,6 +351,30 @@ public class TermVectorAnnotatorTest {
vectors.get(1).value());
}
+ /**
+ * Spans are UTF-16 offsets, so a supplementary-plane character occupies two positions:
+ * {@code "𝕏 x 𝕏"} tokenizes to spans of width two around the surrogate pairs, and both
+ * occurrences group under one term whose spans still cover the original text exactly.
+ */
+ @Test
+ void testSupplementaryPlaneTokensKeepUtf16Offsets() {
+ final String text = "𝕏 x 𝕏";
+ final Document document = new TermVectorAnnotator()
+ .annotate(documentWithTokens(text));
+
+ final List<Annotation<TermVector>> vectors =
+ document.get(TermVectorAnnotator.TERM_VECTORS);
+ assertEquals(2, vectors.size());
+ assertEquals(new TermVector("𝕏", 2, List.of(new Span(0, 2), new Span(5, 7))),
+ vectors.get(0).value());
+ assertEquals(new TermVector("x", 1, List.of(new Span(3, 4))), vectors.get(1).value());
+ for (final Annotation<TermVector> vector : vectors) {
+ for (final Span span : vector.value().spans()) {
+ assertEquals(vector.value().term(), span.getCoveredText(text).toString());
+ }
+ }
+ }
+
@Test
void testEmptyTokenLayerYieldsPresentButEmptyLayer() {
final Document document = new TermVectorAnnotator()
diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java
index a00fba96b..291b2068c 100644
--- a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java
+++ b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java
@@ -17,7 +17,6 @@
package opennlp.tools.termvector;
-import java.util.ArrayList;
import java.util.List;
import java.util.Set;
@@ -28,7 +27,6 @@ import opennlp.tools.document.Document;
import opennlp.tools.document.DocumentAnalyzer;
import opennlp.tools.document.Layers;
import opennlp.tools.document.TokenizerAnnotator;
-import opennlp.tools.tokenize.Tokenizer;
import opennlp.tools.util.Span;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -37,40 +35,11 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
/**
* Wires the {@link TermVectorAnnotator} into a {@link DocumentAnalyzer} behind a
* {@link TokenizerAnnotator}: the token layer goes in, the term vector layer comes out,
- * and nothing else about the document changes. The tokenizer is a deterministic stand-in
- * defined here, so every expected span follows directly from the input text.
+ * and nothing else about the document changes. The tokenizer is the shared deterministic
+ * single-space fixture, so every expected span follows directly from the input text.
*/
public class TermVectorPipelineTest {
- /**
- * A deterministic tokenizer that splits on single space characters and keeps all
- * other characters, including sentence-final periods, attached to their token. Only
- * the span-producing method is implemented because the adapter calls no other method.
- */
- private static final Tokenizer SPACE_TOKENIZER = new Tokenizer() {
-
- @Override
- public String[] tokenize(String s) {
- throw new UnsupportedOperationException("the adapter only calls tokenizePos");
- }
-
- @Override
- public Span[] tokenizePos(String s) {
- final List<Span> spans = new ArrayList<>();
- int start = -1;
- for (int i = 0; i <= s.length(); i++) {
- final boolean boundary = i == s.length() || s.charAt(i) == ' ';
- if (boundary && start >= 0) {
- spans.add(new Span(start, i));
- start = -1;
- } else if (!boundary && start < 0) {
- start = i;
- }
- }
- return spans.toArray(new Span[0]);
- }
- };
-
/**
* Runs tokenizer plus term vector roll-up over a text with repeated tokens and reads
* the aggregated layer back, span by span, in original text coordinates.
@@ -78,7 +47,7 @@ public class TermVectorPipelineTest {
@Test
void testTokenizerAndTermVectorPipeline() {
final DocumentAnalyzer analyzer = DocumentAnalyzer.builder()
- .add(new TokenizerAnnotator(SPACE_TOKENIZER))
+ .add(new TokenizerAnnotator(SingleSpaceTokens.TOKENIZER))
.add(new TermVectorAnnotator())
.build();
@@ -113,7 +82,7 @@ public class TermVectorPipelineTest {
@Test
void testScoringOnlyPipeline() {
final DocumentAnalyzer analyzer = DocumentAnalyzer.builder()
- .add(new TokenizerAnnotator(SPACE_TOKENIZER))
+ .add(new TokenizerAnnotator(SingleSpaceTokens.TOKENIZER))
.add(new TermVectorAnnotator(TermVectorAnnotator.Mode.SCORING_ONLY))
.build();
diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorTest.java b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorTest.java
index 286b7a2c3..8448f465d 100644
--- a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorTest.java
+++ b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorTest.java
@@ -75,6 +75,15 @@ public class TermVectorTest {
assertThrows(IllegalArgumentException.class, () -> new TermVector("dog", 1, null));
}
+ @Test
+ void testNullSpanElementIsRejected() {
+ final List<Span> spans = new ArrayList<>();
+ spans.add(new Span(0, 3));
+ spans.add(null);
+ assertThrows(IllegalArgumentException.class, () -> new TermVector("dog", 2, spans));
+ assertThrows(IllegalArgumentException.class, () -> TermVector.withSpans("dog", spans));
+ }
+
@ParameterizedTest
@ValueSource(ints = {0, -1, Integer.MIN_VALUE})
void testFrequencyBelowOneIsRejected(int frequency) {