You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by kr...@apache.org on 2026/08/16 01:12:59 UTC

(opennlp) 06/06: OPENNLP-1897: Review round: reject null span elements, finalize the annotator, share the space-split test fixture, pin supplementary-plane offsets

This is an automated email from the ASF dual-hosted git repository.

krickert pushed a commit to branch OPENNLP-1897-term-vectors
in repository https://gitbox.apache.org/repos/asf/opennlp.git

commit 9829831e0ac35940a0ce6b947e5fb2c469ab3906
Author: Kristian Rickert <kr...@gmail.com>
AuthorDate: Sat Aug 15 21:08:37 2026 -0400

    OPENNLP-1897: Review round: reject null span elements, finalize the annotator, share the space-split test fixture, pin supplementary-plane offsets
---
 .../java/opennlp/tools/termvector/TermVector.java  | 15 +++-
 .../tools/termvector/TermVectorAnnotator.java      |  2 +-
 .../tools/termvector/SingleSpaceTokens.java        | 90 ++++++++++++++++++++++
 .../tools/termvector/TermVectorAnnotatorTest.java  | 53 ++++++++-----
 .../tools/termvector/TermVectorPipelineTest.java   | 39 +---------
 .../opennlp/tools/termvector/TermVectorTest.java   |  9 +++
 6 files changed, 147 insertions(+), 61 deletions(-)

diff --git a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVector.java b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVector.java
index 676bf0131..42c654257 100644
--- a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVector.java
+++ b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVector.java
@@ -43,8 +43,9 @@ import opennlp.tools.util.Span;
  * @param term The term string. Must not be {@code null}.
  * @param frequency The number of occurrences in the document. Must be at least one.
  * @param spans The occurrence spans in original text coordinates, one per occurrence,
- *              or an empty list for a scoring-only vector. Must not be {@code null} and,
- *              when non-empty, must contain exactly {@code frequency} spans.
+ *              or an empty list for a scoring-only vector. Must not be {@code null} or
+ *              contain {@code null} and, when non-empty, must hold exactly
+ *              {@code frequency} spans.
  *
  * @since 3.0.0
  */
@@ -55,8 +56,9 @@ public record TermVector(String term, int frequency, List<Span> spans) {
    * Validates the term vector and detaches the span list from the caller's input.
    *
    * @throws IllegalArgumentException Thrown if {@code term} is {@code null},
-   *         {@code frequency} is below one, {@code spans} is {@code null}, or a
-   *         non-empty {@code spans} list does not hold exactly {@code frequency} spans.
+   *         {@code frequency} is below one, {@code spans} is or contains {@code null},
+   *         or a non-empty {@code spans} list does not hold exactly {@code frequency}
+   *         spans.
    */
   public TermVector {
     if (term == null) {
@@ -68,6 +70,11 @@ public record TermVector(String term, int frequency, List<Span> spans) {
     if (spans == null) {
       throw new IllegalArgumentException("spans must not be null");
     }
+    for (final Span span : spans) {
+      if (span == null) {
+        throw new IllegalArgumentException("spans must not contain null");
+      }
+    }
     if (!spans.isEmpty() && spans.size() != frequency) {
       throw new IllegalArgumentException("a full term vector holds one span per "
           + "occurrence: frequency is " + frequency + " but spans holds " + spans.size());
diff --git a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java
index 27282db16..bec73882f 100644
--- a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java
+++ b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java
@@ -63,7 +63,7 @@ import opennlp.tools.util.normalizer.OffsetAwareNormalizer;
  *
  * @since 3.0.0
  */
-public class TermVectorAnnotator implements DocumentAnnotator {
+public final class TermVectorAnnotator implements DocumentAnnotator {
 
   /**
    * The key of the term vector layer this annotator provides: a document-scoped layer
diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java b/opennlp-api/src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java
new file mode 100644
index 000000000..4f7beb464
--- /dev/null
+++ b/opennlp-api/src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.termvector;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import opennlp.tools.document.Annotation;
+import opennlp.tools.tokenize.Tokenizer;
+import opennlp.tools.util.Span;
+
+/**
+ * Deterministic single-space tokenization shared by the term vector tests: splits on
+ * single space characters and keeps all other characters, including sentence-final
+ * periods, attached to their token. Runs of spaces yield no empty tokens, so every
+ * expected span follows directly from the input text.
+ */
+final class SingleSpaceTokens {
+
+  /**
+   * A {@link Tokenizer} view of {@link #spans(String)}. Only the span-producing method
+   * is implemented because the annotator adapter calls no other method.
+   */
+  static final Tokenizer TOKENIZER = new Tokenizer() {
+
+    @Override
+    public String[] tokenize(String s) {
+      throw new UnsupportedOperationException("the adapter only calls tokenizePos");
+    }
+
+    @Override
+    public Span[] tokenizePos(String s) {
+      return spans(s).toArray(new Span[0]);
+    }
+  };
+
+  private SingleSpaceTokens() {
+  }
+
+  /**
+   * Computes the token spans of a text split on single space characters.
+   *
+   * @param text The text to split.
+   * @return One span per token, in text order.
+   */
+  static List<Span> spans(String text) {
+    final List<Span> spans = new ArrayList<>();
+    int start = -1;
+    for (int i = 0; i <= text.length(); i++) {
+      final boolean boundary = i == text.length() || text.charAt(i) == ' ';
+      if (boundary && start >= 0) {
+        spans.add(new Span(start, i));
+        start = -1;
+      } else if (!boundary && start < 0) {
+        start = i;
+      }
+    }
+    return spans;
+  }
+
+  /**
+   * Builds a token layer from {@link #spans(String)}, each token valued with its covered
+   * text.
+   *
+   * @param text The text to split.
+   * @return One annotation per token, in text order.
+   */
+  static List<Annotation<String>> tokens(String text) {
+    final List<Annotation<String>> tokens = new ArrayList<>();
+    for (final Span span : spans(text)) {
+      tokens.add(new Annotation<>(span, text.substring(span.getStart(), span.getEnd())));
+    }
+    return tokens;
+  }
+}
diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java
index 03eeb130b..e928d5dfc 100644
--- a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java
+++ b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java
@@ -18,7 +18,6 @@
 package opennlp.tools.termvector;
 
 import java.io.Serial;
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Set;
@@ -148,30 +147,18 @@ public class TermVectorAnnotatorTest {
    * A plain normalizer that deletes every digit, so a token made up of digits alone
    * normalizes to the empty string on the per-token path.
    */
-  private static final CharSequenceNormalizer PLAIN_DROP_DIGITS =
-      text -> text.toString().replaceAll("\\d", "");
-
-  /**
-   * Splits a text on single space characters into a token layer, skipping the empty
-   * tokens a collapsed run would produce, so double spaces yield no tokens.
-   */
-  private static List<Annotation<String>> splitOnSpaces(String text) {
-    final List<Annotation<String>> tokens = new ArrayList<>();
-    int start = -1;
-    for (int i = 0; i <= text.length(); i++) {
-      final boolean boundary = i == text.length() || text.charAt(i) == ' ';
-      if (boundary && start >= 0) {
-        tokens.add(new Annotation<>(new Span(start, i), text.substring(start, i)));
-        start = -1;
-      } else if (!boundary && start < 0) {
-        start = i;
+  private static final CharSequenceNormalizer PLAIN_DROP_DIGITS = text -> {
+    final StringBuilder out = new StringBuilder(text.length());
+    for (int i = 0; i < text.length(); i++) {
+      if (!Character.isDigit(text.charAt(i))) {
+        out.append(text.charAt(i));
       }
     }
-    return tokens;
-  }
+    return out.toString();
+  };
 
   private static Document documentWithTokens(String text) {
-    return Document.of(text).with(Layers.TOKENS, splitOnSpaces(text));
+    return Document.of(text).with(Layers.TOKENS, SingleSpaceTokens.tokens(text));
   }
 
   @Test
@@ -364,6 +351,30 @@ public class TermVectorAnnotatorTest {
         vectors.get(1).value());
   }
 
+  /**
+   * Spans are UTF-16 offsets, so a supplementary-plane character occupies two positions:
+   * {@code "𝕏 x 𝕏"} tokenizes to spans of width two around the surrogate pairs, and both
+   * occurrences group under one term whose spans still cover the original text exactly.
+   */
+  @Test
+  void testSupplementaryPlaneTokensKeepUtf16Offsets() {
+    final String text = "𝕏 x 𝕏";
+    final Document document = new TermVectorAnnotator()
+        .annotate(documentWithTokens(text));
+
+    final List<Annotation<TermVector>> vectors =
+        document.get(TermVectorAnnotator.TERM_VECTORS);
+    assertEquals(2, vectors.size());
+    assertEquals(new TermVector("𝕏", 2, List.of(new Span(0, 2), new Span(5, 7))),
+        vectors.get(0).value());
+    assertEquals(new TermVector("x", 1, List.of(new Span(3, 4))), vectors.get(1).value());
+    for (final Annotation<TermVector> vector : vectors) {
+      for (final Span span : vector.value().spans()) {
+        assertEquals(vector.value().term(), span.getCoveredText(text).toString());
+      }
+    }
+  }
+
   @Test
   void testEmptyTokenLayerYieldsPresentButEmptyLayer() {
     final Document document = new TermVectorAnnotator()
diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java
index a00fba96b..291b2068c 100644
--- a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java
+++ b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java
@@ -17,7 +17,6 @@
 
 package opennlp.tools.termvector;
 
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
 
@@ -28,7 +27,6 @@ import opennlp.tools.document.Document;
 import opennlp.tools.document.DocumentAnalyzer;
 import opennlp.tools.document.Layers;
 import opennlp.tools.document.TokenizerAnnotator;
-import opennlp.tools.tokenize.Tokenizer;
 import opennlp.tools.util.Span;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -37,40 +35,11 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
 /**
  * Wires the {@link TermVectorAnnotator} into a {@link DocumentAnalyzer} behind a
  * {@link TokenizerAnnotator}: the token layer goes in, the term vector layer comes out,
- * and nothing else about the document changes. The tokenizer is a deterministic stand-in
- * defined here, so every expected span follows directly from the input text.
+ * and nothing else about the document changes. The tokenizer is the shared deterministic
+ * single-space fixture, so every expected span follows directly from the input text.
  */
 public class TermVectorPipelineTest {
 
-  /**
-   * A deterministic tokenizer that splits on single space characters and keeps all
-   * other characters, including sentence-final periods, attached to their token. Only
-   * the span-producing method is implemented because the adapter calls no other method.
-   */
-  private static final Tokenizer SPACE_TOKENIZER = new Tokenizer() {
-
-    @Override
-    public String[] tokenize(String s) {
-      throw new UnsupportedOperationException("the adapter only calls tokenizePos");
-    }
-
-    @Override
-    public Span[] tokenizePos(String s) {
-      final List<Span> spans = new ArrayList<>();
-      int start = -1;
-      for (int i = 0; i <= s.length(); i++) {
-        final boolean boundary = i == s.length() || s.charAt(i) == ' ';
-        if (boundary && start >= 0) {
-          spans.add(new Span(start, i));
-          start = -1;
-        } else if (!boundary && start < 0) {
-          start = i;
-        }
-      }
-      return spans.toArray(new Span[0]);
-    }
-  };
-
   /**
    * Runs tokenizer plus term vector roll-up over a text with repeated tokens and reads
    * the aggregated layer back, span by span, in original text coordinates.
@@ -78,7 +47,7 @@ public class TermVectorPipelineTest {
   @Test
   void testTokenizerAndTermVectorPipeline() {
     final DocumentAnalyzer analyzer = DocumentAnalyzer.builder()
-        .add(new TokenizerAnnotator(SPACE_TOKENIZER))
+        .add(new TokenizerAnnotator(SingleSpaceTokens.TOKENIZER))
         .add(new TermVectorAnnotator())
         .build();
 
@@ -113,7 +82,7 @@ public class TermVectorPipelineTest {
   @Test
   void testScoringOnlyPipeline() {
     final DocumentAnalyzer analyzer = DocumentAnalyzer.builder()
-        .add(new TokenizerAnnotator(SPACE_TOKENIZER))
+        .add(new TokenizerAnnotator(SingleSpaceTokens.TOKENIZER))
         .add(new TermVectorAnnotator(TermVectorAnnotator.Mode.SCORING_ONLY))
         .build();
 
diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorTest.java b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorTest.java
index 286b7a2c3..8448f465d 100644
--- a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorTest.java
+++ b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorTest.java
@@ -75,6 +75,15 @@ public class TermVectorTest {
     assertThrows(IllegalArgumentException.class, () -> new TermVector("dog", 1, null));
   }
 
+  @Test
+  void testNullSpanElementIsRejected() {
+    final List<Span> spans = new ArrayList<>();
+    spans.add(new Span(0, 3));
+    spans.add(null);
+    assertThrows(IllegalArgumentException.class, () -> new TermVector("dog", 2, spans));
+    assertThrows(IllegalArgumentException.class, () -> TermVector.withSpans("dog", spans));
+  }
+
   @ParameterizedTest
   @ValueSource(ints = {0, -1, Integer.MIN_VALUE})
   void testFrequencyBelowOneIsRejected(int frequency) {