You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by rz...@apache.org on 2026/10/04 16:51:02 UTC
(opennlp) branch main updated: (fix) GH-1379: Morpheme accepts a surface that does not match its span (#1384)
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/main by this push:
new a368de9cd (fix) GH-1379: Morpheme accepts a surface that does not match its span (#1384)
a368de9cd is described below
commit a368de9cdd8a1b0c7c97eb9baae47170f8c565d0
Author: Kristian Rickert <kr...@apache.org>
AuthorDate: Sun Oct 4 09:50:56 2026 -0700
(fix) GH-1379: Morpheme accepts a surface that does not match its span (#1384)
* Morpheme accepts a surface that does not match its span
The constructor now rejects a surface whose length differs from its
span and features that contain null, and reports a null and an empty
surface separately. Callers passing such values get an
IllegalArgumentException.
Assisted-by: Claude Code 2.1.288
Co-authored-by: Kristian Rickert <kr...@apache.org>
Co-authored-by: Claude <no...@anthropic.com>
Reviewed-by: Kristian Rickert <kr...@apache.org>
Signed-off-by: Kristian Rickert <kr...@apache.org>
* Use ArgumentChecks for the Morpheme null checks
Review feedback on #1384. The messages do not change.
Assisted-by: Claude Code 2.1.288
Co-authored-by: Kristian Rickert <kr...@apache.org>
Co-authored-by: Claude <no...@anthropic.com>
Reviewed-by: Kristian Rickert <kr...@apache.org>
Signed-off-by: Kristian Rickert <kr...@apache.org>
---------
Signed-off-by: Kristian Rickert <kr...@apache.org>
Co-authored-by: Claude <no...@anthropic.com>
---
.../opennlp/tools/tokenize/lattice/Morpheme.java | 29 ++++++++++++++--------
.../tokenize/lattice/LatticeTokenizerTest.java | 8 +++++-
2 files changed, 26 insertions(+), 11 deletions(-)
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/Morpheme.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/Morpheme.java
index a6694ddc5..3da471adc 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/Morpheme.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/Morpheme.java
@@ -19,6 +19,7 @@ package opennlp.tools.tokenize.lattice;
import java.util.List;
+import opennlp.tools.util.ArgumentChecks;
import opennlp.tools.util.Span;
/**
@@ -33,8 +34,10 @@ import opennlp.tools.util.Span;
*
* @param span The location of the morpheme in the original text. Must not be
* {@code null}.
- * @param surface The covered text. Must not be {@code null} or empty.
- * @param features The dictionary feature columns. Must not be {@code null}.
+ * @param surface The covered text. Must not be {@code null} or empty, and its length
+ * must equal the span length.
+ * @param features The dictionary feature columns. Must not be {@code null} and must
+ * not contain {@code null}.
* @param unknown Whether the morpheme came from unknown-word handling rather than a
* lexicon entry.
*
@@ -46,18 +49,24 @@ public record Morpheme(Span span, String surface, List<String> features,
/**
* Validates the morpheme.
*
- * @throws IllegalArgumentException Thrown if {@code span}, {@code surface}, or
- * {@code features} is {@code null}, or {@code surface} is empty.
+ * @throws IllegalArgumentException Thrown if {@code span}, {@code surface},
+ * {@code features}, or a feature is {@code null}; {@code surface} is empty;
+ * or the surface and span lengths differ.
*/
public Morpheme {
- if (span == null) {
- throw new IllegalArgumentException("span must not be null");
+ ArgumentChecks.requireNonNullArg(span, "span");
+ ArgumentChecks.requireNonNullArg(surface, "surface");
+ if (surface.isEmpty()) {
+ throw new IllegalArgumentException("surface must not be empty");
}
- if (surface == null || surface.isEmpty()) {
- throw new IllegalArgumentException("surface must not be null or empty");
+ if (surface.length() != span.length()) {
+ throw new IllegalArgumentException("surface length must match span length");
}
- if (features == null) {
- throw new IllegalArgumentException("features must not be null");
+ ArgumentChecks.requireNonNullArg(features, "features");
+ for (final String feature : features) {
+ if (feature == null) {
+ throw new IllegalArgumentException("features must not contain null");
+ }
}
features = List.copyOf(features);
}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java
index 44d01bd7e..08cd32052 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java
@@ -23,6 +23,7 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import org.junit.jupiter.api.AfterEach;
@@ -743,7 +744,8 @@ public class LatticeTokenizerTest {
/**
* Verifies the {@link Morpheme} contract every segmentation result is built from: a
- * {@code null} span, a {@code null} or empty surface, and {@code null} features are
+ * {@code null} span, a {@code null} or empty surface, a surface whose length differs
+ * from the span length, {@code null} features, and a {@code null} feature entry are
* all rejected, and the feature list is copied so a later change to the caller's list
* cannot be seen through the morpheme.
*/
@@ -756,8 +758,12 @@ public class LatticeTokenizerTest {
() -> new Morpheme(span, null, List.of("noun"), false));
Assertions.assertThrows(IllegalArgumentException.class,
() -> new Morpheme(span, "", List.of("noun"), false));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new Morpheme(span, "\u6771\u6771", List.of("noun"), false));
Assertions.assertThrows(IllegalArgumentException.class,
() -> new Morpheme(span, "\u6771", null, false));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new Morpheme(span, "\u6771", Arrays.asList("noun", null), false));
final List<String> features = new ArrayList<>(List.of("noun"));
final Morpheme morpheme = new Morpheme(span, "\u6771", features, false);