You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by rz...@apache.org on 2026/10/02 08:37:30 UTC
(opennlp) branch minor-dedup-aligned-text-composition created (now 5dbc5e144)
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a change to branch minor-dedup-aligned-text-composition
in repository https://gitbox.apache.org/repos/asf/opennlp.git
at 5dbc5e144 Minor: Compose AlignedText stages with AlignedText.andThen
This branch includes the following new commits:
new 5dbc5e144 Minor: Compose AlignedText stages with AlignedText.andThen
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(opennlp) 01/01: Minor: Compose AlignedText stages with AlignedText.andThen
Posted by rz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch minor-dedup-aligned-text-composition
in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 5dbc5e1446be6934bcd63b4ce06478653cbf7093
Author: Richard Zowalla <rz...@apache.org>
AuthorDate: Fri Oct 2 10:20:10 2026 +0200
Minor: Compose AlignedText stages with AlignedText.andThen
Add AlignedText.andThen in opennlp-api to compose a normalization stage.
Whitespace, LineBreakPreserving, ParagraphPreserving and Quote normalizers,
AlignedAggregateCharSequenceNormalizer and AbstractDL.normalizeInputAligned
use it instead of inline composition.
Describe WordpieceEncoder's per-character MappedText in its Javadoc.
---
.../opennlp/tools/tokenize/WordpieceEncoder.java | 3 ++
.../opennlp/tools/util/normalizer/AlignedText.java | 20 +++++++++
.../tools/util/normalizer/AlignedTextTest.java | 50 ++++++++++++++++++++++
.../src/main/java/opennlp/dl/AbstractDL.java | 11 +----
.../AlignedAggregateCharSequenceNormalizer.java | 12 +++---
...PreservingWhitespaceCharSequenceNormalizer.java | 5 +--
...PreservingWhitespaceCharSequenceNormalizer.java | 4 +-
.../normalizer/QuoteCharSequenceNormalizer.java | 5 +--
.../WhitespaceCharSequenceNormalizer.java | 5 +--
9 files changed, 88 insertions(+), 27 deletions(-)
diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java
index 9e620c3f5..324c93668 100644
--- a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java
+++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java
@@ -441,6 +441,9 @@ public final class WordpieceEncoder implements SubwordTokenizer {
/**
* The normalized text with the original-text range for each character. Characters inserted by
* the pipeline (isolation spaces) use an empty range at the insertion point.
+ *
+ * <p>Positions are appended one code point at a time, each with its own original range, and the
+ * vocabulary lookup reads the characters and ranges of single positions directly.</p>
*/
private static final class MappedText {
private char[] chars;
diff --git a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/AlignedText.java b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/AlignedText.java
index 002b67231..81b39f212 100644
--- a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/AlignedText.java
+++ b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/AlignedText.java
@@ -47,6 +47,26 @@ public record AlignedText(CharSequence original, CharSequence normalized, Alignm
return normalized.toString();
}
+ /**
+ * Composes this result with a stage that further normalizes this result's normalized text.
+ *
+ * <p>If this maps {@code original -> middle} and {@code next} maps {@code middle -> final}, the
+ * result keeps this {@link #original()}, takes the {@link #normalized()} text of {@code next}, and
+ * maps between them with {@link Alignment#andThen(Alignment)}.</p>
+ *
+ * @param next The next stage, produced from this result's normalized text. Must not be
+ * {@code null}.
+ * @return The composed result.
+ * @throws IllegalArgumentException Thrown if {@code next} is {@code null}, or if its alignment's
+ * original length does not equal this alignment's normalized length.
+ */
+ public AlignedText andThen(AlignedText next) {
+ if (next == null) {
+ throw new IllegalArgumentException("next must not be null");
+ }
+ return new AlignedText(original, next.normalized(), alignment.andThen(next.alignment()));
+ }
+
/**
* Maps a span of the normalized text back to the tightest span of the original text.
*
diff --git a/opennlp-api/src/test/java/opennlp/tools/util/normalizer/AlignedTextTest.java b/opennlp-api/src/test/java/opennlp/tools/util/normalizer/AlignedTextTest.java
index 37a8a7571..b3088f3c5 100644
--- a/opennlp-api/src/test/java/opennlp/tools/util/normalizer/AlignedTextTest.java
+++ b/opennlp-api/src/test/java/opennlp/tools/util/normalizer/AlignedTextTest.java
@@ -22,6 +22,7 @@ import opennlp.tools.util.Span;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
public class AlignedTextTest {
@@ -90,4 +91,53 @@ public class AlignedTextTest {
assertSpan(1, 3, aligned.toOriginalSpan(1, 2));
}
+ @Test
+ void andThenKeepsFirstOriginalAndLastNormalized() {
+ final StringBuilder original = new StringBuilder(" a b ");
+ final AlignedText collapsed = CharClass.whitespace().collapseAligned(original);
+ final AlignedText composed =
+ collapsed.andThen(CharClass.whitespace().trimAligned(collapsed.normalized()));
+ assertSame(original, composed.original());
+ assertEquals("a b", composed.normalizedString());
+ assertSpan(2, 3, composed.toOriginalSpan(0, 1)); // "a"
+ assertSpan(3, 6, composed.toOriginalSpan(1, 2)); // collapsed run between the letters
+ assertSpan(6, 7, composed.toOriginalSpan(2, 3)); // "b"
+ assertSpan(2, 3, composed.toNormalizedSpan(6, 7));
+ }
+
+ @Test
+ void andThenMapsSupplementaryCharactersThroughBothStages() {
+ // U+1D400 MATHEMATICAL BOLD CAPITAL A is two UTF-16 units and is removed by the second stage.
+ final String original = "x 𝐀 y";
+ final AlignedText collapsed = CharClass.whitespace().collapseAligned(original);
+ final AlignedText removed = CharClass.of(CodePointSet.of(0x1D400), ' ')
+ .removeAllAligned(collapsed.normalized());
+ final AlignedText composed = collapsed.andThen(removed);
+ assertEquals("x y", composed.normalizedString());
+ assertSpan(5, 6, composed.toOriginalSpan(2, 3)); // second space follows the removed pair
+ assertSpan(6, 7, composed.toOriginalSpan(3, 4)); // "y"
+ }
+
+ @Test
+ void andThenWithIdentityStageKeepsAlignment() {
+ final AlignedText collapsed = CharClass.whitespace().collapseAligned("a b");
+ final AlignedText composed =
+ collapsed.andThen(CharClass.whitespace().trimAligned(collapsed.normalized()));
+ assertEquals("a b", composed.normalizedString());
+ assertSpan(1, 3, composed.toOriginalSpan(1, 2));
+ }
+
+ @Test
+ void andThenRejectsNull() {
+ final AlignedText collapsed = CharClass.whitespace().collapseAligned("a b");
+ assertThrows(IllegalArgumentException.class, () -> collapsed.andThen(null));
+ }
+
+ @Test
+ void andThenRejectsStageThatDoesNotLineUp() {
+ final AlignedText collapsed = CharClass.whitespace().collapseAligned("a b");
+ final AlignedText unrelated = CharClass.whitespace().collapseAligned("a b");
+ assertThrows(IllegalArgumentException.class, () -> collapsed.andThen(unrelated));
+ }
+
}
diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java
index 919bc5155..dc318e4a0 100644
--- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java
+++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java
@@ -428,21 +428,14 @@ public abstract class AbstractDL implements AutoCloseable {
// if that ever changes.
AlignedText result = identityAligned(text, text);
if (normalizeWhitespace) {
- result = compose(result, WHITESPACE.normalizeAligned(result.normalized()));
+ result = result.andThen(WHITESPACE.normalizeAligned(result.normalized()));
}
if (normalizeDashes) {
- result = compose(result, DASHES.normalizeAligned(result.normalized()));
+ result = result.andThen(DASHES.normalizeAligned(result.normalized()));
}
return result;
}
- // Threads a fold stage onto the running alignment: accumulated maps original -> current and next
- // maps current -> next.normalized(), so the composition maps original -> next.normalized().
- private static AlignedText compose(final AlignedText accumulated, final AlignedText next) {
- return new AlignedText(accumulated.original(), next.normalized(),
- accumulated.alignment().andThen(next.alignment()));
- }
-
// An AlignedText whose alignment is the identity, for the case where no length-changing fold was
// applied so the folded text has the same length and offsets as the original.
private static AlignedText identityAligned(final String original, final String normalized) {
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AlignedAggregateCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AlignedAggregateCharSequenceNormalizer.java
index 1f7a8c14f..4f05c1fe3 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AlignedAggregateCharSequenceNormalizer.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AlignedAggregateCharSequenceNormalizer.java
@@ -57,14 +57,14 @@ final class AlignedAggregateCharSequenceNormalizer implements OffsetAwareNormali
}
// Normalize the input to a String once so the stored original and the per-stage alignment
// lengths agree even for a CharSequence whose length() differs from its toString().
+ // The first stage is re-wrapped so the stored original is exactly the input, whatever original
+ // a user-supplied stage reports.
final String input = text.toString();
- AlignedText stage = steps[0].normalizeAligned(input);
- Alignment alignment = stage.alignment();
+ final AlignedText first = steps[0].normalizeAligned(input);
+ AlignedText result = new AlignedText(input, first.normalized(), first.alignment());
for (int i = 1; i < steps.length; i++) {
- final AlignedText next = steps[i].normalizeAligned(stage.normalized());
- alignment = alignment.andThen(next.alignment());
- stage = next;
+ result = result.andThen(steps[i].normalizeAligned(result.normalized()));
}
- return new AlignedText(input, stage.normalized(), alignment);
+ return result;
}
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/LineBreakPreservingWhitespaceCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/LineBreakPreservingWhitespaceCharSequenceNormalizer.java
index d79317ff0..689c9e72e 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/LineBreakPreservingWhitespaceCharSequenceNormalizer.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/LineBreakPreservingWhitespaceCharSequenceNormalizer.java
@@ -51,11 +51,10 @@ public class LineBreakPreservingWhitespaceCharSequenceNormalizer implements Offs
return WHITESPACE.trim(WHITESPACE.collapsePreserving(text, LINE_BREAKS, NEWLINE));
}
+ /** {@inheritDoc} */
@Override
public AlignedText normalizeAligned(CharSequence text) {
final AlignedText collapsed = WHITESPACE.collapsePreservingAligned(text, LINE_BREAKS, NEWLINE);
- final AlignedText trimmed = WHITESPACE.trimAligned(collapsed.normalized());
- return new AlignedText(text, trimmed.normalized(),
- collapsed.alignment().andThen(trimmed.alignment()));
+ return collapsed.andThen(WHITESPACE.trimAligned(collapsed.normalized()));
}
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ParagraphPreservingWhitespaceCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ParagraphPreservingWhitespaceCharSequenceNormalizer.java
index 44ae476de..81e3b8185 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ParagraphPreservingWhitespaceCharSequenceNormalizer.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ParagraphPreservingWhitespaceCharSequenceNormalizer.java
@@ -69,8 +69,6 @@ public class ParagraphPreservingWhitespaceCharSequenceNormalizer implements Offs
public AlignedText normalizeAligned(CharSequence text) {
final AlignedText collapsed =
WHITESPACE.collapseParagraphPreservingAligned(text, LINE_BREAKS, NEWLINE);
- final AlignedText trimmed = WHITESPACE.trimAligned(collapsed.normalized());
- return new AlignedText(text, trimmed.normalized(),
- collapsed.alignment().andThen(trimmed.alignment()));
+ return collapsed.andThen(WHITESPACE.trimAligned(collapsed.normalized()));
}
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/QuoteCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/QuoteCharSequenceNormalizer.java
index 3279dd03e..188c93d90 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/QuoteCharSequenceNormalizer.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/QuoteCharSequenceNormalizer.java
@@ -67,11 +67,10 @@ public class QuoteCharSequenceNormalizer implements OffsetAwareNormalizer {
return DOUBLE.normalize(SINGLE.normalize(text));
}
+ /** {@inheritDoc} */
@Override
public AlignedText normalizeAligned(CharSequence text) {
final AlignedText single = SINGLE.normalizeAligned(text);
- final AlignedText both = DOUBLE.normalizeAligned(single.normalized());
- return new AlignedText(text, both.normalized(),
- single.alignment().andThen(both.alignment()));
+ return single.andThen(DOUBLE.normalizeAligned(single.normalized()));
}
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/WhitespaceCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/WhitespaceCharSequenceNormalizer.java
index 4a6e1d26b..98107aa57 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/WhitespaceCharSequenceNormalizer.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/WhitespaceCharSequenceNormalizer.java
@@ -44,11 +44,10 @@ public class WhitespaceCharSequenceNormalizer implements OffsetAwareNormalizer {
return WHITESPACE.trim(WHITESPACE.collapse(text));
}
+ /** {@inheritDoc} */
@Override
public AlignedText normalizeAligned(CharSequence text) {
final AlignedText collapsed = WHITESPACE.collapseAligned(text);
- final AlignedText trimmed = WHITESPACE.trimAligned(collapsed.normalized());
- return new AlignedText(text, trimmed.normalized(),
- collapsed.alignment().andThen(trimmed.alignment()));
+ return collapsed.andThen(WHITESPACE.trimAligned(collapsed.normalized()));
}
}