You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jz...@apache.org on 2026/10/04 17:46:39 UTC
(opennlp) branch main updated: OPENNLP-1934: Add explicit tokenizer character policies (#1349)
This is an automated email from the ASF dual-hosted git repository.
jzonthemtn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/main by this push:
new 197933cf9 OPENNLP-1934: Add explicit tokenizer character policies (#1349)
197933cf9 is described below
commit 197933cf976d9fb29a2813b99b17e62c0ba522fd
Author: Kristian Rickert <kr...@apache.org>
AuthorDate: Sun Oct 4 10:46:33 2026 -0700
OPENNLP-1934: Add explicit tokenizer character policies (#1349)
Add TokenizerCharacterPolicy, an immutable Predicate<CharSequence> built
from explicit letter, digit and mark sets, with an ASCII preset. Accept
marks after letters or digits, including repeated marks, but reject a
leading mark. Support supplementary characters and return false for
malformed UTF-16.
Require at least one letter or digit. Reject null sets, overlapping
categories, whitespace and surrogate code points. Include tests for
validation, character rules and the manual example.
The policy does not yet configure tokenizer training or inference. The
existing regex configuration is unchanged.
Assisted-by: Claude Code 2.1.281
Reviewed-by: Kristian Rickert <kr...@apache.org>
Signed-off-by: Kristian Rickert <kr...@apache.org>
Co-authored-by: Claude <no...@anthropic.com>
---
.../tools/tokenize/TokenizerCharacterPolicy.java | 174 ++++++++++++++
.../tokenize/TokenizerCharacterPolicyTest.java | 256 +++++++++++++++++++++
opennlp-docs/src/docbkx/tokenizer.xml | 30 +++
3 files changed, 460 insertions(+)
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerCharacterPolicy.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerCharacterPolicy.java
new file mode 100644
index 000000000..1e1892023
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerCharacterPolicy.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.tokenize;
+
+import java.util.function.Predicate;
+
+import opennlp.tools.util.ArgumentChecks;
+import opennlp.tools.util.normalizer.CodePointSet;
+import opennlp.tools.util.normalizer.UnicodeWhitespace;
+
+/**
+ * An immutable policy for testing token eligibility against explicit character sets.
+ *
+ * <p>A token consists of one or more letters or digits. Each letter or digit may be followed by
+ * zero or more marks. A mark cannot start a token.
+ * The supplied sets express a caller's tokenization policy; this class does not infer Unicode
+ * character categories.</p>
+ *
+ * <p>This predicate tests a complete candidate without finding token boundaries or changing
+ * its text.</p>
+ */
+public final class TokenizerCharacterPolicy implements Predicate<CharSequence> {
+
+ private static final String CODE_POINT_FORMAT = "U+%04X";
+
+ private static final TokenizerCharacterPolicy ASCII = new TokenizerCharacterPolicy(
+ CodePointSet.ofRange('A', 'Z').union(CodePointSet.ofRange('a', 'z')),
+ CodePointSet.ofRange('0', '9'), CodePointSet.of());
+
+ private final CodePointSet letters;
+ private final CodePointSet digits;
+ private final CodePointSet marks;
+
+ private TokenizerCharacterPolicy(CodePointSet letters, CodePointSet digits, CodePointSet marks) {
+ this.letters = letters;
+ this.digits = digits;
+ this.marks = marks;
+ }
+
+ /**
+ * Creates a policy from explicit, pairwise-disjoint code point sets. Whitespace separates
+ * tokens and so cannot be part of one; every other non-surrogate code point is left to the
+ * caller.
+ *
+ * @param letters Code points treated as letters.
+ * @param digits Code points treated as digits.
+ * @param marks Code points permitted after a letter, a digit, or another mark.
+ * @return The immutable policy.
+ * @throws IllegalArgumentException Thrown if a set is {@code null}, {@code letters} and
+ * {@code digits} are both empty, a code point occurs in more than one set, or a set
+ * contains a surrogate code point or {@link UnicodeWhitespace} member.
+ */
+ public static TokenizerCharacterPolicy of(
+ CodePointSet letters, CodePointSet digits, CodePointSet marks) {
+ ArgumentChecks.requireNonNullArg(letters, "letters");
+ ArgumentChecks.requireNonNullArg(digits, "digits");
+ ArgumentChecks.requireNonNullArg(marks, "marks");
+ if (letters.isEmpty() && digits.isEmpty()) {
+ throw new IllegalArgumentException("At least one letter or digit is required");
+ }
+ validateSet("letters", letters);
+ validateSet("digits", digits);
+ validateSet("marks", marks);
+ requireDisjoint("letters", letters, "digits", digits);
+ requireDisjoint("letters", letters, "marks", marks);
+ requireDisjoint("digits", digits, "marks", marks);
+ return new TokenizerCharacterPolicy(letters, digits, marks);
+ }
+
+ /** {@return a policy for ASCII letters and digits, with no marks} */
+ public static TokenizerCharacterPolicy ascii() {
+ return ASCII;
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * <p>Returns {@code true} if the entire input is a non-empty token accepted by this policy.
+ * Malformed UTF-16 input (an unpaired or reversed surrogate) returns {@code false}.</p>
+ *
+ * @throws IllegalArgumentException Thrown if {@code input} is {@code null}.
+ */
+ @Override
+ public boolean test(CharSequence input) {
+ ArgumentChecks.requireNonNullArg(input, "input");
+ if (input.isEmpty()) {
+ return false;
+ }
+
+ for (int offset = 0; offset < input.length();) {
+ final boolean first = offset == 0;
+ char unit = input.charAt(offset);
+ final int codePoint;
+ if (Character.isHighSurrogate(unit)) {
+ if (offset + 1 >= input.length() || !Character.isLowSurrogate(input.charAt(offset + 1))) {
+ return false;
+ }
+ codePoint = Character.toCodePoint(unit, input.charAt(offset + 1));
+ offset += 2;
+ } else if (Character.isLowSurrogate(unit)) {
+ return false;
+ } else {
+ codePoint = unit;
+ offset++;
+ }
+
+ if (!letters.contains(codePoint) && !digits.contains(codePoint)
+ && (first || !marks.contains(codePoint))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /** {@return the immutable letter set} */
+ public CodePointSet getLetters() {
+ return letters;
+ }
+
+ /** {@return the immutable digit set} */
+ public CodePointSet getDigits() {
+ return digits;
+ }
+
+ /** {@return the immutable mark set} */
+ public CodePointSet getMarks() {
+ return marks;
+ }
+
+ /** Throws if {@code set} contains a surrogate code point or {@link UnicodeWhitespace} member. */
+ private static void validateSet(String name, CodePointSet set) {
+ for (int codePoint = Character.MIN_SURROGATE; codePoint <= Character.MAX_SURROGATE;
+ codePoint++) {
+ if (set.contains(codePoint)) {
+ throw new IllegalArgumentException(name + " contains the surrogate code point "
+ + String.format(CODE_POINT_FORMAT, codePoint));
+ }
+ }
+ for (int codePoint : UnicodeWhitespace.codePoints()) {
+ if (set.contains(codePoint)) {
+ throw new IllegalArgumentException(name + " contains the whitespace code point "
+ + String.format(CODE_POINT_FORMAT, codePoint));
+ }
+ }
+ }
+
+ /** Throws if {@code first} and {@code second} share a code point. */
+ private static void requireDisjoint(
+ String firstName, CodePointSet first, String secondName, CodePointSet second) {
+ CodePointSet smaller = first.size() <= second.size() ? first : second;
+ CodePointSet larger = smaller == first ? second : first;
+ for (int codePoint : smaller.toArray()) {
+ if (larger.contains(codePoint)) {
+ throw new IllegalArgumentException(
+ firstName + " and " + secondName + " overlap at "
+ + String.format(CODE_POINT_FORMAT, codePoint));
+ }
+ }
+ }
+}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/TokenizerCharacterPolicyTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/TokenizerCharacterPolicyTest.java
new file mode 100644
index 000000000..853a35860
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/TokenizerCharacterPolicyTest.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.tokenize;
+
+import java.util.function.Predicate;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.util.normalizer.CodePointSet;
+import opennlp.tools.util.normalizer.UnicodeWhitespace;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class TokenizerCharacterPolicyTest {
+
+ private static final int DESERET_LETTER = 0x10400;
+ private static final int MUSICAL_MARK = 0x1D165;
+ private static final int MATHEMATICAL_DIGIT = 0x1D7D8;
+ private static final int ACUTE_ACCENT = 0x0301;
+
+ private static final CodePointSet A = CodePointSet.of('a');
+ private static final CodePointSet SEVEN = CodePointSet.of('7');
+ private static final CodePointSet EMPTY = CodePointSet.of();
+
+ private static final TokenizerCharacterPolicy GRAMMAR = TokenizerCharacterPolicy.of(
+ CodePointSet.of('a', 'b'), CodePointSet.of('7', '8'), CodePointSet.of(ACUTE_ACCENT));
+
+ @Test
+ void testPolicyIsACharSequencePredicate() {
+ assertInstanceOf(Predicate.class, TokenizerCharacterPolicy.ascii());
+ }
+
+ @Test
+ void testAsciiPresetHasExactSets() {
+ TokenizerCharacterPolicy policy = TokenizerCharacterPolicy.ascii();
+
+ assertEquals(52, policy.getLetters().size());
+ assertEquals(10, policy.getDigits().size());
+ assertTrue(policy.getMarks().isEmpty());
+ for (int cp = 0; cp < 128; cp++) {
+ boolean letter = cp >= 'A' && cp <= 'Z' || cp >= 'a' && cp <= 'z';
+ boolean digit = cp >= '0' && cp <= '9';
+ assertEquals(letter, policy.getLetters().contains(cp), String.format("letter U+%04X", cp));
+ assertEquals(digit, policy.getDigits().contains(cp), String.format("digit U+%04X", cp));
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"\u00E9", "e\u0301", "\uD801\uDC00"})
+ void testAsciiPresetRejectsNonAsciiInput(String input) {
+ assertFalse(TokenizerCharacterPolicy.ascii().test(input));
+ }
+
+ @Test
+ void testManualExampleAcceptsDecomposedCafe() {
+ TokenizerCharacterPolicy policy = TokenizerCharacterPolicy.of(
+ CodePointSet.ofRange('A', 'Z').union(CodePointSet.ofRange('a', 'z')),
+ CodePointSet.ofRange('0', '9'),
+ CodePointSet.of(0x0300, ACUTE_ACCENT));
+
+ assertTrue(policy.test("Cafe\u0301"));
+ }
+
+ @Test
+ void testGettersReturnSuppliedSets() {
+ CodePointSet letters = CodePointSet.of('a', DESERET_LETTER);
+ CodePointSet digits = CodePointSet.of('7', MATHEMATICAL_DIGIT);
+ CodePointSet marks = CodePointSet.of(ACUTE_ACCENT, MUSICAL_MARK);
+
+ TokenizerCharacterPolicy policy = TokenizerCharacterPolicy.of(letters, digits, marks);
+
+ assertEquals(letters, policy.getLetters());
+ assertEquals(digits, policy.getDigits());
+ assertEquals(marks, policy.getMarks());
+ }
+
+ @Test
+ void testLetterOnlyAndDigitOnlyPoliciesAreValid() {
+ assertTrue(TokenizerCharacterPolicy.of(A, EMPTY, EMPTY).test("aaa"));
+ assertTrue(TokenizerCharacterPolicy.of(EMPTY, SEVEN, EMPTY).test("777"));
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ "a, true",
+ "7, true",
+ "a\u0301\u0301, true",
+ "a\u03017, true",
+ "7a\u0301, true",
+ "a7b\u03018, true",
+ "'', false",
+ "\u0301, false",
+ "7\u0301, true",
+ "a7\u0301, true",
+ "7\u0301\u0301b, true",
+ "a\u0301-, false",
+ "a-b, false"})
+ void testGrammarAcrossCategoryTransitions(String input, boolean expected) {
+ assertEquals(expected, GRAMMAR.test(input), input);
+ }
+
+ @Test
+ void testSupplementaryBasesAndMarks() {
+ TokenizerCharacterPolicy policy = TokenizerCharacterPolicy.of(
+ CodePointSet.of(DESERET_LETTER), CodePointSet.of(MATHEMATICAL_DIGIT),
+ CodePointSet.of(MUSICAL_MARK));
+
+ assertTrue(policy.test(codePoints(DESERET_LETTER, MUSICAL_MARK, MATHEMATICAL_DIGIT)));
+ assertTrue(policy.test(codePoints(MATHEMATICAL_DIGIT, MUSICAL_MARK)));
+ assertFalse(policy.test(codePoints(MUSICAL_MARK, MATHEMATICAL_DIGIT)));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"\uD800", "\uDC00", "a\uD800", "\uDC00a", "\uDC00\uD800", "\uD800\uD800",
+ "a\uD800b"})
+ void testRejectsUnpairedOrReversedSurrogates(String input) {
+ assertFalse(TokenizerCharacterPolicy.ascii().test(input));
+ }
+
+ @Test
+ void testRejectsNullInput() {
+ assertEquals("input must not be null", assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.ascii().test(null)).getMessage());
+ }
+
+ @Test
+ void testAcceptsGeneralCharSequenceAndLongInput() {
+ StringBuilder input = new StringBuilder(100_000);
+ for (int i = 0; i < 100_000; i++) {
+ input.append(i % 2 == 0 ? 'a' : '7');
+ }
+
+ assertTrue(TokenizerCharacterPolicy.ascii().test(input));
+ input.setCharAt(input.length() - 1, '-');
+ assertFalse(TokenizerCharacterPolicy.ascii().test(input));
+ }
+
+ /**
+ * Supplies a null value for each category in turn.
+ *
+ * @return Category sets with one null argument, and the expected message.
+ */
+ static Stream<Arguments> nullSets() {
+ return Stream.of(
+ Arguments.of(null, EMPTY, EMPTY, "letters must not be null"),
+ Arguments.of(A, null, EMPTY, "digits must not be null"),
+ Arguments.of(A, EMPTY, null, "marks must not be null"));
+ }
+
+ @ParameterizedTest
+ @MethodSource("nullSets")
+ void testFactoryRejectsNullSets(CodePointSet letters, CodePointSet digits, CodePointSet marks,
+ String message) {
+ assertEquals(message, assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(letters, digits, marks)).getMessage());
+ }
+
+ @Test
+ void testFactoryRequiresABaseCharacter() {
+ assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(
+ EMPTY, EMPTY, CodePointSet.of(ACUTE_ACCENT)));
+ }
+
+ /**
+ * Supplies an overlap between each pair of categories.
+ *
+ * @return Category sets with a shared code point.
+ */
+ static Stream<Arguments> overlappingSets() {
+ return Stream.of(
+ Arguments.of(A, A, EMPTY),
+ Arguments.of(A, SEVEN, A),
+ Arguments.of(A, SEVEN, SEVEN));
+ }
+
+ @ParameterizedTest
+ @MethodSource("overlappingSets")
+ void testFactoryRejectsOverlappingCategories(
+ CodePointSet letters, CodePointSet digits, CodePointSet marks) {
+ assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(letters, digits, marks));
+ }
+
+ @Test
+ void testFactoryMessagesNameTheOffendingCodePoint() {
+ IllegalArgumentException overlap = assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(A, A, EMPTY));
+ IllegalArgumentException surrogate = assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(A, SEVEN, CodePointSet.of(Character.MIN_SURROGATE)));
+ IllegalArgumentException whitespace = assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(A, SEVEN, CodePointSet.of(' ')));
+
+ assertTrue(overlap.getMessage().contains("U+0061"), overlap.getMessage());
+ assertTrue(surrogate.getMessage().contains("U+D800"), surrogate.getMessage());
+ assertTrue(whitespace.getMessage().contains("U+0020"), whitespace.getMessage());
+ }
+
+ @Test
+ void testFactoryRejectsSurrogatesInEveryCategory() {
+ for (int surrogate = Character.MIN_SURROGATE;
+ surrogate <= Character.MAX_SURROGATE; surrogate++) {
+ int cp = surrogate;
+ assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(CodePointSet.of(cp), SEVEN, EMPTY));
+ assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(A, CodePointSet.of(cp), EMPTY));
+ assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(A, SEVEN, CodePointSet.of(cp)));
+ }
+ }
+
+ @Test
+ void testFactoryRejectsEveryUnicodeWhitespaceInEveryCategory() {
+ for (int whitespace : UnicodeWhitespace.codePoints()) {
+ assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(CodePointSet.of(whitespace), SEVEN, EMPTY));
+ assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(A, CodePointSet.of(whitespace), EMPTY));
+ assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(A, SEVEN, CodePointSet.of(whitespace)));
+ }
+ }
+
+ private static String codePoints(int... codePoints) {
+ StringBuilder value = new StringBuilder();
+ for (int codePoint : codePoints) {
+ value.appendCodePoint(codePoint);
+ }
+ return value.toString();
+ }
+}
diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml
index a05e53fb6..34a702429 100644
--- a/opennlp-docs/src/docbkx/tokenizer.xml
+++ b/opennlp-docs/src/docbkx/tokenizer.xml
@@ -204,6 +204,36 @@ double[] tokenProbs = tokenizer.getTokenProbabilities();]]>
</section>
</section>
+ <section xml:id="tools.tokenizer.character-policy">
+ <title>Explicit token character policies</title>
+ <para>
+ <code>TokenizerCharacterPolicy</code> tests token eligibility from caller-supplied
+ <code>CodePointSet</code> values for letters, digits, and marks. A mark may follow a
+ letter, a digit, or another mark, but cannot start a token. The policy does
+ not infer Unicode character categories, so callers choose the exact script
+ coverage. The predicate checks a complete candidate without finding token
+ boundaries or changing its text.
+ This policy does not yet configure tokenizer training or inference.
+ </para>
+ <programlisting language="java">
+<![CDATA[TokenizerCharacterPolicy policy = TokenizerCharacterPolicy.of(
+ CodePointSet.ofRange('A', 'Z').union(CodePointSet.ofRange('a', 'z')),
+ CodePointSet.ofRange('0', '9'),
+ CodePointSet.of(0x0300, 0x0301));
+
+boolean word = policy.test("Cafe\u0301");]]>
+ </programlisting>
+ <para>
+ Use <code>TokenizerCharacterPolicy.ascii()</code> for the fixed ASCII letter and digit
+ preset. Policy sets must be pairwise disjoint and cannot contain surrogate code points
+ or whitespace. The getters return the immutable sets. Eligibility is still a
+ model-specific decision: accepting every Unicode letter would incorrectly bypass
+ segmentation for some writing systems. See
+ <link xlink:href="https://www.unicode.org/reports/tr29/#Word_Boundaries">Unicode word boundaries</link>
+ for the distinction between character classes and language-specific segmentation.
+ </para>
+ </section>
+
<section xml:id="tools.tokenizer.whitespace.instances">
<title>Independent whitespace tokenizers</title>
<para>