You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by kr...@apache.org on 2026/09/30 23:35:48 UTC
(opennlp) 01/01: OPENNLP-1934: Add explicit tokenizer character policies
This is an automated email from the ASF dual-hosted git repository.
krickert pushed a commit to branch feat/OPENNLP-1934-tokenizer-policy
in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit e8206f207cd1074de360edb3626e386e493561ff
Author: Kristian Rickert <kr...@apache.org>
AuthorDate: Tue Sep 29 01:56:57 2026 -0400
OPENNLP-1934: Add explicit tokenizer character policies
Add TokenizerCharacterPolicy, an immutable Predicate<CharSequence> built
from explicit letter, digit and mark sets, with an ASCII preset. Accept
marks after letters or digits, including repeated marks, but reject a
leading mark. Support supplementary characters and return false for
malformed UTF-16.
Require at least one letter or digit. Reject null sets, overlapping
categories, whitespace and surrogate code points. Include tests for
validation, character rules and the manual example.
The policy does not yet configure tokenizer training or inference. The
existing regex configuration is unchanged.
Assisted-by: Claude Code 2.1.281
Co-authored-by: Kristian Rickert <kr...@apache.org>
Co-authored-by: Claude <no...@anthropic.com>
Reviewed-by: Kristian Rickert <kr...@apache.org>
Signed-off-by: Kristian Rickert <kr...@apache.org>
---
.../tools/tokenize/TokenizerCharacterPolicy.java | 175 ++++++++++++++
.../tokenize/TokenizerCharacterPolicyTest.java | 254 +++++++++++++++++++++
opennlp-docs/src/docbkx/tokenizer.xml | 30 +++
3 files changed, 459 insertions(+)
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerCharacterPolicy.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerCharacterPolicy.java
new file mode 100644
index 000000000..a7f343a8a
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerCharacterPolicy.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.tokenize;
+
+import java.util.function.Predicate;
+
+import opennlp.tools.util.normalizer.CodePointSet;
+import opennlp.tools.util.normalizer.UnicodeWhitespace;
+
+/**
+ * An immutable policy for testing token eligibility against explicit character sets.
+ *
+ * <p>A token consists of one or more letters or digits. Each letter or digit may be followed by
+ * zero or more marks. A mark cannot start a token.
+ * The supplied sets express a caller's tokenization policy; this class does not infer Unicode
+ * character categories.</p>
+ *
+ * <p>This predicate tests a complete candidate without finding token boundaries or changing
+ * its text.</p>
+ */
+public final class TokenizerCharacterPolicy implements Predicate<CharSequence> {
+
+ private static final String CODE_POINT_FORMAT = "U+%04X";
+
+ private static final TokenizerCharacterPolicy ASCII = new TokenizerCharacterPolicy(
+ CodePointSet.ofRange('A', 'Z').union(CodePointSet.ofRange('a', 'z')),
+ CodePointSet.ofRange('0', '9'), CodePointSet.of());
+
+ private final CodePointSet letters;
+ private final CodePointSet digits;
+ private final CodePointSet marks;
+
+ private TokenizerCharacterPolicy(CodePointSet letters, CodePointSet digits, CodePointSet marks) {
+ this.letters = letters;
+ this.digits = digits;
+ this.marks = marks;
+ }
+
+ /**
+ * Creates a policy from explicit, pairwise-disjoint code point sets. Whitespace separates
+ * tokens and so cannot be part of one; every other non-surrogate code point is left to the
+ * caller.
+ *
+ * @param letters Code points treated as letters.
+ * @param digits Code points treated as digits.
+ * @param marks Code points permitted after a letter, a digit, or another mark.
+ * @return The immutable policy.
+ * @throws IllegalArgumentException Thrown if a set is {@code null}, {@code letters} and
+ * {@code digits} are both empty, a code point occurs in more than one set, or a set
+ * contains a surrogate code point or {@link UnicodeWhitespace} member.
+ */
+ public static TokenizerCharacterPolicy of(
+ CodePointSet letters, CodePointSet digits, CodePointSet marks) {
+ if (letters == null || digits == null || marks == null) {
+ throw new IllegalArgumentException("letters, digits and marks must not be null");
+ }
+ if (letters.isEmpty() && digits.isEmpty()) {
+ throw new IllegalArgumentException("At least one letter or digit is required");
+ }
+ validateSet("letters", letters);
+ validateSet("digits", digits);
+ validateSet("marks", marks);
+ requireDisjoint("letters", letters, "digits", digits);
+ requireDisjoint("letters", letters, "marks", marks);
+ requireDisjoint("digits", digits, "marks", marks);
+ return new TokenizerCharacterPolicy(letters, digits, marks);
+ }
+
+ /** {@return a policy for ASCII letters and digits, with no marks} */
+ public static TokenizerCharacterPolicy ascii() {
+ return ASCII;
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * <p>Returns {@code true} if the entire input is a non-empty token accepted by this policy.
+ * Malformed UTF-16 input (an unpaired or reversed surrogate) returns {@code false}.</p>
+ *
+ * @throws IllegalArgumentException Thrown if {@code input} is {@code null}.
+ */
+ @Override
+ public boolean test(CharSequence input) {
+ if (input == null) {
+ throw new IllegalArgumentException("input must not be null");
+ }
+ if (input.isEmpty()) {
+ return false;
+ }
+
+ for (int offset = 0; offset < input.length();) {
+ final boolean first = offset == 0;
+ char unit = input.charAt(offset);
+ final int codePoint;
+ if (Character.isHighSurrogate(unit)) {
+ if (offset + 1 >= input.length() || !Character.isLowSurrogate(input.charAt(offset + 1))) {
+ return false;
+ }
+ codePoint = Character.toCodePoint(unit, input.charAt(offset + 1));
+ offset += 2;
+ } else if (Character.isLowSurrogate(unit)) {
+ return false;
+ } else {
+ codePoint = unit;
+ offset++;
+ }
+
+ if (!letters.contains(codePoint) && !digits.contains(codePoint)
+ && (first || !marks.contains(codePoint))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /** {@return the immutable letter set} */
+ public CodePointSet getLetters() {
+ return letters;
+ }
+
+ /** {@return the immutable digit set} */
+ public CodePointSet getDigits() {
+ return digits;
+ }
+
+ /** {@return the immutable mark set} */
+ public CodePointSet getMarks() {
+ return marks;
+ }
+
+ /** Throws if {@code set} contains a surrogate code point or {@link UnicodeWhitespace} member. */
+ private static void validateSet(String name, CodePointSet set) {
+ for (int codePoint = Character.MIN_SURROGATE; codePoint <= Character.MAX_SURROGATE;
+ codePoint++) {
+ if (set.contains(codePoint)) {
+ throw new IllegalArgumentException(name + " contains the surrogate code point "
+ + String.format(CODE_POINT_FORMAT, codePoint));
+ }
+ }
+ for (int codePoint : UnicodeWhitespace.codePoints()) {
+ if (set.contains(codePoint)) {
+ throw new IllegalArgumentException(name + " contains the whitespace code point "
+ + String.format(CODE_POINT_FORMAT, codePoint));
+ }
+ }
+ }
+
+ /** Throws if {@code first} and {@code second} share a code point. */
+ private static void requireDisjoint(
+ String firstName, CodePointSet first, String secondName, CodePointSet second) {
+ CodePointSet smaller = first.size() <= second.size() ? first : second;
+ CodePointSet larger = smaller == first ? second : first;
+ for (int codePoint : smaller.toArray()) {
+ if (larger.contains(codePoint)) {
+ throw new IllegalArgumentException(
+ firstName + " and " + secondName + " overlap at "
+ + String.format(CODE_POINT_FORMAT, codePoint));
+ }
+ }
+ }
+}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/TokenizerCharacterPolicyTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/TokenizerCharacterPolicyTest.java
new file mode 100644
index 000000000..e1387842c
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/TokenizerCharacterPolicyTest.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.tokenize;
+
+import java.util.function.Predicate;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.util.normalizer.CodePointSet;
+import opennlp.tools.util.normalizer.UnicodeWhitespace;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class TokenizerCharacterPolicyTest {
+
+ private static final int DESERET_LETTER = 0x10400;
+ private static final int MUSICAL_MARK = 0x1D165;
+ private static final int MATHEMATICAL_DIGIT = 0x1D7D8;
+ private static final int ACUTE_ACCENT = 0x0301;
+
+ private static final CodePointSet A = CodePointSet.of('a');
+ private static final CodePointSet SEVEN = CodePointSet.of('7');
+ private static final CodePointSet EMPTY = CodePointSet.of();
+
+ private static final TokenizerCharacterPolicy GRAMMAR = TokenizerCharacterPolicy.of(
+ CodePointSet.of('a', 'b'), CodePointSet.of('7', '8'), CodePointSet.of(ACUTE_ACCENT));
+
+ @Test
+ void testPolicyIsACharSequencePredicate() {
+ assertInstanceOf(Predicate.class, TokenizerCharacterPolicy.ascii());
+ }
+
+ @Test
+ void testAsciiPresetHasExactSets() {
+ TokenizerCharacterPolicy policy = TokenizerCharacterPolicy.ascii();
+
+ assertEquals(52, policy.getLetters().size());
+ assertEquals(10, policy.getDigits().size());
+ assertTrue(policy.getMarks().isEmpty());
+ for (int cp = 0; cp < 128; cp++) {
+ boolean letter = cp >= 'A' && cp <= 'Z' || cp >= 'a' && cp <= 'z';
+ boolean digit = cp >= '0' && cp <= '9';
+ assertEquals(letter, policy.getLetters().contains(cp), String.format("letter U+%04X", cp));
+ assertEquals(digit, policy.getDigits().contains(cp), String.format("digit U+%04X", cp));
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"\u00E9", "e\u0301", "\uD801\uDC00"})
+ void testAsciiPresetRejectsNonAsciiInput(String input) {
+ assertFalse(TokenizerCharacterPolicy.ascii().test(input));
+ }
+
+ @Test
+ void testManualExampleAcceptsDecomposedCafe() {
+ TokenizerCharacterPolicy policy = TokenizerCharacterPolicy.of(
+ CodePointSet.ofRange('A', 'Z').union(CodePointSet.ofRange('a', 'z')),
+ CodePointSet.ofRange('0', '9'),
+ CodePointSet.of(0x0300, ACUTE_ACCENT));
+
+ assertTrue(policy.test("Cafe\u0301"));
+ }
+
+ @Test
+ void testGettersReturnSuppliedSets() {
+ CodePointSet letters = CodePointSet.of('a', DESERET_LETTER);
+ CodePointSet digits = CodePointSet.of('7', MATHEMATICAL_DIGIT);
+ CodePointSet marks = CodePointSet.of(ACUTE_ACCENT, MUSICAL_MARK);
+
+ TokenizerCharacterPolicy policy = TokenizerCharacterPolicy.of(letters, digits, marks);
+
+ assertEquals(letters, policy.getLetters());
+ assertEquals(digits, policy.getDigits());
+ assertEquals(marks, policy.getMarks());
+ }
+
+ @Test
+ void testLetterOnlyAndDigitOnlyPoliciesAreValid() {
+ assertTrue(TokenizerCharacterPolicy.of(A, EMPTY, EMPTY).test("aaa"));
+ assertTrue(TokenizerCharacterPolicy.of(EMPTY, SEVEN, EMPTY).test("777"));
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ "a, true",
+ "7, true",
+ "a\u0301\u0301, true",
+ "a\u03017, true",
+ "7a\u0301, true",
+ "a7b\u03018, true",
+ "'', false",
+ "\u0301, false",
+ "7\u0301, true",
+ "a7\u0301, true",
+ "7\u0301\u0301b, true",
+ "a\u0301-, false",
+ "a-b, false"})
+ void testGrammarAcrossCategoryTransitions(String input, boolean expected) {
+ assertEquals(expected, GRAMMAR.test(input), input);
+ }
+
+ @Test
+ void testSupplementaryBasesAndMarks() {
+ TokenizerCharacterPolicy policy = TokenizerCharacterPolicy.of(
+ CodePointSet.of(DESERET_LETTER), CodePointSet.of(MATHEMATICAL_DIGIT),
+ CodePointSet.of(MUSICAL_MARK));
+
+ assertTrue(policy.test(codePoints(DESERET_LETTER, MUSICAL_MARK, MATHEMATICAL_DIGIT)));
+ assertTrue(policy.test(codePoints(MATHEMATICAL_DIGIT, MUSICAL_MARK)));
+ assertFalse(policy.test(codePoints(MUSICAL_MARK, MATHEMATICAL_DIGIT)));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"\uD800", "\uDC00", "a\uD800", "\uDC00a", "\uDC00\uD800", "\uD800\uD800",
+ "a\uD800b"})
+ void testRejectsUnpairedOrReversedSurrogates(String input) {
+ assertFalse(TokenizerCharacterPolicy.ascii().test(input));
+ }
+
+ @Test
+ void testRejectsNullInput() {
+ assertThrows(IllegalArgumentException.class, () -> TokenizerCharacterPolicy.ascii().test(null));
+ }
+
+ @Test
+ void testAcceptsGeneralCharSequenceAndLongInput() {
+ StringBuilder input = new StringBuilder(100_000);
+ for (int i = 0; i < 100_000; i++) {
+ input.append(i % 2 == 0 ? 'a' : '7');
+ }
+
+ assertTrue(TokenizerCharacterPolicy.ascii().test(input));
+ input.setCharAt(input.length() - 1, '-');
+ assertFalse(TokenizerCharacterPolicy.ascii().test(input));
+ }
+
+ /**
+ * Supplies a null value for each category in turn.
+ *
+ * @return Category sets with one null argument.
+ */
+ static Stream<Arguments> nullSets() {
+ return Stream.of(
+ Arguments.of(null, EMPTY, EMPTY),
+ Arguments.of(A, null, EMPTY),
+ Arguments.of(A, EMPTY, null));
+ }
+
+ @ParameterizedTest
+ @MethodSource("nullSets")
+ void testFactoryRejectsNullSets(CodePointSet letters, CodePointSet digits, CodePointSet marks) {
+ assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(letters, digits, marks));
+ }
+
+ @Test
+ void testFactoryRequiresABaseCharacter() {
+ assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(
+ EMPTY, EMPTY, CodePointSet.of(ACUTE_ACCENT)));
+ }
+
+ /**
+ * Supplies an overlap between each pair of categories.
+ *
+ * @return Category sets with a shared code point.
+ */
+ static Stream<Arguments> overlappingSets() {
+ return Stream.of(
+ Arguments.of(A, A, EMPTY),
+ Arguments.of(A, SEVEN, A),
+ Arguments.of(A, SEVEN, SEVEN));
+ }
+
+ @ParameterizedTest
+ @MethodSource("overlappingSets")
+ void testFactoryRejectsOverlappingCategories(
+ CodePointSet letters, CodePointSet digits, CodePointSet marks) {
+ assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(letters, digits, marks));
+ }
+
+ @Test
+ void testFactoryMessagesNameTheOffendingCodePoint() {
+ IllegalArgumentException overlap = assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(A, A, EMPTY));
+ IllegalArgumentException surrogate = assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(A, SEVEN, CodePointSet.of(Character.MIN_SURROGATE)));
+ IllegalArgumentException whitespace = assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(A, SEVEN, CodePointSet.of(' ')));
+
+ assertTrue(overlap.getMessage().contains("U+0061"), overlap.getMessage());
+ assertTrue(surrogate.getMessage().contains("U+D800"), surrogate.getMessage());
+ assertTrue(whitespace.getMessage().contains("U+0020"), whitespace.getMessage());
+ }
+
+ @Test
+ void testFactoryRejectsSurrogatesInEveryCategory() {
+ for (int surrogate = Character.MIN_SURROGATE;
+ surrogate <= Character.MAX_SURROGATE; surrogate++) {
+ int cp = surrogate;
+ assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(CodePointSet.of(cp), SEVEN, EMPTY));
+ assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(A, CodePointSet.of(cp), EMPTY));
+ assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(A, SEVEN, CodePointSet.of(cp)));
+ }
+ }
+
+ @Test
+ void testFactoryRejectsEveryUnicodeWhitespaceInEveryCategory() {
+ for (int whitespace : UnicodeWhitespace.codePoints()) {
+ assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(CodePointSet.of(whitespace), SEVEN, EMPTY));
+ assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(A, CodePointSet.of(whitespace), EMPTY));
+ assertThrows(IllegalArgumentException.class,
+ () -> TokenizerCharacterPolicy.of(A, SEVEN, CodePointSet.of(whitespace)));
+ }
+ }
+
+ private static String codePoints(int... codePoints) {
+ StringBuilder value = new StringBuilder();
+ for (int codePoint : codePoints) {
+ value.appendCodePoint(codePoint);
+ }
+ return value.toString();
+ }
+}
diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml
index 003048c4d..4afec916a 100644
--- a/opennlp-docs/src/docbkx/tokenizer.xml
+++ b/opennlp-docs/src/docbkx/tokenizer.xml
@@ -204,6 +204,36 @@ double[] tokenProbs = tokenizer.getTokenProbabilities();]]>
</section>
</section>
+ <section xml:id="tools.tokenizer.character-policy">
+ <title>Explicit token character policies</title>
+ <para>
+ <code>TokenizerCharacterPolicy</code> tests token eligibility from caller-supplied
+ <code>CodePointSet</code> values for letters, digits, and marks. A mark may follow a
+ letter, a digit, or another mark, but cannot start a token. The policy does
+ not infer Unicode character categories, so callers choose the exact script
+ coverage. The predicate checks a complete candidate without finding token
+ boundaries or changing its text.
+ This policy does not yet configure tokenizer training or inference.
+ </para>
+ <programlisting language="java">
+<![CDATA[TokenizerCharacterPolicy policy = TokenizerCharacterPolicy.of(
+ CodePointSet.ofRange('A', 'Z').union(CodePointSet.ofRange('a', 'z')),
+ CodePointSet.ofRange('0', '9'),
+ CodePointSet.of(0x0300, 0x0301));
+
+boolean word = policy.test("Cafe\u0301");]]>
+ </programlisting>
+ <para>
+ Use <code>TokenizerCharacterPolicy.ascii()</code> for the fixed ASCII letter and digit
+ preset. Policy sets must be pairwise disjoint and cannot contain surrogate code points
+ or whitespace. The getters return the immutable sets. Eligibility is still a
+ model-specific decision: accepting every Unicode letter would incorrectly bypass
+ segmentation for some writing systems. See
+ <link xlink:href="https://www.unicode.org/reports/tr29/#Word_Boundaries">Unicode word boundaries</link>
+ for the distinction between character classes and language-specific segmentation.
+ </para>
+ </section>
+
<section xml:id="tools.tokenizer.whitespace.instances">
<title>Independent whitespace tokenizers</title>
<para>