You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by kr...@apache.org on 2026/07/21 10:40:09 UTC
(opennlp) 04/12: OPENNLP-1894: Usage example and edge-case tests for the lattice and unigram segmenters, corrected javadoc
This is an automated email from the ASF dual-hosted git repository.
krickert pushed a commit to branch OPENNLP-1894-lattice-cjk
in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 8730659ac821636d97444e5a0b449ef8d69c8429
Author: Kristian Rickert <kr...@gmail.com>
AuthorDate: Thu Jul 16 01:20:06 2026 -0400
OPENNLP-1894: Usage example and edge-case tests for the lattice and unigram segmenters, corrected javadoc
---
.../tools/tokenize/lattice/LatticeTokenizer.java | 6 +-
.../tools/tokenize/lattice/MecabDictionary.java | 48 ++++-
.../tokenize/lattice/MecabDictionaryInstaller.java | 57 +++++-
.../tools/tokenize/lattice/UnigramSegmenter.java | 13 +-
.../tokenize/lattice/LatticeTokenizerTest.java | 149 ++++++++++++++++
.../tokenize/lattice/LatticeUsageExampleTest.java | 198 +++++++++++++++++++++
.../tokenize/lattice/UnigramSegmenterTest.java | 67 +++++++
7 files changed, 529 insertions(+), 9 deletions(-)
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java
index c811fd326..cb610f74f 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java
@@ -225,7 +225,8 @@ public class LatticeTokenizer implements Tokenizer {
}
}
if (candidates.isEmpty()) {
- // no entry and no template: a single-character fallback keeps the lattice alive
+ // Neither the lexicon nor the character's category produced a candidate here, so a
+ // single-character entry from the DEFAULT template keeps the lattice connected.
final List<WordEntry> fallback = dictionary.unknownEntries("DEFAULT");
if (fallback != null) {
for (final WordEntry entry : fallback) {
@@ -251,7 +252,8 @@ public class LatticeTokenizer implements Tokenizer {
final int lengths = category.length();
for (int length = 1; length <= lengths && position + length <= to; length++) {
if (category.group() && position + length == runEnd) {
- continue; // already emitted as the grouped run
+ // This length coincides with the grouped run emitted above; skip the duplicate.
+ continue;
}
for (final WordEntry entry : templates) {
candidates.add(new Node(position, position + length, entry, true));
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java
index 2e20b8c2d..908bc75f2 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java
@@ -293,7 +293,7 @@ public final class MecabDictionary {
}
}
- /** @return The longest surface form in the lexicon, bounding prefix enumeration. */
+ /** @return The length in characters of the longest surface form in the lexicon. */
int maxSurfaceLength() {
return maxSurfaceLength;
}
@@ -331,6 +331,14 @@ public final class MecabDictionary {
return unknownEntries.get(category);
}
+ /**
+ * Reads a required dictionary file into its lines, accepting LF and CRLF endings.
+ *
+ * @param file The file to read.
+ * @param charset The encoding to decode with.
+ * @return The lines without their line terminators. Never {@code null}.
+ * @throws IOException Thrown if the file is missing or reading fails.
+ */
private static List<String> readLines(Path file, Charset charset) throws IOException {
if (!Files.exists(file)) {
throw new IOException("required dictionary file is missing: " + file);
@@ -351,11 +359,25 @@ public final class MecabDictionary {
return lines;
}
+ /**
+ * Removes a trailing {@code #} comment from a {@code char.def} line.
+ *
+ * @param line The raw line.
+ * @return The line up to but excluding the first {@code #}, or the whole line when
+ * there is none.
+ */
private static String stripComment(String line) {
final int hash = line.indexOf('#');
return hash < 0 ? line : line.substring(0, hash);
}
+ /**
+ * Splits a lexicon line at every comma. Surfaces containing commas are not
+ * representable, matching the plain-text lexicon format.
+ *
+ * @param line The line to split.
+ * @return The fields in order, empty fields included. Never {@code null}.
+ */
private static List<String> splitCsv(String line) {
final List<String> fields = new ArrayList<>();
int start = 0;
@@ -368,6 +390,12 @@ public final class MecabDictionary {
return fields;
}
+ /**
+ * Splits a line into its whitespace-separated fields.
+ *
+ * @param line The line to split.
+ * @return The non-empty fields in order. Never {@code null}.
+ */
private static String[] splitWhitespace(String line) {
final List<String> parts = new ArrayList<>();
int start = -1;
@@ -384,6 +412,15 @@ public final class MecabDictionary {
return parts.toArray(new String[0]);
}
+ /**
+ * Parses a decimal integer field, reporting the file and line on failure.
+ *
+ * @param text The field text.
+ * @param file The file being read, for the error message.
+ * @param lineNumber The line being read, for the error message.
+ * @return The parsed value.
+ * @throws IOException Thrown if the field is not a valid integer.
+ */
private static int parseInt(String text, String file, int lineNumber)
throws IOException {
try {
@@ -393,6 +430,15 @@ public final class MecabDictionary {
}
}
+ /**
+ * Parses a {@code 0x}-prefixed hexadecimal code point from {@code char.def}.
+ *
+ * @param text The field text including the {@code 0x} prefix.
+ * @param file The file being read, for the error message.
+ * @param lineNumber The line being read, for the error message.
+ * @return The parsed code point.
+ * @throws IOException Thrown if the field is not a valid hexadecimal code point.
+ */
private static int parseCodePoint(String text, Path file, int lineNumber)
throws IOException {
try {
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java
index 33f3ef58d..9bbbde790 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java
@@ -34,9 +34,11 @@ import java.util.zip.GZIPInputStream;
* built in.
*
* <p>The installer reads gzip-compressed tar archives, the format the common
- * distributions use, and extracts only the files a {@link MecabDictionary} reads:
- * {@code *.csv}, {@code *.def}, and {@code dicrc}. Entries are flattened to their base
- * names, which also means no archive path can escape the target directory.</p>
+ * distributions use, and extracts only the dictionary payload: the {@code *.csv}
+ * lexicon files and {@code *.def} definition files that a {@link MecabDictionary}
+ * reads, plus the {@code dicrc} configuration file distributions ship alongside them.
+ * Entries are flattened to their base names, which also means no archive path can
+ * escape the target directory.</p>
*
* @since 3.0.0
*/
@@ -49,7 +51,7 @@ public final class MecabDictionaryInstaller {
private static final int TAR_TYPE_OFFSET = 156;
private MecabDictionaryInstaller() {
- // static installer only
+ // This class exposes only static methods and is never instantiated.
}
/**
@@ -122,6 +124,15 @@ public final class MecabDictionaryInstaller {
return extracted;
}
+ /**
+ * Fills one tar block from the stream.
+ *
+ * @param in The tar stream.
+ * @param block The block buffer to fill completely.
+ * @return {@code true} when a full block was read, {@code false} at a clean end of
+ * stream before any byte of the block.
+ * @throws IOException Thrown if the stream ends inside the block or reading fails.
+ */
private static boolean readBlock(InputStream in, byte[] block) throws IOException {
int filled = 0;
while (filled < block.length) {
@@ -137,6 +148,12 @@ public final class MecabDictionaryInstaller {
return true;
}
+ /**
+ * Recognizes the all-zero block that terminates a tar archive.
+ *
+ * @param block The block to inspect.
+ * @return {@code true} when every byte is zero.
+ */
private static boolean isEndBlock(byte[] block) {
for (final byte b : block) {
if (b != 0) {
@@ -146,6 +163,12 @@ public final class MecabDictionaryInstaller {
return true;
}
+ /**
+ * Reads the NUL-terminated entry name from a tar header block.
+ *
+ * @param header The header block.
+ * @return The entry name. Never {@code null}.
+ */
private static String headerName(byte[] header) {
int end = 0;
while (end < TAR_NAME_LENGTH && header[end] != 0) {
@@ -154,6 +177,13 @@ public final class MecabDictionaryInstaller {
return new String(header, 0, end, StandardCharsets.UTF_8);
}
+ /**
+ * Reads the octal entry size from a tar header block.
+ *
+ * @param header The header block.
+ * @return The entry size in bytes.
+ * @throws IOException Thrown if the size field holds a non-octal digit.
+ */
private static long headerSize(byte[] header) throws IOException {
long size = 0;
for (int i = TAR_SIZE_OFFSET; i < TAR_SIZE_OFFSET + TAR_SIZE_LENGTH; i++) {
@@ -169,16 +199,35 @@ public final class MecabDictionaryInstaller {
return size;
}
+ /**
+ * Strips any directory prefix from an archive entry name.
+ *
+ * @param name The entry name as stored in the archive.
+ * @return The part after the last {@code /}, or the whole name when there is none.
+ */
private static String baseName(String name) {
final int slash = name.lastIndexOf('/');
return slash < 0 ? name : name.substring(slash + 1);
}
+ /**
+ * Computes the padding after an entry: tar content is stored in whole blocks.
+ *
+ * @param size The entry size in bytes.
+ * @return The number of padding bytes up to the next block boundary.
+ */
private static long padding(long size) {
final long remainder = size % TAR_BLOCK;
return remainder == 0 ? 0 : TAR_BLOCK - remainder;
}
+ /**
+ * Consumes and discards an exact number of bytes from the stream.
+ *
+ * @param in The stream to read from.
+ * @param bytes The number of bytes to discard.
+ * @throws IOException Thrown if the stream ends before that many bytes were read.
+ */
private static void skip(InputStream in, long bytes) throws IOException {
long remaining = bytes;
final byte[] buffer = new byte[8192];
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java
index 3d1ca9884..ea3d188bc 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java
@@ -167,7 +167,8 @@ public class UnigramSegmenter implements Tokenizer {
}
node.logProbability = Math.log(entry.getValue()) - logTotal;
}
- // rarer than any listed word: a fraction of a single count
+ // Charge an unlisted character half of one count out of the total, which makes it
+ // rarer than any listed word: every listed count is at least one.
final double unknown = Math.log(0.5) - logTotal;
return new UnigramSegmenter(root, unknown);
}
@@ -216,7 +217,8 @@ public class UnigramSegmenter implements Tokenizer {
if (best[i] == Double.NEGATIVE_INFINITY) {
continue;
}
- // the single-character fallback keeps every position reachable
+ // A single-character step at the unknown log-probability keeps every position
+ // reachable even where no lexicon word matches.
final double fallback = best[i] + unknownLogProbability;
if (fallback > best[i + 1]) {
best[i + 1] = fallback;
@@ -247,6 +249,13 @@ public class UnigramSegmenter implements Tokenizer {
}
}
+ /**
+ * Finds the first whitespace character in a lexicon line.
+ *
+ * @param text The line to scan.
+ * @return The index of the first whitespace character, or {@code -1} when the line
+ * contains none.
+ */
private static int whitespaceIndex(String text) {
for (int i = 0; i < text.length(); i++) {
if (StringUtil.isWhitespace(text.charAt(i))) {
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java
index 1c3f40a8d..b643786a9 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java
@@ -128,6 +128,155 @@ public class LatticeTokenizerTest {
Assertions.assertEquals(0, tokenizer.analyze("").size());
}
+ /**
+ * Verifies that empty input yields empty results from every view of the tokenizer.
+ */
+ @Test
+ void testEmptyInputYieldsEmptyResults() {
+ Assertions.assertArrayEquals(new String[0], tokenizer.tokenize(""));
+ Assertions.assertArrayEquals(new Span[0], tokenizer.tokenizePos(""));
+ }
+
+ /**
+ * Verifies single-character input for a listed surface and for an unlisted kanji:
+ * both come back as exactly one morpheme covering {@code [0, 1)}, and only the
+ * unlisted one is marked unknown.
+ */
+ @Test
+ void testSingleCharacterInput() {
+ Assertions.assertArrayEquals(new String[] {"に"}, tokenizer.tokenize("に"));
+ Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, tokenizer.tokenizePos("に"));
+ Assertions.assertFalse(tokenizer.analyze("に").get(0).unknown());
+
+ final List<Morpheme> unknown = tokenizer.analyze("峠");
+ Assertions.assertEquals(1, unknown.size());
+ Assertions.assertEquals("峠", unknown.get(0).surface());
+ Assertions.assertEquals(new Span(0, 1), unknown.get(0).span());
+ Assertions.assertTrue(unknown.get(0).unknown());
+ }
+
+ /**
+ * Verifies input made entirely of characters absent from both the lexicon and the
+ * {@code char.def} mappings: they fall into the DEFAULT category, whose grouping
+ * setting joins the whole same-category run into one unknown morpheme carrying the
+ * DEFAULT template's features.
+ */
+ @Test
+ void testEntirelyUnknownInputGroupsIntoOneDefaultMorpheme() {
+ final List<Morpheme> morphemes = tokenizer.analyze("①②③");
+ Assertions.assertEquals(1, morphemes.size());
+ Assertions.assertEquals("①②③", morphemes.get(0).surface());
+ Assertions.assertEquals(new Span(0, 3), morphemes.get(0).span());
+ Assertions.assertTrue(morphemes.get(0).unknown());
+ Assertions.assertEquals(List.of("symbol", "unknown"), morphemes.get(0).features());
+ }
+
+ /**
+ * Verifies a mixed run of known and unknown text: the lexicon words around an
+ * unmapped character are kept intact, the unmapped character becomes its own
+ * unknown morpheme, and every span stays in original text coordinates.
+ */
+ @Test
+ void testMixedKnownAndUnknownRuns() {
+ final String text = "東京①に行く";
+ Assertions.assertArrayEquals(
+ new String[] {"東京", "①", "に", "行く"},
+ tokenizer.tokenize(text));
+ Assertions.assertArrayEquals(new Span[] {
+ new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)},
+ tokenizer.tokenizePos(text));
+ final List<Morpheme> morphemes = tokenizer.analyze(text);
+ Assertions.assertFalse(morphemes.get(0).unknown());
+ Assertions.assertTrue(morphemes.get(1).unknown());
+ Assertions.assertFalse(morphemes.get(2).unknown());
+ }
+
+ /**
+ * Verifies that spans keep original text coordinates when the interesting content
+ * does not start at position zero because of leading whitespace.
+ */
+ @Test
+ void testSpansStayOriginalAfterLeadingWhitespace() {
+ final String text = " 東京都に行く";
+ Assertions.assertArrayEquals(
+ new String[] {"東京", "都", "に", "行く"},
+ tokenizer.tokenize(text));
+ Assertions.assertArrayEquals(new Span[] {
+ new Span(2, 4), new Span(4, 5), new Span(5, 6), new Span(6, 8)},
+ tokenizer.tokenizePos(text));
+ }
+
+ /**
+ * Verifies that a lexicon row with fewer than the four mandatory columns is
+ * rejected at load time.
+ */
+ @Test
+ void testShortLexiconRowFailsLoud(@TempDir Path broken) throws IOException {
+ Files.write(broken.resolve("lexicon.csv"),
+ "東,0,0\n".getBytes(StandardCharsets.UTF_8));
+ Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken));
+ }
+
+ /**
+ * Verifies that a non-numeric cost column in a lexicon row is rejected at load
+ * time.
+ */
+ @Test
+ void testNonNumericLexiconCostFailsLoud(@TempDir Path broken) throws IOException {
+ Files.write(broken.resolve("lexicon.csv"),
+ "東,0,0,abc,noun\n".getBytes(StandardCharsets.UTF_8));
+ Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken));
+ }
+
+ /**
+ * Verifies that a {@code matrix.def} data line with the wrong number of fields is
+ * rejected at load time.
+ */
+ @Test
+ void testMalformedMatrixLineFailsLoud(@TempDir Path broken) throws IOException {
+ Files.write(broken.resolve("lexicon.csv"),
+ "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8));
+ Files.write(broken.resolve("matrix.def"), "1 1\n0 0\n".getBytes(StandardCharsets.UTF_8));
+ Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken));
+ }
+
+ /**
+ * Verifies that a {@code char.def} code point mapping without a category name is
+ * rejected at load time.
+ */
+ @Test
+ void testCharDefMappingWithoutCategoryFailsLoud(@TempDir Path broken)
+ throws IOException {
+ Files.write(broken.resolve("lexicon.csv"),
+ "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8));
+ Files.write(broken.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8));
+ Files.write(broken.resolve("char.def"),
+ "DEFAULT 0 1 0\n0x4E00..0x9FFF\n".getBytes(StandardCharsets.UTF_8));
+ Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken));
+ }
+
+ /**
+ * Verifies the fail-loud path when a loadable dictionary cannot cover the input: the
+ * {@code unk.def} has no DEFAULT template, so a character with neither a lexicon
+ * entry nor a category template stops segmentation with an exception instead of
+ * being dropped silently.
+ */
+ @Test
+ void testMissingDefaultTemplateFailsLoudAtTokenizeTime(@TempDir Path partial)
+ throws IOException {
+ Files.write(partial.resolve("lexicon.csv"),
+ "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8));
+ Files.write(partial.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8));
+ Files.write(partial.resolve("char.def"),
+ "DEFAULT 0 1 0\nKANJI 0 0 2\n0x4E00..0x9FFF KANJI\n"
+ .getBytes(StandardCharsets.UTF_8));
+ Files.write(partial.resolve("unk.def"),
+ "KANJI,0,0,8000,noun\n".getBytes(StandardCharsets.UTF_8));
+ final LatticeTokenizer limited =
+ new LatticeTokenizer(MecabDictionary.load(partial));
+ Assertions.assertThrows(IllegalStateException.class, () -> limited.analyze("①"));
+ }
+
@Test
void testMalformedDictionariesFailLoud(@TempDir Path broken) throws IOException {
Files.write(broken.resolve("lexicon.csv"),
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java
new file mode 100644
index 000000000..a5e2a39d1
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.tokenize.lattice;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.zip.GZIPOutputStream;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import opennlp.tools.util.Span;
+
+/**
+ * Demonstrates the intended end-to-end usage of this package with miniature,
+ * project-authored data: a mecab-format dictionary archive is installed with
+ * {@link MecabDictionaryInstaller}, loaded as a {@link MecabDictionary}, and segmented
+ * with a {@link LatticeTokenizer}; a plain frequency lexicon is loaded and segmented
+ * with a {@link UnigramSegmenter}. Everything is written to a temporary directory by
+ * the test itself; no external dictionary or lexicon data and no network access are
+ * involved.
+ */
+public class LatticeUsageExampleTest {
+
+ /**
+ * Appends one ustar file entry to a growing tar image: a 512-byte header block
+ * followed by the content padded to a block boundary.
+ *
+ * @param tar The tar image under construction. Must not be {@code null}.
+ * @param name The entry name including any directory prefix. Must not be
+ * {@code null}, empty, or longer than the 100-byte header name field.
+ * @param content The entry content bytes. Must not be {@code null}.
+ * @throws IOException Thrown if writing to the in-memory stream fails.
+ * @throws IllegalArgumentException Thrown if {@code name} does not fit the header.
+ */
+ private static void tarEntry(ByteArrayOutputStream tar, String name, byte[] content)
+ throws IOException {
+ final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8);
+ if (nameBytes.length == 0 || nameBytes.length > 100) {
+ throw new IllegalArgumentException(
+ "entry name must be 1..100 bytes, got " + nameBytes.length);
+ }
+ final byte[] header = new byte[512];
+ System.arraycopy(nameBytes, 0, header, 0, nameBytes.length);
+ final byte[] mode = "0000644".getBytes(StandardCharsets.US_ASCII);
+ System.arraycopy(mode, 0, header, 100, mode.length);
+ final String size = String.format("%011o", content.length);
+ System.arraycopy(size.getBytes(StandardCharsets.US_ASCII), 0, header, 124, 11);
+ header[156] = '0';
+ for (int i = 148; i < 156; i++) {
+ header[i] = ' ';
+ }
+ int checksum = 0;
+ for (final byte b : header) {
+ checksum += b & 0xFF;
+ }
+ final String checksumText = String.format("%06o", checksum);
+ System.arraycopy(checksumText.getBytes(StandardCharsets.US_ASCII), 0, header, 148, 6);
+ header[154] = 0;
+ header[155] = ' ';
+ tar.write(header);
+ tar.write(content);
+ final int padding = (512 - content.length % 512) % 512;
+ tar.write(new byte[padding]);
+ }
+
+ /**
+ * Builds a gzip-compressed tar archive from name and content pairs, the layout a
+ * dictionary distribution ships in.
+ *
+ * @param entries The entries as {@code {name, content}} pairs. Must not be
+ * {@code null}.
+ * @return The compressed archive bytes. Never {@code null}.
+ * @throws IOException Thrown if writing to the in-memory streams fails.
+ */
+ private static byte[] gzippedTar(String[][] entries) throws IOException {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ for (final String[] entry : entries) {
+ tarEntry(tar, entry[0], entry[1].getBytes(StandardCharsets.UTF_8));
+ }
+ tar.write(new byte[1024]);
+ final ByteArrayOutputStream compressed = new ByteArrayOutputStream();
+ try (GZIPOutputStream gzip = new GZIPOutputStream(compressed)) {
+ gzip.write(tar.toByteArray());
+ }
+ return compressed.toByteArray();
+ }
+
+ /**
+ * Walks the full mecab-format flow: package a miniature Japanese dictionary as a
+ * {@code tar.gz} archive, install it from a file URI, load it, and tokenize. The
+ * segmentation must pick the cheaper path (Tokyo plus the metropolis suffix) over
+ * the competing reading (east plus Kyoto), the spans must be in original text
+ * coordinates, and the morphemes must carry the dictionary's feature columns.
+ */
+ @Test
+ void testInstallLoadAndTokenizeAMecabFormatDictionary(@TempDir Path work)
+ throws IOException {
+ // A minimal but complete dictionary: one lexicon file plus the three definition
+ // files every mecab-format distribution contains, wrapped like a release archive.
+ final byte[] archive = gzippedTar(new String[][] {
+ {"mini-dict-0.1/lexicon.csv", String.join("\n",
+ "東京,0,0,3000,noun,proper",
+ "京都,0,0,3000,noun,proper",
+ "東,0,0,6000,noun,common",
+ "都,0,0,4000,noun,suffix",
+ "に,0,0,1000,particle,case",
+ "行く,0,0,3000,verb,base",
+ "")},
+ {"mini-dict-0.1/matrix.def", "1 1\n0 0 0\n"},
+ {"mini-dict-0.1/char.def", String.join("\n",
+ "DEFAULT 0 1 0",
+ "KANJI 0 0 2",
+ "HIRAGANA 0 1 0",
+ "",
+ "0x3041..0x3096 HIRAGANA",
+ "0x4E00..0x9FFF KANJI",
+ "")},
+ {"mini-dict-0.1/unk.def", String.join("\n",
+ "DEFAULT,0,0,10000,symbol,unknown",
+ "KANJI,0,0,8000,noun,unknown",
+ "HIRAGANA,0,0,9000,particle,unknown",
+ "")},
+ {"mini-dict-0.1/README", "not a dictionary payload file"}});
+ final Path archiveFile = work.resolve("mini-dict-0.1.tar.gz");
+ Files.write(archiveFile, archive);
+
+ // Install: fetch the archive from the user-chosen location and unpack the payload.
+ final Path dictionaryDirectory = work.resolve("dictionary");
+ final int extracted =
+ MecabDictionaryInstaller.install(archiveFile.toUri(), dictionaryDirectory);
+ Assertions.assertEquals(4, extracted);
+
+ // Load and tokenize; both views must agree and stay in original coordinates.
+ final LatticeTokenizer tokenizer =
+ new LatticeTokenizer(MecabDictionary.load(dictionaryDirectory));
+ final String text = "東京都に行く";
+ Assertions.assertArrayEquals(
+ new String[] {"東京", "都", "に", "行く"},
+ tokenizer.tokenize(text));
+ Assertions.assertArrayEquals(new Span[] {
+ new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)},
+ tokenizer.tokenizePos(text));
+
+ // The analyze view adds the dictionary's feature columns to every morpheme.
+ final List<Morpheme> morphemes = tokenizer.analyze(text);
+ Assertions.assertEquals(4, morphemes.size());
+ Assertions.assertEquals("東京", morphemes.get(0).surface());
+ Assertions.assertEquals(List.of("noun", "proper"), morphemes.get(0).features());
+ Assertions.assertFalse(morphemes.get(0).unknown());
+ }
+
+ /**
+ * Walks the frequency-lexicon flow: write a miniature word-count lexicon to a file,
+ * load it, and segment. The segmentation must recover the listed multi-character
+ * words with spans in original text coordinates.
+ */
+ @Test
+ void testLoadAndSegmentWithAFrequencyLexicon(@TempDir Path work) throws IOException {
+ // One word, its count, and an optional tag per line, whitespace separated.
+ final Path lexicon = work.resolve("words.txt");
+ Files.write(lexicon, String.join("\n",
+ "我 5000 r",
+ "来到 2000 v",
+ "北京 3000 ns",
+ "天安门 1200 ns",
+ "").getBytes(StandardCharsets.UTF_8));
+
+ final UnigramSegmenter segmenter = UnigramSegmenter.load(lexicon);
+ final String text = "我来到北京天安门";
+ Assertions.assertArrayEquals(
+ new String[] {"我", "来到", "北京", "天安门"},
+ segmenter.tokenize(text));
+ Assertions.assertArrayEquals(new Span[] {
+ new Span(0, 1), new Span(1, 3), new Span(3, 5), new Span(5, 8)},
+ segmenter.tokenizePos(text));
+ }
+}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java
index a6104c195..f086eafa2 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java
@@ -82,6 +82,73 @@ public class UnigramSegmenterTest {
Assertions.assertEquals(0, segmenter.tokenizePos(" ").length);
}
+ /**
+ * Verifies that empty input yields empty results from both views of the segmenter.
+ */
+ @Test
+ void testEmptyInputYieldsEmptyResults() {
+ Assertions.assertArrayEquals(new String[0], segmenter.tokenize(""));
+ Assertions.assertArrayEquals(new Span[0], segmenter.tokenizePos(""));
+ }
+
+ /**
+ * Verifies single-character input for a listed word and for a character the
+ * lexicon does not know: both come back as exactly one token covering
+ * {@code [0, 1)}.
+ */
+ @Test
+ void testSingleCharacterInput() {
+ Assertions.assertArrayEquals(new String[] {"我"}, segmenter.tokenize("我"));
+ Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("我"));
+ Assertions.assertArrayEquals(new String[] {"爱"}, segmenter.tokenize("爱"));
+ Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("爱"));
+ }
+
+ /**
+ * Verifies input made entirely of characters absent from the lexicon: every
+ * character becomes its own single-character token, since only the unknown
+ * fallback is available.
+ */
+ @Test
+ void testEntirelyUnknownInputFallsBackToSingleCharacters() {
+ Assertions.assertArrayEquals(
+ new String[] {"x", "y", "z"},
+ segmenter.tokenize("xyz"));
+ Assertions.assertArrayEquals(new Span[] {
+ new Span(0, 1), new Span(1, 2), new Span(2, 3)},
+ segmenter.tokenizePos("xyz"));
+ }
+
+ /**
+ * Verifies a mixed run of known and unknown text inside one whitespace-free
+ * stretch: the unknown character becomes a single token while the listed words
+ * around it, including the longest listed compound, stay intact.
+ */
+ @Test
+ void testMixedKnownAndUnknownRuns() {
+ Assertions.assertArrayEquals(
+ new String[] {"我", "爱", "清华大学"},
+ segmenter.tokenize("我爱清华大学"));
+ Assertions.assertArrayEquals(new Span[] {
+ new Span(0, 1), new Span(1, 2), new Span(2, 6)},
+ segmenter.tokenizePos("我爱清华大学"));
+ }
+
+ /**
+ * Verifies that spans keep original text coordinates when the content does not
+ * start at position zero because of leading whitespace.
+ */
+ @Test
+ void testSpansStayOriginalAfterLeadingWhitespace() {
+ final String text = " 我来到北京";
+ Assertions.assertArrayEquals(
+ new String[] {"我", "来到", "北京"},
+ segmenter.tokenize(text));
+ Assertions.assertArrayEquals(new Span[] {
+ new Span(2, 3), new Span(3, 5), new Span(5, 7)},
+ segmenter.tokenizePos(text));
+ }
+
@Test
void testMalformedLexiconsFailLoud() {
Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load(