You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by rz...@apache.org on 2026/10/08 09:14:11 UTC
(opennlp) branch main updated: GH-1441: Report malformed lattice dictionary content as InvalidFormatException (#1443)
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/main by this push:
new 11a136cc6 GH-1441: Report malformed lattice dictionary content as InvalidFormatException (#1443)
11a136cc6 is described below
commit 11a136cc64f0e5014bd69dbf1947839824dd1dc7
Author: Richard Zowalla <13...@users.noreply.github.com>
AuthorDate: Thu Oct 8 11:14:06 2026 +0200
GH-1441: Report malformed lattice dictionary content as InvalidFormatException (#1443)
MecabDictionary, CategoryTable and UnigramSegmenter now throw
InvalidFormatException for malformed content; missing files, read
failures and size limits stay plain IOException.
---
.../tools/tokenize/lattice/CategoryTable.java | 12 +--
.../tools/tokenize/lattice/MecabDictionary.java | 91 ++++++++++++----------
.../tools/tokenize/lattice/UnigramSegmenter.java | 21 +++--
.../tokenize/lattice/LatticeTokenizerTest.java | 8 +-
.../tokenize/lattice/UnigramSegmenterTest.java | 6 +-
5 files changed, 76 insertions(+), 62 deletions(-)
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/CategoryTable.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/CategoryTable.java
index 85db6e237..61328be47 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/CategoryTable.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/CategoryTable.java
@@ -17,7 +17,6 @@
package opennlp.tools.tokenize.lattice;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.IdentityHashMap;
@@ -26,6 +25,7 @@ import java.util.Locale;
import java.util.Map;
import opennlp.tools.tokenize.lattice.MecabDictionary.Category;
+import opennlp.tools.util.InvalidFormatException;
/**
* The {@code char.def} code point to category mappings over the Unicode code point
@@ -191,9 +191,9 @@ final class CategoryTable {
* @param categories The categories the {@code char.def} category section defined,
* keyed by name.
* @return The table. Not {@code null}.
- * @throws IOException Thrown if a mapping names a category that was not defined.
+ * @throws InvalidFormatException Thrown if a mapping names a category that was not defined.
*/
- CategoryTable build(Map<String, Category> categories) throws IOException {
+ CategoryTable build(Map<String, Category> categories) throws InvalidFormatException {
// Cut the supplementary ranges at every boundary they introduce, so that each
// resulting elementary interval is covered by a single winning range and the
// table stays sorted and non-overlapping for binary search.
@@ -260,11 +260,11 @@ final class CategoryTable {
* shared category-name arrays.
* @param codePoint A code point the mapping covers, for the error message.
* @return The resolved assignment. Not {@code null}.
- * @throws IOException Thrown if any named category was not defined.
+ * @throws InvalidFormatException Thrown if any named category was not defined.
*/
private CategoryAssignment resolve(String[] names, Map<String, Category> categories,
Map<String[], CategoryAssignment> resolvedAssignments, int codePoint)
- throws IOException {
+ throws InvalidFormatException {
final CategoryAssignment cached = resolvedAssignments.get(names);
if (cached != null) {
return cached;
@@ -273,7 +273,7 @@ final class CategoryTable {
for (int i = 0; i < names.length; i++) {
resolved[i] = categories.get(names[i]);
if (resolved[i] == null) {
- throw new IOException(String.format(Locale.ROOT,
+ throw new InvalidFormatException(String.format(Locale.ROOT,
MecabDictionary.CHAR_DEF + " declaration at U+%04X names the"
+ " undefined category %s", codePoint, names[i]));
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java
index 8f7799bfa..32d36a859 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java
@@ -34,6 +34,7 @@ import java.util.Map;
import java.util.Objects;
import opennlp.tools.tokenize.lattice.CategoryTable.CategoryAssignment;
+import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.ParamChecks;
import opennlp.tools.util.ResourceLimits;
import opennlp.tools.util.StringUtil;
@@ -191,7 +192,9 @@ public final class MecabDictionary {
*
* @param directory The unpacked dictionary directory. Must not be {@code null}.
* @return The loaded dictionary. Never {@code null}.
- * @throws IOException Thrown if reading fails or a file is malformed.
+ * @throws InvalidFormatException Thrown if a file is malformed.
+ * @throws IOException Thrown if reading fails, a required file is missing, or a size
+ * limit is exceeded.
* @throws IllegalArgumentException Thrown if {@code directory} is {@code null}.
*/
public static MecabDictionary load(Path directory) throws IOException {
@@ -207,9 +210,10 @@ public final class MecabDictionary {
* @param charset The encoding the distribution uses, for example UTF-8 or EUC-JP.
* Must not be {@code null}.
* @return The loaded dictionary. Never {@code null}.
- * @throws IOException Thrown if reading fails, a required file is missing, a file is
- * malformed, or a lexicon entry's context ids are outside the
- * {@code matrix.def} dimensions.
+ * @throws InvalidFormatException Thrown if a file is malformed or a lexicon entry's
+ * context ids are outside the {@code matrix.def} dimensions.
+ * @throws IOException Thrown if reading fails, a required file is missing, or a size
+ * limit is exceeded.
* @throws IllegalArgumentException Thrown if a parameter is {@code null}.
*/
public static MecabDictionary load(Path directory, Charset charset) throws IOException {
@@ -226,20 +230,20 @@ public final class MecabDictionary {
try (BufferedReader reader = Files.newBufferedReader(matrixFile, charset)) {
final String rawHeader = reader.readLine();
if (rawHeader == null) {
- throw new IOException("empty " + MATRIX_DEF + " under " + directory);
+ throw new InvalidFormatException("empty " + MATRIX_DEF + " under " + directory);
}
final String headerLine = StringUtil.trimUnicodeWhitespace(rawHeader);
if (headerLine.isEmpty()) {
- throw new IOException("empty " + MATRIX_DEF + " under " + directory);
+ throw new InvalidFormatException("empty " + MATRIX_DEF + " under " + directory);
}
final String[] header = StringUtil.splitOnUnicodeWhitespace(headerLine);
if (header.length != 2) {
- throw new IOException("malformed " + MATRIX_DEF + " header: " + headerLine);
+ throw new InvalidFormatException("malformed " + MATRIX_DEF + " header: " + headerLine);
}
leftSize = parseInt(header[0], MATRIX_DEF, 1);
rightSize = parseInt(header[1], MATRIX_DEF, 1);
if (leftSize < 1 || rightSize < 1) {
- throw new IOException(MATRIX_DEF + " dimensions must be positive, got "
+ throw new InvalidFormatException(MATRIX_DEF + " dimensions must be positive, got "
+ leftSize + " " + rightSize);
}
if (leftSize > ResourceLimits.MAX_ENTRIES
@@ -271,31 +275,31 @@ public final class MecabDictionary {
}
final String[] fields = StringUtil.splitOnUnicodeWhitespace(line);
if (fields.length != 3) {
- throw new IOException("malformed " + MATRIX_DEF + " line " + lineNumber);
+ throw new InvalidFormatException("malformed " + MATRIX_DEF + " line " + lineNumber);
}
final int right = parseInt(fields[0], MATRIX_DEF, lineNumber);
final int left = parseInt(fields[1], MATRIX_DEF, lineNumber);
if (right < 0 || right >= leftSize || left < 0 || left >= rightSize) {
- throw new IOException("malformed " + MATRIX_DEF + " line " + lineNumber
+ throw new InvalidFormatException("malformed " + MATRIX_DEF + " line " + lineNumber
+ ": context ids " + right + " " + left
+ " are outside the declared dimensions " + leftSize + " " + rightSize);
}
final int cost = parseInt(fields[2], MATRIX_DEF, lineNumber);
if (cost < Short.MIN_VALUE || cost > Short.MAX_VALUE) {
- throw new IOException("malformed " + MATRIX_DEF + " line " + lineNumber
+ throw new InvalidFormatException("malformed " + MATRIX_DEF + " line " + lineNumber
+ ": connection cost " + cost + " is outside the 16-bit range the"
+ " format defines");
}
final int index = right * rightSize + left;
if (filled.get(index)) {
- throw new IOException("duplicate " + MATRIX_DEF + " entry " + right + " "
+ throw new InvalidFormatException("duplicate " + MATRIX_DEF + " entry " + right + " "
+ left + " at line " + lineNumber);
}
costs[index] = (short) cost;
filled.set(index);
}
if (filled.cardinality() != cellCount) {
- throw new IOException(MATRIX_DEF + " declares " + leftSize + " x " + rightSize
+ throw new InvalidFormatException(MATRIX_DEF + " declares " + leftSize + " x " + rightSize
+ " connection costs but only " + filled.cardinality()
+ " pairs are listed");
}
@@ -314,7 +318,7 @@ public final class MecabDictionary {
readLexicon(csv, charset, lexicon, leftSize, rightSize, entryCount);
}
if (lexicon.isEmpty()) {
- throw new IOException("no lexicon entries found under " + directory);
+ throw new InvalidFormatException("no lexicon entries found under " + directory);
}
final Map<String, Category> categories = new HashMap<>();
@@ -326,7 +330,7 @@ public final class MecabDictionary {
readLexicon(unkFile, charset, unknown, leftSize, rightSize, new int[] {0});
for (final String category : unknown.keySet()) {
if (!categories.containsKey(category)) {
- throw new IOException(
+ throw new InvalidFormatException(
UNK_DEF + " names the undefined category " + category + ": " + unkFile);
}
}
@@ -348,9 +352,10 @@ public final class MecabDictionary {
* ids.
* @param entryCount A one-element running total of entries read so far, shared across
* the lexicon files of one load.
- * @throws IOException Thrown if the file is missing, an entry is malformed or has
- * an empty surface, an entry's context id is outside the matrix dimensions,
- * or the running entry count exceeds {@link ResourceLimits#MAX_ENTRIES}.
+ * @throws InvalidFormatException Thrown if an entry is malformed, has an empty surface,
+ * a context id outside the matrix dimensions or a cost outside the 16-bit range.
+ * @throws IOException Thrown if the file is missing, reading fails, or the running entry
+ * count exceeds {@link ResourceLimits#MAX_ENTRIES}.
*/
private static void readLexicon(Path file, Charset charset,
Map<String, List<WordEntry>> target, int leftSize, int rightSize, int[] entryCount)
@@ -409,10 +414,10 @@ public final class MecabDictionary {
* @param charset The encoding to decode with.
* @param categories Receives the defined categories, keyed by name.
* @param categoryTable Receives the code point to category name mappings.
- * @throws IOException Thrown if the file is missing, a line is malformed, a code
- * point is outside the Unicode range, a range descends, a category is
- * duplicated, category count or length exceeds the MeCab format, or the file
- * defines no {@code DEFAULT} category.
+ * @throws InvalidFormatException Thrown if a line is malformed, a code point is outside
+ * the Unicode range, a range descends, a category is duplicated, category count or
+ * length exceeds the MeCab format, or the file defines no {@code DEFAULT} category.
+ * @throws IOException Thrown if the file is missing or reading fails.
*/
private static void readCharacterDefinition(Path file, Charset charset,
Map<String, Category> categories, CategoryTable.Builder categoryTable)
@@ -443,35 +448,35 @@ public final class MecabDictionary {
to = from;
}
if (fields.length < 2) {
- throw new IOException(
+ throw new InvalidFormatException(
"mapping without category at " + file + " line " + lineNumber);
}
if (from > to) {
- throw new IOException("code point range descends at " + file + " line "
+ throw new InvalidFormatException("code point range descends at " + file + " line "
+ lineNumber);
}
categoryTable.map(from, to, Arrays.copyOfRange(fields, 1, fields.length));
} else {
if (fields.length < 4) {
- throw new IOException(
+ throw new InvalidFormatException(
"malformed category at " + file + " line " + lineNumber);
}
if (!isFlag(fields[1]) || !isFlag(fields[2])) {
- throw new IOException(
+ throw new InvalidFormatException(
"malformed category flag at " + file + " line " + lineNumber);
}
final int length = parseInt(fields[3], file.toString(), lineNumber);
if (length < 0 || length > MAX_CATEGORY_LENGTH) {
- throw new IOException(
+ throw new InvalidFormatException(
"category LENGTH must be between 0 and " + MAX_CATEGORY_LENGTH + " at "
+ file + " line " + lineNumber);
}
if (categories.containsKey(fields[0])) {
- throw new IOException("duplicate " + CHAR_DEF + " category "
+ throw new InvalidFormatException("duplicate " + CHAR_DEF + " category "
+ fields[0] + " at line " + lineNumber);
}
if (categories.size() >= MAX_CATEGORY_COUNT) {
- throw new IOException(CHAR_DEF + " defines " + (categories.size() + 1)
+ throw new InvalidFormatException(CHAR_DEF + " defines " + (categories.size() + 1)
+ " categories; MeCab supports at most " + MAX_CATEGORY_COUNT);
}
final Category category = new Category(categories.size(), fields[0],
@@ -481,7 +486,7 @@ public final class MecabDictionary {
}
}
if (!categories.containsKey(DEFAULT_CATEGORY)) {
- throw new IOException(
+ throw new InvalidFormatException(
CHAR_DEF + " defines no " + DEFAULT_CATEGORY + " category: " + file);
}
}
@@ -577,9 +582,9 @@ public final class MecabDictionary {
* @param detail What is wrong with the line, or {@code null} for no detail.
* @return The exception to throw. Never {@code null}.
*/
- private static IOException malformedEntry(Path file, int lineNumber, String detail) {
+ private static InvalidFormatException malformedEntry(Path file, int lineNumber, String detail) {
final String message = "malformed entry at " + file + " line " + lineNumber;
- return new IOException(detail == null ? message : message + ": " + detail);
+ return new InvalidFormatException(detail == null ? message : message + ": " + detail);
}
/**
@@ -592,10 +597,10 @@ public final class MecabDictionary {
* @param file The file the line came from, for error messages.
* @param lineNumber The line's position in the file, for error messages.
* @return The fields in order, empty fields included. Never {@code null}.
- * @throws IOException Thrown if a quoted field is unterminated.
+ * @throws InvalidFormatException Thrown if a quoted field is unterminated.
*/
private static List<String> splitCsv(String line, Path file, int lineNumber)
- throws IOException {
+ throws InvalidFormatException {
final List<String> fields = new ArrayList<>();
final StringBuilder field = new StringBuilder();
boolean inQuotes = false;
@@ -635,14 +640,14 @@ public final class MecabDictionary {
* @param file The file being read, for the error message.
* @param lineNumber The line being read, for the error message.
* @return The parsed value.
- * @throws IOException Thrown if the field is not a valid integer.
+ * @throws InvalidFormatException Thrown if the field is not a valid integer.
*/
private static int parseInt(String text, String file, int lineNumber)
- throws IOException {
+ throws InvalidFormatException {
try {
return Integer.parseInt(StringUtil.trimUnicodeWhitespace(text));
} catch (NumberFormatException e) {
- throw new IOException("malformed number in " + file + " line " + lineNumber, e);
+ throw new InvalidFormatException("malformed number in " + file + " line " + lineNumber, e);
}
}
@@ -653,21 +658,21 @@ public final class MecabDictionary {
* @param file The file being read, for the error message.
* @param lineNumber The line being read, for the error message.
* @return The parsed code point, which may be in a supplementary plane.
- * @throws IOException Thrown if the field does not start with the {@code 0x} prefix in
- * either case, is not a valid hexadecimal number, or names a value no Unicode code
- * point has.
+ * @throws InvalidFormatException Thrown if the field does not start with the {@code 0x}
+ * prefix in either case, is not a valid hexadecimal number, or names a value no
+ * Unicode code point has.
*/
private static int parseCodePoint(String text, Path file, int lineNumber)
- throws IOException {
+ throws InvalidFormatException {
final String trimmed = StringUtil.trimUnicodeWhitespace(text);
if (!trimmed.regionMatches(true, 0, HEX_PREFIX, 0, HEX_PREFIX.length())) {
- throw new IOException("code point without " + HEX_PREFIX + " prefix in " + file
+ throw new InvalidFormatException("code point without " + HEX_PREFIX + " prefix in " + file
+ " line " + lineNumber);
}
try {
return HexCodePoints.parseCodePoint(trimmed, HEX_PREFIX.length(), trimmed.length());
} catch (IllegalArgumentException e) {
- throw new IOException("malformed code point in " + file + " line " + lineNumber, e);
+ throw new InvalidFormatException("malformed code point in " + file + " line " + lineNumber, e);
}
}
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java
index 8cd3c202a..761798be3 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java
@@ -34,6 +34,7 @@ import java.util.List;
import java.util.Map;
import opennlp.tools.tokenize.Tokenizer;
+import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.ParamChecks;
import opennlp.tools.util.ResourceLimits;
import opennlp.tools.util.Span;
@@ -167,7 +168,8 @@ public final class UnigramSegmenter implements Tokenizer {
*
* @param lexicon The lexicon file. Must not be {@code null}.
* @return The segmenter. Not {@code null}.
- * @throws IOException Thrown if reading fails or the lexicon is empty or malformed.
+ * @throws InvalidFormatException Thrown if the lexicon is empty or malformed.
+ * @throws IOException Thrown if reading fails or the lexicon exceeds a size limit.
* @throws IllegalArgumentException Thrown if {@code lexicon} is {@code null}.
*/
public static UnigramSegmenter load(Path lexicon) throws IOException {
@@ -181,7 +183,8 @@ public final class UnigramSegmenter implements Tokenizer {
* line. Must not be {@code null}.
* @param charset The lexicon encoding. Must not be {@code null}.
* @return The segmenter. Not {@code null}.
- * @throws IOException Thrown if reading fails or the lexicon is empty or malformed.
+ * @throws InvalidFormatException Thrown if the lexicon is empty or malformed.
+ * @throws IOException Thrown if reading fails or the lexicon exceeds a size limit.
* @throws IllegalArgumentException Thrown if a parameter is {@code null}.
*/
public static UnigramSegmenter load(Path lexicon, Charset charset) throws IOException {
@@ -198,7 +201,8 @@ public final class UnigramSegmenter implements Tokenizer {
* @param lexiconStream The lexicon content. Must not be {@code null}. Not closed.
* @param charset The lexicon encoding. Must not be {@code null}.
* @return The segmenter. Not {@code null}.
- * @throws IOException Thrown if reading fails or the lexicon is empty or malformed.
+ * @throws InvalidFormatException Thrown if the lexicon is empty or malformed.
+ * @throws IOException Thrown if reading fails or the lexicon exceeds a size limit.
* @throws IllegalArgumentException Thrown if a parameter is {@code null}.
*/
public static UnigramSegmenter load(InputStream lexiconStream, Charset charset)
@@ -213,7 +217,8 @@ public final class UnigramSegmenter implements Tokenizer {
* @param charset The lexicon encoding. Must not be {@code null}.
* @param maxEntries The inclusive limit on distinct lexicon entries.
* @return The segmenter. Not {@code null}.
- * @throws IOException Thrown if reading fails or the lexicon is empty or malformed.
+ * @throws InvalidFormatException Thrown if the lexicon is empty or malformed.
+ * @throws IOException Thrown if reading fails or the lexicon exceeds a size limit.
* @throws IllegalArgumentException Thrown if a parameter is invalid.
*/
static UnigramSegmenter load(InputStream lexiconStream, Charset charset, int maxEntries)
@@ -239,17 +244,17 @@ public final class UnigramSegmenter implements Tokenizer {
}
final String[] fields = StringUtil.splitOnUnicodeWhitespace(line);
if (fields.length < 2) {
- throw new IOException("lexicon line " + lineNumber + " has no count");
+ throw new InvalidFormatException("lexicon line " + lineNumber + " has no count");
}
final String word = fields[0];
final long count;
try {
count = Long.parseLong(fields[1]);
} catch (NumberFormatException e) {
- throw new IOException("malformed count at lexicon line " + lineNumber, e);
+ throw new InvalidFormatException("malformed count at lexicon line " + lineNumber, e);
}
if (count <= 0) {
- throw new IOException("count must be positive at lexicon line " + lineNumber);
+ throw new InvalidFormatException("count must be positive at lexicon line " + lineNumber);
}
if (!counts.containsKey(word) && counts.size() >= maxEntries) {
throw new IOException(
@@ -263,7 +268,7 @@ public final class UnigramSegmenter implements Tokenizer {
}
}
if (counts.isEmpty()) {
- throw new IOException("the lexicon lists no words");
+ throw new InvalidFormatException("the lexicon lists no words");
}
final WordTrieBuilder root = new WordTrieBuilder();
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java
index 08cd32052..c86042f6e 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java
@@ -35,6 +35,7 @@ import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
import org.junit.jupiter.params.provider.ValueSource;
+import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.ResourceLimits;
import opennlp.tools.util.Span;
import opennlp.tools.util.WhitespaceMode;
@@ -354,7 +355,7 @@ public class LatticeTokenizerTest {
WhitespaceMode.setActive(mode);
write(broken, LEXICON_CSV, "東,0,0,3000,noun\n");
write(broken, MATRIX_DEF, "1\u001C1\n0 0 0\n");
- final IOException e = Assertions.assertThrows(IOException.class,
+ final InvalidFormatException e = Assertions.assertThrows(InvalidFormatException.class,
() -> MecabDictionary.load(broken));
Assertions.assertEquals("malformed matrix.def header: 1\u001C1", e.getMessage());
}
@@ -401,6 +402,7 @@ public class LatticeTokenizerTest {
() -> MecabDictionary.load(broken));
Assertions.assertEquals("required dictionary file is missing: "
+ broken.resolve(MATRIX_DEF), e.getMessage());
+ Assertions.assertFalse(e instanceof InvalidFormatException);
}
/**
@@ -413,7 +415,7 @@ public class LatticeTokenizerTest {
write(broken, MATRIX_DEF, UNIT_MATRIX);
write(broken, CHAR_DEF, "KANJI 0 0 2\n0x4E00..0x9FFF KANJI\n");
write(broken, UNK_DEF, "KANJI,0,0,8000,noun\n");
- final IOException e = Assertions.assertThrows(IOException.class,
+ final InvalidFormatException e = Assertions.assertThrows(InvalidFormatException.class,
() -> MecabDictionary.load(broken));
Assertions.assertEquals("char.def defines no DEFAULT category: "
+ broken.resolve(CHAR_DEF), e.getMessage());
@@ -488,7 +490,7 @@ public class LatticeTokenizerTest {
write(broken, MATRIX_DEF, UNIT_MATRIX);
write(broken, CHAR_DEF, DEFAULT_CATEGORY_LINE + "\nLATIN 1 1 0\n" + range + " LATIN\n");
write(broken, UNK_DEF, DEFAULT_UNKNOWN_TEMPLATE + "\n");
- final IOException e = Assertions.assertThrows(IOException.class,
+ final InvalidFormatException e = Assertions.assertThrows(InvalidFormatException.class,
() -> MecabDictionary.load(broken));
Assertions.assertEquals("code point without 0x prefix in " + broken.resolve(CHAR_DEF)
+ " line 3", e.getMessage());
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java
index a1f4675d6..e76f174de 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java
@@ -32,6 +32,7 @@ import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
import org.junit.jupiter.params.provider.ValueSource;
+import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.Span;
import opennlp.tools.util.WhitespaceMode;
@@ -176,7 +177,7 @@ public class UnigramSegmenterTest {
@ParameterizedTest(name = "lexicon content \"{0}\"")
@ValueSource(strings = {"word\n", "word abc\n", "word 0\n", "\n\n"})
void testMalformedLexiconsFailLoud(String lexicon) {
- Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load(
+ Assertions.assertThrows(InvalidFormatException.class, () -> UnigramSegmenter.load(
new ByteArrayInputStream(lexicon.getBytes(StandardCharsets.UTF_8)),
StandardCharsets.UTF_8));
}
@@ -189,6 +190,7 @@ public class UnigramSegmenterTest {
new ByteArrayInputStream(lexicon.getBytes(StandardCharsets.UTF_8)),
StandardCharsets.UTF_8, 1));
Assertions.assertEquals("lexicon entry count exceeds safe limit of 1", e.getMessage());
+ Assertions.assertFalse(e instanceof InvalidFormatException);
}
@Test
@@ -284,7 +286,7 @@ public class UnigramSegmenterTest {
void testInformationSeparatorDoesNotSeparateLexiconFields(WhitespaceMode mode) {
WhitespaceMode.setActive(mode);
final String lexicon = "我\u001C5000\n";
- final IOException e = Assertions.assertThrows(IOException.class,
+ final InvalidFormatException e = Assertions.assertThrows(InvalidFormatException.class,
() -> UnigramSegmenter.load(
new ByteArrayInputStream(lexicon.getBytes(StandardCharsets.UTF_8)),
StandardCharsets.UTF_8));