You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by ma...@apache.org on 2026/09/01 05:24:12 UTC
(opennlp) branch opennlp-2.x updated: [2.x] OPENNLP-1906: Sentence detector abbreviation veto is quadratic in document length (#1261)
This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a commit to branch opennlp-2.x
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/opennlp-2.x by this push:
new aaa2df673 [2.x] OPENNLP-1906: Sentence detector abbreviation veto is quadratic in document length (#1261)
aaa2df673 is described below
commit aaa2df67383440d11e73835f3c5669c509a5d901
Author: Kristian Rickert <kr...@gmail.com>
AuthorDate: Tue Sep 1 01:24:07 2026 -0400
[2.x] OPENNLP-1906: Sentence detector abbreviation veto is quadratic in document length (#1261)
* OPENNLP-1906: Sentence detector abbreviation veto is quadratic in document length
SentenceDetectorME.isAcceptableBreak searched the whole text once per
abbreviation dictionary entry for every end-of-sentence candidate, and
for a case-insensitive dictionary, which every shipped abbreviation list
is, it also converted and lower-cased the whole text on each of those
calls. Cost per document therefore grew as candidates x entries x
document length, which is quadratic in document length. Measured with a
200 entry dictionary, one sentPosDetect pass took 546 ms at 100 000
characters, 2 066 ms at 200 000 and 8 222 ms at 400 000, that is a
factor of about 3.8 per doubling.
Backport of #1208 to opennlp-2.x.
---
.../SentenceDetectorMEAbbreviationBenchmark.java | 208 +++++++++
.../tools/sentdetect/SentenceDetectorME.java | 213 +++++++--
.../LegacyAbbreviationSentenceDetectorME.java | 111 +++++
...tenceDetectorMEAbbreviationEquivalenceTest.java | 481 +++++++++++++++++++++
4 files changed, 982 insertions(+), 31 deletions(-)
diff --git a/opennlp-tools/src/jmh/java/opennlp/tools/sentdetect/SentenceDetectorMEAbbreviationBenchmark.java b/opennlp-tools/src/jmh/java/opennlp/tools/sentdetect/SentenceDetectorMEAbbreviationBenchmark.java
new file mode 100644
index 000000000..851020eae
--- /dev/null
+++ b/opennlp-tools/src/jmh/java/opennlp/tools/sentdetect/SentenceDetectorMEAbbreviationBenchmark.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.sentdetect;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Locale;
+import java.util.concurrent.TimeUnit;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import opennlp.tools.dictionary.Dictionary;
+import opennlp.tools.formats.ResourceAsStreamFactory;
+import opennlp.tools.util.InputStreamFactory;
+import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.PlainTextByLineStream;
+import opennlp.tools.util.StringList;
+import opennlp.tools.util.TrainingParameters;
+
+/**
+ * JMH benchmark for the abbreviation veto of
+ * {@link SentenceDetectorME#isAcceptableBreak(CharSequence, int, int)}.
+ * <p>
+ * One op is one {@link SentenceDetectorME#sentPosDetect(CharSequence)} call over a document of
+ * {@code documentChars} characters. Three variants run over the same input:
+ * <ul>
+ * <li>{@code noDictionary} is the floor, the veto is switched off entirely, so what is left is
+ * the shared cost of scanning and of the maxent evaluation per candidate;</li>
+ * <li>{@code legacyVeto} is {@link LegacyAbbreviationSentenceDetectorME}, which holds the
+ * previous full-text scan;</li>
+ * <li>{@code boundedWindowVeto} is the current implementation.</li>
+ * </ul>
+ * The {@code documentChars} axis is the point of the benchmark: the previous implementation
+ * searched the whole text once per dictionary entry per candidate, so its cost per document grows
+ * quadratically, while the bounded window makes it grow linearly. The {@code dictionaryEntries}
+ * axis exposes the second factor of that product.
+ */
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 3, time = 2)
+@Measurement(iterations = 5, time = 2)
+@Fork(2)
+@Threads(1)
+public class SentenceDetectorMEAbbreviationBenchmark {
+
+ /**
+ * The abbreviations of the shipped English dictionary. They occur in the benchmark document.
+ */
+ private static final String[] PRESENT_ABBREVIATIONS = {"Mr.", "Mrs.", "Ms.", "tel."};
+
+ @State(Scope.Benchmark)
+ public static class DocumentState {
+
+ /**
+ * The document length in characters. Doubling it is what separates a quadratic veto from a
+ * linear one.
+ */
+ @Param({"12500", "25000", "50000", "100000"})
+ public int documentChars;
+
+ /**
+ * The number of dictionary entries. The shipped German list has 215, the shipped English one
+ * that the runtime module tests against has 4.
+ */
+ @Param({"10", "200"})
+ public int dictionaryEntries;
+
+ SentenceModel model;
+ String document;
+ SentenceDetectorME noDictionary;
+ SentenceDetectorME legacy;
+ SentenceDetectorME boundedWindow;
+
+ @Setup(Level.Trial)
+ public void prepare() throws IOException {
+ model = trainModel();
+ document = buildDocument(documentChars);
+ final Dictionary dictionary = buildDictionary(dictionaryEntries);
+ noDictionary = new SentenceDetectorME(model, (Dictionary) null);
+ legacy = new LegacyAbbreviationSentenceDetectorME(model, dictionary);
+ boundedWindow = new SentenceDetectorME(model, dictionary);
+ }
+ }
+
+ @Benchmark
+ public void noDictionary(DocumentState state, Blackhole bh) {
+ bh.consume(state.noDictionary.sentPosDetect(state.document));
+ }
+
+ @Benchmark
+ public void legacyVeto(DocumentState state, Blackhole bh) {
+ bh.consume(state.legacy.sentPosDetect(state.document));
+ }
+
+ @Benchmark
+ public void boundedWindowVeto(DocumentState state, Blackhole bh) {
+ bh.consume(state.boundedWindow.sentPosDetect(state.document));
+ }
+
+ /*
+ * ------------------------------------------------------------------------------------------
+ * Fixtures.
+ * ------------------------------------------------------------------------------------------
+ */
+
+ /**
+ * @return A model trained on the bundled English samples, without an abbreviation dictionary,
+ * so all three variants share one feature generator and differ only in the veto.
+ * @throws IOException Thrown if the training samples cannot be read.
+ */
+ private static SentenceModel trainModel() throws IOException {
+ final InputStreamFactory in = new ResourceAsStreamFactory(
+ SentenceDetectorMEAbbreviationBenchmark.class,
+ "/opennlp/tools/sentdetect/Sentences.txt");
+ final ObjectStream<SentenceSample> samples = new SentenceSampleStream(
+ new PlainTextByLineStream(in, StandardCharsets.UTF_8));
+ return SentenceDetectorME.train("eng", samples,
+ new SentenceDetectorFactory("eng", true, null, null),
+ TrainingParameters.defaultParams());
+ }
+
+ /**
+ * Builds a document of about {@code chars} characters that contains abbreviations at a
+ * realistic rate, so the veto is reached often rather than in a corner.
+ *
+ * @param chars The minimum document length in characters.
+ * @return The document.
+ */
+ private static String buildDocument(int chars) {
+ final String paragraph =
+ "Mr. Smith left the building at noon. She told me he lived in Edinburgh. "
+ + "Mrs. Clark called tel. 555 1234 and asked for Ms. Adams. "
+ + "The driver got badly injured near the old bridge. "
+ + "OpenNLP provides tools for natural language processing. "
+ + "I wrote him a letter right away and posted it the same day. ";
+ final StringBuilder document = new StringBuilder(chars + paragraph.length());
+ while (document.length() < chars) {
+ document.append(paragraph);
+ }
+ return document.toString();
+ }
+
+ /**
+ * Builds a case-insensitive dictionary of {@code entries} abbreviations, as the shipped
+ * dictionaries are. The first entries occur in the document; the rest are synthetic and do not,
+ * which is the realistic case, since a dictionary covers a language and a document uses a
+ * handful of its entries.
+ *
+ * @param entries The number of entries to produce.
+ * @return The dictionary.
+ */
+ private static Dictionary buildDictionary(int entries) {
+ final Dictionary dictionary = new Dictionary(false);
+ for (String abbreviation : PRESENT_ABBREVIATIONS) {
+ dictionary.put(new StringList(abbreviation));
+ }
+ for (int i = PRESENT_ABBREVIATIONS.length; i < entries; i++) {
+ dictionary.put(new StringList(String.format(Locale.ROOT, "zq%d.", i)));
+ }
+ return dictionary;
+ }
+
+ /**
+ * Quick local iteration only: {@code forks(0)} disables JVM fork isolation
+ * (unlike {@code mvn} with the {@code jmh} profile).
+ * Use the Maven-invoked configuration for publishable numbers.
+ */
+ public static void main(String[] args) throws Exception {
+ Options opt = new OptionsBuilder()
+ .include(SentenceDetectorMEAbbreviationBenchmark.class.getSimpleName())
+ .forks(0)
+ .warmupIterations(2)
+ .measurementIterations(3)
+ .build();
+ new Runner(opt).run();
+ }
+}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java
index bf2cd7c28..23b40703d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java
@@ -21,9 +21,12 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
import opennlp.tools.dictionary.Dictionary;
import opennlp.tools.ml.ArrayMath;
@@ -80,9 +83,10 @@ public class SentenceDetectorME implements SentenceDetector, Probabilistic {
private final List<Double> sentProbs = new ArrayList<>();
/**
- * The {@link Dictionary abbreviation dictionary} if available (may be {@code null}).
+ * The abbreviation dictionary index that backs {@link #isAcceptableBreak(CharSequence, int, int)}.
+ * It is {@code null} if no abbreviation dictionary is available for the underlying model.
*/
- private final Dictionary abbDict;
+ private final AbbreviationIndex abbIndex;
protected final boolean useTokenEnd;
@@ -110,10 +114,12 @@ public class SentenceDetectorME implements SentenceDetector, Probabilistic {
*
* @param model The {@link SentenceModel} to be used.
* @param abbDict The {@link Dictionary} to be used. It must fit the language of the {@code model}.
+ * Its entries are read once, here; later changes to the {@code abbDict} instance do not
+ * affect this detector.
*/
public SentenceDetectorME(SentenceModel model, Dictionary abbDict) {
this.model = model.getMaxentModel();
- this.abbDict = abbDict;
+ this.abbIndex = AbbreviationIndex.of(abbDict);
SentenceDetectorFactory sdFactory = model.getFactory();
cgen = sdFactory.getSDContextGenerator();
scanner = sdFactory.getEndOfSentenceScanner();
@@ -139,7 +145,7 @@ public class SentenceDetectorME implements SentenceDetector, Probabilistic {
getAbbreviations(model.getAbbreviations()), customEOSCharacters);
scanner = factory.createEndOfSentenceScanner(customEOSCharacters);
}
- abbDict = model.getAbbreviations();
+ abbIndex = AbbreviationIndex.of(model.getAbbreviations());
useTokenEnd = model.useTokenEnd();
}
@@ -336,47 +342,192 @@ public class SentenceDetectorME implements SentenceDetector, Probabilistic {
* <p>Note: The implementation always returns {@code true} if no
* abbreviation dictionary is available for the underlying model.</p>
*
+ * <p>Only a bounded region of {@code s} is consulted. An abbreviation occurrence starting at
+ * {@code p} with length {@code L} can veto the break only when it starts at or before the
+ * candidate and reaches it, that is when
+ * {@code candidateIndex - L <= p <= candidateIndex}. Positions outside that window are
+ * irrelevant by construction, so the decision costs time proportional to the longest
+ * dictionary entry rather than to the length of {@code s}.</p>
+ *
+ * <p>A {@code candidateIndex} past the end of {@code s} cannot carry an abbreviation, so it
+ * is accepted rather than raising an {@link IndexOutOfBoundsException}. At
+ * {@code candidateIndex == s.length()}, the decision matches the previous implementation when
+ * the match is not at {@code fromIndex}; only the match-at-{@code fromIndex} case is an
+ * accept-instead-of-throw relaxation.</p>
+ *
* @param s the {@link CharSequence} in which the break occurred.
* @param fromIndex the start of the segment currently being evaluated.
- * @param candidateIndex the index of the candidate sentence ending.
+ * @param candidateIndex the index of the candidate sentence ending. Must be greater than or
+ * equal to {@code fromIndex} and a valid index into {@code s}.
* @return {@code true} if the break is acceptable, {@code false} otherwise.
*/
protected boolean isAcceptableBreak(CharSequence s, int fromIndex, int candidateIndex) {
- if (abbDict == null)
- return true;
-
- final String text = s.toString();
- final boolean caseSensitive = abbDict.isCaseSensitive();
- final String searchText = caseSensitive ? text : StringUtil.toLowerCase(text);
- for (StringList abb : abbDict) {
- final String abbToken = caseSensitive ? abb.getToken(0)
- : StringUtil.toLowerCase(abb.getToken(0));
- final int tokenLength = abbToken.length();
- int tokenStartPos = searchText.indexOf(abbToken, fromIndex);
- while (tokenStartPos != -1) {
- if (tokenStartPos > candidateIndex) {
- break; // past candidate position, no point searching further
- }
- if (tokenStartPos == fromIndex
- && searchText.substring(tokenStartPos, candidateIndex + 1).equals(abbToken)) {
- return false; // full abbreviation match at segment start -> no acceptable break
- }
- final char prevChar = s.charAt(tokenStartPos == fromIndex ? tokenStartPos : tokenStartPos - 1);
- if (tokenStartPos + tokenLength >= candidateIndex
+ return abbIndex == null || abbIndex.allowsBreak(s, fromIndex, candidateIndex);
+ }
+
+ /**
+ * An immutable, length-bucketed index over an abbreviation {@link Dictionary}. It answers
+ * {@link #isAcceptableBreak(CharSequence, int, int)} by enumerating the few text positions
+ * that can carry a relevant abbreviation and asking a hash set what is there, instead of
+ * searching the whole text once per dictionary entry.
+ */
+ private static final class AbbreviationIndex {
+
+ /**
+ * The dictionary entries, folded to lower case unless the dictionary is case-sensitive.
+ * Only the first token of a multi-token entry participates, as before.
+ */
+ private final Set<String> entries;
+
+ /**
+ * The distinct entry lengths, ascending. These are the only window sizes worth probing.
+ */
+ private final int[] entryLengths;
+
+ /**
+ * The longest entry length, that is how far past a candidate a relevant match can reach.
+ */
+ private final int maxEntryLength;
+
+ /**
+ * Whether the dictionary matches case-sensitively.
+ */
+ private final boolean caseSensitive;
+
+ /**
+ * @param abbDict The {@link Dictionary} to index, may be {@code null}.
+ * @return An index over {@code abbDict}, or {@code null} if {@code abbDict} is {@code null}.
+ */
+ static AbbreviationIndex of(Dictionary abbDict) {
+ return abbDict == null ? null : new AbbreviationIndex(abbDict);
+ }
+
+ /**
+ * Reads {@code abbDict} once, so this instance is immutable and safe to share.
+ *
+ * @param abbDict The {@link Dictionary} to index. Must not be {@code null}.
+ */
+ private AbbreviationIndex(Dictionary abbDict) {
+ caseSensitive = abbDict.isCaseSensitive();
+ final Set<String> tokens = new HashSet<>();
+ final SortedSet<Integer> lengths = new TreeSet<>();
+ for (StringList abb : abbDict) {
+ final String token = caseSensitive ? abb.getToken(0)
+ : StringUtil.toLowerCase(abb.getToken(0));
+ tokens.add(token);
+ lengths.add(token.length());
+ }
+ entries = tokens;
+ entryLengths = lengths.stream().mapToInt(Integer::intValue).toArray();
+ maxEntryLength = entryLengths.length == 0 ? 0 : entryLengths[entryLengths.length - 1];
+ }
+
+ /**
+ * @param s The text in which the break occurred.
+ * @param fromIndex The start of the segment currently being evaluated.
+ * @param candidateIndex The index of the candidate sentence ending.
+ * @return {@code true} if a break at {@code candidateIndex} is allowed.
+ */
+ boolean allowsBreak(CharSequence s, int fromIndex, int candidateIndex) {
+ final int textLength = s.length();
+ if (entryLengths.length == 0 || candidateIndex < fromIndex || candidateIndex < 0
+ || candidateIndex > textLength) {
+ return true;
+ }
+ // Occurrences before the segment start do not participate.
+ final int scanStart = StrictMath.max(0, fromIndex);
+ // The window holds every position a relevant occurrence can start at, plus the longest
+ // entry so that the text of an occurrence starting at the candidate is covered too.
+ final int windowStart = codePointStart(s,
+ StrictMath.max(scanStart, candidateIndex - maxEntryLength));
+ final int windowEnd = codePointEnd(s,
+ StrictMath.min(textLength, candidateIndex + maxEntryLength));
+ // Case folding is applied to the window only. It is per code point, so it preserves
+ // indices and yields exactly the characters a folding of the whole text would.
+ final String window = caseSensitive
+ ? s.subSequence(windowStart, windowEnd).toString()
+ : toLowerCase(s, windowStart, windowEnd);
+
+ for (final int tokenLength : entryLengths) {
+ for (int pos = StrictMath.max(scanStart, candidateIndex - tokenLength);
+ pos <= candidateIndex; pos++) {
+ final int endPos = pos + tokenLength;
+ if (endPos > textLength) {
+ break; // the entry no longer fits, and it fits even less further right
+ }
+ if (!entries.contains(window.substring(pos - windowStart, endPos - windowStart))) {
+ continue;
+ }
+ if (pos == fromIndex && endPos == candidateIndex + 1) {
+ return false; // full abbreviation match at segment start -> no acceptable break
+ }
+ final char prevChar = s.charAt(pos == fromIndex ? pos : pos - 1);
/*
* Note:
* Skip abbreviation candidate if regular characters exist directly before it,
* That is, any letter or digit except: a whitespace, an apostrophe, or an opening round bracket.
* This prevents mismatches from overlaps close to an actual sentence end.
*/
- && (Character.isWhitespace(prevChar) || isApostrophe(prevChar) || prevChar == '(')) {
- return false; // in case of a valid abbreviation: the (sentence) break is not accepted
+ if (Character.isWhitespace(prevChar) || isApostrophe(prevChar) || prevChar == '(') {
+ return false; // in case of a valid abbreviation: the (sentence) break is not accepted
+ }
}
- // Try next occurrence of this abbreviation in the text
- tokenStartPos = searchText.indexOf(abbToken, tokenStartPos + 1);
}
+ return true; // no abbreviation(s) at given positions: valid sentence boundary
+ }
+
+ /**
+ * Lower-cases {@code [from, to)} exactly as {@link StringUtil#toLowerCase(CharSequence)}
+ * lower-cases a whole text: per code point via {@link Character#toLowerCase(int)}.
+ *
+ * <p>Folding only the window is correct only because that mapping is 1:1 in {@code char}
+ * count, so {@code pos - windowStart} taken from the unfolded text still indexes the folded
+ * window. {@link String#toLowerCase()} must not be used here: full case mapping can expand
+ * a code point (for example {@code İ} / U+0130) and would corrupt every offset in the window.</p>
+ *
+ * @param s The text to read from.
+ * @param from The first index to fold, at a code point boundary.
+ * @param to The index to stop at, at a code point boundary.
+ * @return The folded characters of {@code [from, to)}.
+ */
+ private static String toLowerCase(CharSequence s, int from, int to) {
+ final StringBuilder folded = new StringBuilder(to - from);
+ int i = from;
+ while (i < to) {
+ final int cp = Character.codePointAt(s, i);
+ folded.appendCodePoint(Character.toLowerCase(cp));
+ i += Character.charCount(cp);
+ }
+ return folded.toString();
+ }
+
+ /**
+ * @param s The text {@code index} refers to.
+ * @param index The index to align.
+ * @return {@code index}, moved one character left if it points at the trailing half of a
+ * surrogate pair, which is the only index a code point walk cannot start at.
+ */
+ private static int codePointStart(CharSequence s, int index) {
+ if (index > 0 && index < s.length() && Character.isLowSurrogate(s.charAt(index))
+ && Character.isHighSurrogate(s.charAt(index - 1))) {
+ return index - 1;
+ }
+ return index;
+ }
+
+ /**
+ * @param s The text {@code index} refers to.
+ * @param index The index to align.
+ * @return {@code index}, moved one character right if it splits a surrogate pair, which is
+ * the only index a code point walk cannot stop at.
+ */
+ private static int codePointEnd(CharSequence s, int index) {
+ if (index > 0 && index < s.length() && Character.isHighSurrogate(s.charAt(index - 1))
+ && Character.isLowSurrogate(s.charAt(index))) {
+ return index + 1;
+ }
+ return index;
}
- return true; // no abbreviation(s) at given positions: valid sentence boundary
}
/**
diff --git a/opennlp-tools/src/test/java/opennlp/tools/sentdetect/LegacyAbbreviationSentenceDetectorME.java b/opennlp-tools/src/test/java/opennlp/tools/sentdetect/LegacyAbbreviationSentenceDetectorME.java
new file mode 100644
index 000000000..a04a208f7
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/sentdetect/LegacyAbbreviationSentenceDetectorME.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.sentdetect;
+
+import opennlp.tools.dictionary.Dictionary;
+import opennlp.tools.util.StringList;
+import opennlp.tools.util.StringUtil;
+
+/**
+ * A {@link SentenceDetectorME} whose abbreviation veto is the full text scan that
+ * {@link SentenceDetectorME#isAcceptableBreak(CharSequence, int, int)} used before it was given a
+ * bounded window. It is the oracle of {@link SentenceDetectorMEAbbreviationEquivalenceTest} and
+ * the baseline of {@code SentenceDetectorMEAbbreviationBenchmark}, which is why it lives in its
+ * own class rather than in either of them.
+ */
+class LegacyAbbreviationSentenceDetectorME extends SentenceDetectorME {
+
+ private final Dictionary abbDict;
+ private int vetoes;
+
+ /**
+ * @param model The {@link SentenceModel} to be used.
+ * @param abbDict The {@link Dictionary} to veto with, may be {@code null}.
+ */
+ LegacyAbbreviationSentenceDetectorME(SentenceModel model, Dictionary abbDict) {
+ super(model, abbDict);
+ this.abbDict = abbDict;
+ }
+
+ @Override
+ protected boolean isAcceptableBreak(CharSequence s, int fromIndex, int candidateIndex) {
+ final boolean acceptable = decide(abbDict, s, fromIndex, candidateIndex);
+ if (!acceptable) {
+ vetoes++;
+ }
+ return acceptable;
+ }
+
+ /**
+ * @return How often this instance vetoed a break, so a comparison run can show that it
+ * exercised the veto at all.
+ */
+ int vetoes() {
+ return vetoes;
+ }
+
+ /**
+ * The pre-rewrite implementation, kept verbatim.
+ *
+ * @param abbDict The abbreviation {@link Dictionary}, may be {@code null}.
+ * @param s The {@link CharSequence} in which the break occurred.
+ * @param fromIndex The start of the segment currently being evaluated.
+ * @param candidateIndex The index of the candidate sentence ending.
+ * @return {@code true} if the break is acceptable, {@code false} otherwise.
+ */
+ static boolean decide(Dictionary abbDict, CharSequence s, int fromIndex, int candidateIndex) {
+ if (abbDict == null)
+ return true;
+
+ final String text = s.toString();
+ final boolean caseSensitive = abbDict.isCaseSensitive();
+ final String searchText = caseSensitive ? text : StringUtil.toLowerCase(text);
+ for (StringList abb : abbDict) {
+ final String abbToken = caseSensitive ? abb.getToken(0)
+ : StringUtil.toLowerCase(abb.getToken(0));
+ final int tokenLength = abbToken.length();
+ int tokenStartPos = searchText.indexOf(abbToken, fromIndex);
+ while (tokenStartPos != -1) {
+ if (tokenStartPos > candidateIndex) {
+ break; // past candidate position, no point searching further
+ }
+ if (tokenStartPos == fromIndex
+ && searchText.substring(tokenStartPos, candidateIndex + 1).equals(abbToken)) {
+ return false; // full abbreviation match at segment start -> no acceptable break
+ }
+ final char prevChar =
+ s.charAt(tokenStartPos == fromIndex ? tokenStartPos : tokenStartPos - 1);
+ if (tokenStartPos + tokenLength >= candidateIndex
+ && (Character.isWhitespace(prevChar) || isApostrophe(prevChar) || prevChar == '(')) {
+ return false; // in case of a valid abbreviation: the (sentence) break is not accepted
+ }
+ // Try next occurrence of this abbreviation in the text
+ tokenStartPos = searchText.indexOf(abbToken, tokenStartPos + 1);
+ }
+ }
+ return true; // no abbreviation(s) at given positions: valid sentence boundary
+ }
+
+ /**
+ * @param c The character to check.
+ * @return {@code true} if the character represents an apostrophe, {@code false} otherwise.
+ */
+ private static boolean isApostrophe(char c) {
+ return c == '\'' || c == '`' || c == '´';
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/sentdetect/SentenceDetectorMEAbbreviationEquivalenceTest.java b/opennlp-tools/src/test/java/opennlp/tools/sentdetect/SentenceDetectorMEAbbreviationEquivalenceTest.java
new file mode 100644
index 000000000..ffd1251c5
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/sentdetect/SentenceDetectorMEAbbreviationEquivalenceTest.java
@@ -0,0 +1,481 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.sentdetect;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Random;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.dictionary.Dictionary;
+import opennlp.tools.util.Span;
+import opennlp.tools.util.StringList;
+
+/**
+ * Proves that the bounded-window abbreviation veto of
+ * {@link SentenceDetectorME#isAcceptableBreak(CharSequence, int, int)} decides exactly what the
+ * previous full-text scan decided.
+ * <p>
+ * {@link LegacyAbbreviationSentenceDetectorME} holds that previous implementation verbatim and
+ * is the oracle here. Agreement is asserted twice over: for every single {@code (fromIndex,
+ * candidateIndex)} pair of a set of hand-written and seeded random texts, and for the sentence
+ * spans a full {@link SentenceDetectorME#sentPosDetect(CharSequence)} run produces on a corpus.
+ */
+public class SentenceDetectorMEAbbreviationEquivalenceTest extends AbstractSentenceDetectorTest {
+
+ private static SentenceModel sentdetectModel;
+
+ /**
+ * A model without an abbreviation dictionary, the shape every sentence model published by the
+ * project has. Detectors built from it never enter the veto at all.
+ */
+ private static SentenceModel modelWithoutAbbreviations;
+ private static String corpus;
+
+ @BeforeAll
+ public static void prepareResources() throws IOException {
+ final SentenceDetectorFactory factory = new SentenceDetectorFactory(
+ "eng", true, loadAbbDictionary(Locale.ENGLISH), null);
+ sentdetectModel = train(factory, Locale.ENGLISH);
+ Assertions.assertNotNull(sentdetectModel);
+ modelWithoutAbbreviations = train(
+ new SentenceDetectorFactory("eng", true, null, null), Locale.ENGLISH);
+ Assertions.assertNull(modelWithoutAbbreviations.getAbbreviations());
+ corpus = readCorpus();
+ Assertions.assertTrue(corpus.length() > 10_000, "corpus too small to be meaningful");
+ }
+
+ /**
+ * The pre-rewrite implementation, the oracle every comparison below is made against.
+ *
+ * @see LegacyAbbreviationSentenceDetectorME#decide
+ */
+ private static boolean legacyIsAcceptableBreak(Dictionary abbDict, CharSequence s,
+ int fromIndex, int candidateIndex) {
+ return LegacyAbbreviationSentenceDetectorME.decide(abbDict, s, fromIndex, candidateIndex);
+ }
+
+ /*
+ * ------------------------------------------------------------------------------------------
+ * Differential harness.
+ * ------------------------------------------------------------------------------------------
+ */
+
+ private static Dictionary dictionaryOf(boolean caseSensitive, String... entries) {
+ final Dictionary dict = new Dictionary(caseSensitive);
+ for (String entry : entries) {
+ dict.put(new StringList(entry));
+ }
+ return dict;
+ }
+
+ /**
+ * Compares old and new for every {@code 0 <= fromIndex <= candidateIndex < text.length()},
+ * that is over the whole contract domain of the method.
+ *
+ * @return The number of decisions compared.
+ */
+ private static long assertAgreesEverywhere(Dictionary dict, String text) {
+ final SentenceDetectorME detector = new SentenceDetectorME(sentdetectModel, dict);
+ long compared = 0;
+ for (int fromIndex = 0; fromIndex < text.length(); fromIndex++) {
+ for (int candidateIndex = fromIndex; candidateIndex < text.length(); candidateIndex++) {
+ final boolean expected = legacyIsAcceptableBreak(dict, text, fromIndex, candidateIndex);
+ final boolean actual = detector.isAcceptableBreak(text, fromIndex, candidateIndex);
+ if (expected != actual) {
+ Assertions.fail(String.format(Locale.ROOT,
+ "disagreement at fromIndex=%d candidateIndex=%d: legacy=%b, new=%b, text=<%s>",
+ fromIndex, candidateIndex, expected, actual, text));
+ }
+ compared++;
+ }
+ }
+ return compared;
+ }
+
+ /**
+ * As {@link #assertAgreesEverywhere}, and additionally requires that the oracle vetoes at
+ * least once, so a case cannot pass by never reaching the interesting branch.
+ */
+ private static long assertAgreesEverywhereAndVetoes(Dictionary dict, String text) {
+ final long compared = assertAgreesEverywhere(dict, text);
+ int vetoes = 0;
+ for (int fromIndex = 0; fromIndex < text.length(); fromIndex++) {
+ for (int candidateIndex = fromIndex; candidateIndex < text.length(); candidateIndex++) {
+ if (!legacyIsAcceptableBreak(dict, text, fromIndex, candidateIndex)) {
+ vetoes++;
+ }
+ }
+ }
+ Assertions.assertTrue(vetoes > 0,
+ "the oracle never vetoes here, so the case proves nothing: " + text);
+ return compared;
+ }
+
+ /*
+ * ------------------------------------------------------------------------------------------
+ * Edge cases.
+ * ------------------------------------------------------------------------------------------
+ */
+
+ @Test
+ void testAbbreviationAtDocumentStart() {
+ final Dictionary dict = dictionaryOf(true, "Mr.", "Dr.");
+ Assertions.assertTrue(assertAgreesEverywhereAndVetoes(dict, "Mr. Smith left.") > 0);
+ Assertions.assertTrue(assertAgreesEverywhereAndVetoes(dict, "Dr.") > 0);
+ }
+
+ @Test
+ void testAbbreviationAtDocumentEnd() {
+ final Dictionary dict = dictionaryOf(true, "etc.", "Mr.");
+ Assertions.assertTrue(assertAgreesEverywhereAndVetoes(dict, "Bring apples, pears, etc.") > 0);
+ // The entry reaches exactly the last index, and one that reaches past the end.
+ Assertions.assertTrue(assertAgreesEverywhere(dict, "Bring apples, pears, etc") > 0);
+ }
+
+ @Test
+ void testOverlappingCandidates() {
+ // "U.S." and "S." overlap, and "..." makes several candidate ends adjacent.
+ final Dictionary dict = dictionaryOf(true, "U.S.", "S.", "e.g.", "g.");
+ Assertions.assertTrue(assertAgreesEverywhereAndVetoes(dict, "The U.S. and e.g. Spain...") > 0);
+ Assertions.assertTrue(assertAgreesEverywhereAndVetoes(dict, "U.S.S.R. is gone.") > 0);
+ }
+
+ @Test
+ void testEmptyDictionary() {
+ final Dictionary dict = dictionaryOf(true);
+ Assertions.assertTrue(assertAgreesEverywhere(dict, "A sentence. Another one.") > 0);
+ final Dictionary insensitive = dictionaryOf(false);
+ Assertions.assertTrue(assertAgreesEverywhere(insensitive, "A sentence. Another one.") > 0);
+ }
+
+ @Test
+ void testNullDictionaryAlwaysAccepts() {
+ final SentenceDetectorME detector = new SentenceDetectorME(sentdetectModel, (Dictionary) null);
+ final String text = "Mr. Smith left. Dr. Jones stayed.";
+ for (int fromIndex = 0; fromIndex < text.length(); fromIndex++) {
+ for (int candidateIndex = fromIndex; candidateIndex < text.length(); candidateIndex++) {
+ Assertions.assertTrue(detector.isAcceptableBreak(text, fromIndex, candidateIndex));
+ }
+ }
+ }
+
+ @Test
+ void testDictionaryEntryNotPresentInText() {
+ final Dictionary dict = dictionaryOf(true, "Zzz.", "Qqq.", "Mr.");
+ Assertions.assertTrue(assertAgreesEverywhere(dict, "Nothing here matches at all.") > 0);
+ }
+
+ @Test
+ void testEntryLongerThanText() {
+ final Dictionary dict = dictionaryOf(true, "averyveryverylongabbreviation.");
+ Assertions.assertTrue(assertAgreesEverywhere(dict, "short.") > 0);
+ }
+
+ @Test
+ void testMultiCharacterAndUnicodeAbbreviations() {
+ final Dictionary dict = dictionaryOf(true, "z.B.", "Abb.", "Straße.", "№.", "ç.");
+ Assertions.assertTrue(assertAgreesEverywhereAndVetoes(dict,
+ "Siehe z.B. Abb. 3 in der Straße. Und №. 7 sowie ç. hier.") > 0);
+ }
+
+ @Test
+ void testSupplementaryCodePointsInTextAndDictionary() {
+ // U+1D400 MATHEMATICAL BOLD CAPITAL A and U+10400 DESERET CAPITAL LONG I, the latter has a
+ // lower case mapping, so it exercises the case-folded window on a surrogate pair.
+ final String bold = new String(Character.toChars(0x1D400));
+ final String deseret = new String(Character.toChars(0x10400));
+ final String deseretLower = new String(Character.toChars(0x10428));
+ final Dictionary sensitive = dictionaryOf(true, bold + ".", deseret + ".", "Mr.");
+ final String text = "A " + bold + ". B " + deseret + ". C " + deseretLower + ". Mr. D.";
+ Assertions.assertTrue(assertAgreesEverywhereAndVetoes(sensitive, text) > 0);
+ final Dictionary insensitive = dictionaryOf(false, bold + ".", deseret + ".", "Mr.");
+ Assertions.assertTrue(assertAgreesEverywhereAndVetoes(insensitive, text) > 0);
+ // A lone high surrogate, so the window can start or end on an unpaired half.
+ final Dictionary loneHalf = dictionaryOf(false, "\uD801.", "Mr.");
+ Assertions.assertTrue(assertAgreesEverywhere(loneHalf, "x \uD801. y " + deseret + ". Mr. z") > 0);
+ }
+
+ @Test
+ void testRepeatedAbbreviations() {
+ final Dictionary dict = dictionaryOf(true, "Mr.");
+ Assertions.assertTrue(assertAgreesEverywhereAndVetoes(dict,
+ "Mr. A, Mr. B, Mr. C, Mr. D, and Mr. E.") > 0);
+ }
+
+ @Test
+ void testCaseInsensitiveDictionary() {
+ // "i̇." is the decomposed lower case of "İ" (LATIN CAPITAL LETTER I WITH DOT
+ // ABOVE), whose single code point lower case mapping is plain "i", so the two do not match.
+ // The point is only that both implementations say so.
+ final Dictionary dict = dictionaryOf(false, "Mr.", "TEL.", "i̇.");
+ Assertions.assertTrue(assertAgreesEverywhereAndVetoes(dict,
+ "mr. Smith, MR. Jones and Tel. 555 plus İ. and I. here.") > 0);
+ }
+
+ @Test
+ void testEmptyStringEntry() {
+ // An empty entry matches at every position; the oracle can still veto through the
+ // preceding-character branch, so the window has to probe length zero as well.
+ final Dictionary onlyEmpty = dictionaryOf(true, "");
+ Assertions.assertFalse(legacyIsAcceptableBreak(onlyEmpty, "a b", 0, 2),
+ "the empty entry has to be able to veto on its own, otherwise this proves nothing");
+ Assertions.assertTrue(assertAgreesEverywhereAndVetoes(onlyEmpty, "a b c d.") > 0);
+ final Dictionary dict = dictionaryOf(true, "", "Mr.");
+ Assertions.assertTrue(assertAgreesEverywhereAndVetoes(dict, "Mr. A said no.") > 0);
+ }
+
+ @Test
+ void testMultiTokenEntryUsesFirstTokenOnly() {
+ final Dictionary dict = new Dictionary(true);
+ dict.put(new StringList("Mr.", "Smith"));
+ dict.put(new StringList("St.", "Louis"));
+ Assertions.assertTrue(assertAgreesEverywhereAndVetoes(dict, "Mr. Smith of St. Louis left.") > 0);
+ }
+
+ @Test
+ void testApostropheAndOpeningBracketPrefixes() {
+ final Dictionary dict = dictionaryOf(true, "Mr.", "cf.");
+ Assertions.assertTrue(assertAgreesEverywhereAndVetoes(dict,
+ "He said 'Mr. X' and (cf. Y) and `Mr. Z` and ´cf. W´ and xMr. V.") > 0);
+ }
+
+ @Test
+ void testSegmentStartInsideAnAbbreviation() {
+ // fromIndex walking through the middle of an abbreviation is what the sentence loop does
+ // after accepting a break, and it is where the "match at segment start" branch fires.
+ final Dictionary dict = dictionaryOf(true, "Mr.", "r.", ".");
+ Assertions.assertTrue(assertAgreesEverywhereAndVetoes(dict, "Mr. Mr. Mr.") > 0);
+ }
+
+ /*
+ * ------------------------------------------------------------------------------------------
+ * Seeded random sweep.
+ * ------------------------------------------------------------------------------------------
+ */
+
+ @Test
+ void testSeededRandomTexts() {
+ final String[] pieces = {
+ "Mr.", "Dr.", "e.g.", "i.e.", "U.S.A.", "etc.", "St.", "Nr.", "z.B.", "vs.",
+ " ", " ", "\n", "\t", " ", "(", ")", "'", "`", "´", "\"",
+ "word", "Word", "WORD", "x", ".", "?", "!", "3.5", "ç", "ä", "Straße",
+ new String(Character.toChars(0x10400)), new String(Character.toChars(0x1D400)),
+ };
+ final Dictionary[] dicts = {
+ dictionaryOf(true, "Mr.", "Dr.", "e.g.", "etc.", "St.", "U.S.A.", ".", "Nr."),
+ dictionaryOf(false, "mr.", "DR.", "E.g.", "etc.", "st.", "u.s.a.", "z.b.", "vs.",
+ "straße", "ç.", new String(Character.toChars(0x10400)) + "."),
+ dictionaryOf(false, "notpresent.", "alsonot."),
+ };
+
+ final Random random = new Random(20260803L);
+ long compared = 0;
+ int vetoing = 0;
+ for (int run = 0; run < 250; run++) {
+ final StringBuilder text = new StringBuilder();
+ while (text.length() < 70) {
+ text.append(pieces[random.nextInt(pieces.length)]);
+ }
+ final String sample = text.toString();
+ for (Dictionary dict : dicts) {
+ compared += assertAgreesEverywhere(dict, sample);
+ if (!legacyIsAcceptableBreak(dict, sample, 0, sample.length() - 1)) {
+ vetoing++;
+ }
+ }
+ }
+ Assertions.assertTrue(compared > 1_500_000L,
+ "the sweep degenerated, only " + compared + " decisions compared");
+ Assertions.assertTrue(vetoing > 0, "no sampled text ever vetoed");
+ }
+
+ /*
+ * ------------------------------------------------------------------------------------------
+ * Corpus level: identical spans out of sentPosDetect.
+ * ------------------------------------------------------------------------------------------
+ */
+
+ /**
+ * Differential check of {@link SentenceDetectorME#sentPosDetect(CharSequence)} against the
+ * legacy veto. The English sentence model is paired with each shipped abbreviation dictionary
+ * (including the German one); this is not German-language coverage.
+ */
+ @Test
+ void testSentPosDetectAgreesOnCorpus() throws IOException {
+ for (Locale locale : new Locale[] {Locale.ENGLISH, Locale.GERMAN}) {
+ assertCorpusRunsAgree(loadAbbDictionary(locale),
+ "the shipped abbreviation dictionary of " + locale + " on the English model");
+ }
+ }
+
+ @Test
+ void testSentPosDetectAgreesOnCaseSensitiveDictionary() {
+ assertCorpusRunsAgree(dictionaryOf(true, "Mr.", "Mrs.", "Ms.", "Dr.", "St.", "etc.",
+ "e.g.", "i.e.", "vs.", "No.", "Jr.", "Sr.", "Prof.", "Inc.", "Ltd."),
+ "a case-sensitive dictionary");
+ }
+
+ /**
+ * The path every user of a published sentence model is on: no abbreviation dictionary at all,
+ * so the veto is never entered. This is the case an optimisation of the veto is most likely to
+ * regress by accident, and the least likely to be noticed.
+ */
+ @Test
+ void testNoDictionaryConfiguredIsUnchangedOnCorpus() {
+ // A model published without an abbreviation dictionary, used through the one argument
+ // constructor, and a model that has one but is asked to ignore it.
+ assertNoDictionaryRunAgrees(modelWithoutAbbreviations,
+ new SentenceDetectorME(modelWithoutAbbreviations));
+ assertNoDictionaryRunAgrees(sentdetectModel,
+ new SentenceDetectorME(sentdetectModel, (Dictionary) null));
+ }
+
+ private static void assertNoDictionaryRunAgrees(SentenceModel model, SentenceDetectorME fixed) {
+ final LegacyAbbreviationSentenceDetectorME legacy =
+ new LegacyAbbreviationSentenceDetectorME(model, null);
+
+ final Span[] expected = legacy.sentPosDetect(corpus);
+ final double[] expectedProbs = legacy.probs();
+ final Span[] actual = fixed.sentPosDetect(corpus);
+ final double[] actualProbs = fixed.probs();
+
+ Assertions.assertTrue(expected.length > 100,
+ "corpus produced too few sentences to be meaningful: " + expected.length);
+ Assertions.assertEquals(0, legacy.vetoes(),
+ "the veto must not be reachable without a dictionary");
+ Assertions.assertArrayEquals(expected, actual);
+ Assertions.assertArrayEquals(expectedProbs, actualProbs, 0.0d);
+ }
+
+ private static void assertCorpusRunsAgree(Dictionary dict, String description) {
+ final SentenceDetectorME fixed = new SentenceDetectorME(sentdetectModel, dict);
+ final LegacyAbbreviationSentenceDetectorME legacy =
+ new LegacyAbbreviationSentenceDetectorME(sentdetectModel, dict);
+
+ final Span[] expected = legacy.sentPosDetect(corpus);
+ final double[] expectedProbs = legacy.probs();
+ final Span[] actual = fixed.sentPosDetect(corpus);
+ final double[] actualProbs = fixed.probs();
+
+ Assertions.assertTrue(expected.length > 100,
+ "corpus produced too few sentences to be meaningful: " + expected.length);
+ Assertions.assertTrue(legacy.vetoes() > 0,
+ "the veto never fired for " + description + ", so this run proves nothing");
+ Assertions.assertArrayEquals(expected, actual, "span mismatch for " + description);
+ Assertions.assertArrayEquals(expectedProbs, actualProbs, 0.0d,
+ "probability mismatch for " + description);
+ }
+
+ /*
+ * ------------------------------------------------------------------------------------------
+ * Documented differences, outside the contract of the method.
+ * ------------------------------------------------------------------------------------------
+ */
+
+ @Test
+ void testCandidateIndexOutsideTextNoLongerThrows() {
+ final Dictionary dict = dictionaryOf(false, "Mr.", "etc.");
+ final SentenceDetectorME detector = new SentenceDetectorME(sentdetectModel, dict);
+
+ // Match at fromIndex with candidateIndex == length: legacy clause A substring-throws;
+ // the bounded window accepts instead.
+ final String matchAtFromIndex = "Mr. x";
+ Assertions.assertThrows(IndexOutOfBoundsException.class,
+ () -> legacyIsAcceptableBreak(dict, matchAtFromIndex, 0, matchAtFromIndex.length()));
+ Assertions.assertTrue(detector.isAcceptableBreak(matchAtFromIndex, 0, matchAtFromIndex.length()));
+
+ // Match elsewhere with candidateIndex == length: legacy short-circuits clause A and
+ // clause B rejects the break. The bounded window must agree (not silently accept).
+ final String matchElsewhere = "Bring apples, pears, etc.";
+ Assertions.assertFalse(legacyIsAcceptableBreak(dict, matchElsewhere, 0, matchElsewhere.length()));
+ Assertions.assertFalse(detector.isAcceptableBreak(matchElsewhere, 0, matchElsewhere.length()));
+
+ // Past the end of the text cannot carry an abbreviation; both accept, and the new path
+ // must not throw.
+ Assertions.assertTrue(detector.isAcceptableBreak(matchAtFromIndex, 0,
+ matchAtFromIndex.length() + 100));
+ }
+
+ @Test
+ void testNegativeFromIndexIsStillUndefined() {
+ final Dictionary dict = dictionaryOf(true, "Mr.");
+ final SentenceDetectorME detector = new SentenceDetectorME(sentdetectModel, dict);
+ // A negative segment start makes the character before the first match unreadable. Both
+ // implementations fail on it the same way when the match sits at the start of the text.
+ final String atStart = "Mr. x";
+ Assertions.assertThrows(IndexOutOfBoundsException.class,
+ () -> legacyIsAcceptableBreak(dict, atStart, -1, 3));
+ Assertions.assertThrows(IndexOutOfBoundsException.class,
+ () -> detector.isAcceptableBreak(atStart, -1, 3));
+ // They part company when the match is too far left to be relevant: the previous
+ // implementation still read index -1 for it, the bounded window never looks there.
+ final String far = "Mr. and something else.";
+ Assertions.assertThrows(IndexOutOfBoundsException.class,
+ () -> legacyIsAcceptableBreak(dict, far, -1, far.length() - 1));
+ Assertions.assertTrue(detector.isAcceptableBreak(far, -1, far.length() - 1));
+ }
+
+ @Test
+ void testCandidateBeforeSegmentStartAcceptsAsBefore() {
+ final Dictionary dict = dictionaryOf(true, "Mr.");
+ final SentenceDetectorME detector = new SentenceDetectorME(sentdetectModel, dict);
+ final String text = "Mr. x";
+ Assertions.assertTrue(legacyIsAcceptableBreak(dict, text, 4, 2));
+ Assertions.assertTrue(detector.isAcceptableBreak(text, 4, 2));
+ }
+
+ /*
+ * ------------------------------------------------------------------------------------------
+ * Helpers.
+ * ------------------------------------------------------------------------------------------
+ */
+
+ private static String readCorpus() throws IOException {
+ final List<String> lines = new ArrayList<>();
+ for (String resource : new String[] {"/opennlp/tools/sentdetect/Sentences.txt",
+ "/opennlp/tools/sentdetect/Sentences_DE.txt"}) {
+ try (InputStream in = SentenceDetectorMEAbbreviationEquivalenceTest.class
+ .getResourceAsStream(resource)) {
+ Assertions.assertNotNull(in, resource + " is not on the test classpath");
+ final String all = new String(in.readAllBytes(), StandardCharsets.UTF_8);
+ for (String line : all.split("\n")) {
+ if (!line.isBlank()) {
+ lines.add(line.strip());
+ }
+ }
+ }
+ }
+ // The shipped training corpora happen to contain almost no abbreviations, which would make
+ // the comparison runs above vacuous. This tail supplies them, in the shape they occur in.
+ lines.add("Mr. Smith called tel. 555 1234 this morning. Mrs. Smith did not.");
+ lines.add("Ms. Adams, Mr. Brown and Mrs. Clark met in the hall.");
+ lines.add("Ask Mr. Brown or, failing that, Ms. Adams (cf. tel. 555 1234).");
+ lines.add("Er wohnt in der S. Bahnstrasse, vgl. S. 12, ca. 30 Minuten entfernt.");
+ lines.add("Das gilt z.B. fuer Bek. 4 ff. und lt. V. 7 ugs. auch sonst.");
+ lines.add("Siehe ca. 20 Stueck, z. B. Bek. 9, S. 3 f. und ff.");
+ return String.join(" ", lines);
+ }
+}