You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by ma...@apache.org on 2026/03/21 09:58:54 UTC
(opennlp) branch OPENNLP-1809 created (now a7933754)
This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a change to branch OPENNLP-1809
in repository https://gitbox.apache.org/repos/asf/opennlp.git
at a7933754 OPENNLP-1809: SentenceDetector misses multi-letter abbreviations at sentence start - adds reproducer & test - fixes the issue in SentenceDetectorME#isAcceptableBreak(..) - refactors some code in other spots
This branch includes the following new commits:
new a7933754 OPENNLP-1809: SentenceDetector misses multi-letter abbreviations at sentence start - adds reproducer & test - fixes the issue in SentenceDetectorME#isAcceptableBreak(..) - refactors some code in other spots
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(opennlp) 01/01: OPENNLP-1809: SentenceDetector misses multi-letter abbreviations at sentence start - adds reproducer & test - fixes the issue in SentenceDetectorME#isAcceptableBreak(..) - refactors some code in other spots
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a commit to branch OPENNLP-1809
in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit a79337545e89396ba5a25893a53078da3d2e88ae
Author: Martin Wiesner <ma...@hs-heilbronn.de>
AuthorDate: Sat Mar 21 10:57:47 2026 +0100
OPENNLP-1809: SentenceDetector misses multi-letter abbreviations at sentence start
- adds reproducer & test
- fixes the issue in SentenceDetectorME#isAcceptableBreak(..)
- refactors some code in other spots
---
.../serializer/DictionaryEntryPersistor.java | 6 ++--
.../tools/sentdetect/SentenceDetectorME.java | 40 ++++++++++++----------
.../sentdetect/SentenceDetectorMEGermanTest.java | 25 ++++++++++++--
.../tools/tokenize/TokenizerFactoryTest.java | 4 +--
.../test/resources/opennlp/tools/lang/abb_DE.xml | 3 ++
5 files changed, 53 insertions(+), 25 deletions(-)
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/dictionary/serializer/DictionaryEntryPersistor.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/dictionary/serializer/DictionaryEntryPersistor.java
index a20891c9..c8e3ad42 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/dictionary/serializer/DictionaryEntryPersistor.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/dictionary/serializer/DictionaryEntryPersistor.java
@@ -226,7 +226,7 @@ public class DictionaryEntryPersistor {
public static boolean create(InputStream in, EntryInserter inserter)
throws IOException {
- DictionaryContenthandler profileContentHandler = new DictionaryContenthandler(inserter);
+ DictionaryContenthandler handler = new DictionaryContenthandler(inserter);
XMLReader xmlReader;
try {
@@ -235,14 +235,14 @@ public class DictionaryEntryPersistor {
// There is a compatibility problem here: JAXP default is false while SAX 2 default is true!
// OpenNLP requires it activated!
xmlReader.setFeature(SAX_FEATURE_NAMESPACES, true);
- xmlReader.setContentHandler(profileContentHandler);
+ xmlReader.setContentHandler(handler);
xmlReader.parse(new InputSource(new UncloseableInputStream(in)));
}
catch (ParserConfigurationException | SAXException e) {
throw new InvalidFormatException("The profile data stream has " +
"an invalid format!", e);
}
- return profileContentHandler.mIsCaseSensitiveDictionary;
+ return handler.mIsCaseSensitiveDictionary;
}
/**
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java
index 9b113e12..55faf79c 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java
@@ -339,27 +339,31 @@ public class SentenceDetectorME implements SentenceDetector, Probabilistic {
if (abbDict == null)
return true;
+ final String text = s.toString();
for (StringList abb : abbDict) {
- final String token = abb.getToken(0);
- final int tokenPosition = s.toString().indexOf(token, fromIndex);
- if (tokenPosition == -1) {
- continue; // skip fast
+ final String abbToken = abb.getToken(0);
+ final int tokenStartPos = text.indexOf(abbToken, fromIndex);
+ if (tokenStartPos == -1) {
+ continue; // skip fast when abb is not present in text
}
-
- final char prevChar = s.charAt(tokenPosition == 0 ? tokenPosition : tokenPosition - 1);
- int tokenLength = token.length();
- if (tokenPosition + tokenLength < candidateIndex || tokenPosition > candidateIndex ||
- /*
- * Note:
- * Skip abbreviation candidate if regular characters exist directly before it,
- * That is, any letter or digit except: a whitespace, an apostrophe, or an opening round bracket.
- * This prevents mismatches from overlaps close to an actual sentence end.
- */
- !(Character.isWhitespace(prevChar) || isApostrophe(prevChar) || prevChar == '(')) {
-
- continue;
+ if (tokenStartPos == 0 && text.substring(tokenStartPos, candidateIndex + 1).equals(abbToken)) {
+ return false; // full abbreviation match at sentence start -> no acceptable break
+ } else {
+ final int tokenLength = abbToken.length();
+ final char prevChar = s.charAt(tokenStartPos == 0 ? tokenStartPos : tokenStartPos - 1);
+ if (tokenStartPos + tokenLength < candidateIndex || tokenStartPos > candidateIndex ||
+ /*
+ * Note:
+ * Skip abbreviation candidate if regular characters exist directly before it,
+ * That is, any letter or digit except: a whitespace, an apostrophe, or an opening round bracket.
+ * This prevents mismatches from overlaps close to an actual sentence end.
+ */
+ !(Character.isWhitespace(prevChar) || isApostrophe(prevChar) || prevChar == '(')) {
+
+ continue;
+ }
+ return false; // in case of a valid abbreviation: the (sentence) break is not accepted
}
- return false; // in case of a valid abbreviation: the (sentence) break is not accepted
}
return true; // no abbreviation(s) at given positions: valid sentence boundary
}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/sentdetect/SentenceDetectorMEGermanTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/sentdetect/SentenceDetectorMEGermanTest.java
index 5417fe17..9d271ce0 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/sentdetect/SentenceDetectorMEGermanTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/sentdetect/SentenceDetectorMEGermanTest.java
@@ -152,10 +152,10 @@ public class SentenceDetectorMEGermanTest extends AbstractSentenceDetectorTest {
}
/*
- * A reproducer and test for OPENNLP-1781.
+ * A reproducer and test for OPENNLP-1781.
*/
@Test
- void testSentDetectWithAbbreviationsAtSentenceStart() {
+ void testSentDetectWithSingleLetterAbbreviationsAtSentenceStart() {
prepareResources(true);
final String sent1 = "S. Träume sind eine Verbindung von Gedanken.";
@@ -169,6 +169,27 @@ public class SentenceDetectorMEGermanTest extends AbstractSentenceDetectorTest {
() -> assertEquals(1, probs.length));
}
+ /*
+ * A reproducer and test for OPENNLP-1809.
+ */
+ @Test
+ void testSentDetectWithMultiLetterAbbreviationsAtSentenceStart() {
+ prepareResources(true);
+
+ final String sent1 = "Bek. Problem: Schlafmangel.";
+ final String sent2 = "Über die letzten Tage hinweg war sie zunehmend müde.";
+
+ String sampleSentences = sent1 + " " + sent2;
+ String[] sents = sentenceDetector.sentDetect(sampleSentences);
+ double[] probs = sentenceDetector.probs();
+
+ assertAll(
+ () -> assertEquals(2, sents.length),
+ () -> assertEquals(sent1, sents[0]),
+ () -> assertEquals(sent2, sents[1]),
+ () -> assertEquals(2, probs.length));
+ }
+
/*
* A reproducer and test for OPENNLP-1767.
* It checks that sentence detection with common abbreviations works correctly,
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/TokenizerFactoryTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/TokenizerFactoryTest.java
index 41ff5d44..de478083 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/TokenizerFactoryTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/TokenizerFactoryTest.java
@@ -210,8 +210,8 @@ public class TokenizerFactoryTest {
Assertions.assertEquals(expectedNumTokens, tokens.length);
String[] sentSplit = sentence
- .replaceAll("'", " '")
- .replaceAll(",", " ,")
+ .replace("'", " '")
+ .replace(",", " ,")
.split(" ");
for (int i = 0; i < sentSplit.length; i++) {
String sElement = sentSplit[i];
diff --git a/opennlp-core/opennlp-runtime/src/test/resources/opennlp/tools/lang/abb_DE.xml b/opennlp-core/opennlp-runtime/src/test/resources/opennlp/tools/lang/abb_DE.xml
index 23e09abc..33c6c875 100644
--- a/opennlp-core/opennlp-runtime/src/test/resources/opennlp/tools/lang/abb_DE.xml
+++ b/opennlp-core/opennlp-runtime/src/test/resources/opennlp/tools/lang/abb_DE.xml
@@ -47,4 +47,7 @@
<entry>
<token>z.B.</token>
</entry>
+ <entry>
+ <token>Bek.</token>
+ </entry>
</dictionary>