You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by ma...@apache.org on 2026/10/03 19:50:28 UTC
(opennlp) branch main updated: Minor: Correct factually wrong Javadoc and manual statements (#1375)
This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/main by this push:
new 4962c2d04 Minor: Correct factually wrong Javadoc and manual statements (#1375)
4962c2d04 is described below
commit 4962c2d04df86f45eb2656937c48b0f2e563dbb2
Author: Richard Zowalla <13...@users.noreply.github.com>
AuthorDate: Sat Oct 3 21:50:23 2026 +0200
Minor: Correct factually wrong Javadoc and manual statements (#1375)
---
.../main/java/opennlp/tools/tokenize/WordpieceEncoder.java | 9 +++++----
.../java/opennlp/tools/formats/ad/ADSentenceStream.java | 5 +++--
.../java/opennlp/tools/sentdetect/SentenceDetectorME.java | 13 ++++++-------
.../src/main/java/opennlp/tools/util/DictionaryCatalog.java | 5 +++--
opennlp-docs/src/docbkx/document.xml | 3 ++-
.../spellcheck/dictionary/FrequencyDictionaryLoader.java | 4 ++--
6 files changed, 21 insertions(+), 18 deletions(-)
diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java
index 9e620c3f5..f86003042 100644
--- a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java
+++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java
@@ -51,10 +51,11 @@ import opennlp.tools.util.StringUtil;
* vocabulary, because each emitted piece must have an id. Vocabulary entries starting with
* {@code ##} are continuation pieces and can match only after the first piece of a word.</p>
*
- * <p>Lower casing applies the <a href="https://www.unicode.org/Public/15.0.0/ucd/SpecialCasing.txt">
- * Unicode full case mapping</a>, including the {@code Final_Sigma}
- * context, so a word-final Greek capital sigma becomes U+03C2 as in the reference
- * implementation.</p>
+ * <p>Lower casing applies the Unicode simple case mapping of {@link Character#toLowerCase(int)},
+ * one code point to one code point, plus the {@code Final_Sigma} context, so a word-final Greek
+ * capital sigma becomes U+03C2 as in the reference implementation. Other
+ * <a href="https://www.unicode.org/Public/15.0.0/ucd/SpecialCasing.txt">SpecialCasing</a>
+ * mappings are not applied; for example U+0130 becomes {@code i} without a combining dot.</p>
*
* <p>A word exceeding the configured maximum number of normalized Unicode code points becomes
* the unknown piece. The default is 100, the value used by the Hugging Face {@code transformers}
diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
index c81f9a54a..84d110a09 100644
--- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
+++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
@@ -1041,10 +1041,11 @@ public class ADSentenceStream extends FilterObjectStream<String, ADSentenceStrea
}
/**
- * Tests whether a line is an opening markup tag with the given name: the name right after the
+ * Checks whether a line is an opening markup tag with the given name: the name right after the
* opening angle bracket, then either the closing angle bracket or whitespace and attributes,
* which contain no closing angle bracket, then the closing angle bracket as the last character.
- * A self-closing tag is not an opening tag.
+ * A slash directly after the name, as in {@code <s/>}, is rejected. A slash after whitespace
+ * counts as part of the attributes, so {@code <s />} and {@code <s id="1"/>} are accepted.
*
* @param line The line.
* @param name The tag name.
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java
index 9e2109b31..09b334c15 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java
@@ -400,16 +400,15 @@ public class SentenceDetectorME implements SentenceDetector, Probabilistic {
* irrelevant by construction, so the decision costs time proportional to the longest
* dictionary entry rather than to the length of {@code s}.</p>
*
- * <p>A {@code candidateIndex} past the end of {@code s} cannot carry an abbreviation, so it
- * is accepted rather than raising an {@link IndexOutOfBoundsException}. At
- * {@code candidateIndex == s.length()}, the decision matches the previous implementation when
- * the match is not at {@code fromIndex}; only the match-at-{@code fromIndex} case is an
- * accept-instead-of-throw relaxation.</p>
+ * <p>A {@code candidateIndex} greater than {@code s.length()} cannot carry an abbreviation, so
+ * the break is accepted rather than raising an {@link IndexOutOfBoundsException}. A
+ * {@code candidateIndex} equal to {@code s.length()} is evaluated like any other index.</p>
*
* @param s the {@link CharSequence} in which the break occurred.
* @param fromIndex the start of the segment currently being evaluated.
- * @param candidateIndex the index of the candidate sentence ending. Must be greater than or
- * equal to {@code fromIndex} and a valid index into {@code s}.
+ * @param candidateIndex the index of the candidate sentence ending. A value below
+ * {@code fromIndex}, negative or greater than {@code s.length()} is accepted as a
+ * break.
* @return {@code true} if the break is acceptable, {@code false} otherwise.
*/
protected boolean isAcceptableBreak(CharSequence s, int fromIndex, int candidateIndex) {
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DictionaryCatalog.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DictionaryCatalog.java
index 5e3a2de01..35eab865d 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DictionaryCatalog.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DictionaryCatalog.java
@@ -115,8 +115,9 @@ public final class DictionaryCatalog {
/**
* Installs a catalog entry into {@code targetDirectory} after checking that remote
* catalog downloads are enabled. The entry is fetched, digest-verified, and unpacked
- * by {@link ResourceInstaller#install(URI, Path, String)}: an archive expands into
- * the directory, and a plain file is stored under its source name.
+ * by {@link ResourceInstaller}: an archive expands into the directory, and a plain
+ * file is stored under the entry's {@code <id>.filename} value when the catalog sets
+ * one, or under the file name of its source URI otherwise.
*
* @param id The entry id. Must not be {@code null}.
* @param targetDirectory The directory to install into; created when absent. Must
diff --git a/opennlp-docs/src/docbkx/document.xml b/opennlp-docs/src/docbkx/document.xml
index 169a34c2f..c508ed46d 100644
--- a/opennlp-docs/src/docbkx/document.xml
+++ b/opennlp-docs/src/docbkx/document.xml
@@ -192,7 +192,8 @@ String language = tagged.get(LANGUAGE).get(0).value(); // "eng", span is null]]>
</tgroup>
</table>
<para>
- The keys of the first four layers in the table live in <code>Layers</code>;
+ The keys of the sentences, tokens, pos and entities layers live in <code>Layers</code>
+ (<code>SENTENCES</code>, <code>TOKENS</code>, <code>POS_TAGS</code> and <code>ENTITIES</code>);
the others live on the annotator that provides them, which is also where an
extension puts its key. Layers that refer to other layers do so by index into
them, which the invariants above keep valid: a coreference layer, for example,
diff --git a/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/dictionary/FrequencyDictionaryLoader.java b/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/dictionary/FrequencyDictionaryLoader.java
index 718699610..4c41fb04a 100644
--- a/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/dictionary/FrequencyDictionaryLoader.java
+++ b/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/dictionary/FrequencyDictionaryLoader.java
@@ -45,7 +45,7 @@ import opennlp.tools.util.StringUtil;
*
* <p>Columns are separated by one or more TAB or space characters, so the space-delimited
* SymSpell reference dictionaries load as they are, as do TAB-delimited files. The count
- * column holds ASCII digits only.</p>
+ * column holds decimal digits in any script, such as ASCII or Devanagari digits.</p>
*
* <p>The loader is encoding-aware (UTF-8 by default) and tolerant of input noise: a
* leading UTF-8 byte-order mark is stripped; blank lines, lines that are entirely
@@ -80,7 +80,7 @@ public final class FrequencyDictionaryLoader {
/** The reason reported for a count with a minus sign. */
private static final String COUNT_NEGATIVE = "count must not be negative";
- /** The reason reported for a count that is not ASCII digits or does not fit in a long. */
+ /** The reason reported for a count that is not decimal digits in any script or does not fit in a long. */
private static final String COUNT_NOT_INTEGER = "count is not an integer";
private final Charset charset;