You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by kr...@apache.org on 2026/09/30 23:17:42 UTC
(opennlp) 04/04: OPENNLP-1935: Move the remaining String.split calls to the StringUtil helpers
This is an automated email from the ASF dual-hosted git repository.
krickert pushed a commit to branch feat/OPENNLP-1935-regex-guard
in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 28f4a68f2ed653e49105027b09704f5a22184c70
Author: Kristian Rickert <kr...@apache.org>
AuthorDate: Wed Sep 30 18:55:51 2026 -0400
OPENNLP-1935: Move the remaining String.split calls to the StringUtil helpers
ConlluStream splits a range id on the literal dash. TokenSampleStream and DownloadUtil split
on whitespace runs under the active whitespace mode. The forbidden list message names the
helpers that exist now. BasicContextGenerator's call is replaced by #1280.
---
dev/forbidden-regex.txt | 2 +-
.../src/main/java/opennlp/tools/formats/conllu/ConlluStream.java | 3 ++-
.../main/java/opennlp/tools/tokenize/lang/en/TokenSampleStream.java | 3 ++-
.../opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java | 2 +-
4 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/dev/forbidden-regex.txt b/dev/forbidden-regex.txt
index 0306a7315..3eecc2937 100644
--- a/dev/forbidden-regex.txt
+++ b/dev/forbidden-regex.txt
@@ -20,7 +20,7 @@
# @opennlp.tools.commons.SuppressForbidden and is exempt. Only production classes are
# checked; tests may use regular expressions.
-@defaultMessage Use a code-point scan or a StringUtil helper (splitOnUnicodeWhitespace, split(CharSequence, char), splitNonEmpty once OPENNLP-1955 is merged) instead of java.util.regex; a member with a user-supplied pattern as its public contract is marked @opennlp.tools.commons.SuppressForbidden (OPENNLP-1935)
+@defaultMessage Use a code-point scan or a StringUtil helper (splitOnUnicodeWhitespace, splitOnWhitespace, split(CharSequence, char), splitNonEmpty) instead of java.util.regex; a member with a user-supplied pattern as its public contract is marked @opennlp.tools.commons.SuppressForbidden (OPENNLP-1935)
java.util.regex.**
java.lang.String#split(java.lang.String)
java.lang.String#split(java.lang.String,int)
diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluStream.java
index 9a2e63336..d87c285ab 100644
--- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluStream.java
+++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluStream.java
@@ -37,6 +37,7 @@ import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.ParagraphStream;
import opennlp.tools.util.PlainTextByLineStream;
+import opennlp.tools.util.StringUtil;
/**
* The CoNNL-U Format is specified
@@ -166,7 +167,7 @@ public class ConlluStream implements ObjectStream<ConlluSentence> {
for (ConlluWordLine line : lines) {
if (line.getId().contains("-")) {
List<String> expandedContractions = new ArrayList<>();
- String[] ids = line.getId().split("-");
+ String[] ids = StringUtil.split(line.getId(), '-');
int start = Integer.parseInt(ids[0]);
int end = Integer.parseInt(ids[1]);
for (long j = start; j <= end; j++) {
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lang/en/TokenSampleStream.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lang/en/TokenSampleStream.java
index 951fca5b5..e68b0c65f 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lang/en/TokenSampleStream.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lang/en/TokenSampleStream.java
@@ -31,6 +31,7 @@ import org.slf4j.LoggerFactory;
import opennlp.tools.tokenize.TokenSample;
import opennlp.tools.util.Span;
+import opennlp.tools.util.StringUtil;
/**
* Class which produces an Iterator<TokenSample> from a file of space delimited token.
@@ -55,7 +56,7 @@ public class TokenSampleStream implements Iterator<TokenSample> {
}
public TokenSample next() {
- String[] tokens = line.split("\\s+");
+ String[] tokens = StringUtil.splitOnWhitespace(line);
if (tokens.length == 0) {
evenq = true;
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java
index 0f6f6800d..fa0426b15 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java
@@ -262,7 +262,7 @@ public class DownloadUtil {
return null;
}
final String trimmed = checksumFileContent.trim();
- return trimmed.isEmpty() ? null : trimmed.split("\\s")[0];
+ return trimmed.isEmpty() ? null : StringUtil.splitOnWhitespace(trimmed)[0];
}
private static void verifyChecksum(Path model, String expectedChecksum) throws IOException {