You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by rz...@apache.org on 2026/10/08 10:00:29 UTC
(opennlp) branch main updated: GH-1436: Document thread-safety limits of WhitespaceTokenizer.INSTANCE and the process-wide modes (#1437)
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/main by this push:
new 5a6347ac2 GH-1436: Document thread-safety limits of WhitespaceTokenizer.INSTANCE and the process-wide modes (#1437)
5a6347ac2 is described below
commit 5a6347ac29bacb7fc680180cb0f8e755cf9c0db4
Author: Richard Zowalla <13...@users.noreply.github.com>
AuthorDate: Thu Oct 8 12:00:23 2026 +0200
GH-1436: Document thread-safety limits of WhitespaceTokenizer.INSTANCE and the process-wide modes (#1437)
WhitespaceTokenizer.INSTANCE must not be reconfigured; callers needing
keepNewLines=true use newInstance(boolean). WhitespaceMode and
CompatibilityMode state that setActive and reset are for tests and
application startup only. Tests no longer mutate the shared instance.
---
.../opennlp/tools/tokenize/WhitespaceTokenizer.java | 8 +++++++-
.../java/opennlp/tools/util/CompatibilityMode.java | 20 ++++++++++++++------
.../main/java/opennlp/tools/util/WhitespaceMode.java | 19 ++++++++++++++-----
.../java/opennlp/tools/tokenize/TokenizerMETest.java | 6 ------
.../tools/tokenize/WhitespaceTokenizerTest.java | 7 ++-----
5 files changed, 37 insertions(+), 23 deletions(-)
diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/WhitespaceTokenizer.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/WhitespaceTokenizer.java
index fb158f6bb..460a32598 100644
--- a/opennlp-api/src/main/java/opennlp/tools/tokenize/WhitespaceTokenizer.java
+++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/WhitespaceTokenizer.java
@@ -28,13 +28,19 @@ import opennlp.tools.util.StringUtil;
* using white spaces.
* <p>
* To obtain an instance of this tokenizer use the static final
- * {@link #INSTANCE} field.
+ * {@link #INSTANCE} field. Use {@link #newInstance(boolean)} for a tokenizer with a
+ * non-default {@link #setKeepNewLines(boolean)} setting.
*/
public class WhitespaceTokenizer extends AbstractTokenizer {
/**
* Use this static reference to retrieve an instance of the
* {@link WhitespaceTokenizer}.
+ * <p>
+ * This instance is shared by every caller in the process, so it must not be reconfigured:
+ * calling {@link #setKeepNewLines(boolean)} on it changes the output of all other users and
+ * is not safely published to other threads. Use {@link #newInstance(boolean)} to obtain a
+ * tokenizer with a different setting.
*/
public static final WhitespaceTokenizer INSTANCE = new WhitespaceTokenizer();
diff --git a/opennlp-api/src/main/java/opennlp/tools/util/CompatibilityMode.java b/opennlp-api/src/main/java/opennlp/tools/util/CompatibilityMode.java
index 6bc7e0fd2..6ed6da6e7 100644
--- a/opennlp-api/src/main/java/opennlp/tools/util/CompatibilityMode.java
+++ b/opennlp-api/src/main/java/opennlp/tools/util/CompatibilityMode.java
@@ -29,9 +29,11 @@ import org.slf4j.LoggerFactory;
* it until it is retrained. Each class that consults the mode documents what differs.
* <p>
* Resolved from the {@value #MODE_PROPERTY} system property when this class is initialized
- * and shared process-wide, so a model is trained and decoded under one mode. Tests and
- * embedders may override the mode via {@link #setActive(CompatibilityMode)} and
- * {@link #reset()}. The mode is independent of {@link WhitespaceMode}.
+ * and shared process-wide, so a model is trained and decoded under one mode. The mode is
+ * read on every call that consults it and is not recorded in a model, so it must be fixed
+ * before any component is used and must match the mode the loaded models were trained under.
+ * {@link #setActive(CompatibilityMode)} and {@link #reset()} are meant for tests and
+ * application startup only. The mode is independent of {@link WhitespaceMode}.
*
* @since 3.0.0
*/
@@ -75,8 +77,12 @@ public enum CompatibilityMode {
/**
* Overrides the active {@link CompatibilityMode} for the whole process, taking precedence
- * over the {@value #MODE_PROPERTY} system property. Intended for tests and embedders;
- * callers pinning a mode temporarily should call {@link #reset()} afterward.
+ * over the {@value #MODE_PROPERTY} system property. Intended for tests and application
+ * startup only; callers pinning a mode temporarily should call {@link #reset()} afterward.
+ * <p>
+ * The switch takes effect immediately for every thread, so a component processing text
+ * concurrently may apply the old mode to one part of that text and the new mode to the rest.
+ * Call this method before any component is created or used.
*
* @param mode The {@link CompatibilityMode} to activate. Must not be {@code null}.
*
@@ -90,7 +96,9 @@ public enum CompatibilityMode {
/**
* Discards any override set via {@link #setActive(CompatibilityMode)} and re-resolves the
- * active mode from the {@value #MODE_PROPERTY} system property.
+ * active mode from the {@value #MODE_PROPERTY} system property. Intended for tests and
+ * application startup only, with the same process-wide effect as
+ * {@link #setActive(CompatibilityMode)}.
*
* @throws IllegalArgumentException Thrown if the property holds a value other than
* {@code LEGACY} or {@code CURRENT} (case-insensitive); the previous mode is retained.
diff --git a/opennlp-api/src/main/java/opennlp/tools/util/WhitespaceMode.java b/opennlp-api/src/main/java/opennlp/tools/util/WhitespaceMode.java
index 48a806270..301754040 100644
--- a/opennlp-api/src/main/java/opennlp/tools/util/WhitespaceMode.java
+++ b/opennlp-api/src/main/java/opennlp/tools/util/WhitespaceMode.java
@@ -29,8 +29,11 @@ import org.slf4j.LoggerFactory;
* OpenNLP's legacy definition from 1.x/2.x.
* <p>
* Resolved from the {@value #MODE_PROPERTY} system property when this class is initialized
- * and shared process-wide, so a model is trained and decoded under one definition. Tests and
- * embedders may override the mode via {@link #setActive(WhitespaceMode)} and {@link #reset()}.
+ * and shared process-wide, so a model is trained and decoded under one definition. The mode is
+ * read on every whitespace check and is not recorded in a model, so it must be fixed before any
+ * component is used and must match the mode the loaded models were trained under.
+ * {@link #setActive(WhitespaceMode)} and {@link #reset()} are meant for tests and application
+ * startup only.
*
* @since 3.0.0
*/
@@ -74,8 +77,12 @@ public enum WhitespaceMode {
/**
* Overrides the active {@link WhitespaceMode} for the whole process, taking precedence
- * over the {@value #MODE_PROPERTY} system property. Intended for tests and embedders;
- * callers pinning a mode temporarily should call {@link #reset()} afterward.
+ * over the {@value #MODE_PROPERTY} system property. Intended for tests and application
+ * startup only; callers pinning a mode temporarily should call {@link #reset()} afterward.
+ * <p>
+ * The switch takes effect immediately for every thread, so a component processing text
+ * concurrently may apply the old mode to one part of that text and the new mode to the rest.
+ * Call this method before any component is created or used.
*
* @param mode The {@link WhitespaceMode} to activate. Must not be {@code null}.
*
@@ -89,7 +96,9 @@ public enum WhitespaceMode {
/**
* Discards any override set via {@link #setActive(WhitespaceMode)} and re-resolves the
- * active mode from the {@value #MODE_PROPERTY} system property.
+ * active mode from the {@value #MODE_PROPERTY} system property. Intended for tests and
+ * application startup only, with the same process-wide effect as
+ * {@link #setActive(WhitespaceMode)}.
*
* @throws IllegalArgumentException If the property holds a value other than
* {@code LEGACY} or {@code UNICODE} (case-insensitive); the previous mode is retained.
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/TokenizerMETest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/TokenizerMETest.java
index 2a1fed18d..6c2e4381a 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/TokenizerMETest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/TokenizerMETest.java
@@ -20,7 +20,6 @@ package opennlp.tools.tokenize;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
-import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -43,11 +42,6 @@ import opennlp.tools.util.TrainingParameters;
*/
public class TokenizerMETest {
- @AfterEach
- void restoreSharedWhitespaceTokenizer() {
- WhitespaceTokenizer.INSTANCE.setKeepNewLines(false);
- }
-
@Test
void testTokenizerSimpleModel() throws IOException {
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WhitespaceTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WhitespaceTokenizerTest.java
index ee5a4459f..20b854e89 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WhitespaceTokenizerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WhitespaceTokenizerTest.java
@@ -66,8 +66,7 @@ public class WhitespaceTokenizerTest {
@Test
void testTokenizationOfStringWithUnixNewLineTokens() {
- WhitespaceTokenizer tokenizer = WhitespaceTokenizer.INSTANCE;
- tokenizer.setKeepNewLines(true);
+ WhitespaceTokenizer tokenizer = WhitespaceTokenizer.newInstance(true);
Assertions.assertEquals(2, tokenizer.tokenize("a\n").length);
Assertions.assertArrayEquals(new String[] {"a", "\n"}, tokenizer.tokenize("a\n"));
@@ -85,8 +84,7 @@ public class WhitespaceTokenizerTest {
@Test
void testTokenizationOfStringWithWindowsNewLineTokens() {
- WhitespaceTokenizer tokenizer = WhitespaceTokenizer.INSTANCE;
- tokenizer.setKeepNewLines(true);
+ WhitespaceTokenizer tokenizer = WhitespaceTokenizer.newInstance(true);
Assertions.assertEquals(3, tokenizer.tokenize("a\r\n").length);
Assertions.assertArrayEquals(new String[] {"a", "\r", "\n"}, tokenizer.tokenize("a\r\n"));
@@ -139,7 +137,6 @@ public class WhitespaceTokenizerTest {
@Test
void testNewInstanceIsNotTheSharedInstance() {
- WhitespaceTokenizer.INSTANCE.setKeepNewLines(false);
WhitespaceTokenizer keeping = WhitespaceTokenizer.newInstance(true);
WhitespaceTokenizer dropping = WhitespaceTokenizer.newInstance(false);
Assertions.assertNotSame(WhitespaceTokenizer.INSTANCE, keeping);