You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by ma...@apache.org on 2026/10/03 11:13:53 UTC
(opennlp) branch main updated: Minor: Share ISO 639 code conversion via LanguageCodeValidator (#1356)
This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/main by this push:
new 36cd43a8c Minor: Share ISO 639 code conversion via LanguageCodeValidator (#1356)
36cd43a8c is described below
commit 36cd43a8c30283980d946cc85857dbb4a086e730
Author: Richard Zowalla <13...@users.noreply.github.com>
AuthorDate: Sat Oct 3 13:13:47 2026 +0200
Minor: Share ISO 639 code conversion via LanguageCodeValidator (#1356)
* Minor: Share ISO 639 code conversion via LanguageCodeValidator
Add LanguageCodeValidator.toIso6391 and toIso6393 in opennlp-api.
StopwordLists drops its private ISO 639-3 to 639-1 map and uses toIso6391.
NormalizationProfiles.forLanguage uses toIso6393, so it now also
resolves the bibliographic codes ger, fre and dut.
* Use StringUtil.toLowerCase in LanguageCodeValidator and align the Javadoc
The conversion methods lower-case through StringUtil.toLowerCase instead of String.toLowerCase(Locale.ROOT).
---------
Signed-off-by: Kristian Rickert <kr...@apache.org>
Co-authored-by: Kristian Rickert <kr...@apache.org>
---
.../opennlp/tools/util/LanguageCodeValidator.java | 130 +++++++++++++++++----
.../java/opennlp/tools/stopword/StopwordLists.java | 60 +---------
.../util/normalizer/NormalizationProfiles.java | 22 +---
.../opennlp/tools/stopword/StopwordListsTest.java | 8 ++
.../tools/util/LanguageCodeValidatorTest.java | 64 ++++++++++
.../util/normalizer/NormalizationProfilesTest.java | 10 ++
6 files changed, 196 insertions(+), 98 deletions(-)
diff --git a/opennlp-api/src/main/java/opennlp/tools/util/LanguageCodeValidator.java b/opennlp-api/src/main/java/opennlp/tools/util/LanguageCodeValidator.java
index 8c8ead1d1..77d5d4163 100644
--- a/opennlp-api/src/main/java/opennlp/tools/util/LanguageCodeValidator.java
+++ b/opennlp-api/src/main/java/opennlp/tools/util/LanguageCodeValidator.java
@@ -17,10 +17,12 @@
package opennlp.tools.util;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Locale;
+import java.util.Map;
+import java.util.MissingResourceException;
import java.util.Set;
-import java.util.stream.Collectors;
/**
* Validates language codes against ISO 639 standards.
@@ -38,6 +40,10 @@ import java.util.stream.Collectors;
* Valid codes are derived from {@link Locale#availableLocales()}
* plus additional ISO 639-2 bibliographic codes and
* {@code "und"} (undetermined).
+ * <p>
+ * Also converts between ISO 639-1 two-letter codes and ISO 639-2/3
+ * three-letter codes, see {@link #toIso6391(String)} and
+ * {@link #toIso6393(String)}.
*
* @see <a href="https://iso639-3.sil.org/">ISO 639-3</a>
*/
@@ -45,38 +51,63 @@ public final class LanguageCodeValidator {
private static final String X_UNSPECIFIED = "x-unspecified";
+ private static final int ISO_639_1_LENGTH = 2;
+
+ /**
+ * ISO 639-2 bibliographic codes that {@link Locale#getISO3Language()} never
+ * returns, mapped to their ISO 639-2/T (terminologic) equivalent.
+ */
+ private static final Map<String, String> BIBLIOGRAPHIC_TO_TERMINOLOGIC = Map.of(
+ "dut", "nld", // Dutch
+ "fre", "fra", // French
+ "ger", "deu"); // German
+
private static final Set<String> VALID_CODES;
+ /**
+ * Maps three-letter ISO 639-2/3 codes, terminologic and bibliographic, to
+ * their ISO 639-1 two-letter equivalent.
+ */
+ private static final Map<String, String> ISO_639_3_TO_ISO_639_1;
+
static {
- VALID_CODES = Locale.availableLocales()
- .flatMap(loc -> {
- final Set<String> codes = new HashSet<>();
- final String lang = loc.getLanguage();
- if (!lang.isEmpty()) {
- codes.add(lang);
- }
- try {
- final String iso3 = loc.getISO3Language();
- if (!iso3.isEmpty()) {
- codes.add(iso3);
- }
- } catch (Exception ignored) {
- // Some locales may not have a 3-letter equivalent
+ final Set<String> validCodes = new HashSet<>();
+ final Map<String, String> toIso6391 = new HashMap<>();
+ for (final Locale locale : Locale.getAvailableLocales()) {
+ final String lang = locale.getLanguage();
+ if (!lang.isEmpty()) {
+ validCodes.add(lang);
+ }
+ try {
+ final String iso3 = locale.getISO3Language();
+ if (!iso3.isEmpty()) {
+ validCodes.add(iso3);
+ if (lang.length() == ISO_639_1_LENGTH) {
+ toIso6391.putIfAbsent(iso3, lang);
}
- return codes.stream();
- })
- .collect(Collectors.toCollection(HashSet::new));
+ }
+ } catch (MissingResourceException ignored) {
+ // locale has no three-letter form; skip it
+ }
+ }
+ VALID_CODES = validCodes;
// ISO 639-2 bibliographic codes not returned by Locale
- VALID_CODES.add("dut"); // Dutch (bibliographic)
- VALID_CODES.add("fre"); // French (bibliographic)
- VALID_CODES.add("ger"); // German (bibliographic)
+ VALID_CODES.addAll(BIBLIOGRAPHIC_TO_TERMINOLOGIC.keySet());
// ISO 639-3 special code for undetermined language
VALID_CODES.add("und");
// OpenNLP-specific special code
VALID_CODES.add(X_UNSPECIFIED);
+
+ for (final Map.Entry<String, String> e : BIBLIOGRAPHIC_TO_TERMINOLOGIC.entrySet()) {
+ final String iso1 = toIso6391.get(e.getValue());
+ if (iso1 != null) {
+ toIso6391.put(e.getKey(), iso1);
+ }
+ }
+ ISO_639_3_TO_ISO_639_1 = Map.copyOf(toIso6391);
}
private LanguageCodeValidator() {
@@ -90,7 +121,7 @@ public final class LanguageCodeValidator {
* Must not be {@code null}.
* @return {@code true} if the code is valid,
* {@code false} otherwise.
- * @throws IllegalArgumentException if {@code languageCode}
+ * @throws IllegalArgumentException Thrown if {@code languageCode}
* is {@code null}.
*/
public static boolean isValid(String languageCode) {
@@ -108,7 +139,7 @@ public final class LanguageCodeValidator {
*
* @param languageCode The language code to validate.
* Must not be {@code null}.
- * @throws IllegalArgumentException if the code is not a valid
+ * @throws IllegalArgumentException Thrown if the code is not a valid
* ISO 639 language code or is {@code null}.
*/
public static void validateLanguageCode(String languageCode) {
@@ -118,4 +149,57 @@ public final class LanguageCodeValidator {
+ "', must be a valid ISO 639 code!");
}
}
+
+ /**
+ * Converts a three-letter ISO 639-2/3 code, terminologic ({@code "nld"}) or
+ * bibliographic ({@code "dut"}), to its ISO 639-1 two-letter equivalent
+ * ({@code "nl"}). The code is lower-cased with
+ * {@link StringUtil#toLowerCase(CharSequence)} first. Two-letter codes and
+ * codes without a two-letter equivalent are returned lower-cased and
+ * otherwise unchanged. The code is not validated; use
+ * {@link #validateLanguageCode(String)} for that.
+ *
+ * @param languageCode The language code to convert. Must not be {@code null}.
+ * @return The ISO 639-1 code, or the lower-cased input if there is none.
+ * @throws IllegalArgumentException Thrown if {@code languageCode} is {@code null}.
+ */
+ public static String toIso6391(String languageCode) {
+ if (languageCode == null) {
+ throw new IllegalArgumentException("languageCode must not be null");
+ }
+ final String lower = StringUtil.toLowerCase(languageCode);
+ if (lower.length() == ISO_639_1_LENGTH) {
+ return lower;
+ }
+ return ISO_639_3_TO_ISO_639_1.getOrDefault(lower, lower);
+ }
+
+ /**
+ * Converts an ISO 639-1 two-letter code ({@code "de"}) or an ISO 639-2
+ * bibliographic code ({@code "ger"}) to its ISO 639-2/3 three-letter
+ * equivalent ({@code "deu"}). The code is lower-cased with
+ * {@link StringUtil#toLowerCase(CharSequence)} first. Other codes, and
+ * two-letter codes without a three-letter equivalent, are returned
+ * lower-cased and otherwise unchanged. The code is not validated; use
+ * {@link #validateLanguageCode(String)} for that.
+ *
+ * @param languageCode The language code to convert. Must not be {@code null}.
+ * @return The ISO 639-2/3 code, or the lower-cased input if there is none.
+ * @throws IllegalArgumentException Thrown if {@code languageCode} is {@code null}.
+ */
+ public static String toIso6393(String languageCode) {
+ if (languageCode == null) {
+ throw new IllegalArgumentException("languageCode must not be null");
+ }
+ final String lower = StringUtil.toLowerCase(languageCode);
+ if (lower.length() == ISO_639_1_LENGTH) {
+ try {
+ final String iso3 = Locale.of(lower).getISO3Language();
+ return iso3.isEmpty() ? lower : iso3;
+ } catch (MissingResourceException ignored) {
+ return lower;
+ }
+ }
+ return BIBLIOGRAPHIC_TO_TERMINOLOGIC.getOrDefault(lower, lower);
+ }
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stopword/StopwordLists.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stopword/StopwordLists.java
index 59f1c1550..1933cf922 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stopword/StopwordLists.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stopword/StopwordLists.java
@@ -23,11 +23,8 @@ import java.io.UncheckedIOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
-import java.util.HashMap;
import java.util.LinkedHashSet;
-import java.util.Locale;
import java.util.Map;
-import java.util.MissingResourceException;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -49,18 +46,6 @@ public final class StopwordLists {
private static final Set<String> SUPPORTED_LANGUAGES;
- /**
- * Maps three-letter ISO 639-2/3 codes to their ISO 639-1 two-letter
- * equivalent. Built once at class-initialization time from the JVM's locale
- * data (terminologic forms such as {@code nld}, {@code fra}, {@code deu})
- * plus the ISO 639-2 bibliographic forms ({@code dut}, {@code fre},
- * {@code ger}) that {@link Locale#getISO3Language()} does not produce, so
- * that {@link #normalizeToIso6391(String)} resolves codes with a single map
- * lookup instead of scanning {@link Locale#getAvailableLocales()} on every
- * call.
- */
- private static final Map<String, String> ISO6393_TO_ISO6391;
-
/**
* Caches the immutable, thread-safe filters loaded from the bundled
* resources, keyed by normalized ISO 639-1 code, so that repeated
@@ -75,28 +60,6 @@ public final class StopwordLists {
Collections.addAll(langs,
"bg", "da", "de", "en", "es", "fi", "fr", "it", "nl", "pt", "ru");
SUPPORTED_LANGUAGES = Collections.unmodifiableSet(langs);
-
- final Map<String, String> iso3 = new HashMap<>();
- // ISO 639-2 bibliographic codes that getISO3Language() never returns.
- iso3.put("dut", "nl"); // Dutch
- iso3.put("fre", "fr"); // French
- iso3.put("ger", "de"); // German
- // Resolve the terminologic three-letter forms once from the JVM locale data.
- for (final Locale locale : Locale.getAvailableLocales()) {
- final String lang = locale.getLanguage();
- if (lang.length() != 2) {
- continue;
- }
- try {
- final String iso3Lang = locale.getISO3Language();
- if (!iso3Lang.isEmpty()) {
- iso3.putIfAbsent(iso3Lang, lang);
- }
- } catch (final MissingResourceException ignored) {
- // locale has no three-letter form; skip it
- }
- }
- ISO6393_TO_ISO6391 = Collections.unmodifiableMap(iso3);
}
private StopwordLists() {
@@ -123,7 +86,7 @@ public final class StopwordLists {
}
LanguageCodeValidator.validateLanguageCode(iso639Code);
- final String normalized = normalizeToIso6391(iso639Code);
+ final String normalized = LanguageCodeValidator.toIso6391(iso639Code);
if (!SUPPORTED_LANGUAGES.contains(normalized)) {
throw new IllegalArgumentException(
@@ -179,25 +142,4 @@ public final class StopwordLists {
}
return new DictionaryStopwordFilter(in, cs, caseSensitive);
}
-
- /**
- * Normalizes an ISO 639-2/3 three-letter code to its ISO 639-1 two-letter
- * equivalent when one is available, otherwise returns the (lower-cased)
- * input unchanged. The caller is responsible for validating the code first
- * via {@link LanguageCodeValidator#validateLanguageCode(String)}.
- * <p>
- * Two-letter inputs are simply lower-cased and returned. Three-letter inputs
- * are resolved with a single lookup against {@link #ISO6393_TO_ISO6391},
- * which is precomputed once at class-initialization time (covering both the
- * terminologic forms produced by {@link Locale#getISO3Language()} and the
- * ISO 639-2 bibliographic forms {@code dut}, {@code fre} and {@code ger}).
- * Unresolved codes are returned lower-cased and unchanged.
- */
- private static String normalizeToIso6391(final String code) {
- final String lower = code.toLowerCase(Locale.ROOT);
- if (lower.length() == 2) {
- return lower;
- }
- return ISO6393_TO_ISO6391.getOrDefault(lower, lower);
- }
}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NormalizationProfiles.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NormalizationProfiles.java
index 70b2f560c..4c3bbf228 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NormalizationProfiles.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NormalizationProfiles.java
@@ -17,14 +17,13 @@
package opennlp.tools.util.normalizer;
import java.util.HashMap;
-import java.util.Locale;
import java.util.Map;
-import java.util.MissingResourceException;
import java.util.Optional;
import java.util.Set;
import opennlp.tools.langdetect.LanguageDetector;
import opennlp.tools.stemmer.snowball.SnowballStemmer;
+import opennlp.tools.util.LanguageCodeValidator;
/**
* A registry of {@link NormalizationProfile}s by language, with detection-based fallback. This is
@@ -36,7 +35,8 @@ import opennlp.tools.stemmer.snowball.SnowballStemmer;
* Norwegian written standards all use {@code NORWEGIAN}).
*
* <p>Profiles are keyed by ISO 639-3 code (what {@link LanguageDetector} produces);
- * {@link #forLanguage(String)} also accepts ISO 639-1 two-letter codes.</p>
+ * {@link #forLanguage(String)} also accepts ISO 639-1 two-letter codes and the ISO 639-2
+ * bibliographic codes {@code dut}, {@code fre} and {@code ger}.</p>
*/
public final class NormalizationProfiles {
@@ -88,8 +88,8 @@ public final class NormalizationProfiles {
/**
* Returns the {@link NormalizationProfile profile} for a language.
*
- * @param language An ISO 639-3 or ISO 639-1 language code; case-insensitive. Must not be
- * {@code null}.
+ * @param language An ISO 639-3, ISO 639-2 bibliographic or ISO 639-1 language code;
+ * case-insensitive. Must not be {@code null}.
* @return The profile, or empty if the language has no Snowball stemmer.
* @throws IllegalArgumentException if {@code language} is {@code null}.
*/
@@ -97,17 +97,7 @@ public final class NormalizationProfiles {
if (language == null) {
throw new IllegalArgumentException("language must not be null");
}
- String code = language.strip().toLowerCase(Locale.ROOT);
- if (code.length() == 2) {
- try {
- final String iso3 = Locale.of(code).getISO3Language();
- if (!iso3.isEmpty()) {
- code = iso3;
- }
- } catch (MissingResourceException ignored) {
- // No ISO 639-3 code for this two-letter code; fall through and look up as given.
- }
- }
+ final String code = LanguageCodeValidator.toIso6393(language.strip());
return Optional.ofNullable(BY_LANGUAGE.get(code));
}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stopword/StopwordListsTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stopword/StopwordListsTest.java
index 067e6288f..98b642d54 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stopword/StopwordListsTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stopword/StopwordListsTest.java
@@ -25,6 +25,7 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.CsvSource;
import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.ValueSource;
@@ -119,6 +120,13 @@ public class StopwordListsTest {
Assertions.assertTrue(en3.isStopword("and"));
}
+ @ParameterizedTest(name = "forLanguage(\"{0}\") shares the filter of \"{1}\"")
+ @CsvSource({"ger, de", "fre, fr", "dut, nl", "deu, de", "fra, fr", "nld, nl"})
+ void testThreeLetterCodeSharesCachedTwoLetterFilter(final String iso3, final String iso1) {
+ Assertions.assertSame(StopwordLists.forLanguage(iso1), StopwordLists.forLanguage(iso3),
+ "'" + iso3 + "' should resolve to the cached '" + iso1 + "' filter");
+ }
+
/**
* The Finnish pronoun and determiner paradigms ship in the Snowball list as
* whitespace-separated columns. OpenNLP's loader treats whitespace on a line
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/LanguageCodeValidatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/LanguageCodeValidatorTest.java
index 6dabe46ec..fb1e85ea1 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/LanguageCodeValidatorTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/LanguageCodeValidatorTest.java
@@ -19,6 +19,8 @@ package opennlp.tools.util;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
class LanguageCodeValidatorTest {
@@ -119,4 +121,66 @@ class LanguageCodeValidatorTest {
Assertions.assertDoesNotThrow(
() -> LanguageCodeValidator.validateLanguageCode("x-unspecified"));
}
+
+ @ParameterizedTest
+ @CsvSource({
+ "nld, nl",
+ "dut, nl",
+ "fra, fr",
+ "fre, fr",
+ "deu, de",
+ "ger, de",
+ "eng, en",
+ "ENG, en",
+ "GER, de",
+ "en, en",
+ "EN, en",
+ "xx, xx",
+ "und, und",
+ "zzz, zzz",
+ "abcd, abcd",
+ "x-unspecified, x-unspecified",
+ })
+ void testToIso6391(String input, String expected) {
+ Assertions.assertEquals(expected, LanguageCodeValidator.toIso6391(input));
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ "de, deu",
+ "DE, deu",
+ "fr, fra",
+ "nl, nld",
+ "nb, nob",
+ "nn, nno",
+ "no, nor",
+ "en, eng",
+ "ger, deu",
+ "fre, fra",
+ "DUT, nld",
+ "deu, deu",
+ "eng, eng",
+ "xx, xx",
+ "und, und",
+ "zzz, zzz",
+ "abcd, abcd",
+ "x-unspecified, x-unspecified",
+ })
+ void testToIso6393(String input, String expected) {
+ Assertions.assertEquals(expected, LanguageCodeValidator.toIso6393(input));
+ }
+
+ @Test
+ void testToIso6391And6393EmptyCode() {
+ Assertions.assertEquals("", LanguageCodeValidator.toIso6391(""));
+ Assertions.assertEquals("", LanguageCodeValidator.toIso6393(""));
+ }
+
+ @Test
+ void testToIso6391And6393NullCode() {
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanguageCodeValidator.toIso6391(null));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanguageCodeValidator.toIso6393(null));
+ }
}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NormalizationProfilesTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NormalizationProfilesTest.java
index 149139847..f0f1207d9 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NormalizationProfilesTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NormalizationProfilesTest.java
@@ -63,6 +63,16 @@ public class NormalizationProfilesTest {
NormalizationProfiles.forLanguage("de").orElseThrow().stemmerAlgorithm());
}
+ @Test
+ void testBibliographicCodesResolveToProfile() {
+ assertEquals(SnowballStemmer.ALGORITHM.GERMAN,
+ NormalizationProfiles.forLanguage("ger").orElseThrow().stemmerAlgorithm());
+ assertEquals(SnowballStemmer.ALGORITHM.FRENCH,
+ NormalizationProfiles.forLanguage("fre").orElseThrow().stemmerAlgorithm());
+ assertEquals(SnowballStemmer.ALGORITHM.DUTCH,
+ NormalizationProfiles.forLanguage("DUT").orElseThrow().stemmerAlgorithm());
+ }
+
@Test
void testNorwegianWrittenStandardsResolveToTheNorwegianProfile() {
// "nb" (Bokmal) and "nn" (Nynorsk) are the standard modern written codes; both convert to the