You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@opennlp.apache.org by "ASF GitHub Bot (JIRA)" <ji...@apache.org> on 2018/05/24 18:18:00 UTC
[jira] [Commented] (OPENNLP-1178) Add language code constants
[ https://issues.apache.org/jira/browse/OPENNLP-1178?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16489523#comment-16489523 ]
ASF GitHub Bot commented on OPENNLP-1178:
-----------------------------------------
jzonthemtn closed pull request #303: OPENNLP-1178: Replacing hardcoded languages with constants.
URL: https://github.com/apache/opennlp/pull/303
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/ParserTrainerTool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/ParserTrainerTool.java
index 2709fd591..c1f6edb64 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/ParserTrainerTool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/ParserTrainerTool.java
@@ -34,6 +34,7 @@
import opennlp.tools.parser.ParserModel;
import opennlp.tools.parser.ParserType;
import opennlp.tools.parser.chunking.Parser;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.ext.ExtensionLoader;
import opennlp.tools.util.model.ArtifactSerializer;
@@ -90,10 +91,10 @@ static HeadRules creaeHeadRules(TrainerToolParams params) throws IOException {
params.getHeadRulesSerializerImpl());
}
else {
- if ("en".equals(params.getLang()) || "eng".equals(params.getLang())) {
+ if ("en".equals(params.getLang()) || LanguageCode.ENGLISH.equals(params.getLang())) {
headRulesSerializer = new opennlp.tools.parser.lang.en.HeadRules.HeadRulesSerializer();
}
- else if ("es".equals(params.getLang()) || "spa".equals(params.getLang())) {
+ else if ("es".equals(params.getLang()) || LanguageCode.SPANISH.equals(params.getLang())) {
headRulesSerializer = new opennlp.tools.parser.lang.es.AncoraSpanishHeadRules.HeadRulesSerializer();
}
else {
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/AbstractSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/AbstractSampleStreamFactory.java
index 33d0f95a2..dc273c1f3 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/AbstractSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/AbstractSampleStreamFactory.java
@@ -18,6 +18,7 @@
package opennlp.tools.formats;
import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.util.LanguageCode;
/**
* Base class for sample stream factories.
@@ -34,7 +35,7 @@ private AbstractSampleStreamFactory() {
}
public String getLang() {
- return "eng";
+ return LanguageCode.ENGLISH.getCode();
}
@SuppressWarnings({"unchecked"})
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStreamFactory.java
index f19328a2f..2630a1bbb 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStreamFactory.java
@@ -27,6 +27,7 @@
import opennlp.tools.cmdline.params.BasicFormatParams;
import opennlp.tools.formats.Conll02NameSampleStream.LANGUAGE;
import opennlp.tools.namefind.NameSample;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.ObjectStream;
/**
@@ -56,11 +57,11 @@ public static void registerFactory() {
Parameters params = ArgumentParser.parse(args, Parameters.class);
LANGUAGE lang;
- if ("nl".equals(params.getLang()) || "nld".equals(params.getLang())) {
+ if ("nl".equals(params.getLang()) || LanguageCode.DUTCH.equals(params.getLang())) {
lang = LANGUAGE.NLD;
language = params.getLang();
}
- else if ("es".equals(params.getLang()) || "spa".equals(params.getLang())) {
+ else if ("es".equals(params.getLang()) || LanguageCode.SPANISH.equals(params.getLang())) {
lang = LANGUAGE.SPA;
language = params.getLang();
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStreamFactory.java
index 599d48a2a..26401dce3 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStreamFactory.java
@@ -27,6 +27,7 @@
import opennlp.tools.cmdline.params.BasicFormatParams;
import opennlp.tools.formats.Conll03NameSampleStream.LANGUAGE;
import opennlp.tools.namefind.NameSample;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.ObjectStream;
public class Conll03NameSampleStreamFactory extends LanguageSampleStreamFactory<NameSample> {
@@ -54,11 +55,11 @@ public static void registerFactory() {
// TODO: support the other languages with this CoNLL.
LANGUAGE lang;
- if ("eng".equals(params.getLang())) {
+ if (LanguageCode.ENGLISH.equals(params.getLang())) {
lang = LANGUAGE.EN;
language = params.getLang();
}
- else if ("deu".equals(params.getLang())) {
+ else if (LanguageCode.GERMAN.equals(params.getLang())) {
lang = LANGUAGE.DE;
language = params.getLang();
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/LanguageCode.java b/opennlp-tools/src/main/java/opennlp/tools/util/LanguageCode.java
new file mode 100644
index 000000000..26bdccefd
--- /dev/null
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/LanguageCode.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util;
+
+import java.util.Objects;
+
+/**
+ * Represents a language using ISO-639-3 codes.
+ *
+ */
+public final class LanguageCode {
+
+ public static final LanguageCode ARABIC = new LanguageCode("aao");
+ public static final LanguageCode BENGALI = new LanguageCode("ben");
+ public static final LanguageCode DUTCH = new LanguageCode("nld");
+ public static final LanguageCode DEUTSCH = new LanguageCode("deu");
+ public static final LanguageCode ENGLISH = new LanguageCode("eng");
+ public static final LanguageCode FRENCH = new LanguageCode("fra");
+ public static final LanguageCode GERMAN = new LanguageCode("deu");
+ public static final LanguageCode HAUSA = new LanguageCode("hau");
+ public static final LanguageCode ITALIAN = new LanguageCode("ita");
+ public static final LanguageCode JAPANESE = new LanguageCode("jpn");
+ public static final LanguageCode KOREAN = new LanguageCode("kor");
+ public static final LanguageCode MALAY = new LanguageCode("mal");
+ public static final LanguageCode PERSIAN = new LanguageCode("prp");
+ public static final LanguageCode PORTUGUESE = new LanguageCode("por");
+ public static final LanguageCode PUNJABI = new LanguageCode("pan");
+ public static final LanguageCode RUSSIAN = new LanguageCode("rus");
+ public static final LanguageCode SPANISH = new LanguageCode("spa");
+ public static final LanguageCode SWAHILI = new LanguageCode("swa");
+
+ private final String code;
+
+ /**
+ * Creates a new <code>code</code>. The code will be converted
+ * to lowercase to conform with ISO 639-3 recommendation of
+ * using lowercase codes.
+ * @param code The language code. It must not be <code>null</code>
+ * and three characters in length.
+ */
+ public LanguageCode(String code) {
+
+ if (code == null || code.length() != 3) {
+ throw new IllegalArgumentException("Code must be not null and 3 characters.");
+ }
+ this.code = code.toLowerCase();
+
+ }
+
+ /**
+ * Gets the language code.
+ */
+ public String getCode() {
+ return code;
+ }
+
+ @Override
+ public String toString() {
+ return code;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+
+ if (obj instanceof LanguageCode) {
+ LanguageCode a = (LanguageCode) obj;
+ return getCode().equals(a.getCode());
+ }
+
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(code);
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerFactoryTest.java
index 42051975b..638d06a8d 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerFactoryTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerFactoryTest.java
@@ -26,6 +26,7 @@
import org.junit.Test;
import opennlp.tools.formats.ResourceAsStreamFactory;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.TrainingParameters;
@@ -47,7 +48,7 @@
private static ChunkerModel trainModel(ModelType type, ChunkerFactory factory)
throws IOException {
- return ChunkerME.train("eng", createSampleStream(),
+ return ChunkerME.train(LanguageCode.ENGLISH.getCode(), createSampleStream(),
TrainingParameters.defaultParams(), factory);
}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerMETest.java b/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerMETest.java
index b7654fba0..c22d10e8b 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerMETest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerMETest.java
@@ -28,6 +28,7 @@
import opennlp.tools.formats.ResourceAsStreamFactory;
import opennlp.tools.namefind.NameFinderME;
import opennlp.tools.util.InsufficientTrainingDataException;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Sequence;
@@ -78,7 +79,8 @@ public void startup() throws IOException {
params.put(TrainingParameters.ITERATIONS_PARAM, 70);
params.put(TrainingParameters.CUTOFF_PARAM, 1);
- ChunkerModel chunkerModel = ChunkerME.train("eng", sampleStream, params, new ChunkerFactory());
+ ChunkerModel chunkerModel = ChunkerME.train(LanguageCode.ENGLISH.getCode(), sampleStream,
+ params, new ChunkerFactory());
this.chunker = new ChunkerME(chunkerModel);
}
@@ -143,7 +145,7 @@ public void testInsufficientData() throws IOException {
params.put(TrainingParameters.ITERATIONS_PARAM, 70);
params.put(TrainingParameters.CUTOFF_PARAM, 1);
- ChunkerME.train("eng", sampleStream, params, new ChunkerFactory());
+ ChunkerME.train(LanguageCode.ENGLISH.getCode(), sampleStream, params, new ChunkerFactory());
}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/cmdline/TokenNameFinderToolTest.java b/opennlp-tools/src/test/java/opennlp/tools/cmdline/TokenNameFinderToolTest.java
index e76814ae8..8cb4023ca 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/cmdline/TokenNameFinderToolTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/cmdline/TokenNameFinderToolTest.java
@@ -36,6 +36,7 @@
import opennlp.tools.namefind.NameSampleDataStream;
import opennlp.tools.namefind.TokenNameFinderFactory;
import opennlp.tools.namefind.TokenNameFinderModel;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.MockInputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
@@ -111,8 +112,8 @@ private File trainModel() throws IOException {
TokenNameFinderFactory nameFinderFactory = new TokenNameFinderFactory();
try (ObjectStream<NameSample> sampleStream = new NameSampleDataStream(lineStream)) {
- model = NameFinderME.train("eng", null, sampleStream, params,
- nameFinderFactory);
+ model = NameFinderME.train(LanguageCode.ENGLISH.getCode(), null, sampleStream,
+ params, nameFinderFactory);
}
File modelFile = File.createTempFile("model", ".bin");
diff --git a/opennlp-tools/src/test/java/opennlp/tools/eval/Conll00ChunkerEval.java b/opennlp-tools/src/test/java/opennlp/tools/eval/Conll00ChunkerEval.java
index 80a0a74e2..bbc5920b9 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/eval/Conll00ChunkerEval.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/eval/Conll00ChunkerEval.java
@@ -34,6 +34,7 @@
import opennlp.tools.chunker.ChunkerFactory;
import opennlp.tools.chunker.ChunkerME;
import opennlp.tools.chunker.ChunkerModel;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.MarkableFileInputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
@@ -59,7 +60,8 @@ private static ChunkerModel train(File trainFile, TrainingParameters params)
new PlainTextByLineStream(
new MarkableFileInputStreamFactory(trainFile), StandardCharsets.UTF_8));
- return ChunkerME.train("eng", samples, params, new ChunkerFactory());
+ return ChunkerME.train(LanguageCode.ENGLISH.getCode(), samples,
+ params, new ChunkerFactory());
}
private static void eval(ChunkerModel model, File testData,
diff --git a/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4NameFinderEval.java b/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4NameFinderEval.java
index 1ce225af9..c45510143 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4NameFinderEval.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4NameFinderEval.java
@@ -42,6 +42,7 @@
import opennlp.tools.namefind.NameSampleTypeFilter;
import opennlp.tools.namefind.TokenNameFinderCrossValidator;
import opennlp.tools.namefind.TokenNameFinderFactory;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.model.ModelUtil;
@@ -67,8 +68,8 @@ private void crossEval(TrainingParameters params, String type, double expectedSc
throws IOException {
try (ObjectStream<NameSample> samples = createNameSampleStream()) {
- TokenNameFinderCrossValidator cv = new TokenNameFinderCrossValidator("eng", null,
- params, new TokenNameFinderFactory());
+ TokenNameFinderCrossValidator cv = new TokenNameFinderCrossValidator(
+ LanguageCode.ENGLISH.getCode(), null, params, new TokenNameFinderFactory());
ObjectStream<NameSample> filteredSamples;
if (type != null) {
@@ -140,8 +141,8 @@ public void evalAllTypesWithPOSNameFinder() throws IOException, URISyntaxExcepti
try (ObjectStream<NameSample> samples = createNameSampleStream()) {
- TokenNameFinderCrossValidator cv = new TokenNameFinderCrossValidator("eng", null,
- params, featureGen, resources);
+ TokenNameFinderCrossValidator cv = new TokenNameFinderCrossValidator(
+ LanguageCode.ENGLISH.getCode(), null, params, featureGen, resources);
ObjectStream<NameSample> filteredSamples;
diff --git a/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4ParserEval.java b/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4ParserEval.java
index 31c7ff766..e722bd1c5 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4ParserEval.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4ParserEval.java
@@ -37,6 +37,7 @@
import opennlp.tools.parser.ParserCrossValidator;
import opennlp.tools.parser.ParserType;
import opennlp.tools.parser.lang.en.HeadRulesTest;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.model.ModelUtil;
@@ -62,7 +63,8 @@
private void crossEval(TrainingParameters params, HeadRules rules, double expectedScore)
throws IOException {
try (ObjectStream<Parse> samples = createParseSampleStream()) {
- ParserCrossValidator cv = new ParserCrossValidator("eng", params, rules, ParserType.CHUNKING);
+ ParserCrossValidator cv = new ParserCrossValidator(LanguageCode.ENGLISH.getCode(),
+ params, rules, ParserType.CHUNKING);
cv.evaluate(samples, 5);
Assert.assertEquals(expectedScore, cv.getFMeasure().getFMeasure(), 0.0001d);
diff --git a/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4PosTaggerEval.java b/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4PosTaggerEval.java
index a373192be..88a5ef9d5 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4PosTaggerEval.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4PosTaggerEval.java
@@ -20,7 +20,6 @@
import java.io.File;
import java.io.IOException;
import java.math.BigInteger;
-
import java.nio.charset.StandardCharsets;
import org.junit.Assert;
@@ -35,6 +34,7 @@
import opennlp.tools.postag.POSSample;
import opennlp.tools.postag.POSTaggerCrossValidator;
import opennlp.tools.postag.POSTaggerFactory;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.model.ModelUtil;
@@ -60,7 +60,8 @@
private void crossEval(TrainingParameters params, double expectedScore)
throws IOException {
try (ObjectStream<POSSample> samples = createPOSSampleStream()) {
- POSTaggerCrossValidator cv = new POSTaggerCrossValidator("eng", params, new POSTaggerFactory());
+ POSTaggerCrossValidator cv = new POSTaggerCrossValidator(LanguageCode.ENGLISH.getCode(),
+ params, new POSTaggerFactory());
cv.evaluate(samples, 5);
Assert.assertEquals(expectedScore, cv.getWordAccuracy(), 0.0001d);
diff --git a/opennlp-tools/src/test/java/opennlp/tools/eval/SourceForgeModelEval.java b/opennlp-tools/src/test/java/opennlp/tools/eval/SourceForgeModelEval.java
index cc86b5e2c..21ad63d49 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/eval/SourceForgeModelEval.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/eval/SourceForgeModelEval.java
@@ -57,6 +57,7 @@
import opennlp.tools.tokenize.WhitespaceTokenizer;
import opennlp.tools.util.FilterObjectStream;
import opennlp.tools.util.InputStreamFactory;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.MarkableFileInputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
@@ -102,7 +103,7 @@ private LeipzigTestSample(String[] text) {
@Override
public String toString() {
- StringBuilder sampleString = new StringBuilder("eng");
+ StringBuilder sampleString = new StringBuilder(LanguageCode.ENGLISH.getCode());
sampleString.append('\t');
diff --git a/opennlp-tools/src/test/java/opennlp/tools/eval/UniversalDependency20Eval.java b/opennlp-tools/src/test/java/opennlp/tools/eval/UniversalDependency20Eval.java
index 3ab1a7d33..e4782cd3f 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/eval/UniversalDependency20Eval.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/eval/UniversalDependency20Eval.java
@@ -33,6 +33,7 @@
import opennlp.tools.lemmatizer.LemmatizerFactory;
import opennlp.tools.lemmatizer.LemmatizerME;
import opennlp.tools.lemmatizer.LemmatizerModel;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.MarkableFileInputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.TrainingParameters;
@@ -76,7 +77,7 @@ public void trainAndEvalSpanishAncora() throws IOException {
TrainingParameters params = ModelUtil.createDefaultTrainingParameters();
params.put("Threads", "4");
- double wordAccuracy = trainAndEval("spa", SPA_ANCORA_TRAIN,
+ double wordAccuracy = trainAndEval(LanguageCode.SPANISH.getCode(), SPA_ANCORA_TRAIN,
params, SPA_ANCORA_DEV);
Assert.assertEquals(0.9057341692068787d, wordAccuracy, ACCURACY_DELTA);
diff --git a/opennlp-tools/src/test/java/opennlp/tools/langdetect/LanguageDetectorEvaluatorTest.java b/opennlp-tools/src/test/java/opennlp/tools/langdetect/LanguageDetectorEvaluatorTest.java
index 5e8b2eae4..2efd7198d 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/langdetect/LanguageDetectorEvaluatorTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/langdetect/LanguageDetectorEvaluatorTest.java
@@ -25,6 +25,7 @@
import org.junit.Test;
import opennlp.tools.cmdline.langdetect.LanguageDetectorEvaluationErrorListener;
+import opennlp.tools.util.LanguageCode;
public class LanguageDetectorEvaluatorTest {
@@ -57,10 +58,10 @@ public void missclassified(LanguageSample reference,
evaluator.evaluateSample(new LanguageSample(new Language("pob"),
"escreve e faz palestras pelo mundo inteiro sobre anjos"));
- evaluator.evaluateSample(new LanguageSample(new Language("fra"),
+ evaluator.evaluateSample(new LanguageSample(new Language(LanguageCode.FRENCH.getCode()),
"escreve e faz palestras pelo mundo inteiro sobre anjos"));
- evaluator.evaluateSample(new LanguageSample(new Language("fra"),
+ evaluator.evaluateSample(new LanguageSample(new Language(LanguageCode.FRENCH.getCode()),
"escreve e faz palestras pelo mundo inteiro sobre anjos"));
diff --git a/opennlp-tools/src/test/java/opennlp/tools/langdetect/LanguageDetectorMETest.java b/opennlp-tools/src/test/java/opennlp/tools/langdetect/LanguageDetectorMETest.java
index e5ee8aa54..99c25e093 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/langdetect/LanguageDetectorMETest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/langdetect/LanguageDetectorMETest.java
@@ -25,6 +25,7 @@
import org.junit.Test;
import opennlp.tools.formats.ResourceAsStreamFactory;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.TrainingParameters;
@@ -47,9 +48,9 @@ public void testPredictLanguages() {
Assert.assertEquals(4, languages.length);
Assert.assertEquals("pob", languages[0].getLang());
- Assert.assertEquals("ita", languages[1].getLang());
- Assert.assertEquals("spa", languages[2].getLang());
- Assert.assertEquals("fra", languages[3].getLang());
+ Assert.assertEquals(LanguageCode.ITALIAN.getCode(), languages[1].getLang());
+ Assert.assertEquals(LanguageCode.SPANISH.getCode(), languages[2].getLang());
+ Assert.assertEquals(LanguageCode.FRENCH.getCode(), languages[3].getLang());
}
@Test
@@ -57,7 +58,7 @@ public void testPredictLanguage() {
LanguageDetector ld = new LanguageDetectorME(this.model);
Language language = ld.predictLanguage("Dove รจ meglio che giochi");
- Assert.assertEquals("ita", language.getLang());
+ Assert.assertEquals(LanguageCode.ITALIAN.getCode(), language.getLang());
}
@Test
diff --git a/opennlp-tools/src/test/java/opennlp/tools/lemmatizer/LemmatizerMETest.java b/opennlp-tools/src/test/java/opennlp/tools/lemmatizer/LemmatizerMETest.java
index f925cb359..f975576cd 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/lemmatizer/LemmatizerMETest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/lemmatizer/LemmatizerMETest.java
@@ -25,6 +25,7 @@
import org.junit.Test;
import opennlp.tools.util.InsufficientTrainingDataException;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.MockInputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
@@ -72,8 +73,8 @@ public void startup() throws IOException {
params.put(TrainingParameters.ITERATIONS_PARAM, 100);
params.put(TrainingParameters.CUTOFF_PARAM, 5);
- LemmatizerModel lemmatizerModel = LemmatizerME.train("eng", sampleStream,
- params, new LemmatizerFactory());
+ LemmatizerModel lemmatizerModel = LemmatizerME.train(LanguageCode.ENGLISH.getCode(),
+ sampleStream, params, new LemmatizerFactory());
this.lemmatizer = new LemmatizerME(lemmatizerModel);
}
@@ -98,7 +99,8 @@ public void testInsufficientData() throws IOException {
params.put(TrainingParameters.ITERATIONS_PARAM, 100);
params.put(TrainingParameters.CUTOFF_PARAM, 5);
- LemmatizerME.train("eng", sampleStream, params, new LemmatizerFactory());
+ LemmatizerME.train(LanguageCode.ENGLISH.getCode(), sampleStream,
+ params, new LemmatizerFactory());
}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/namefind/NameFinderMETest.java b/opennlp-tools/src/test/java/opennlp/tools/namefind/NameFinderMETest.java
index 740e7d168..6ba21bd17 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/namefind/NameFinderMETest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/namefind/NameFinderMETest.java
@@ -24,6 +24,7 @@
import org.junit.Test;
import opennlp.tools.ml.model.SequenceClassificationModel;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.MockInputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
@@ -68,7 +69,8 @@ public void testNameFinder() throws Exception {
params.put(TrainingParameters.ITERATIONS_PARAM, 70);
params.put(TrainingParameters.CUTOFF_PARAM, 1);
- TokenNameFinderModel nameFinderModel = NameFinderME.train("eng", null, sampleStream,
+ TokenNameFinderModel nameFinderModel = NameFinderME.train(LanguageCode.ENGLISH.getCode(),
+ null, sampleStream,
params, TokenNameFinderFactory.create(null, null, Collections.emptyMap(), new BioCodec()));
TokenNameFinder nameFinder = new NameFinderME(nameFinderModel);
@@ -126,7 +128,8 @@ public void testNameFinderWithTypes() throws Exception {
params.put(TrainingParameters.ITERATIONS_PARAM, 70);
params.put(TrainingParameters.CUTOFF_PARAM, 1);
- TokenNameFinderModel nameFinderModel = NameFinderME.train("eng", null, sampleStream,
+ TokenNameFinderModel nameFinderModel = NameFinderME.train(LanguageCode.ENGLISH.getCode(),
+ null, sampleStream,
params, TokenNameFinderFactory.create(null, null, Collections.emptyMap(), new BioCodec()));
NameFinderME nameFinder = new NameFinderME(nameFinderModel);
@@ -170,8 +173,9 @@ public void testOnlyWithNames() throws Exception {
params.put(TrainingParameters.ITERATIONS_PARAM, 70);
params.put(TrainingParameters.CUTOFF_PARAM, 1);
- TokenNameFinderModel nameFinderModel = NameFinderME.train("eng", null, sampleStream,
- params, TokenNameFinderFactory.create(null, null, Collections.emptyMap(), new BioCodec()));
+ TokenNameFinderModel nameFinderModel = NameFinderME.train(LanguageCode.ENGLISH.getCode(),
+ null, sampleStream,
+ params, TokenNameFinderFactory.create(null, null, Collections.emptyMap(), new BioCodec()));
NameFinderME nameFinder = new NameFinderME(nameFinderModel);
@@ -200,8 +204,9 @@ public void testOnlyWithNamesTypeOverride() throws Exception {
params.put(TrainingParameters.ITERATIONS_PARAM, 70);
params.put(TrainingParameters.CUTOFF_PARAM, 1);
- TokenNameFinderModel nameFinderModel = NameFinderME.train("eng", TYPE_OVERRIDE, sampleStream,
- params, TokenNameFinderFactory.create(null, null, Collections.emptyMap(), new BioCodec()));
+ TokenNameFinderModel nameFinderModel = NameFinderME.train(LanguageCode.ENGLISH.getCode(),
+ TYPE_OVERRIDE, sampleStream, params,
+ TokenNameFinderFactory.create(null, null, Collections.emptyMap(), new BioCodec()));
NameFinderME nameFinder = new NameFinderME(nameFinderModel);
@@ -235,7 +240,8 @@ public void testOnlyWithNamesWithTypes() throws Exception {
params.put(TrainingParameters.ITERATIONS_PARAM, 70);
params.put(TrainingParameters.CUTOFF_PARAM, 1);
- TokenNameFinderModel nameFinderModel = NameFinderME.train("eng", null, sampleStream,
+ TokenNameFinderModel nameFinderModel = NameFinderME.train(LanguageCode.ENGLISH.getCode(),
+ null, sampleStream,
params, TokenNameFinderFactory.create(null, null, Collections.emptyMap(), new BioCodec()));
NameFinderME nameFinder = new NameFinderME(nameFinderModel);
@@ -271,7 +277,8 @@ public void testOnlyWithEntitiesWithTypes() throws Exception {
params.put(TrainingParameters.ITERATIONS_PARAM, 70);
params.put(TrainingParameters.CUTOFF_PARAM, 1);
- TokenNameFinderModel nameFinderModel = NameFinderME.train("eng", null, sampleStream,
+ TokenNameFinderModel nameFinderModel = NameFinderME.train(LanguageCode.ENGLISH.getCode(),
+ null, sampleStream,
params, TokenNameFinderFactory.create(null, null, Collections.emptyMap(), new BioCodec()));
NameFinderME nameFinder = new NameFinderME(nameFinderModel);
@@ -322,7 +329,8 @@ public void testNameFinderWithMultipleTypes() throws Exception {
params.put(TrainingParameters.ITERATIONS_PARAM, 70);
params.put(TrainingParameters.CUTOFF_PARAM, 1);
- TokenNameFinderModel nameFinderModel = NameFinderME.train("eng", null, sampleStream,
+ TokenNameFinderModel nameFinderModel = NameFinderME.train(LanguageCode.ENGLISH.getCode(),
+ null, sampleStream,
params, TokenNameFinderFactory.create(null, null, Collections.emptyMap(), new BioCodec()));
NameFinderME nameFinder = new NameFinderME(nameFinderModel);
diff --git a/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderCrossValidatorTest.java b/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderCrossValidatorTest.java
index 1f4b5a636..10a842be9 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderCrossValidatorTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderCrossValidatorTest.java
@@ -29,6 +29,7 @@
import opennlp.tools.formats.ResourceAsStreamFactory;
import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.InsufficientTrainingDataException;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.TrainingParameters;
@@ -57,8 +58,9 @@ public void testWithNullResources() throws Exception {
mlParams.put(TrainingParameters.ALGORITHM_PARAM,
ModelType.MAXENT.toString());
- TokenNameFinderCrossValidator cv = new TokenNameFinderCrossValidator("eng",
- TYPE, mlParams, null, (TokenNameFinderEvaluationMonitor)null);
+ TokenNameFinderCrossValidator cv = new TokenNameFinderCrossValidator(
+ LanguageCode.ENGLISH.getCode(), TYPE, mlParams, null,
+ (TokenNameFinderEvaluationMonitor)null);
cv.evaluate(sampleStream, 2);
@@ -88,8 +90,8 @@ public void testWithNameEvaluationErrorListener() throws Exception {
NameEvaluationErrorListener listener = new NameEvaluationErrorListener(out);
Map<String, Object> resources = Collections.emptyMap();
- TokenNameFinderCrossValidator cv = new TokenNameFinderCrossValidator("eng",
- TYPE, mlParams, null, resources, listener);
+ TokenNameFinderCrossValidator cv = new TokenNameFinderCrossValidator(
+ LanguageCode.ENGLISH.getCode(), TYPE, mlParams, null, resources, listener);
cv.evaluate(sampleStream, 2);
@@ -113,7 +115,8 @@ public void testWithInsufficientData() throws Exception {
mlParams.put(TrainingParameters.ALGORITHM_PARAM,
ModelType.MAXENT.toString());
- TokenNameFinderCrossValidator cv = new TokenNameFinderCrossValidator("eng",
+ TokenNameFinderCrossValidator cv = new TokenNameFinderCrossValidator(
+ LanguageCode.ENGLISH.getCode(),
TYPE, mlParams, null, (TokenNameFinderEvaluationMonitor)null);
cv.evaluate(sampleStream, 2);
diff --git a/opennlp-tools/src/test/java/opennlp/tools/parser/chunking/ParserTest.java b/opennlp-tools/src/test/java/opennlp/tools/parser/chunking/ParserTest.java
index 277a4e5eb..3cc5496b0 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/parser/chunking/ParserTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/parser/chunking/ParserTest.java
@@ -27,6 +27,7 @@
import opennlp.tools.parser.ParserFactory;
import opennlp.tools.parser.ParserModel;
import opennlp.tools.parser.ParserTestUtil;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.TrainingParameters;
@@ -45,7 +46,7 @@ public void testChunkingParserTraining() throws Exception {
ObjectStream<Parse> parseSamples = ParserTestUtil.openTestTrainingData();
HeadRules headRules = ParserTestUtil.createTestHeadRules();
- ParserModel model = Parser.train("eng", parseSamples, headRules,
+ ParserModel model = Parser.train(LanguageCode.ENGLISH.getCode(), parseSamples, headRules,
TrainingParameters.defaultParams());
opennlp.tools.parser.Parser parser = ParserFactory.create(model);
diff --git a/opennlp-tools/src/test/java/opennlp/tools/parser/treeinsert/ParserTest.java b/opennlp-tools/src/test/java/opennlp/tools/parser/treeinsert/ParserTest.java
index a8c00155c..f177432e5 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/parser/treeinsert/ParserTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/parser/treeinsert/ParserTest.java
@@ -27,6 +27,7 @@
import opennlp.tools.parser.ParserFactory;
import opennlp.tools.parser.ParserModel;
import opennlp.tools.parser.ParserTestUtil;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.ObjectStream;
/**
@@ -44,7 +45,8 @@ public void testTreeInsertParserTraining() throws Exception {
ObjectStream<Parse> parseSamples = ParserTestUtil.openTestTrainingData();
HeadRules headRules = ParserTestUtil.createTestHeadRules();
- ParserModel model = Parser.train("eng", parseSamples, headRules, 100, 0);
+ ParserModel model = Parser.train(LanguageCode.ENGLISH.getCode(), parseSamples,
+ headRules, 100, 0);
opennlp.tools.parser.Parser parser = ParserFactory.create(model);
diff --git a/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerFactoryTest.java
index 6f3443b5c..c616c0279 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerFactoryTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerFactoryTest.java
@@ -32,6 +32,7 @@
import opennlp.tools.util.BaseToolFactory;
import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.InvalidFormatException;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.TrainingParameters;
@@ -52,7 +53,7 @@
private static POSModel trainPOSModel(POSTaggerFactory factory)
throws IOException {
- return POSTaggerME.train("eng", createSampleStream(),
+ return POSTaggerME.train(LanguageCode.ENGLISH.getCode(), createSampleStream(),
TrainingParameters.defaultParams(), factory);
}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerMETest.java b/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerMETest.java
index 065fe1567..cdacba872 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerMETest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerMETest.java
@@ -26,6 +26,7 @@
import opennlp.tools.formats.ResourceAsStreamFactory;
import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.InsufficientTrainingDataException;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.TrainingParameters;
@@ -54,7 +55,7 @@ public static POSModel trainPOSModel(ModelType type) throws IOException {
params.put(TrainingParameters.ITERATIONS_PARAM, 100);
params.put(TrainingParameters.CUTOFF_PARAM, 5);
- return POSTaggerME.train("eng", createSampleStream(), params,
+ return POSTaggerME.train(LanguageCode.ENGLISH.getCode(), createSampleStream(), params,
new POSTaggerFactory());
}
@@ -101,7 +102,7 @@ public void insufficientTestData() throws IOException {
params.put(TrainingParameters.ITERATIONS_PARAM, 100);
params.put(TrainingParameters.CUTOFF_PARAM, 5);
- POSTaggerME.train("eng", stream, params, new POSTaggerFactory());
+ POSTaggerME.train(LanguageCode.ENGLISH.getCode(), stream, params, new POSTaggerFactory());
}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/sentdetect/SDEventStreamTest.java b/opennlp-tools/src/test/java/opennlp/tools/sentdetect/SDEventStreamTest.java
index 138e915ff..e90d22c13 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/sentdetect/SDEventStreamTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/sentdetect/SDEventStreamTest.java
@@ -24,6 +24,7 @@
import opennlp.tools.ml.model.Event;
import opennlp.tools.sentdetect.lang.Factory;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.ObjectStreamUtils;
import opennlp.tools.util.Span;
@@ -45,8 +46,8 @@ public void testEventOutcomes() throws IOException {
Factory factory = new Factory();
ObjectStream<Event> eventStream = new SDEventStream(sampleStream,
- factory.createSentenceContextGenerator("eng"),
- factory.createEndOfSentenceScanner("eng"));
+ factory.createSentenceContextGenerator(LanguageCode.ENGLISH.getCode()),
+ factory.createEndOfSentenceScanner(LanguageCode.ENGLISH.getCode()));
Assert.assertEquals(SentenceDetectorME.NO_SPLIT, eventStream.read().getOutcome());
Assert.assertEquals(SentenceDetectorME.SPLIT, eventStream.read().getOutcome());
diff --git a/opennlp-tools/src/test/java/opennlp/tools/sentdetect/SentenceDetectorFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/sentdetect/SentenceDetectorFactoryTest.java
index 06de8991f..66a198cc1 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/sentdetect/SentenceDetectorFactoryTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/sentdetect/SentenceDetectorFactoryTest.java
@@ -34,6 +34,7 @@
import opennlp.tools.sentdetect.DummySentenceDetectorFactory.DummySDContextGenerator;
import opennlp.tools.sentdetect.lang.Factory;
import opennlp.tools.util.InputStreamFactory;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.TrainingParameters;
@@ -55,8 +56,8 @@
private static SentenceModel train(SentenceDetectorFactory factory)
throws IOException {
- return SentenceDetectorME.train("eng", createSampleStream(), factory,
- TrainingParameters.defaultParams());
+ return SentenceDetectorME.train(LanguageCode.ENGLISH.getCode(),
+ createSampleStream(), factory, TrainingParameters.defaultParams());
}
private static Dictionary loadAbbDictionary() throws IOException {
@@ -72,8 +73,8 @@ public void testDefault() throws IOException {
Dictionary dic = loadAbbDictionary();
char[] eos = {'.', '?'};
- SentenceModel sdModel = train(new SentenceDetectorFactory("eng", true, dic,
- eos));
+ SentenceModel sdModel = train(new SentenceDetectorFactory(LanguageCode.ENGLISH.getCode(),
+ true, dic, eos));
SentenceDetectorFactory factory = sdModel.getFactory();
Assert.assertTrue(factory.getSDContextGenerator() instanceof DefaultSDContextGenerator);
@@ -97,8 +98,8 @@ public void testNullDict() throws IOException {
Dictionary dic = null;
char[] eos = {'.', '?'};
- SentenceModel sdModel = train(new SentenceDetectorFactory("eng", true,
- dic, eos));
+ SentenceModel sdModel = train(new SentenceDetectorFactory(LanguageCode.ENGLISH.getCode(),
+ true, dic, eos));
SentenceDetectorFactory factory = sdModel.getFactory();
Assert.assertNull(factory.getAbbreviationDictionary());
@@ -124,8 +125,8 @@ public void testDefaultEOS() throws IOException {
Dictionary dic = null;
char[] eos = null;
- SentenceModel sdModel = train(new SentenceDetectorFactory("eng", true,
- dic, eos));
+ SentenceModel sdModel = train(new SentenceDetectorFactory(LanguageCode.ENGLISH.getCode(),
+ true, dic, eos));
SentenceDetectorFactory factory = sdModel.getFactory();
Assert.assertNull(factory.getAbbreviationDictionary());
@@ -154,8 +155,8 @@ public void testDummyFactory() throws IOException {
Dictionary dic = loadAbbDictionary();
char[] eos = {'.', '?'};
- SentenceModel sdModel = train(new DummySentenceDetectorFactory("eng", true,
- dic, eos));
+ SentenceModel sdModel = train(new DummySentenceDetectorFactory(
+ LanguageCode.ENGLISH.getCode(), true, dic, eos));
SentenceDetectorFactory factory = sdModel.getFactory();
Assert.assertTrue(factory.getAbbreviationDictionary() instanceof DummyDictionary);
@@ -185,8 +186,8 @@ public void testCreateDummyFactory() throws IOException {
char[] eos = {'.', '?'};
SentenceDetectorFactory factory = SentenceDetectorFactory.create(
- DummySentenceDetectorFactory.class.getCanonicalName(), "spa", false,
- dic, eos);
+ DummySentenceDetectorFactory.class.getCanonicalName(), LanguageCode.SPANISH.getCode(),
+ false, dic, eos);
Assert.assertTrue(factory.getAbbreviationDictionary() instanceof DummyDictionary);
Assert.assertTrue(factory.getSDContextGenerator() instanceof DummySDContextGenerator);
diff --git a/opennlp-tools/src/test/java/opennlp/tools/sentdetect/SentenceDetectorMETest.java b/opennlp-tools/src/test/java/opennlp/tools/sentdetect/SentenceDetectorMETest.java
index 87ced1b84..72c4414ca 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/sentdetect/SentenceDetectorMETest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/sentdetect/SentenceDetectorMETest.java
@@ -27,6 +27,7 @@
import opennlp.tools.formats.ResourceAsStreamFactory;
import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.InsufficientTrainingDataException;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;
import opennlp.tools.util.TrainingParameters;
@@ -46,13 +47,14 @@ public void testSentenceDetector() throws IOException {
mlParams.put(TrainingParameters.ITERATIONS_PARAM, 100);
mlParams.put(TrainingParameters.CUTOFF_PARAM, 0);
- SentenceDetectorFactory factory = new SentenceDetectorFactory("eng", true, null, null);
+ SentenceDetectorFactory factory = new SentenceDetectorFactory(LanguageCode.ENGLISH.getCode(),
+ true, null, null);
SentenceModel sentdetectModel = SentenceDetectorME.train(
- "eng", new SentenceSampleStream(new PlainTextByLineStream(in,
- StandardCharsets.UTF_8)), factory, mlParams);
+ LanguageCode.ENGLISH.getCode(), new SentenceSampleStream(new PlainTextByLineStream(in,
+ StandardCharsets.UTF_8)), factory, mlParams);
- Assert.assertEquals("eng", sentdetectModel.getLanguage());
+ Assert.assertEquals(LanguageCode.ENGLISH.getCode(), sentdetectModel.getLanguage());
SentenceDetectorME sentDetect = new SentenceDetectorME(sentdetectModel);
@@ -146,9 +148,10 @@ public void testInsufficientData() throws IOException {
mlParams.put(TrainingParameters.ITERATIONS_PARAM, 100);
mlParams.put(TrainingParameters.CUTOFF_PARAM, 0);
- SentenceDetectorFactory factory = new SentenceDetectorFactory("eng", true, null, null);
+ SentenceDetectorFactory factory = new SentenceDetectorFactory(LanguageCode.ENGLISH.getCode(),
+ true, null, null);
- SentenceDetectorME.train("eng",
+ SentenceDetectorME.train(LanguageCode.ENGLISH.getCode(),
new SentenceSampleStream(
new PlainTextByLineStream(in, StandardCharsets.UTF_8)), factory, mlParams);
diff --git a/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerFactoryTest.java
index a916a32af..4be9ef642 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerFactoryTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerFactoryTest.java
@@ -33,6 +33,7 @@
import opennlp.tools.tokenize.DummyTokenizerFactory.DummyDictionary;
import opennlp.tools.tokenize.lang.Factory;
import opennlp.tools.util.InputStreamFactory;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.TrainingParameters;
@@ -66,7 +67,7 @@ private static Dictionary loadAbbDictionary() throws IOException {
public void testDefault() throws IOException {
Dictionary dic = loadAbbDictionary();
- final String lang = "spa";
+ final String lang = LanguageCode.SPANISH.getCode();
TokenizerModel model = train(new TokenizerFactory(lang, dic, false, null));
@@ -99,7 +100,7 @@ public void testDefault() throws IOException {
public void testNullDict() throws IOException {
Dictionary dic = null;
- final String lang = "spa";
+ final String lang = LanguageCode.SPANISH.getCode();
TokenizerModel model = train(new TokenizerFactory(lang, dic, false, null));
@@ -132,7 +133,7 @@ public void testNullDict() throws IOException {
public void testCustomPatternAndAlphaOpt() throws IOException {
Dictionary dic = null;
- final String lang = "spa";
+ final String lang = LanguageCode.SPANISH.getCode();
String pattern = "^[0-9A-Za-z]+$";
TokenizerModel model = train(new TokenizerFactory(lang, dic, true,
@@ -166,7 +167,7 @@ public void testCustomPatternAndAlphaOpt() throws IOException {
public void testDummyFactory() throws IOException {
Dictionary dic = loadAbbDictionary();
- final String lang = "spa";
+ final String lang = LanguageCode.SPANISH.getCode();
String pattern = "^[0-9A-Za-z]+$";
TokenizerModel model = train(new DummyTokenizerFactory(lang, dic, true,
@@ -198,7 +199,7 @@ public void testDummyFactory() throws IOException {
@Test
public void testCreateDummyFactory() throws IOException {
Dictionary dic = loadAbbDictionary();
- final String lang = "spa";
+ final String lang = LanguageCode.SPANISH.getCode();
String pattern = "^[0-9A-Za-z]+$";
TokenizerFactory factory = TokenizerFactory.create(
diff --git a/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerMETest.java b/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerMETest.java
index a634b070f..aeb7f0572 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerMETest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerMETest.java
@@ -26,6 +26,7 @@
import opennlp.tools.formats.ResourceAsStreamFactory;
import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.InsufficientTrainingDataException;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.TrainingParameters;
@@ -87,7 +88,8 @@ public void testInsufficientData() throws IOException {
mlParams.put(TrainingParameters.ITERATIONS_PARAM, 100);
mlParams.put(TrainingParameters.CUTOFF_PARAM, 5);
- TokenizerME.train(samples, TokenizerFactory.create(null, "eng", null, true, null), mlParams);
+ TokenizerME.train(samples, TokenizerFactory.create(null, LanguageCode.ENGLISH.getCode(),
+ null, true, null), mlParams);
}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerTestUtil.java b/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerTestUtil.java
index 1d43f2255..76a1dd666 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerTestUtil.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerTestUtil.java
@@ -25,6 +25,7 @@
import opennlp.tools.formats.ResourceAsStreamFactory;
import opennlp.tools.util.CollectionObjectStream;
import opennlp.tools.util.InputStreamFactory;
+import opennlp.tools.util.LanguageCode;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;
@@ -58,7 +59,7 @@ static TokenizerModel createSimpleMaxentTokenModel() throws IOException {
mlParams.put(TrainingParameters.CUTOFF_PARAM, 0);
return TokenizerME.train(new CollectionObjectStream<>(samples),
- TokenizerFactory.create(null, "eng", null, true, null), mlParams);
+ TokenizerFactory.create(null, LanguageCode.ENGLISH.getCode(), null, true, null), mlParams);
}
static TokenizerModel createMaxentTokenModel() throws IOException {
@@ -73,7 +74,8 @@ static TokenizerModel createMaxentTokenModel() throws IOException {
mlParams.put(TrainingParameters.ITERATIONS_PARAM, 100);
mlParams.put(TrainingParameters.CUTOFF_PARAM, 0);
- return TokenizerME.train(samples, TokenizerFactory.create(null, "eng", null, true, null), mlParams);
+ return TokenizerME.train(samples, TokenizerFactory.create(null,
+ LanguageCode.ENGLISH.getCode(), null, true, null), mlParams);
}
}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/LanguageCodeTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/LanguageCodeTest.java
new file mode 100644
index 000000000..1822b4276
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/util/LanguageCodeTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class LanguageCodeTest {
+
+ @Test
+ public void equalsTest() {
+
+ Assert.assertEquals(new LanguageCode("eng"), LanguageCode.ENGLISH);
+ Assert.assertNotEquals(new LanguageCode("not"), LanguageCode.ENGLISH);
+ Assert.assertNotEquals(new LanguageCode("not"), LanguageCode.ENGLISH);
+
+ }
+
+}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
> Add language code constants
> ---------------------------
>
> Key: OPENNLP-1178
> URL: https://issues.apache.org/jira/browse/OPENNLP-1178
> Project: OpenNLP
> Issue Type: Task
> Reporter: Jeff Zemerick
> Assignee: Jeff Zemerick
> Priority: Minor
>
> There are many places in the code where languages are referenced by hardcoded strings such as "eng". Add a class that contains constants for the language codes and replace hardcoded string with the constant values. Use ISO 639-2 language codes.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)