You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by ma...@apache.org on 2025/01/18 14:17:13 UTC
(opennlp) 01/01: OPENNLP-1695: Add more tests for classes in formats package - introduces AbstractSampleStreamFactoryTest as common base class - reduces code duplication in format factory classes - adds several many new test classes for format factories (+ sample data) - adds two more Evalita samples taken from Appendix of: https://www.evalita.it/wp-content/uploads/2021/11/Guidelines_evalita09_NER.pdf - adds two OntoNotes samples from the public, official v5.0 release notes (sec 6.4 + 6.8), see: https://ca [...]
This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a commit to branch OPENNLP-1695-Add-more-tests-for-classes-in-formats-package
in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 90e3bc4ffb9a3a013eca81021f9fa29fcf885872
Author: Martin Wiesner <ma...@hs-heilbronn.de>
AuthorDate: Thu Jan 16 12:09:51 2025 +0100
OPENNLP-1695: Add more tests for classes in formats package
- introduces AbstractSampleStreamFactoryTest as common base class
- reduces code duplication in format factory classes
- adds several many new test classes for format factories (+ sample data)
- adds two more Evalita samples taken from Appendix of: https://www.evalita.it/wp-content/uploads/2021/11/Guidelines_evalita09_NER.pdf
- adds two OntoNotes samples from the public, official v5.0 release notes (sec 6.4 + 6.8), see: https://catalog.ldc.upenn.edu/docs/LDC2013T19/OntoNotes-Release-5.0.pdf
---
.../opennlp/tools/chunker/ChunkSampleStream.java | 12 +-
.../tools/formats/AbstractSampleStreamFactory.java | 48 +++++++
.../tools/formats/BioNLP2004NameSampleStream.java | 38 +++---
.../formats/BioNLP2004NameSampleStreamFactory.java | 30 ++---
.../tools/formats/ChunkerSampleStreamFactory.java | 28 +---
.../formats/Conll02NameSampleStreamFactory.java | 13 +-
.../formats/Conll03NameSampleStreamFactory.java | 12 +-
.../formats/ConllXPOSSampleStreamFactory.java | 26 ++--
.../formats/ConllXSentenceSampleStreamFactory.java | 16 ++-
.../formats/ConllXTokenSampleStreamFactory.java | 14 +-
.../tools/formats/DocumentSampleStreamFactory.java | 27 +---
.../tools/formats/EvalitaNameSampleStream.java | 59 +++++----
.../formats/EvalitaNameSampleStreamFactory.java | 13 +-
.../LanguageDetectorSampleStreamFactory.java | 31 +----
.../formats/LemmatizerSampleStreamFactory.java | 28 +---
.../tools/formats/NameSampleDataStreamFactory.java | 29 +----
.../tools/formats/ParseSampleStreamFactory.java | 26 +---
.../tools/formats/SentenceSampleStreamFactory.java | 28 +---
.../tools/formats/TokenSampleStreamFactory.java | 28 +---
.../tools/formats/TwentyNewsgroupSampleStream.java | 9 ++
.../TwentyNewsgroupSampleStreamFactory.java | 49 +++----
.../tools/formats/WordTagSampleStreamFactory.java | 27 +---
.../conllu/ConlluLemmaSampleStreamFactory.java | 15 ++-
.../conllu/ConlluPOSSampleStreamFactory.java | 16 ++-
.../tools/formats/conllu/ConlluSentence.java | 4 +-
.../conllu/ConlluSentenceSampleStreamFactory.java | 16 +--
.../conllu/ConlluTokenSampleStreamFactory.java | 17 ++-
.../convert/NameToSentenceSampleStreamFactory.java | 9 +-
.../convert/NameToTokenSampleStreamFactory.java | 11 +-
.../convert/POSToSentenceSampleStreamFactory.java | 11 +-
.../convert/POSToTokenSampleStreamFactory.java | 11 +-
.../convert/ParseToPOSSampleStreamFactory.java | 19 ++-
.../ParseToSentenceSampleStreamFactory.java | 18 +--
.../convert/ParseToTokenSampleStreamFactory.java | 17 +--
.../frenchtreebank/ConstitDocumentHandler.java | 3 +-
.../ConstitParseSampleStreamFactory.java | 19 +--
.../IrishSentenceBankSentenceStreamFactory.java | 21 +--
.../IrishSentenceBankTokenSampleStreamFactory.java | 21 +--
.../LeipzigLanguageSampleStreamFactory.java | 32 ++---
.../tools/formats/leipzig/SampleShuffleStream.java | 17 ++-
.../tools/formats/leipzig/SampleSkipStream.java | 14 +-
.../letsmt/LetsmtSentenceStreamFactory.java | 22 ++--
.../java/opennlp/tools/formats/masc/Masc.java} | 15 +--
.../opennlp/tools/formats/masc/MascDocument.java | 1 -
.../masc/MascNamedEntitySampleStreamFactory.java | 53 ++++----
.../formats/masc/MascPOSSampleStreamFactory.java | 52 ++++----
.../opennlp/tools/formats/masc/MascSentence.java | 4 +-
.../masc/MascSentenceSampleStreamFactory.java | 53 ++++----
.../java/opennlp/tools/formats/masc/MascToken.java | 3 +
.../formats/masc/MascTokenSampleStreamFactory.java | 54 ++++----
.../java/opennlp/tools/formats/masc/MascWord.java | 3 +
.../formats/moses/MosesSentenceSampleStream.java | 12 +-
.../moses/MosesSentenceSampleStreamFactory.java | 35 ++---
.../formats/muc/Muc6NameSampleStreamFactory.java | 28 ++--
.../opennlp/tools/formats/muc/MucElementNames.java | 6 +-
.../tools/formats/muc/MucNameSampleStream.java | 2 +-
.../formats/nkjp/NKJPSegmentationDocument.java | 58 +++++----
.../formats/nkjp/NKJPSentenceSampleStream.java | 4 +-
.../nkjp/NKJPSentenceSampleStreamFactory.java | 23 ++--
.../tools/formats/nkjp/NKJPTextDocument.java | 6 +-
.../ontonotes/OntoNotesNameSampleStream.java | 111 ++++++++--------
.../OntoNotesNameSampleStreamFactory.java | 25 +++-
.../ontonotes/OntoNotesPOSSampleStreamFactory.java | 19 ++-
.../ontonotes/OntoNotesParseSampleStream.java | 2 +-
.../OntoNotesParseSampleStreamFactory.java | 30 +++--
.../tools/namefind/NameSampleDataStream.java | 10 +-
.../opennlp/tools/parser/ParseSampleStream.java | 4 +
.../opennlp/tools/postag/WordTagSampleStream.java | 12 +-
.../formats/AbstractSampleStreamFactoryTest.java | 68 ++++++++++
.../tools/formats/AbstractSampleStreamTest.java | 3 +-
.../BioNLP2004NameSampleStreamFactoryTest.java | 106 +++++++++++++++
.../formats/ChunkerSampleStreamFactoryTest.java | 79 ++++++++++++
.../Conll02NameSampleStreamFactoryTest.java | 127 ++++++++++++++++++
.../Conll03NameSampleStreamFactoryTest.java | 127 ++++++++++++++++++
.../formats/ConllXPOSSampleStreamFactoryTest.java | 78 +++++++++++
.../ConllXSentenceSampleStreamFactoryTest.java | 99 ++++++++++++++
.../ConllXTokenSampleStreamFactoryTest.java | 98 ++++++++++++++
.../EvalitaNameSampleStreamFactoryTest.java | 107 ++++++++++++++++
.../tools/formats/EvalitaNameSampleStreamTest.java | 88 ++++++++++---
.../LanguageDetectorSampleStreamFactoryTest.java | 79 ++++++++++++
.../formats/LemmatizerSampleStreamFactoryTest.java | 79 ++++++++++++
.../formats/NameSampleDataStreamFactoryTest.java | 85 ++++++++++++
.../formats/ParseSampleStreamFactoryTest.java | 80 ++++++++++++
.../formats/SentenceSampleStreamFactoryTest.java | 79 ++++++++++++
.../formats/TokenSampleStreamFactoryTest.java | 79 ++++++++++++
.../TwentyNewsgroupSampleStreamFactoryTest.java | 142 +++++++++++++++++++++
.../formats/WordTagSampleStreamFactoryTest.java | 83 ++++++++++++
.../conllu/ConlluLemmaSampleStreamFactoryTest.java | 113 ++++++++++++++++
.../conllu/ConlluPOSSampleStreamFactoryTest.java | 113 ++++++++++++++++
.../ConlluSentenceSampleStreamFactoryTest.java | 99 ++++++++++++++
.../conllu/ConlluTokenSampleStreamFactoryTest.java | 82 ++++++++++++
.../NameToSentenceSampleStreamFactoryTest.java | 101 +++++++++++++++
.../NameToTokenSampleStreamFactoryTest.java | 101 +++++++++++++++
.../POSToSentenceSampleStreamFactoryTest.java | 101 +++++++++++++++
.../convert/POSToTokenSampleStreamFactoryTest.java | 101 +++++++++++++++
.../convert/ParseToPOSSampleStreamFactoryTest.java | 81 ++++++++++++
.../ParseToSentenceSampleStreamFactoryTest.java | 101 +++++++++++++++
.../ParseToTokenSampleStreamFactoryTest.java | 101 +++++++++++++++
.../ConstitParseSampleStreamFactoryTest.java | 93 ++++++++++++++
...IrishSentenceBankSentenceStreamFactoryTest.java | 83 ++++++++++++
...shSentenceBankTokenSampleStreamFactoryTest.java | 83 ++++++++++++
.../LeipzigLanguageSampleStreamFactoryTest.java | 110 ++++++++++++++++
.../letsmt/LetsmtSentenceStreamFactoryTest.java | 84 ++++++++++++
.../MascNamedEntitySampleStreamFactoryTest.java | 102 +++++++++++++++
.../masc/MascPOSSampleStreamFactoryTest.java | 102 +++++++++++++++
.../masc/MascSentenceSampleStreamFactoryTest.java | 102 +++++++++++++++
.../masc/MascTokenSampleStreamFactoryTest.java | 102 +++++++++++++++
.../MosesSentenceSampleStreamFactoryTest.java | 82 ++++++++++++
.../muc/Muc6NameSampleStreamFactoryTest.java | 105 +++++++++++++++
.../nkjp/NKJPSentenceSampleStreamFactoryTest.java | 102 +++++++++++++++
.../OntoNotesNameSampleStreamFactoryTest.java | 96 ++++++++++++++
.../OntoNotesPOSSampleStreamFactoryTest.java | 96 ++++++++++++++
.../OntoNotesParseSampleStreamFactoryTest.java | 102 +++++++++++++++
.../20newsgroup/sci.electronics/52794.sample | 59 +++++++++
.../opennlp/tools/formats/bionlp2004-01.sample | 33 +++++
.../opennlp/tools/formats/chunker-01.sample | 16 +++
...lita-ner-it.sample => evalita-ner-it-01.sample} | 0
.../opennlp/tools/formats/evalita-ner-it-02.sample | 29 +++++
.../opennlp/tools/formats/evalita-ner-it-03.sample | 22 ++++
.../tools/formats/evalita-ner-it-broken.sample | 2 +
.../tools/formats/evalita-ner-it-incorrect.sample | 3 +
.../opennlp/tools/formats/lang-detect-01.sample | 1 +
.../opennlp/tools/formats/lemma-01.sample | 1 +
.../opennlp/tools/formats/moses/moses-tiny.sample | 3 +
.../opennlp/tools/formats/muc/LDC2003T13.sgm | 73 +++++++++++
.../opennlp/tools/formats/name-data-01.sample | 1 +
.../formats/ontonotes/ontonotes-sample-01.name | 9 ++
.../formats/ontonotes/ontonotes-sample-02.parse | 29 +++++
.../opennlp/tools/formats/parse-01.sample | 1 +
.../opennlp/tools/formats/sentences-01.sample | 2 +
.../opennlp/tools/formats/tokens-01.sample | 1 +
.../opennlp/tools/formats/word-tags-01.sample | 1 +
132 files changed, 5273 insertions(+), 766 deletions(-)
diff --git a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleStream.java
index 2bc23eed..53f6e979 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleStream.java
@@ -31,8 +31,8 @@ import opennlp.tools.util.ObjectStream;
* Parses the conll 2000 shared task shallow parser training data.
* <p>
* Data format is specified on the conll page:<br>
- * <a href="http://www.cnts.ua.ac.be/conll2000/chunking/">
- * http://www.cnts.ua.ac.be/conll2000/chunking/</a>
+ * <a href="https://www.cnts.ua.ac.be/conll2000/chunking/">
+ * https://www.cnts.ua.ac.be/conll2000/chunking/</a>
*/
public class ChunkSampleStream extends FilterObjectStream<String, ChunkSample> {
@@ -57,7 +57,7 @@ public class ChunkSampleStream extends FilterObjectStream<String, ChunkSample> {
for (String line = samples.read(); line != null && !line.isEmpty(); line = samples.read()) {
String[] parts = line.split(" ");
if (parts.length != 3) {
- logger.error("Skipping corrupt line: {}", line);
+ logger.warn("Skipping corrupt line: {}", line);
}
else {
toks.add(parts[0]);
@@ -66,11 +66,11 @@ public class ChunkSampleStream extends FilterObjectStream<String, ChunkSample> {
}
}
- if (toks.size() > 0) {
+ if (!toks.isEmpty()) {
return new ChunkSample(toks.toArray(new String[0]),
tags.toArray(new String[0]), preds.toArray(new String[0]));
+ } else {
+ return null;
}
-
- return null;
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/AbstractSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/AbstractSampleStreamFactory.java
index 663980ac..da9a36d1 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/AbstractSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/AbstractSampleStreamFactory.java
@@ -17,7 +17,15 @@
package opennlp.tools.formats;
+import java.io.IOException;
+
+import opennlp.tools.cmdline.ArgumentParser;
+import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.params.BasicFormatParams;
+import opennlp.tools.util.InputStreamFactory;
+import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.PlainTextByLineStream;
/**
* Base class for sample stream factories.
@@ -40,4 +48,44 @@ public abstract class AbstractSampleStreamFactory<T,P> implements ObjectStreamFa
public Class<P> getParameters() {
return params;
}
+
+ /**
+ * Creates an {@link ObjectStream} for the specified arguments and
+ * the generic type {@code P}.
+ *
+ * @param args A set of command line arguments.
+ * @return The created {@link ObjectStream} instance.
+ */
+ protected <P extends BasicFormatParams> ObjectStream<String> readData(String[] args,
+ Class<P> parametersClass) {
+ P params = validateBasicFormatParameters(args, parametersClass);
+ ObjectStream<String> lineStream = null;
+ try {
+ InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
+ lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
+ } catch (IOException ex) {
+ CmdLineUtil.handleCreateObjectStreamError(ex);
+ }
+ return lineStream;
+ }
+
+ /**
+ * Validates the specified arguments ({@code args}) given the
+ * context the generic type {@code P} which provides at least all
+ * {@link BasicFormatParams}.
+ *
+ * @implNote Additional checks for the basic {@code -data} argument are conducted, that is
+ * wether the file exists or not.
+ *
+ * @param args A set of command line arguments.
+ * @return The parsed (basic format) parameter instance.
+ */
+ protected <P extends BasicFormatParams> P validateBasicFormatParameters(String[] args, Class<P> clazz) {
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
+ P params = ArgumentParser.parse(args, clazz);
+ CmdLineUtil.checkInputFile("Data", params.getData());
+ return params;
+ }
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java
index e2ad4ef4..5bdff932 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java
@@ -45,25 +45,24 @@ import opennlp.tools.util.StringUtil;
* <p>
* Data can be found on this
* <a href="http://www.geniaproject.org/shared-tasks/bionlp-jnlpba-shared-task-2004">website</a>,
- * or in
- * <a href="https://github.com/spyysalo/jnlpba">this repository</a>.
+ * or in this
+ * <a href="https://github.com/spyysalo/jnlpba">GitHub repository</a>.
* <p>
- * The BioNLP/NLPBA 2004 data were originally published here:
- * <p>
- * <a href="http://www-tsujii.is.s.u-tokyo.ac.jp/GENIA/ERtask/report.html">
- * http://www-tsujii.is.s.u-tokyo.ac.jp/GENIA/ERtask/report.html</a>,
+ * The BioNLP/NLPBA 2004 data were originally published
+ * <a href="http://www-tsujii.is.s.u-tokyo.ac.jp/GENIA/ERtask/report.html">here</a>,
* <p>
* yet this page was gone when last checked in December 2022.
* <p>
- * It looks like this repo contains a copy of the data located on the original page:
- * The BioNLP 2004 seems to be related to http://www.geniaproject.org/shared-tasks/bionlp-jnlpba-shared-task-2004
- * <p>
* <b>Note:</b>
* Do not use this class, internal use only!
*/
@Internal
public class BioNLP2004NameSampleStream implements ObjectStream<NameSample> {
+ private static final String CODEC_TAG_O = "O";
+ private static final String CODEC_TAG_B = "B-";
+ private static final String CODEC_TAG_I = "I-";
+
public static final int GENERATE_DNA_ENTITIES = 0x01;
public static final int GENERATE_PROTEIN_ENTITIES = 0x01 << 1;
public static final int GENERATE_CELLTYPE_ENTITIES = 0x01 << 2;
@@ -96,7 +95,6 @@ public class BioNLP2004NameSampleStream implements ObjectStream<NameSample> {
boolean isClearAdaptiveData = false;
// Empty line indicates end of sentence
-
String line;
while ((line = lineStream.read()) != null && !StringUtil.isEmpty(line.trim())) {
@@ -121,7 +119,7 @@ public class BioNLP2004NameSampleStream implements ObjectStream<NameSample> {
}
}
- if (sentence.size() > 0) {
+ if (!sentence.isEmpty()) {
// convert name tags into spans
List<Span> names = new ArrayList<>();
@@ -133,34 +131,32 @@ public class BioNLP2004NameSampleStream implements ObjectStream<NameSample> {
String tag = tags.get(i);
if (tag.endsWith("DNA") && (types & GENERATE_DNA_ENTITIES) == 0)
- tag = "O";
+ tag = CODEC_TAG_O;
if (tag.endsWith("protein") && (types & GENERATE_PROTEIN_ENTITIES) == 0)
- tag = "O";
+ tag = CODEC_TAG_O;
if (tag.endsWith("cell_type") && (types & GENERATE_CELLTYPE_ENTITIES) == 0)
- tag = "O";
+ tag = CODEC_TAG_O;
if (tag.endsWith("cell_line") && (types & GENERATE_CELLTYPE_ENTITIES) == 0)
- tag = "O";
+ tag = CODEC_TAG_O;
if (tag.endsWith("RNA") && (types & GENERATE_RNA_ENTITIES) == 0)
- tag = "O";
+ tag = CODEC_TAG_O;
- if (tag.startsWith("B-")) {
+ if (tag.startsWith(CODEC_TAG_B)) {
if (beginIndex != -1) {
names.add(new Span(beginIndex, endIndex, tags.get(beginIndex).substring(2)));
- beginIndex = -1;
- endIndex = -1;
}
beginIndex = i;
endIndex = i + 1;
}
- else if (tag.startsWith("I-")) {
+ else if (tag.startsWith(CODEC_TAG_I)) {
endIndex++;
}
- else if (tag.equals("O")) {
+ else if (tag.equals(CODEC_TAG_O)) {
if (beginIndex != -1) {
names.add(new Span(beginIndex, endIndex, tags.get(beginIndex).substring(2)));
beginIndex = -1;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactory.java
index 422cd4c6..0b7bfe3c 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactory.java
@@ -19,7 +19,6 @@ package opennlp.tools.formats;
import java.io.IOException;
-import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
@@ -30,46 +29,46 @@ import opennlp.tools.util.ObjectStream;
/**
* @see BioNLP2004NameSampleStream
*/
-public class BioNLP2004NameSampleStreamFactory<P> extends AbstractSampleStreamFactory<NameSample, P> {
+public class BioNLP2004NameSampleStreamFactory extends
+ AbstractSampleStreamFactory<NameSample, BioNLP2004NameSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
@ParameterDescription(valueName = "DNA,protein,cell_type,cell_line,RNA")
String getTypes();
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(NameSample.class,
- "bionlp2004", new BioNLP2004NameSampleStreamFactory<>(Parameters.class));
+ "bionlp2004", new BioNLP2004NameSampleStreamFactory(Parameters.class));
}
- protected BioNLP2004NameSampleStreamFactory(Class<P> params) {
+ protected BioNLP2004NameSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<NameSample> create(String[] args) {
-
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
int typesToGenerate = 0;
-
- if (params.getTypes().contains("DNA")) {
+ String types = params.getTypes();
+ if (types.contains("DNA")) {
typesToGenerate = typesToGenerate |
BioNLP2004NameSampleStream.GENERATE_DNA_ENTITIES;
}
- else if (params.getTypes().contains("protein")) {
+ if (types.contains("protein")) {
typesToGenerate = typesToGenerate |
BioNLP2004NameSampleStream.GENERATE_PROTEIN_ENTITIES;
}
- else if (params.getTypes().contains("cell_type")) {
+ if (types.contains("cell_type")) {
typesToGenerate = typesToGenerate |
BioNLP2004NameSampleStream.GENERATE_CELLTYPE_ENTITIES;
}
- else if (params.getTypes().contains("cell_line")) {
+ if (types.contains("cell_line")) {
typesToGenerate = typesToGenerate |
BioNLP2004NameSampleStream.GENERATE_CELLLINE_ENTITIES;
}
- else if (params.getTypes().contains("RNA")) {
+ if (types.contains("RNA")) {
typesToGenerate = typesToGenerate |
BioNLP2004NameSampleStream.GENERATE_RNA_ENTITIES;
}
@@ -77,8 +76,9 @@ public class BioNLP2004NameSampleStreamFactory<P> extends AbstractSampleStreamFa
try {
return new BioNLP2004NameSampleStream(
CmdLineUtil.createInputStreamFactory(params.getData()), typesToGenerate);
- } catch (IOException e) {
- throw new IllegalStateException(e);
+ } catch (IOException ex) {
+ CmdLineUtil.handleCreateObjectStreamError(ex);
}
+ return null;
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ChunkerSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ChunkerSampleStreamFactory.java
index 36f8b58e..8925d196 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ChunkerSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ChunkerSampleStreamFactory.java
@@ -17,49 +17,33 @@
package opennlp.tools.formats;
-import java.io.IOException;
-
import opennlp.tools.chunker.ChunkSample;
import opennlp.tools.chunker.ChunkSampleStream;
-import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
* Factory producing OpenNLP {@link ChunkSampleStream}s.
*/
-public class ChunkerSampleStreamFactory<P> extends AbstractSampleStreamFactory<ChunkSample, P> {
+public class ChunkerSampleStreamFactory extends
+ AbstractSampleStreamFactory<ChunkSample, ChunkerSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(ChunkSample.class,
- StreamFactoryRegistry.DEFAULT_FORMAT, new ChunkerSampleStreamFactory<>(Parameters.class));
+ StreamFactoryRegistry.DEFAULT_FORMAT, new ChunkerSampleStreamFactory(Parameters.class));
}
- protected ChunkerSampleStreamFactory(Class<P> params) {
+ protected ChunkerSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<ChunkSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
-
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new ChunkSampleStream(lineStream);
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStreamFactory.java
index d417df0c..91cca667 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStreamFactory.java
@@ -19,7 +19,6 @@ package opennlp.tools.formats;
import java.io.IOException;
-import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
@@ -37,9 +36,10 @@ import opennlp.tools.util.ObjectStream;
* @see Conll02NameSampleStream
*/
@Internal
-public class Conll02NameSampleStreamFactory<P> extends LanguageSampleStreamFactory<NameSample, P> {
+public class Conll02NameSampleStreamFactory extends
+ LanguageSampleStreamFactory<NameSample, Conll02NameSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
@ParameterDescription(valueName = "spa|nld")
String getLang();
@@ -49,17 +49,17 @@ public class Conll02NameSampleStreamFactory<P> extends LanguageSampleStreamFacto
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(NameSample.class,
- "conll02", new Conll02NameSampleStreamFactory<>(Parameters.class));
+ "conll02", new Conll02NameSampleStreamFactory(Parameters.class));
}
- protected Conll02NameSampleStreamFactory(Class<P> params) {
+ protected Conll02NameSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<NameSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
LANGUAGE lang;
if ("nl".equals(params.getLang()) || "nld".equals(params.getLang())) {
@@ -93,7 +93,6 @@ public class Conll02NameSampleStreamFactory<P> extends LanguageSampleStreamFacto
Conll02NameSampleStream.GENERATE_MISC_ENTITIES;
}
-
try {
return new Conll02NameSampleStream(lang,
CmdLineUtil.createInputStreamFactory(params.getData()), typesToGenerate);
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStreamFactory.java
index dae580cb..d1a61509 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStreamFactory.java
@@ -19,7 +19,6 @@ package opennlp.tools.formats;
import java.io.IOException;
-import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
@@ -32,9 +31,10 @@ import opennlp.tools.util.ObjectStream;
/**
* @see Conll03NameSampleStream
*/
-public class Conll03NameSampleStreamFactory<P> extends LanguageSampleStreamFactory<NameSample, P> {
+public class Conll03NameSampleStreamFactory extends
+ LanguageSampleStreamFactory<NameSample, Conll03NameSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
@ParameterDescription(valueName = "eng|deu")
String getLang();
@@ -44,17 +44,17 @@ public class Conll03NameSampleStreamFactory<P> extends LanguageSampleStreamFacto
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(NameSample.class,
- "conll03", new Conll03NameSampleStreamFactory<>(Parameters.class));
+ "conll03", new Conll03NameSampleStreamFactory(Parameters.class));
}
- protected Conll03NameSampleStreamFactory(Class<P> params) {
+ protected Conll03NameSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<NameSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
// TODO: support the other languages with this CoNLL.
LANGUAGE lang;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXPOSSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXPOSSampleStreamFactory.java
index 18b48b5a..a7a586d9 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXPOSSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXPOSSampleStreamFactory.java
@@ -18,13 +18,10 @@
package opennlp.tools.formats;
import java.io.IOException;
-import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
-import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
-import opennlp.tools.cmdline.TerminateToolException;
import opennlp.tools.cmdline.params.BasicFormatParams;
import opennlp.tools.commons.Internal;
import opennlp.tools.postag.POSSample;
@@ -35,40 +32,35 @@ import opennlp.tools.util.ObjectStream;
* <b>Note:</b>
* Do not use this class, internal use only!
*
+ * @see POSSample
* @see ConllXPOSSampleStream
*/
@Internal
-public class ConllXPOSSampleStreamFactory<P> extends AbstractSampleStreamFactory<POSSample, P> {
+public class ConllXPOSSampleStreamFactory extends
+ AbstractSampleStreamFactory<POSSample, ConllXPOSSampleStreamFactory.Parameters> {
public static final String CONLLX_FORMAT = "conllx";
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(POSSample.class,
- CONLLX_FORMAT, new ConllXPOSSampleStreamFactory<>(Parameters.class));
+ CONLLX_FORMAT, new ConllXPOSSampleStreamFactory(Parameters.class));
}
- protected ConllXPOSSampleStreamFactory(Class<P> params) {
+ protected ConllXPOSSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<POSSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- InputStreamFactory inFactory =
- CmdLineUtil.createInputStreamFactory(params.getData());
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
try {
+ InputStreamFactory inFactory = CmdLineUtil.createInputStreamFactory(params.getData());
return new ConllXPOSSampleStream(inFactory, StandardCharsets.UTF_8);
- } catch (UnsupportedEncodingException e) {
- // this shouldn't happen
- throw new TerminateToolException(-1, "UTF-8 encoding is not supported: " + e.getMessage(), e);
- }
- catch (IOException e) {
- // That will throw an exception
+ } catch (IOException e) {
CmdLineUtil.handleCreateObjectStreamError(e);
return null;
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXSentenceSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXSentenceSampleStreamFactory.java
index 505f94f2..2d622e0f 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXSentenceSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXSentenceSampleStreamFactory.java
@@ -29,29 +29,31 @@ import opennlp.tools.util.ObjectStream;
/**
* <b>Note:</b>
* Do not use this class, internal use only!
+ *
+ * @see SentenceSample
+ * @see POSToSentenceSampleStream
*/
@Internal
-public class ConllXSentenceSampleStreamFactory<P> extends
- DetokenizerSampleStreamFactory<SentenceSample, P> {
+public class ConllXSentenceSampleStreamFactory extends
+ DetokenizerSampleStreamFactory<SentenceSample, ConllXSentenceSampleStreamFactory.Parameters> {
- interface Parameters extends ConllXPOSSampleStreamFactory.Parameters, DetokenizerParameter {
+ public interface Parameters extends ConllXPOSSampleStreamFactory.Parameters, DetokenizerParameter {
// TODO: make chunk size configurable
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(SentenceSample.class,
ConllXPOSSampleStreamFactory.CONLLX_FORMAT,
- new ConllXSentenceSampleStreamFactory<>(Parameters.class));
+ new ConllXSentenceSampleStreamFactory(Parameters.class));
}
- protected ConllXSentenceSampleStreamFactory(Class<P> params) {
+ protected ConllXSentenceSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<SentenceSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
ObjectStream<POSSample> posSampleStream = StreamFactoryRegistry.getFactory(POSSample.class,
ConllXPOSSampleStreamFactory.CONLLX_FORMAT).create(
ArgumentParser.filter(args, ConllXPOSSampleStreamFactory.Parameters.class));
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXTokenSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXTokenSampleStreamFactory.java
index c894be0a..81c1eb21 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXTokenSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXTokenSampleStreamFactory.java
@@ -29,25 +29,29 @@ import opennlp.tools.util.ObjectStream;
/**
* <b>Note:</b>
* Do not use this class, internal use only!
+ *
+ * @see TokenSample
+ * @see POSToTokenSampleStream
*/
@Internal
-public class ConllXTokenSampleStreamFactory<P> extends DetokenizerSampleStreamFactory<TokenSample, P> {
+public class ConllXTokenSampleStreamFactory extends
+ DetokenizerSampleStreamFactory<TokenSample, ConllXTokenSampleStreamFactory.Parameters> {
- interface Parameters extends ConllXPOSSampleStreamFactory.Parameters, DetokenizerParameter {
+ public interface Parameters extends ConllXPOSSampleStreamFactory.Parameters, DetokenizerParameter {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(TokenSample.class,
- ConllXPOSSampleStreamFactory.CONLLX_FORMAT, new ConllXTokenSampleStreamFactory<>(Parameters.class));
+ ConllXPOSSampleStreamFactory.CONLLX_FORMAT, new ConllXTokenSampleStreamFactory(Parameters.class));
}
- protected ConllXTokenSampleStreamFactory(Class<P> params) {
+ protected ConllXTokenSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<TokenSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
ObjectStream<POSSample> samples = StreamFactoryRegistry.getFactory(POSSample.class,
ConllXPOSSampleStreamFactory.CONLLX_FORMAT).create(
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/DocumentSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/DocumentSampleStreamFactory.java
index 2ecf5664..1775eaa4 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/DocumentSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/DocumentSampleStreamFactory.java
@@ -17,48 +17,33 @@
package opennlp.tools.formats;
-import java.io.IOException;
-
-import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
import opennlp.tools.doccat.DocumentSample;
import opennlp.tools.doccat.DocumentSampleStream;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
* Factory producing OpenNLP {@link DocumentSampleStream}s.
*/
-public class DocumentSampleStreamFactory<P> extends AbstractSampleStreamFactory<DocumentSample, P> {
+public class DocumentSampleStreamFactory extends
+ AbstractSampleStreamFactory<DocumentSample, DocumentSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(DocumentSample.class,
- StreamFactoryRegistry.DEFAULT_FORMAT, new DocumentSampleStreamFactory<>(Parameters.class));
+ StreamFactoryRegistry.DEFAULT_FORMAT, new DocumentSampleStreamFactory(Parameters.class));
}
- protected DocumentSampleStreamFactory(Class<P> params) {
+ protected DocumentSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<DocumentSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new DocumentSampleStream(lineStream);
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java
index 76574500..a1382c5d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java
@@ -32,7 +32,7 @@ import opennlp.tools.util.Span;
import opennlp.tools.util.StringUtil;
/**
- * Parser for the Italian NER training files of the Evalita 2007 and 2009 NER shared tasks.
+ * Parser for the Italian NER training files of the Evalita 2007 and 2009 NER shared tasks.
* <p>
* The data does not contain article boundaries,
* adaptive data will be cleared for every sentence.
@@ -46,12 +46,12 @@ import opennlp.tools.util.StringUtil;
* 2. The Entity type tag: PER (for Person), ORG (for Organization),
* GPE (for Geo-Political Entity), or LOC (for Location).
* <p>
- * Each file consists of four columns separated by a blank, containing
- * respectively the token, the Elsnet PoS-tag, the Adige news story to
- * which the token belongs, and the Named Entity tag.
+ * Each file consists of four columns separated by a blank, containing respectively the token, the
+ * <a href="https://www.evalita.it/wp-content/uploads/2021/11/elsnet-tagset-IT.pdf">Elsnet</a>
+ * PoS-tag, the Adige news story to which the token belongs, and the Named Entity tag.
* <p>
* Data can be found on this
- * <a href="http://www.evalita.it">web site</a>.
+ * <a href="https://www.evalita.it">web site</a>.
* <p>
* <b>Note:</b>
* Do not use this class, internal use only!
@@ -59,6 +59,15 @@ import opennlp.tools.util.StringUtil;
@Internal
public class EvalitaNameSampleStream implements ObjectStream<NameSample> {
+ public static final String DOCSTART = "-DOCSTART-";
+ private static final String CODEC_TAG_O = "O";
+ private static final String CODEC_TAG_B = "B-";
+ private static final String CODEC_TAG_I = "I-";
+ private static final String ENT_TYPE_PER = "PER"; // Person
+ private static final String ENT_TYPE_LOC = "LOC"; // Location
+ private static final String ENT_TYPE_GPE = "GPE"; // Geo-Political Entity
+ private static final String ENT_TYPE_ORG = "ORG"; // Organization
+
public enum LANGUAGE {
IT
}
@@ -68,8 +77,6 @@ public class EvalitaNameSampleStream implements ObjectStream<NameSample> {
public static final int GENERATE_LOCATION_ENTITIES = 0x01 << 2;
public static final int GENERATE_GPE_ENTITIES = 0x01 << 3;
- public static final String DOCSTART = "-DOCSTART-";
-
private final LANGUAGE lang;
private final ObjectStream<String> lineStream;
@@ -82,7 +89,7 @@ public class EvalitaNameSampleStream implements ObjectStream<NameSample> {
}
public EvalitaNameSampleStream(LANGUAGE lang, InputStreamFactory in, int types) throws IOException {
- this(lang, new PlainTextByLineStream(in, StandardCharsets.UTF_8),types);
+ this(lang, new PlainTextByLineStream(in, StandardCharsets.UTF_8), types);
}
private static Span extract(int begin, int end, String beginTag) throws InvalidFormatException {
@@ -90,17 +97,16 @@ public class EvalitaNameSampleStream implements ObjectStream<NameSample> {
String type = beginTag.substring(2);
type = switch (type) {
- case "PER" -> "person";
- case "LOC" -> "location";
- case "GPE" -> "gpe";
- case "ORG" -> "organization";
+ case ENT_TYPE_PER -> "person";
+ case ENT_TYPE_LOC -> "location";
+ case ENT_TYPE_GPE -> "gpe";
+ case ENT_TYPE_ORG -> "organization";
default -> throw new InvalidFormatException("Unknown type: " + type);
};
return new Span(begin, end, type);
}
-
@Override
public NameSample read() throws IOException {
@@ -110,7 +116,6 @@ public class EvalitaNameSampleStream implements ObjectStream<NameSample> {
boolean isClearAdaptiveData = false;
// Empty line indicates end of sentence
-
String line;
while ((line = lineStream.read()) != null && !StringUtil.isEmpty(line)) {
@@ -140,7 +145,7 @@ public class EvalitaNameSampleStream implements ObjectStream<NameSample> {
if (LANGUAGE.IT.equals(lang))
isClearAdaptiveData = true;
- if (sentence.size() > 0) {
+ if (!sentence.isEmpty()) {
// convert name tags into spans
List<Span> names = new ArrayList<>();
@@ -151,33 +156,31 @@ public class EvalitaNameSampleStream implements ObjectStream<NameSample> {
String tag = tags.get(i);
- if (tag.endsWith("PER") && (types & GENERATE_PERSON_ENTITIES) == 0)
- tag = "O";
+ if (tag.endsWith(ENT_TYPE_PER) && (types & GENERATE_PERSON_ENTITIES) == 0)
+ tag = CODEC_TAG_O;
- if (tag.endsWith("ORG") && (types & GENERATE_ORGANIZATION_ENTITIES) == 0)
- tag = "O";
+ if (tag.endsWith(ENT_TYPE_ORG) && (types & GENERATE_ORGANIZATION_ENTITIES) == 0)
+ tag = CODEC_TAG_O;
- if (tag.endsWith("LOC") && (types & GENERATE_LOCATION_ENTITIES) == 0)
- tag = "O";
+ if (tag.endsWith(ENT_TYPE_LOC) && (types & GENERATE_LOCATION_ENTITIES) == 0)
+ tag = CODEC_TAG_O;
- if (tag.endsWith("GPE") && (types & GENERATE_GPE_ENTITIES) == 0)
- tag = "O";
+ if (tag.endsWith(ENT_TYPE_GPE) && (types & GENERATE_GPE_ENTITIES) == 0)
+ tag = CODEC_TAG_O;
- if (tag.startsWith("B-")) {
+ if (tag.startsWith(CODEC_TAG_B)) {
if (beginIndex != -1) {
names.add(extract(beginIndex, endIndex, tags.get(beginIndex)));
- beginIndex = -1;
- endIndex = -1;
}
beginIndex = i;
endIndex = i + 1;
}
- else if (tag.startsWith("I-")) {
+ else if (tag.startsWith(CODEC_TAG_I)) {
endIndex++;
}
- else if (tag.equals("O")) {
+ else if (tag.equals(CODEC_TAG_O)) {
if (beginIndex != -1) {
names.add(extract(beginIndex, endIndex, tags.get(beginIndex)));
beginIndex = -1;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStreamFactory.java
index 7fa9db40..066861f3 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStreamFactory.java
@@ -19,7 +19,6 @@ package opennlp.tools.formats;
import java.io.IOException;
-import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
@@ -37,9 +36,10 @@ import opennlp.tools.util.ObjectStream;
* @see EvalitaNameSampleStream
*/
@Internal
-public class EvalitaNameSampleStreamFactory<P> extends LanguageSampleStreamFactory<NameSample, P> {
+public class EvalitaNameSampleStreamFactory extends
+ LanguageSampleStreamFactory<NameSample, EvalitaNameSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
@ParameterDescription(valueName = "it")
String getLang();
@@ -49,17 +49,16 @@ public class EvalitaNameSampleStreamFactory<P> extends LanguageSampleStreamFacto
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(NameSample.class,
- "evalita", new EvalitaNameSampleStreamFactory<>(Parameters.class));
+ "evalita", new EvalitaNameSampleStreamFactory(Parameters.class));
}
- protected EvalitaNameSampleStreamFactory(Class<P> params) {
+ protected EvalitaNameSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<NameSample> create(String[] args) {
-
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
LANGUAGE lang;
if ("it".equals(params.getLang())) {
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/LanguageDetectorSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/LanguageDetectorSampleStreamFactory.java
index bda9d482..9aedc4bb 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/LanguageDetectorSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/LanguageDetectorSampleStreamFactory.java
@@ -17,51 +17,34 @@
package opennlp.tools.formats;
-import java.io.IOException;
-
-import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
-import opennlp.tools.doccat.DocumentSampleStream;
import opennlp.tools.langdetect.LanguageDetectorSampleStream;
import opennlp.tools.langdetect.LanguageSample;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
- * Factory producing OpenNLP {@link DocumentSampleStream}s.
+ * Factory producing OpenNLP {@link LanguageDetectorSampleStream lang detector sample streams}.
*/
-public class LanguageDetectorSampleStreamFactory<P>
- extends AbstractSampleStreamFactory<LanguageSample, P> {
+public class LanguageDetectorSampleStreamFactory extends
+ AbstractSampleStreamFactory<LanguageSample, LanguageDetectorSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(LanguageSample.class,
StreamFactoryRegistry.DEFAULT_FORMAT,
- new LanguageDetectorSampleStreamFactory<>(Parameters.class));
+ new LanguageDetectorSampleStreamFactory(Parameters.class));
}
- protected LanguageDetectorSampleStreamFactory(Class<P> params) {
+ protected LanguageDetectorSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<LanguageSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new LanguageDetectorSampleStream(lineStream);
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/LemmatizerSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/LemmatizerSampleStreamFactory.java
index dfb137e7..f2209170 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/LemmatizerSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/LemmatizerSampleStreamFactory.java
@@ -17,49 +17,33 @@
package opennlp.tools.formats;
-import java.io.IOException;
-
-import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
import opennlp.tools.lemmatizer.LemmaSample;
import opennlp.tools.lemmatizer.LemmaSampleStream;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
* Factory producing OpenNLP {@link LemmaSampleStream}s.
*/
-public class LemmatizerSampleStreamFactory<P> extends AbstractSampleStreamFactory<LemmaSample, P> {
+public class LemmatizerSampleStreamFactory extends
+ AbstractSampleStreamFactory<LemmaSample, LemmatizerSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(LemmaSample.class,
- StreamFactoryRegistry.DEFAULT_FORMAT, new LemmatizerSampleStreamFactory<>(Parameters.class));
+ StreamFactoryRegistry.DEFAULT_FORMAT, new LemmatizerSampleStreamFactory(Parameters.class));
}
- protected LemmatizerSampleStreamFactory(Class<P> params) {
+ protected LemmatizerSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<LemmaSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
-
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new LemmaSampleStream(lineStream);
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/NameSampleDataStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/NameSampleDataStreamFactory.java
index 508359bb..efc786e1 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/NameSampleDataStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/NameSampleDataStreamFactory.java
@@ -17,50 +17,35 @@
package opennlp.tools.formats;
-import java.io.IOException;
-
-import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
+import opennlp.tools.commons.Internal;
import opennlp.tools.namefind.NameSample;
import opennlp.tools.namefind.NameSampleDataStream;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
* Factory producing OpenNLP {@link NameSampleDataStream}s.
*/
-public class NameSampleDataStreamFactory<P> extends AbstractSampleStreamFactory<NameSample, P> {
+@Internal
+public class NameSampleDataStreamFactory extends
+ AbstractSampleStreamFactory<NameSample, NameSampleDataStreamFactory.Parameters> {
public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(NameSample.class,
- StreamFactoryRegistry.DEFAULT_FORMAT, new NameSampleDataStreamFactory<>(Parameters.class));
+ StreamFactoryRegistry.DEFAULT_FORMAT, new NameSampleDataStreamFactory(Parameters.class));
}
- protected NameSampleDataStreamFactory(Class<P> params) {
+ protected NameSampleDataStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<NameSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
-
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
-
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new NameSampleDataStream(lineStream);
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ParseSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ParseSampleStreamFactory.java
index 6ed45743..2ce9ec36 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ParseSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ParseSampleStreamFactory.java
@@ -17,49 +17,33 @@
package opennlp.tools.formats;
-import java.io.IOException;
-
-import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
import opennlp.tools.parser.Parse;
import opennlp.tools.parser.ParseSampleStream;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
* Factory producing OpenNLP {@link ParseSampleStream}s.
*/
-public class ParseSampleStreamFactory<P> extends AbstractSampleStreamFactory<Parse, P> {
+public class ParseSampleStreamFactory extends
+ AbstractSampleStreamFactory<Parse, ParseSampleStreamFactory.Parameters> {
public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(Parse.class,
- StreamFactoryRegistry.DEFAULT_FORMAT, new ParseSampleStreamFactory<>(Parameters.class));
+ StreamFactoryRegistry.DEFAULT_FORMAT, new ParseSampleStreamFactory(Parameters.class));
}
- protected ParseSampleStreamFactory(Class<P> params) {
+ protected ParseSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<Parse> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
-
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new ParseSampleStream(lineStream);
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/SentenceSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/SentenceSampleStreamFactory.java
index e002bbb1..61b8e6bb 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/SentenceSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/SentenceSampleStreamFactory.java
@@ -17,50 +17,34 @@
package opennlp.tools.formats;
-import java.io.IOException;
-
-import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
import opennlp.tools.sentdetect.SentenceSample;
import opennlp.tools.sentdetect.SentenceSampleStream;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
* Factory producing OpenNLP {@link SentenceSampleStream}s.
*/
-public class SentenceSampleStreamFactory<P> extends AbstractSampleStreamFactory<SentenceSample, P> {
+public class SentenceSampleStreamFactory extends
+ AbstractSampleStreamFactory<SentenceSample, SentenceSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(SentenceSample.class,
- StreamFactoryRegistry.DEFAULT_FORMAT, new SentenceSampleStreamFactory<>(Parameters.class));
+ StreamFactoryRegistry.DEFAULT_FORMAT, new SentenceSampleStreamFactory(Parameters.class));
}
- protected SentenceSampleStreamFactory(Class<P> params) {
+ protected SentenceSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<SentenceSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
-
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new SentenceSampleStream(lineStream);
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/TokenSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/TokenSampleStreamFactory.java
index ffbd1e6c..75dc62d4 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/TokenSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/TokenSampleStreamFactory.java
@@ -17,49 +17,33 @@
package opennlp.tools.formats;
-import java.io.IOException;
-
-import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
import opennlp.tools.tokenize.TokenSample;
import opennlp.tools.tokenize.TokenSampleStream;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
* Factory producing OpenNLP {@link TokenSampleStream}s.
*/
-public class TokenSampleStreamFactory<P> extends LanguageSampleStreamFactory<TokenSample, P> {
+public class TokenSampleStreamFactory extends
+ LanguageSampleStreamFactory<TokenSample, TokenSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(TokenSample.class,
- StreamFactoryRegistry.DEFAULT_FORMAT, new TokenSampleStreamFactory<>(Parameters.class));
+ StreamFactoryRegistry.DEFAULT_FORMAT, new TokenSampleStreamFactory(Parameters.class));
}
- protected TokenSampleStreamFactory(Class<P> params) {
+ protected TokenSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<TokenSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
-
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new TokenSampleStream(lineStream);
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/TwentyNewsgroupSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/TwentyNewsgroupSampleStream.java
index eb3ab1df..c69319ce 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/TwentyNewsgroupSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/TwentyNewsgroupSampleStream.java
@@ -28,6 +28,15 @@ import opennlp.tools.doccat.DocumentSample;
import opennlp.tools.tokenize.Tokenizer;
import opennlp.tools.util.ObjectStream;
+/**
+ * An {@link ObjectStream} implementation for the Twenty Newsgroups text corpus.
+ * <p>
+ * The document collection was created and donated by: <i>Tom Mitchell,
+ * School of Computer Science, Carnegie Mellon University</i>.
+ * <p>
+ * Details and the data can be found via this DOI:
+ * <a href="https://dx.doi.org/10.24432/C5C323">10.24432/C5C323</a>.
+ */
public class TwentyNewsgroupSampleStream implements ObjectStream<DocumentSample> {
private final Tokenizer tokenizer;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/TwentyNewsgroupSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/TwentyNewsgroupSampleStreamFactory.java
index edf3d5d3..f8c84dda 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/TwentyNewsgroupSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/TwentyNewsgroupSampleStreamFactory.java
@@ -26,35 +26,50 @@ import opennlp.tools.cmdline.TerminateToolException;
import opennlp.tools.cmdline.params.EncodingParameter;
import opennlp.tools.doccat.DocumentSample;
import opennlp.tools.tokenize.SimpleTokenizer;
+import opennlp.tools.tokenize.ThreadSafeTokenizerME;
import opennlp.tools.tokenize.Tokenizer;
-import opennlp.tools.tokenize.TokenizerME;
import opennlp.tools.tokenize.TokenizerModel;
import opennlp.tools.tokenize.WhitespaceTokenizer;
import opennlp.tools.util.ObjectStream;
-public class TwentyNewsgroupSampleStreamFactory<P> extends AbstractSampleStreamFactory<DocumentSample,P> {
+public class TwentyNewsgroupSampleStreamFactory extends
+ AbstractSampleStreamFactory<DocumentSample, TwentyNewsgroupSampleStreamFactory.Parameters> {
+
+ public interface Parameters extends EncodingParameter {
+ @ArgumentParser.ParameterDescription(valueName = "dataDir",
+ description = "dir containing the 20newsgroup folders")
+ File getDataDir();
+
+ @ArgumentParser.ParameterDescription(valueName = "modelFile")
+ @ArgumentParser.OptionalParameter
+ File getTokenizerModel();
+
+ @ArgumentParser.ParameterDescription(valueName = "name")
+ @ArgumentParser.OptionalParameter
+ String getRuleBasedTokenizer();
+ }
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(DocumentSample.class,
"20newsgroup",
- new TwentyNewsgroupSampleStreamFactory<>(TwentyNewsgroupSampleStreamFactory.Parameters.class));
+ new TwentyNewsgroupSampleStreamFactory(TwentyNewsgroupSampleStreamFactory.Parameters.class));
}
- protected TwentyNewsgroupSampleStreamFactory(Class<P> params) {
+ protected TwentyNewsgroupSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<DocumentSample> create(String[] args) {
-
- TwentyNewsgroupSampleStreamFactory.Parameters params =
- ArgumentParser.parse(args, TwentyNewsgroupSampleStreamFactory.Parameters.class);
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
+ Parameters params = ArgumentParser.parse(args, Parameters.class);
Tokenizer tokenizer = WhitespaceTokenizer.INSTANCE;
-
if (params.getTokenizerModel() != null) {
try {
- tokenizer = new TokenizerME(new TokenizerModel(params.getTokenizerModel()));
+ tokenizer = new ThreadSafeTokenizerME(new TokenizerModel(params.getTokenizerModel()));
} catch (IOException e) {
throw new TerminateToolException(-1, "Failed to load tokenizer model!", e);
}
@@ -74,24 +89,10 @@ public class TwentyNewsgroupSampleStreamFactory<P> extends AbstractSampleStreamF
}
try {
- return new TwentyNewsgroupSampleStream(
- tokenizer, params.getDataDir().toPath());
+ return new TwentyNewsgroupSampleStream(tokenizer, params.getDataDir().toPath());
} catch (IOException e) {
throw new TerminateToolException(-1, "IO error while opening sample data: " + e.getMessage(), e);
}
}
- interface Parameters extends EncodingParameter {
- @ArgumentParser.ParameterDescription(valueName = "dataDir",
- description = "dir containing the 20newsgroup folders")
- File getDataDir();
-
- @ArgumentParser.ParameterDescription(valueName = "modelFile")
- @ArgumentParser.OptionalParameter
- File getTokenizerModel();
-
- @ArgumentParser.ParameterDescription(valueName = "name")
- @ArgumentParser.OptionalParameter
- String getRuleBasedTokenizer();
- }
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/WordTagSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/WordTagSampleStreamFactory.java
index 4972b4d1..6b92c41a 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/WordTagSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/WordTagSampleStreamFactory.java
@@ -17,52 +17,37 @@
package opennlp.tools.formats;
-import java.io.IOException;
-
-import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
import opennlp.tools.commons.Internal;
import opennlp.tools.postag.POSSample;
import opennlp.tools.postag.WordTagSampleStream;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
* <b>Note:</b>
* Do not use this class, internal use only!
*/
@Internal
-public class WordTagSampleStreamFactory<P> extends AbstractSampleStreamFactory<POSSample, P> {
+public class WordTagSampleStreamFactory extends
+ AbstractSampleStreamFactory<POSSample, WordTagSampleStreamFactory.Parameters> {
public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(POSSample.class,
- StreamFactoryRegistry.DEFAULT_FORMAT, new WordTagSampleStreamFactory<>(Parameters.class));
+ StreamFactoryRegistry.DEFAULT_FORMAT, new WordTagSampleStreamFactory(Parameters.class));
}
- protected WordTagSampleStreamFactory(Class<P> params) {
+ protected WordTagSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<POSSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
-
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new WordTagSampleStream(lineStream);
}
+
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluLemmaSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluLemmaSampleStreamFactory.java
index c7b1f77b..2dedf86c 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluLemmaSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluLemmaSampleStreamFactory.java
@@ -32,11 +32,15 @@ import opennlp.tools.util.ObjectStream;
/**
* <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see LemmaSample
+ * @see ConlluLemmaSampleStream
*/
@Internal
-public class ConlluLemmaSampleStreamFactory<P> extends AbstractSampleStreamFactory<LemmaSample, P> {
+public class ConlluLemmaSampleStreamFactory extends
+ AbstractSampleStreamFactory<LemmaSample, ConlluLemmaSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
@ArgumentParser.ParameterDescription(valueName = "tagset",
description = "u|x u for unified tags and x for language-specific part-of-speech tags")
@ArgumentParser.OptionalParameter(defaultValue = "u")
@@ -46,15 +50,15 @@ public class ConlluLemmaSampleStreamFactory<P> extends AbstractSampleStreamFacto
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(LemmaSample.class,
ConlluPOSSampleStreamFactory.CONLLU_FORMAT,
- new ConlluLemmaSampleStreamFactory<>(Parameters.class));
+ new ConlluLemmaSampleStreamFactory(Parameters.class));
}
- protected ConlluLemmaSampleStreamFactory(Class<P> params) {
+ protected ConlluLemmaSampleStreamFactory(Class<Parameters> params) {
super(params);
}
public ObjectStream<LemmaSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
ConlluTagset tagset = switch (params.getTagset()) {
case "u" -> ConlluTagset.U;
@@ -68,7 +72,6 @@ public class ConlluLemmaSampleStreamFactory<P> extends AbstractSampleStreamFacto
try {
return new ConlluLemmaSampleStream(new ConlluStream(inFactory), tagset);
} catch (IOException e) {
- // That will throw an exception
CmdLineUtil.handleCreateObjectStreamError(e);
}
return null;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluPOSSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluPOSSampleStreamFactory.java
index 6601dbb6..9a40b0a1 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluPOSSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluPOSSampleStreamFactory.java
@@ -32,13 +32,17 @@ import opennlp.tools.util.ObjectStream;
/**
* <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see POSSample
+ * @see ConlluPOSSampleStream
*/
@Internal
-public class ConlluPOSSampleStreamFactory<P> extends AbstractSampleStreamFactory<POSSample, P> {
+public class ConlluPOSSampleStreamFactory extends
+ AbstractSampleStreamFactory<POSSample, ConlluPOSSampleStreamFactory.Parameters> {
public static final String CONLLU_FORMAT = "conllu";
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
@ArgumentParser.ParameterDescription(valueName = "tagset",
description = "u|x u for unified tags and x for language-specific part-of-speech tags")
@ArgumentParser.OptionalParameter(defaultValue = "u")
@@ -47,15 +51,16 @@ public class ConlluPOSSampleStreamFactory<P> extends AbstractSampleStreamFactory
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(POSSample.class,
- CONLLU_FORMAT, new ConlluPOSSampleStreamFactory<>(Parameters.class));
+ CONLLU_FORMAT, new ConlluPOSSampleStreamFactory(Parameters.class));
}
- protected ConlluPOSSampleStreamFactory(Class<P> params) {
+ protected ConlluPOSSampleStreamFactory(Class<Parameters> params) {
super(params);
}
+ @Override
public ObjectStream<POSSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
ConlluTagset tagset = switch (params.getTagset()) {
case "u" -> ConlluTagset.U;
@@ -69,7 +74,6 @@ public class ConlluPOSSampleStreamFactory<P> extends AbstractSampleStreamFactory
try {
return new ConlluPOSSampleStream(new ConlluStream(inFactory), tagset);
} catch (IOException e) {
- // That will throw an exception
CmdLineUtil.handleCreateObjectStreamError(e);
}
return null;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluSentence.java b/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluSentence.java
index 695534d1..2150be30 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluSentence.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluSentence.java
@@ -44,9 +44,7 @@ public class ConlluSentence {
public ConlluSentence(List<ConlluWordLine> wordLines, String sentenceIdComment, String textComment,
boolean newDocument, String documentId, boolean newParagraph, String paragraphId,
Map<Locale, String> textLang, String translit) {
- this.wordLines = wordLines;
- this.sentenceIdComment = sentenceIdComment;
- this.textComment = textComment;
+ this(wordLines, sentenceIdComment, textComment);
this.newDocument = newDocument;
this.documentId = documentId;
this.newParagraph = newParagraph;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluSentenceSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluSentenceSampleStreamFactory.java
index 3b1164ca..6e665759 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluSentenceSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluSentenceSampleStreamFactory.java
@@ -30,15 +30,16 @@ import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
/**
- * <b>Note:</b>
- * Do not use this class, internal use only!
+ * <b>Note:</b> Do not use this class, internal use only!
*
+ * @see SentenceSample
* @see ConlluSentenceSampleStream
*/
@Internal
-public class ConlluSentenceSampleStreamFactory<P> extends AbstractSampleStreamFactory<SentenceSample, P> {
+public class ConlluSentenceSampleStreamFactory extends
+ AbstractSampleStreamFactory<SentenceSample, ConlluSentenceSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
@ArgumentParser.ParameterDescription(valueName = "sentencesPerSample",
description = "number of sentences per sample")
String getSentencesPerSample();
@@ -47,16 +48,16 @@ public class ConlluSentenceSampleStreamFactory<P> extends AbstractSampleStreamFa
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(SentenceSample.class,
ConlluPOSSampleStreamFactory.CONLLU_FORMAT,
- new ConlluSentenceSampleStreamFactory<>(ConlluSentenceSampleStreamFactory.Parameters.class));
+ new ConlluSentenceSampleStreamFactory(ConlluSentenceSampleStreamFactory.Parameters.class));
}
- protected ConlluSentenceSampleStreamFactory(Class<P> params) {
+ protected ConlluSentenceSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<SentenceSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
InputStreamFactory inFactory =
CmdLineUtil.createInputStreamFactory(params.getData());
@@ -65,7 +66,6 @@ public class ConlluSentenceSampleStreamFactory<P> extends AbstractSampleStreamFa
return new ConlluSentenceSampleStream(new ConlluStream(inFactory),
Integer.parseInt(params.getSentencesPerSample()));
} catch (IOException e) {
- // That will throw an exception
CmdLineUtil.handleCreateObjectStreamError(e);
}
return null;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluTokenSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluTokenSampleStreamFactory.java
index 5f813a65..ac3b18d4 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluTokenSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluTokenSampleStreamFactory.java
@@ -19,7 +19,6 @@ package opennlp.tools.formats.conllu;
import java.io.IOException;
-import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
@@ -30,30 +29,31 @@ import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
/**
- * <b>Note:</b>
- * Do not use this class, internal use only!
+ * <b>Note:</b> Do not use this class, internal use only!
*
+ * @see TokenSample
* @see ConlluTokenSampleStream
*/
@Internal
-public class ConlluTokenSampleStreamFactory<P> extends AbstractSampleStreamFactory<TokenSample, P> {
+public class ConlluTokenSampleStreamFactory extends
+ AbstractSampleStreamFactory<TokenSample, ConlluTokenSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(TokenSample.class,
ConlluPOSSampleStreamFactory.CONLLU_FORMAT,
- new ConlluTokenSampleStreamFactory<>(ConlluTokenSampleStreamFactory.Parameters.class));
+ new ConlluTokenSampleStreamFactory(ConlluTokenSampleStreamFactory.Parameters.class));
}
- protected ConlluTokenSampleStreamFactory(Class<P> params) {
+ protected ConlluTokenSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<TokenSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
InputStreamFactory inFactory =
CmdLineUtil.createInputStreamFactory(params.getData());
@@ -61,7 +61,6 @@ public class ConlluTokenSampleStreamFactory<P> extends AbstractSampleStreamFacto
try {
return new ConlluTokenSampleStream(new ConlluStream(inFactory));
} catch (IOException e) {
- // That will throw an exception
CmdLineUtil.handleCreateObjectStreamError(e);
}
return null;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/NameToSentenceSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/NameToSentenceSampleStreamFactory.java
index ad5354e4..7297126b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/NameToSentenceSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/NameToSentenceSampleStreamFactory.java
@@ -34,23 +34,24 @@ import opennlp.tools.util.ObjectStream;
* @see NameToSentenceSampleStream
*/
@Internal
-public class NameToSentenceSampleStreamFactory<P> extends DetokenizerSampleStreamFactory<SentenceSample, P> {
+public class NameToSentenceSampleStreamFactory extends
+ DetokenizerSampleStreamFactory<SentenceSample, NameToSentenceSampleStreamFactory.Parameters> {
interface Parameters extends NameSampleDataStreamFactory.Parameters, DetokenizerParameter {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(SentenceSample.class,
- "namefinder", new NameToSentenceSampleStreamFactory<>(Parameters.class));
+ "namefinder", new NameToSentenceSampleStreamFactory(Parameters.class));
}
- protected NameToSentenceSampleStreamFactory(Class<P> params) {
+ protected NameToSentenceSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<SentenceSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
ObjectStream<NameSample> nameSampleStream = StreamFactoryRegistry.getFactory(
NameSample.class, StreamFactoryRegistry.DEFAULT_FORMAT).create(
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/NameToTokenSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/NameToTokenSampleStreamFactory.java
index 4d2c10bc..54392331 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/NameToTokenSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/NameToTokenSampleStreamFactory.java
@@ -34,23 +34,24 @@ import opennlp.tools.util.ObjectStream;
* @see NameToTokenSampleStream
*/
@Internal
-public class NameToTokenSampleStreamFactory<P> extends DetokenizerSampleStreamFactory<TokenSample, P> {
+public class NameToTokenSampleStreamFactory extends
+ DetokenizerSampleStreamFactory<TokenSample, NameToTokenSampleStreamFactory.Parameters> {
- interface Parameters extends NameSampleDataStreamFactory.Parameters, DetokenizerParameter {
+ public interface Parameters extends NameSampleDataStreamFactory.Parameters, DetokenizerParameter {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(TokenSample.class,
- "namefinder", new NameToTokenSampleStreamFactory<>(Parameters.class));
+ "namefinder", new NameToTokenSampleStreamFactory(Parameters.class));
}
- protected NameToTokenSampleStreamFactory(Class<P> params) {
+ protected NameToTokenSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<TokenSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
ObjectStream<NameSample> nameSampleStream = StreamFactoryRegistry.getFactory(
NameSample.class, StreamFactoryRegistry.DEFAULT_FORMAT).create(
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/POSToSentenceSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/POSToSentenceSampleStreamFactory.java
index 25162683..8aaf104f 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/POSToSentenceSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/POSToSentenceSampleStreamFactory.java
@@ -34,23 +34,24 @@ import opennlp.tools.util.ObjectStream;
* @see POSToSentenceSampleStream
*/
@Internal
-public class POSToSentenceSampleStreamFactory<P> extends DetokenizerSampleStreamFactory<SentenceSample, P> {
+public class POSToSentenceSampleStreamFactory extends
+ DetokenizerSampleStreamFactory<SentenceSample, POSToSentenceSampleStreamFactory.Parameters> {
- interface Parameters extends WordTagSampleStreamFactory.Parameters, DetokenizerParameter {
+ public interface Parameters extends WordTagSampleStreamFactory.Parameters, DetokenizerParameter {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(SentenceSample.class,
- "pos", new POSToSentenceSampleStreamFactory<>(Parameters.class));
+ "pos", new POSToSentenceSampleStreamFactory(Parameters.class));
}
- protected POSToSentenceSampleStreamFactory(Class<P> params) {
+ protected POSToSentenceSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<SentenceSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
ObjectStream<POSSample> posSampleStream = StreamFactoryRegistry.getFactory(POSSample.class,
StreamFactoryRegistry.DEFAULT_FORMAT).create(
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/POSToTokenSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/POSToTokenSampleStreamFactory.java
index e5750a33..5f7eb51d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/POSToTokenSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/POSToTokenSampleStreamFactory.java
@@ -34,23 +34,24 @@ import opennlp.tools.util.ObjectStream;
* @see POSToTokenSampleStream
*/
@Internal
-public class POSToTokenSampleStreamFactory<P> extends DetokenizerSampleStreamFactory<TokenSample, P> {
+public class POSToTokenSampleStreamFactory extends
+ DetokenizerSampleStreamFactory<TokenSample, POSToTokenSampleStreamFactory.Parameters> {
- interface Parameters extends WordTagSampleStreamFactory.Parameters, DetokenizerParameter {
+ public interface Parameters extends WordTagSampleStreamFactory.Parameters, DetokenizerParameter {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(TokenSample.class,
- "pos", new POSToTokenSampleStreamFactory<>(Parameters.class));
+ "pos", new POSToTokenSampleStreamFactory(Parameters.class));
}
- protected POSToTokenSampleStreamFactory(Class<P> params) {
+ protected POSToTokenSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<TokenSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
ObjectStream<POSSample> posSampleStream = StreamFactoryRegistry.getFactory(POSSample.class,
StreamFactoryRegistry.DEFAULT_FORMAT).create(
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToPOSSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToPOSSampleStreamFactory.java
index 73f12a3c..eb9855e9 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToPOSSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToPOSSampleStreamFactory.java
@@ -19,6 +19,7 @@ package opennlp.tools.formats.convert;
import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.params.DetokenizerParameter;
import opennlp.tools.commons.Internal;
import opennlp.tools.formats.LanguageSampleStreamFactory;
import opennlp.tools.formats.ParseSampleStreamFactory;
@@ -34,17 +35,25 @@ import opennlp.tools.util.ObjectStream;
*/
@Internal
public class ParseToPOSSampleStreamFactory
- extends LanguageSampleStreamFactory<POSSample, ParseSampleStreamFactory.Parameters> {
+ extends LanguageSampleStreamFactory<POSSample, ParseToPOSSampleStreamFactory.Parameters> {
+ public interface Parameters extends ParseSampleStreamFactory.Parameters, DetokenizerParameter {
+ }
+
private ParseToPOSSampleStreamFactory() {
- super(ParseSampleStreamFactory.Parameters.class);
+ super(ParseToPOSSampleStreamFactory.Parameters.class);
+ }
+
+ public static void registerFactory() {
+ StreamFactoryRegistry.registerFactory(POSSample.class,
+ "parse", new ParseToPOSSampleStreamFactory());
}
@Override
public ObjectStream<POSSample> create(String[] args) {
ParseSampleStreamFactory.Parameters params =
- ArgumentParser.parse(args, ParseSampleStreamFactory.Parameters.class);
+ validateBasicFormatParameters(args, ParseSampleStreamFactory.Parameters.class);
ObjectStream<Parse> parseSampleStream = StreamFactoryRegistry.getFactory(Parse.class,
StreamFactoryRegistry.DEFAULT_FORMAT).create(
@@ -53,8 +62,4 @@ public class ParseToPOSSampleStreamFactory
return new ParseToPOSSampleStream(parseSampleStream);
}
- public static void registerFactory() {
- StreamFactoryRegistry.registerFactory(POSSample.class,
- "parse", new ParseToPOSSampleStreamFactory());
- }
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToSentenceSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToSentenceSampleStreamFactory.java
index cfa253b7..15648a90 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToSentenceSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToSentenceSampleStreamFactory.java
@@ -32,19 +32,24 @@ import opennlp.tools.util.ObjectStream;
* Do not use this class, internal use only!
*/
@Internal
-public class ParseToSentenceSampleStreamFactory extends DetokenizerSampleStreamFactory
- <SentenceSample, ParseToSentenceSampleStreamFactory.Parameters> {
+public class ParseToSentenceSampleStreamFactory extends
+ DetokenizerSampleStreamFactory<SentenceSample, ParseToSentenceSampleStreamFactory.Parameters> {
- interface Parameters extends ParseSampleStreamFactory.Parameters, DetokenizerParameter {
+ public interface Parameters extends ParseSampleStreamFactory.Parameters, DetokenizerParameter {
}
private ParseToSentenceSampleStreamFactory() {
super(Parameters.class);
}
+ public static void registerFactory() {
+ StreamFactoryRegistry.registerFactory(SentenceSample.class,
+ "parse", new ParseToSentenceSampleStreamFactory());
+ }
+
@Override
public ObjectStream<SentenceSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
ObjectStream<Parse> parseSampleStream = StreamFactoryRegistry.getFactory(Parse.class,
StreamFactoryRegistry.DEFAULT_FORMAT).create(
@@ -54,8 +59,5 @@ public class ParseToSentenceSampleStreamFactory extends DetokenizerSampleStreamF
new ParseToPOSSampleStream(parseSampleStream), 30);
}
- public static void registerFactory() {
- StreamFactoryRegistry.registerFactory(SentenceSample.class,
- "parse", new ParseToSentenceSampleStreamFactory());
- }
+
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToTokenSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToTokenSampleStreamFactory.java
index 1a39c2c1..4d0d0728 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToTokenSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToTokenSampleStreamFactory.java
@@ -33,19 +33,24 @@ import opennlp.tools.util.ObjectStream;
* Do not use this class, internal use only!
*/
@Internal
-public class ParseToTokenSampleStreamFactory extends DetokenizerSampleStreamFactory<TokenSample,
- ParseToTokenSampleStreamFactory.Parameters> {
+public class ParseToTokenSampleStreamFactory extends
+ DetokenizerSampleStreamFactory<TokenSample, ParseToTokenSampleStreamFactory.Parameters> {
- interface Parameters extends ParseSampleStreamFactory.Parameters, DetokenizerParameter {
+ public interface Parameters extends ParseSampleStreamFactory.Parameters, DetokenizerParameter {
}
private ParseToTokenSampleStreamFactory() {
super(Parameters.class);
}
+ public static void registerFactory() {
+ StreamFactoryRegistry.registerFactory(TokenSample.class,
+ "parse", new ParseToTokenSampleStreamFactory());
+ }
+
@Override
public ObjectStream<TokenSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
ObjectStream<Parse> parseSampleStream = StreamFactoryRegistry.getFactory(Parse.class,
StreamFactoryRegistry.DEFAULT_FORMAT).create(
@@ -55,8 +60,4 @@ public class ParseToTokenSampleStreamFactory extends DetokenizerSampleStreamFact
new ParseToPOSSampleStream(parseSampleStream));
}
- public static void registerFactory() {
- StreamFactoryRegistry.registerFactory(TokenSample.class,
- "parse", new ParseToTokenSampleStreamFactory());
- }
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitDocumentHandler.java b/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitDocumentHandler.java
index a724d77b..f2f57f94 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitDocumentHandler.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitDocumentHandler.java
@@ -34,7 +34,6 @@ class ConstitDocumentHandler extends DefaultHandler {
private static final String SENT_ELEMENT_NAME = "SENT";
private static final String WORD_ELEMENT_NAME = "w";
-
private static final String SENT_TYPE_NAME = "S";
private final List<Parse> parses;
@@ -139,7 +138,7 @@ class ConstitDocumentHandler extends DefaultHandler {
if (WORD_ELEMENT_NAME.equals(qName)) {
String token = tokenBuffer.toString().trim();
- if (token.length() > 0) {
+ if (!token.isEmpty()) {
cons.add(new Constituent(AbstractBottomUpParser.TOK_NODE,
new Span(offset, offset + token.length())));
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStreamFactory.java
index 11cd0108..d68b9be6 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStreamFactory.java
@@ -28,9 +28,9 @@ import opennlp.tools.parser.Parse;
import opennlp.tools.util.ObjectStream;
/**
- * <b>Note:</b>
- * Do not use this class, internal use only!
+ * <b>Note:</b> Do not use this class, internal use only!
*
+ * @see Parse
* @see ConstitParseSampleStream
*/
@Internal
@@ -45,18 +45,19 @@ public class ConstitParseSampleStreamFactory
super(Parameters.class);
}
+ public static void registerFactory() {
+ StreamFactoryRegistry.registerFactory(Parse.class, "frenchtreebank",
+ new ConstitParseSampleStreamFactory());
+ }
+
@Override
public ObjectStream<Parse> create(String[] args) {
-
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
Parameters params = ArgumentParser.parse(args, Parameters.class);
-
-
return new ConstitParseSampleStream(new FileToByteArraySampleStream(
new DirectorySampleStream(params.getData(), null, false)));
}
- public static void registerFactory() {
- StreamFactoryRegistry.registerFactory(Parse.class, "frenchtreebank",
- new ConstitParseSampleStreamFactory());
- }
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankSentenceStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankSentenceStreamFactory.java
index 3decd8e0..5c1036f1 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankSentenceStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankSentenceStreamFactory.java
@@ -19,39 +19,40 @@ package opennlp.tools.formats.irishsentencebank;
import java.io.IOException;
-import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.sentdetect.SentenceSample;
import opennlp.tools.util.ObjectStream;
/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see SentenceSample
* @see IrishSentenceBankSentenceStream
*/
-public class IrishSentenceBankSentenceStreamFactory<P>
- extends AbstractSampleStreamFactory<SentenceSample, P> {
+@Internal
+public class IrishSentenceBankSentenceStreamFactory extends
+ AbstractSampleStreamFactory<SentenceSample, IrishSentenceBankSentenceStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(SentenceSample.class,
- "irishsentencebank", new IrishSentenceBankSentenceStreamFactory<>(
+ "irishsentencebank", new IrishSentenceBankSentenceStreamFactory(
IrishSentenceBankSentenceStreamFactory.Parameters.class));
}
- protected IrishSentenceBankSentenceStreamFactory(Class<P> params) {
+ protected IrishSentenceBankSentenceStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<SentenceSample> create(String[] args) {
-
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
IrishSentenceBankDocument isbDoc = null;
try {
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankTokenSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankTokenSampleStreamFactory.java
index 213c0b8a..b41d5741 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankTokenSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankTokenSampleStreamFactory.java
@@ -19,39 +19,40 @@ package opennlp.tools.formats.irishsentencebank;
import java.io.IOException;
-import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.DetokenizerSampleStreamFactory;
import opennlp.tools.tokenize.TokenSample;
import opennlp.tools.util.ObjectStream;
/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see TokenSample
* @see IrishSentenceBankTokenSampleStream
*/
-public class IrishSentenceBankTokenSampleStreamFactory<P>
- extends DetokenizerSampleStreamFactory<TokenSample, P> {
+@Internal
+public class IrishSentenceBankTokenSampleStreamFactory extends
+ DetokenizerSampleStreamFactory<TokenSample, IrishSentenceBankTokenSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(TokenSample.class,
- "irishsentencebank", new IrishSentenceBankTokenSampleStreamFactory<>(
+ "irishsentencebank", new IrishSentenceBankTokenSampleStreamFactory(
IrishSentenceBankTokenSampleStreamFactory.Parameters.class));
}
- protected IrishSentenceBankTokenSampleStreamFactory(Class<P> params) {
+ protected IrishSentenceBankTokenSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<TokenSample> create(String[] args) {
-
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
IrishSentenceBankDocument isbDoc = null;
try {
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStreamFactory.java
index 4b3b769d..789904f0 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStreamFactory.java
@@ -38,49 +38,49 @@ import opennlp.tools.util.ObjectStream;
* @see LeipzigLanguageSampleStream
*/
@Internal
-public class LeipzigLanguageSampleStreamFactory<P>
- extends AbstractSampleStreamFactory<LanguageSample, P> {
+public class LeipzigLanguageSampleStreamFactory extends
+ AbstractSampleStreamFactory<LanguageSample, LeipzigLanguageSampleStreamFactory.Parameters> {
- interface Parameters extends EncodingParameter {
+ public interface Parameters extends EncodingParameter {
@ParameterDescription(valueName = "sentencesDir",
description = "dir with Leipzig sentences to be used")
File getSentencesDir();
@ParameterDescription(valueName = "sentencesPerSample",
description = "number of sentences per sample")
- String getSentencesPerSample();
+ Integer getSentencesPerSample();
@ParameterDescription(valueName = "samplesPerLanguage",
description = "number of samples per language")
- String getSamplesPerLanguage();
+ Integer getSamplesPerLanguage();
@ParameterDescription(valueName = "samplesToSkip",
description = "number of samples to skip before returning")
@OptionalParameter(defaultValue = "0")
- String getSamplesToSkip();
+ Integer getSamplesToSkip();
}
- protected LeipzigLanguageSampleStreamFactory(Class<P> params) {
+ protected LeipzigLanguageSampleStreamFactory(Class<Parameters> params) {
super(params);
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(LanguageSample.class,
- "leipzig", new LeipzigLanguageSampleStreamFactory<>(Parameters.class));
+ "leipzig", new LeipzigLanguageSampleStreamFactory(Parameters.class));
}
@Override
public ObjectStream<LanguageSample> create(String[] args) {
-
- Parameters params = ArgumentParser.parse(args, Parameters.class);
- File sentencesFileDir = params.getSentencesDir();
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
+ Parameters p = ArgumentParser.parse(args, Parameters.class);
+ File sentencesFileDir = p.getSentencesDir();
try {
- return new SampleSkipStream<>(new SampleShuffleStream<>(
- new LeipzigLanguageSampleStream(sentencesFileDir,
- Integer.parseInt(params.getSentencesPerSample()),
- Integer.parseInt(params.getSamplesPerLanguage()) + Integer.parseInt(params.getSamplesToSkip()))),
- Integer.parseInt(params.getSamplesToSkip()));
+ return new SampleSkipStream<>(new SampleShuffleStream<>(new LeipzigLanguageSampleStream(
+ sentencesFileDir, p.getSentencesPerSample(),
+ p.getSamplesPerLanguage() + p.getSamplesToSkip())), p.getSamplesToSkip());
} catch (IOException e) {
throw new TerminateToolException(-1, "IO error while opening sample data.", e);
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/SampleShuffleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/SampleShuffleStream.java
index 417d202e..1e3d5e69 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/SampleShuffleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/SampleShuffleStream.java
@@ -24,33 +24,40 @@ import java.util.Iterator;
import java.util.List;
import java.util.Random;
+import opennlp.tools.commons.Sample;
import opennlp.tools.util.ObjectStream;
+/**
+ * A specialization of {@link ObjectStream} that shuffles samples.
+ * @param <T> The template parameter which represents
+ * the {@link Sample} type.
+ */
class SampleShuffleStream<T> implements ObjectStream<T> {
private final List<T> bufferedSamples = new ArrayList<>();
private Iterator<T> sampleIt;
+ /**
+ * Initializes a {@link SampleShuffleStream} with the specified parameters.
+ *
+ * @param samples The {@link ObjectStream} to process.
+ * @throws IOException Thrown if IO errors occurred during skip operation.
+ */
SampleShuffleStream(ObjectStream<T> samples) throws IOException {
-
T sample;
while ((sample = samples.read()) != null) {
bufferedSamples.add(sample);
}
-
Collections.shuffle(bufferedSamples, new Random(23));
-
reset();
}
@Override
public T read() throws IOException {
-
if (sampleIt.hasNext()) {
return sampleIt.next();
}
-
return null;
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/SampleSkipStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/SampleSkipStream.java
index 2bfff5b8..e5f72e03 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/SampleSkipStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/SampleSkipStream.java
@@ -19,13 +19,26 @@ package opennlp.tools.formats.leipzig;
import java.io.IOException;
+import opennlp.tools.commons.Sample;
import opennlp.tools.util.ObjectStream;
+/**
+ * A specialization of {@link ObjectStream} that skips a number of samples.
+ * @param <T> The template parameter which represents
+ * the {@link Sample} type.
+ */
class SampleSkipStream<T> implements ObjectStream<T> {
private final ObjectStream<T> samples;
private final int samplesToSkip;
+ /**
+ * Initializes a {@link SampleSkipStream} with the specified parameters.
+ *
+ * @param samples The {@link ObjectStream} to process.
+ * @param samplesToSkip The number of samples to skip. Must be greater than {@code 0}.
+ * @throws IOException Thrown if IO errors occurred during skip operation.
+ */
SampleSkipStream(ObjectStream<T> samples, int samplesToSkip) throws IOException {
this.samples = samples;
this.samplesToSkip = samplesToSkip;
@@ -46,7 +59,6 @@ class SampleSkipStream<T> implements ObjectStream<T> {
private void skipSamples() throws IOException {
int i = 0;
-
while (i < samplesToSkip && (samples.read()) != null) {
i++;
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/letsmt/LetsmtSentenceStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/letsmt/LetsmtSentenceStreamFactory.java
index e3474257..1077ab37 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/letsmt/LetsmtSentenceStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/letsmt/LetsmtSentenceStreamFactory.java
@@ -25,6 +25,7 @@ import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.TerminateToolException;
import opennlp.tools.cmdline.params.BasicFormatParams;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.sentdetect.SentenceSample;
import opennlp.tools.tokenize.DetokenizationDictionary;
@@ -33,11 +34,16 @@ import opennlp.tools.tokenize.DictionaryDetokenizer;
import opennlp.tools.util.ObjectStream;
/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see SentenceSample
* @see LetsmtSentenceStream
*/
-public class LetsmtSentenceStreamFactory<P> extends AbstractSampleStreamFactory<SentenceSample, P> {
+@Internal
+public class LetsmtSentenceStreamFactory extends
+ AbstractSampleStreamFactory<SentenceSample, LetsmtSentenceStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
@ArgumentParser.ParameterDescription(valueName = "dictionary",
description = "specifies the file with detokenizer dictionary.")
@ArgumentParser.OptionalParameter
@@ -45,21 +51,17 @@ public class LetsmtSentenceStreamFactory<P> extends AbstractSampleStreamFactory<
}
public static void registerFactory() {
- StreamFactoryRegistry.registerFactory(SentenceSample.class,
- "letsmt", new LetsmtSentenceStreamFactory<>(
- LetsmtSentenceStreamFactory.Parameters.class));
+ StreamFactoryRegistry.registerFactory(SentenceSample.class, "letsmt",
+ new LetsmtSentenceStreamFactory(LetsmtSentenceStreamFactory.Parameters.class));
}
- protected LetsmtSentenceStreamFactory(Class<P> params) {
+ protected LetsmtSentenceStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<SentenceSample> create(String[] args) {
-
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
LetsmtDocument letsmtDoc = null;
try {
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/AbstractSampleStreamTest.java b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/Masc.java
similarity index 71%
copy from opennlp-tools/src/test/java/opennlp/tools/formats/AbstractSampleStreamTest.java
copy to opennlp-tools/src/main/java/opennlp/tools/formats/masc/Masc.java
index c154f21d..8b79b83d 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/formats/AbstractSampleStreamTest.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/Masc.java
@@ -15,14 +15,13 @@
* limitations under the License.
*/
-package opennlp.tools.formats;
+package opennlp.tools.formats.masc;
-import opennlp.tools.util.InputStreamFactory;
-
-
-public abstract class AbstractSampleStreamTest extends AbstractFormatTest {
+/**
+ * A simple marker interface for classes that support or refer to
+ * the {@link #MASC_FORMAT}.
+ */
+public interface Masc {
- protected InputStreamFactory getFactory(String resource) {
- return new ResourceAsStreamFactory(AbstractSampleStreamTest.class, FORMATS_BASE_DIR + resource);
- }
+ String MASC_FORMAT = "masc";
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascDocument.java b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascDocument.java
index e4e1177c..7af48a76 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascDocument.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascDocument.java
@@ -40,7 +40,6 @@ import org.xml.sax.SAXException;
import opennlp.tools.util.Span;
import opennlp.tools.util.XmlUtil;
-
public class MascDocument {
private static final Logger logger = LoggerFactory.getLogger(MascDocument.class);
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascNamedEntitySampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascNamedEntitySampleStreamFactory.java
index 1adf3535..ab85f88d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascNamedEntitySampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascNamedEntitySampleStreamFactory.java
@@ -24,29 +24,49 @@ import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.namefind.NameSample;
import opennlp.tools.util.ObjectStream;
-public class MascNamedEntitySampleStreamFactory<P> extends AbstractSampleStreamFactory<NameSample, P> {
- public static final String MASC_FORMAT = "masc";
+/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see NameSample
+ * @see MascPOSSampleStream
+ */
+@Internal
+public class MascNamedEntitySampleStreamFactory extends
+ AbstractSampleStreamFactory<NameSample, MascNamedEntitySampleStreamFactory.Parameters>
+ implements Masc {
+
+ public interface Parameters extends BasicFormatParams {
- protected MascNamedEntitySampleStreamFactory(Class<P> params) {
+ @ArgumentParser.ParameterDescription(valueName = "recurrentSearch",
+ description = "search through files recursively")
+ Boolean getRecurrentSearch();
+
+ @ArgumentParser.ParameterDescription(valueName = "fileFilterString",
+ description = "only include files which contain a given string in their name")
+ String getFileFilter();
+
+ }
+
+ protected MascNamedEntitySampleStreamFactory(Class<Parameters> params) {
super(params);
}
public static void registerFactory() {
- StreamFactoryRegistry.registerFactory(NameSample.class,
- MASC_FORMAT,
- new MascNamedEntitySampleStreamFactory<>(
- MascNamedEntitySampleStreamFactory.Parameters.class));
+ StreamFactoryRegistry.registerFactory(NameSample.class, MASC_FORMAT,
+ new MascNamedEntitySampleStreamFactory(MascNamedEntitySampleStreamFactory.Parameters.class));
}
@Override
public ObjectStream<NameSample> create(String[] args) {
- MascNamedEntitySampleStreamFactory.Parameters params =
- ArgumentParser.parse(args, MascNamedEntitySampleStreamFactory.Parameters.class);
-
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
+ Parameters params = ArgumentParser.parse(args, Parameters.class);
try {
FileFilter fileFilter = pathname -> pathname.getName().contains(params.getFileFilter());
@@ -57,17 +77,4 @@ public class MascNamedEntitySampleStreamFactory<P> extends AbstractSampleStreamF
}
return null;
}
-
- interface Parameters extends BasicFormatParams {
-
- @ArgumentParser.ParameterDescription(valueName = "recurrentSearch",
- description = "search through files recursively")
- boolean getRecurrentSearch();
-
- @ArgumentParser.ParameterDescription(valueName = "fileFilterString",
- description = "only include files which contain a given string in their name")
- String getFileFilter();
-
- }
-
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascPOSSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascPOSSampleStreamFactory.java
index 4e910c68..b3bcedde 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascPOSSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascPOSSampleStreamFactory.java
@@ -24,29 +24,49 @@ import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.postag.POSSample;
import opennlp.tools.util.ObjectStream;
-public class MascPOSSampleStreamFactory<P> extends AbstractSampleStreamFactory<POSSample, P> {
- public static final String MASC_FORMAT = "masc";
+/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see POSSample
+ * @see MascPOSSampleStream
+ */
+@Internal
+public class MascPOSSampleStreamFactory extends
+ AbstractSampleStreamFactory<POSSample, MascPOSSampleStreamFactory.Parameters>
+ implements Masc {
+
+ public interface Parameters extends BasicFormatParams {
+
+ @ArgumentParser.ParameterDescription(valueName = "recurrentSearch",
+ description = "search through files recursively")
+ Boolean getRecurrentSearch();
+
+ @ArgumentParser.ParameterDescription(valueName = "fileFilterString",
+ description = "only include files which contain a given string in their name")
+ String getFileFilter();
+
+ }
- protected MascPOSSampleStreamFactory(Class<P> params) {
+ protected MascPOSSampleStreamFactory(Class<Parameters> params) {
super(params);
}
public static void registerFactory() {
- StreamFactoryRegistry.registerFactory(POSSample.class,
- MASC_FORMAT,
- new MascPOSSampleStreamFactory<>(
- MascPOSSampleStreamFactory.Parameters.class));
+ StreamFactoryRegistry.registerFactory(POSSample.class, MASC_FORMAT,
+ new MascPOSSampleStreamFactory(Parameters.class));
}
@Override
public ObjectStream<POSSample> create(String[] args) {
- MascPOSSampleStreamFactory.Parameters params =
- ArgumentParser.parse(args, MascPOSSampleStreamFactory.Parameters.class);
-
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
+ Parameters params = ArgumentParser.parse(args, Parameters.class);
try {
FileFilter fileFilter = pathname -> pathname.getName().contains(params.getFileFilter());
@@ -59,17 +79,5 @@ public class MascPOSSampleStreamFactory<P> extends AbstractSampleStreamFactory<P
return null;
}
- interface Parameters extends BasicFormatParams {
-
- @ArgumentParser.ParameterDescription(valueName = "recurrentSearch",
- description = "search through files recursively")
- boolean getRecurrentSearch();
-
- @ArgumentParser.ParameterDescription(valueName = "fileFilterString",
- description = "only include files which contain a given string in their name")
- String getFileFilter();
-
- }
-
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascSentence.java b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascSentence.java
index 05d661e8..808bb65c 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascSentence.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascSentence.java
@@ -18,6 +18,7 @@
package opennlp.tools.formats.masc;
import java.io.IOException;
+import java.io.Serial;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -34,8 +35,9 @@ import opennlp.tools.util.Span;
public class MascSentence extends Span {
- private static final Logger logger = LoggerFactory.getLogger(MascSentence.class);
+ @Serial
private static final long serialVersionUID = 6295507533472650848L;
+ private static final Logger logger = LoggerFactory.getLogger(MascSentence.class);
/**
* A helper class to extract the extract a quark from the corpus file even if it is beyond the
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascSentenceSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascSentenceSampleStreamFactory.java
index d44bf780..9b6eeb43 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascSentenceSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascSentenceSampleStreamFactory.java
@@ -24,28 +24,52 @@ import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.sentdetect.SentenceSample;
import opennlp.tools.util.ObjectStream;
-public class MascSentenceSampleStreamFactory<P> extends AbstractSampleStreamFactory<SentenceSample, P> {
+/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see SentenceSample
+ * @see MascSentenceSampleStream
+ */
+@Internal
+public class MascSentenceSampleStreamFactory extends
+ AbstractSampleStreamFactory<SentenceSample, MascSentenceSampleStreamFactory.Parameters>
+ implements Masc {
+
+ public interface Parameters extends BasicFormatParams {
+ @ArgumentParser.ParameterDescription(valueName = "sentencesPerSample",
+ description = "number of sentences per sample")
+ String getSentencesPerSample();
+
+ @ArgumentParser.ParameterDescription(valueName = "recurrentSearch",
+ description = "search through files recursively")
+ Boolean getRecurrentSearch();
+
+ @ArgumentParser.ParameterDescription(valueName = "fileFilterString",
+ description = "only include files which contain a given string in their name")
+ String getFileFilter();
- public static final String MASC_FORMAT = "masc";
+ }
- protected MascSentenceSampleStreamFactory(Class<P> params) {
+ protected MascSentenceSampleStreamFactory(Class<Parameters> params) {
super(params);
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(SentenceSample.class, MASC_FORMAT,
- new MascSentenceSampleStreamFactory<>(MascSentenceSampleStreamFactory.Parameters.class));
+ new MascSentenceSampleStreamFactory(Parameters.class));
}
@Override
public ObjectStream<SentenceSample> create(String[] args) {
- MascSentenceSampleStreamFactory.Parameters params =
- ArgumentParser.parse(args, MascSentenceSampleStreamFactory.Parameters.class);
-
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
+ Parameters params = ArgumentParser.parse(args, Parameters.class);
try {
FileFilter fileFilter = pathname -> pathname.getName().contains(params.getFileFilter());
@@ -59,20 +83,5 @@ public class MascSentenceSampleStreamFactory<P> extends AbstractSampleStreamFact
return null;
}
- interface Parameters extends BasicFormatParams {
- @ArgumentParser.ParameterDescription(valueName = "sentencesPerSample",
- description = "number of sentences per sample")
- String getSentencesPerSample();
-
- @ArgumentParser.ParameterDescription(valueName = "recurrentSearch",
- description = "search through files recursively")
- boolean getRecurrentSearch();
-
- @ArgumentParser.ParameterDescription(valueName = "fileFilterString",
- description = "only include files which contain a given string in their name")
- String getFileFilter();
-
- }
-
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascToken.java b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascToken.java
index 5fa96eeb..aef52dc0 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascToken.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascToken.java
@@ -17,6 +17,8 @@
package opennlp.tools.formats.masc;
+import java.io.Serial;
+
import opennlp.tools.util.Span;
/**
@@ -24,6 +26,7 @@ import opennlp.tools.util.Span;
*/
public class MascToken extends Span {
+ @Serial
private static final long serialVersionUID = -780646706788037041L;
private final String pos;
private final String base;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascTokenSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascTokenSampleStreamFactory.java
index c58eb133..8a70c0b7 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascTokenSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascTokenSampleStreamFactory.java
@@ -24,29 +24,52 @@ import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.tokenize.TokenSample;
import opennlp.tools.util.ObjectStream;
-public class MascTokenSampleStreamFactory<P> extends AbstractSampleStreamFactory<TokenSample, P> {
+/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see TokenSample
+ * @see MascTokenSampleStream
+ */
+@Internal
+public class MascTokenSampleStreamFactory extends
+ AbstractSampleStreamFactory<TokenSample, MascTokenSampleStreamFactory.Parameters>
+ implements Masc {
+
+ public interface Parameters extends BasicFormatParams {
+ @ArgumentParser.ParameterDescription(valueName = "sentencesPerSample",
+ description = "number of sentences per sample")
+ String getSentencesPerSample();
+
+ @ArgumentParser.ParameterDescription(valueName = "recurrentSearch",
+ description = "search through files recursively")
+ Boolean getRecurrentSearch();
+
+ @ArgumentParser.ParameterDescription(valueName = "fileFilterString",
+ description = "only include files which contain a given string in their name")
+ String getFileFilter();
- public static final String MASC_FORMAT = "masc";
+ }
- protected MascTokenSampleStreamFactory(Class<P> params) {
+ protected MascTokenSampleStreamFactory(Class<Parameters> params) {
super(params);
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(TokenSample.class, MASC_FORMAT,
- new MascTokenSampleStreamFactory<>(MascTokenSampleStreamFactory.Parameters.class));
+ new MascTokenSampleStreamFactory(Parameters.class));
}
-
@Override
public ObjectStream<TokenSample> create(String[] args) {
- MascTokenSampleStreamFactory.Parameters params =
- ArgumentParser.parse(args, MascTokenSampleStreamFactory.Parameters.class);
-
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
+ Parameters params = ArgumentParser.parse(args, Parameters.class);
try {
FileFilter fileFilter = pathname -> pathname.getName().contains(params.getFileFilter());
@@ -59,20 +82,5 @@ public class MascTokenSampleStreamFactory<P> extends AbstractSampleStreamFactory
return null;
}
- interface Parameters extends BasicFormatParams {
- @ArgumentParser.ParameterDescription(valueName = "sentencesPerSample",
- description = "number of sentences per sample")
- String getSentencesPerSample();
-
- @ArgumentParser.ParameterDescription(valueName = "recurrentSearch",
- description = "search through files recursively")
- boolean getRecurrentSearch();
-
- @ArgumentParser.ParameterDescription(valueName = "fileFilterString",
- description = "only include files which contain a given string in their name")
- String getFileFilter();
-
- }
-
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascWord.java b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascWord.java
index 916f6d2d..f351927b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascWord.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascWord.java
@@ -17,10 +17,13 @@
package opennlp.tools.formats.masc;
+import java.io.Serial;
+
import opennlp.tools.util.Span;
public class MascWord extends Span {
+ @Serial
private static final long serialVersionUID = 2133473549058189775L;
private final int id;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/moses/MosesSentenceSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/moses/MosesSentenceSampleStream.java
index cd7b10f8..e0f8bc2f 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/moses/MosesSentenceSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/moses/MosesSentenceSampleStream.java
@@ -27,6 +27,13 @@ import opennlp.tools.util.FilterObjectStream;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.Span;
+/**
+ * Moses is a statistical machine translation system that allows you
+ * to automatically train translation models for any language pair.
+ * <p>
+ * Details are found on the <a href="https://www2.statmt.org/moses/">
+ * official website</a>.
+ */
public class MosesSentenceSampleStream extends FilterObjectStream<String, SentenceSample> {
/**
@@ -58,9 +65,8 @@ public class MosesSentenceSampleStream extends FilterObjectStream<String, Senten
sentencesString.append(' ');
}
- if (sentenceSpans.size() > 0) {
- return new SentenceSample(sentencesString.toString(),
- sentenceSpans.toArray(new Span[0]));
+ if (!sentenceSpans.isEmpty()) {
+ return new SentenceSample(sentencesString.toString(), sentenceSpans.toArray(new Span[0]));
}
return null;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/moses/MosesSentenceSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/moses/MosesSentenceSampleStreamFactory.java
index 6e942c93..529f2179 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/moses/MosesSentenceSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/moses/MosesSentenceSampleStreamFactory.java
@@ -17,49 +17,40 @@
package opennlp.tools.formats.moses;
-import java.io.IOException;
-
-import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.sentdetect.SentenceSample;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
* Factory producing OpenNLP {@link MosesSentenceSampleStream} objects.
+ * <p>
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see SentenceSample
+ * @see MosesSentenceSampleStream
*/
-public class MosesSentenceSampleStreamFactory<P> extends AbstractSampleStreamFactory<SentenceSample, P> {
+@Internal
+public class MosesSentenceSampleStreamFactory extends
+ AbstractSampleStreamFactory<SentenceSample, MosesSentenceSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(SentenceSample.class,
- "moses", new MosesSentenceSampleStreamFactory<>(Parameters.class));
+ "moses", new MosesSentenceSampleStreamFactory(Parameters.class));
}
- protected MosesSentenceSampleStreamFactory(Class<P> params) {
+ protected MosesSentenceSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<SentenceSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
-
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new MosesSentenceSampleStream(lineStream);
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/Muc6NameSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/Muc6NameSampleStreamFactory.java
index 28030beb..4acb31d1 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/Muc6NameSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/Muc6NameSampleStreamFactory.java
@@ -23,21 +23,33 @@ import java.nio.charset.StandardCharsets;
import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
import opennlp.tools.cmdline.params.BasicFormatParams;
import opennlp.tools.cmdline.tokenizer.TokenizerModelLoader;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.formats.DirectorySampleStream;
import opennlp.tools.formats.convert.FileToStringSampleStream;
import opennlp.tools.namefind.NameSample;
+import opennlp.tools.tokenize.ThreadSafeTokenizerME;
import opennlp.tools.tokenize.Tokenizer;
-import opennlp.tools.tokenize.TokenizerME;
import opennlp.tools.tokenize.TokenizerModel;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.StringUtil;
+/**
+ * @see NameSample
+ * @see MucNameSampleStream
+ */
+@Internal
public class Muc6NameSampleStreamFactory
extends AbstractSampleStreamFactory<NameSample, Muc6NameSampleStreamFactory.Parameters> {
+ public interface Parameters extends BasicFormatParams {
+ @ParameterDescription(valueName = "modelFile")
+ File getTokenizerModel();
+ }
+
protected Muc6NameSampleStreamFactory() {
super(Parameters.class);
}
@@ -49,11 +61,16 @@ public class Muc6NameSampleStreamFactory
@Override
public ObjectStream<NameSample> create(String[] args) {
-
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
Parameters params = ArgumentParser.parse(args, Parameters.class);
+ if (!params.getData().isDirectory() || !params.getData().exists()) {
+ throw new TerminateToolException(-1, "The specified data directory is not valid!");
+ }
TokenizerModel tokenizerModel = new TokenizerModelLoader().load(params.getTokenizerModel());
- Tokenizer tokenizer = new TokenizerME(tokenizerModel);
+ Tokenizer tokenizer = new ThreadSafeTokenizerME(tokenizerModel);
ObjectStream<String> mucDocStream = new FileToStringSampleStream(
new DirectorySampleStream(params.getData(),
@@ -62,9 +79,4 @@ public class Muc6NameSampleStreamFactory
return new MucNameSampleStream(tokenizer, mucDocStream);
}
-
- interface Parameters extends BasicFormatParams {
- @ParameterDescription(valueName = "modelFile")
- File getTokenizerModel();
- }
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucElementNames.java b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucElementNames.java
index 23eb09f2..d3ac523d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucElementNames.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucElementNames.java
@@ -31,10 +31,8 @@ class MucElementNames {
static {
CONTENT_ELEMENTS = Set.of(
- MucElementNames.HEADLINE_ELEMENT,
- MucElementNames.DATELINE_ELEMENT,
- MucElementNames.DD_ELEMENT,
- MucElementNames.SENTENCE_ELEMENT);
+ MucElementNames.HEADLINE_ELEMENT, MucElementNames.DATELINE_ELEMENT,
+ MucElementNames.DD_ELEMENT, MucElementNames.SENTENCE_ELEMENT);
}
private MucElementNames() {
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucNameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucNameSampleStream.java
index f9a8bbc0..caf63a33 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucNameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucNameSampleStream.java
@@ -64,7 +64,7 @@ public class MucNameSampleStream extends FilterObjectStream<String, NameSample>
}
}
- if (storedSamples.size() > 0) {
+ if (!storedSamples.isEmpty()) {
return storedSamples.remove(0);
}
else {
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSegmentationDocument.java b/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSegmentationDocument.java
index 08f615b1..3cbadd82 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSegmentationDocument.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSegmentationDocument.java
@@ -32,6 +32,7 @@ import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import org.w3c.dom.Document;
+import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
@@ -41,6 +42,13 @@ import opennlp.tools.util.XmlUtil;
public class NKJPSegmentationDocument {
+ private static final String XML_ID = "xml:id";
+ private static final String SEG = "seg";
+ private static final String CHOICE = "choice";
+ private static final String NKJP_PAREN = "nkjp:paren";
+ private static final String NKJP_REJECTED = "nkjp:rejected";
+ private static final String NKJP_NPS = "nkjp:nps";
+
public static class Pointer {
final String doc;
final String id;
@@ -62,8 +70,7 @@ public class NKJPSegmentationDocument {
@Override
public String toString() {
- return doc + "#string-range(" + id + "," + offset
- + "," + length + ")";
+ return doc + "#string-range(" + id + "," + offset + "," + length + ")";
}
}
@@ -103,8 +110,8 @@ public class NKJPSegmentationDocument {
Node sentnode = nl.item(i);
String sentid = null;
- if (sentnode.getAttributes().getNamedItem("xml:id") != null) {
- sentid = sentnode.getAttributes().getNamedItem("xml:id").getTextContent();
+ if (sentnode.getAttributes().getNamedItem(XML_ID) != null) {
+ sentid = sentnode.getAttributes().getNamedItem(XML_ID).getTextContent();
}
Map<String, Pointer> segments = new LinkedHashMap<>();
@@ -112,18 +119,17 @@ public class NKJPSegmentationDocument {
for (int j = 0; j < segnl.getLength(); j++) {
Node n = segnl.item(j);
- if (n.getNodeName().equals("seg")) {
+ if (n.getNodeName().equals(SEG)) {
String segid = xmlID(n);
Pointer pointer = fromSeg(n);
segments.put(segid, pointer);
- } else if (n.getNodeName().equals("choice")) {
-
+ } else if (n.getNodeName().equals(CHOICE)) {
NodeList choices = n.getChildNodes();
-
for (int k = 0; k < choices.getLength(); k++) {
- if (choices.item(k).getNodeName().equals("nkjp:paren")) {
- if (!checkRejectedParen(choices.item(k))) {
- NodeList paren_segs = (NodeList) SEG_NODES_ONLY.evaluate(choices.item(k),
+ Node choice = choices.item(k);
+ if (choice.getNodeName().equals(NKJP_PAREN)) {
+ if (!checkRejectedParen(choice)) {
+ NodeList paren_segs = (NodeList) SEG_NODES_ONLY.evaluate(choice,
XPathConstants.NODESET);
for (int l = 0; l < paren_segs.getLength(); l++) {
@@ -132,17 +138,16 @@ public class NKJPSegmentationDocument {
segments.put(segid, pointer);
}
}
- } else if (choices.item(k).getNodeName().equals("seg")) {
- if (!checkRejected(choices.item(k))) {
- String segid = xmlID(choices.item(k));
- Pointer pointer = fromSeg(choices.item(k));
+ } else if (choice.getNodeName().equals(SEG)) {
+ if (!checkRejected(choice)) {
+ String segid = xmlID(choice);
+ Pointer pointer = fromSeg(choice);
segments.put(segid, pointer);
}
}
}
}
}
-
sentences.put(sentid, segments);
}
@@ -154,14 +159,14 @@ public class NKJPSegmentationDocument {
}
static boolean checkRejected(Node n) {
- if (n.getAttributes() == null) {
+ NamedNodeMap attrs = n.getAttributes();
+ if (attrs == null) {
return false;
}
- if (n.getAttributes().getNamedItem("nkjp:rejected") == null) {
+ if (attrs.getNamedItem(NKJP_REJECTED) == null) {
return false;
}
- String rejected = n.getAttributes().getNamedItem("nkjp:rejected").getTextContent();
- return rejected.equals("true");
+ return attrs.getNamedItem(NKJP_REJECTED).getTextContent().equals("true");
}
static boolean checkRejectedParen(Node n) {
@@ -170,7 +175,7 @@ public class NKJPSegmentationDocument {
} else {
for (int i = 0; i < n.getChildNodes().getLength(); i++) {
Node m = n.getChildNodes().item(i);
- if (m.getNodeName().equals("seg")) {
+ if (m.getNodeName().equals(SEG)) {
if (!checkRejected(m)) {
return false;
}
@@ -181,13 +186,14 @@ public class NKJPSegmentationDocument {
}
static String xmlID(Node n) throws IOException {
- if (n.getAttributes() == null || n.getAttributes().getLength() < 1) {
+ NamedNodeMap attr = n.getAttributes();
+ if (attr == null || attr.getLength() < 1) {
throw new IOException("Missing required attributes");
}
String id = null;
- if (n.getAttributes().getNamedItem("xml:id") != null) {
- id = n.getAttributes().getNamedItem("xml:id").getTextContent();
+ if (attr.getNamedItem(XML_ID) != null) {
+ id = attr.getNamedItem(XML_ID).getTextContent();
}
if (id == null) {
@@ -207,8 +213,8 @@ public class NKJPSegmentationDocument {
ptr = n.getAttributes().getNamedItem("corresp").getTextContent();
}
String spacing = "";
- if (n.getAttributes().getNamedItem("nkjp:nps") != null) {
- spacing = n.getAttributes().getNamedItem("nkjp:nps").getTextContent();
+ if (n.getAttributes().getNamedItem(NKJP_NPS) != null) {
+ spacing = n.getAttributes().getNamedItem(NKJP_NPS).getTextContent();
}
if (ptr == null) {
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSentenceSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSentenceSampleStream.java
index 03ed8348..b71a4c50 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSentenceSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSentenceSampleStream.java
@@ -28,8 +28,8 @@ import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.Span;
public class NKJPSentenceSampleStream implements ObjectStream<SentenceSample> {
- private final NKJPSegmentationDocument segments;
+ private final NKJPSegmentationDocument segments;
private final NKJPTextDocument text;
private Iterator<Map.Entry<String, Map<String, NKJPSegmentationDocument.Pointer>>> segmentIt;
@@ -89,7 +89,7 @@ public class NKJPSentenceSampleStream implements ObjectStream<SentenceSample> {
}
// end of stream is reached, indicate that with null return value
- if (sentenceSpans.size() == 0) {
+ if (sentenceSpans.isEmpty()) {
return null;
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSentenceSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSentenceSampleStreamFactory.java
index d126d21e..da9e5989 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSentenceSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSentenceSampleStreamFactory.java
@@ -24,13 +24,22 @@ import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.sentdetect.SentenceSample;
import opennlp.tools.util.ObjectStream;
-public class NKJPSentenceSampleStreamFactory<P> extends AbstractSampleStreamFactory<SentenceSample, P> {
+/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see SentenceSample
+ * @see NKJPSentenceSampleStream
+ */
+@Internal
+public class NKJPSentenceSampleStreamFactory extends
+ AbstractSampleStreamFactory<SentenceSample, NKJPSentenceSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
@ArgumentParser.ParameterDescription(valueName = "text",
description = "file containing NKJP text")
File getTextFile();
@@ -38,20 +47,16 @@ public class NKJPSentenceSampleStreamFactory<P> extends AbstractSampleStreamFact
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(SentenceSample.class, "nkjp",
- new NKJPSentenceSampleStreamFactory<>(NKJPSentenceSampleStreamFactory.Parameters.class));
+ new NKJPSentenceSampleStreamFactory(NKJPSentenceSampleStreamFactory.Parameters.class));
}
- protected NKJPSentenceSampleStreamFactory(Class<P> params) {
+ protected NKJPSentenceSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<SentenceSample> create(String[] args) {
-
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
-
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
CmdLineUtil.checkInputFile("Text", params.getTextFile());
NKJPSegmentationDocument segDoc = null;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPTextDocument.java b/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPTextDocument.java
index 3c20e704..205914eb 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPTextDocument.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPTextDocument.java
@@ -43,14 +43,14 @@ import opennlp.tools.util.XmlUtil;
* The National corpus of Polish (NKJP) format.
* <p>
* Information about the format are found on this
- * <a href="http://nkjp.pl/index.php?page=0%26lang=1">web site</a>.
+ * <a href="https://nkjp.pl/index.php?page=0%26lang=1">web site</a>.
* <p>
* A 1-million word corpus can be found on this
- * <a href="http://nkjp.pl/index.php?page=14%26lang=1">
+ * <a href="https://nkjp.pl/index.php?page=14%26lang=1">
* web site</a>.
* <p>
* The NKJP schema can be found
- * <a href="http://nlp.ipipan.waw.pl/TEI4NKJP/">here</a>.
+ * <a href="https://nlp.ipipan.waw.pl/TEI4NKJP/">here</a>.
*/
public class NKJPTextDocument {
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStream.java
index 8dc09af8..130e7386 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStream.java
@@ -35,10 +35,18 @@ import opennlp.tools.util.Span;
import opennlp.tools.util.StringUtil;
/**
- * Name Sample Stream parser for the OntoNotes 4.0 corpus.
+ * Name Sample Stream parser for the OntoNotes 4.0 named entity files.
*/
public class OntoNotesNameSampleStream extends FilterObjectStream<String, NameSample> {
+ private static final String TAG_DOC_OPEN = "<DOC";
+ private static final String TAG_DOC_CLOSE = "</DOC>";
+ private static final String TAG_ENAMEX_OPEN = "<ENAMEX";
+ private static final String TAG_ENAMEX_CLOSE = "</ENAMEX>";
+ private static final String TYPE = "TYPE=\"";
+ private static final String SYMBOL_CLOSE = ">";
+ private static final String SYMBOL_OPEN = "<";
+
private final Map<String, String> tokenConversionMap;
private final List<NameSample> nameSamples = new LinkedList<>();
@@ -69,21 +77,18 @@ public class OntoNotesNameSampleStream extends FilterObjectStream<String, NameSa
StringBuilder convertedToken = new StringBuilder(token);
- int startTagEndIndex = convertedToken.indexOf(">");
-
+ int startTagEndIndex = convertedToken.indexOf(SYMBOL_CLOSE);
if (token.contains("=\"") && startTagEndIndex != -1) {
convertedToken.delete(0, startTagEndIndex + 1);
}
- int endTagBeginIndex = convertedToken.indexOf("<");
- int endTagEndIndex = convertedToken.indexOf(">");
-
+ int endTagBeginIndex = convertedToken.indexOf(SYMBOL_OPEN);
+ int endTagEndIndex = convertedToken.indexOf(SYMBOL_CLOSE);
if (endTagBeginIndex != -1 && endTagEndIndex != -1) {
convertedToken.delete(endTagBeginIndex, endTagEndIndex + 1);
}
String cleanedToken = convertedToken.toString();
-
if (tokenConversionMap.get(cleanedToken) != null) {
cleanedToken = tokenConversionMap.get(cleanedToken);
}
@@ -96,74 +101,64 @@ public class OntoNotesNameSampleStream extends FilterObjectStream<String, NameSa
if (nameSamples.isEmpty()) {
String doc = samples.read();
-
if (doc != null) {
- BufferedReader docIn = new BufferedReader(new StringReader(doc));
-
boolean clearAdaptiveData = true;
-
String line;
- while ((line = docIn.readLine()) != null) {
-
- if (line.startsWith("<DOC")) {
- continue;
- }
-
- if (line.equals("</DOC>")) {
- break;
- }
-
- String[] tokens = WhitespaceTokenizer.INSTANCE.tokenize(line);
+ try (BufferedReader docIn = new BufferedReader(new StringReader(doc))) {
+ while ((line = docIn.readLine()) != null) {
- List<Span> entities = new LinkedList<>();
- List<String> cleanedTokens = new ArrayList<>(tokens.length);
-
- int tokenIndex = 0;
- int entityBeginIndex = -1;
- String entityType = null;
- boolean insideStartEnmaxTag = false;
- for (String token : tokens) {
-
- // Split here, next part of tag is in new token
- if (token.startsWith("<ENAMEX")) {
- insideStartEnmaxTag = true;
+ if (line.startsWith(TAG_DOC_OPEN)) {
continue;
}
+ if (line.equals(TAG_DOC_CLOSE)) {
+ break;
+ }
- if (insideStartEnmaxTag) {
-
- String typeBegin = "TYPE=\"";
+ String[] tokens = WhitespaceTokenizer.INSTANCE.tokenize(line);
+ List<Span> entities = new LinkedList<>();
+ List<String> cleanedTokens = new ArrayList<>(tokens.length);
- if (token.startsWith(typeBegin)) {
+ int tokenIndex = 0;
+ int entityBeginIndex = -1;
+ String entityType = null;
+ boolean insideStartEnmaxTag = false;
+ for (String token : tokens) {
- int typeEnd = token.indexOf("\"", typeBegin.length());
+ // Split here, next part of tag is in new token
+ if (token.startsWith(TAG_ENAMEX_OPEN)) {
+ insideStartEnmaxTag = true;
+ continue;
+ }
- entityType = StringUtil.toLowerCase(token.substring(typeBegin.length(), typeEnd));
+ if (insideStartEnmaxTag) {
+ String typeBegin = TYPE;
+ if (token.startsWith(typeBegin)) {
+ int typeEnd = token.indexOf("\"", typeBegin.length());
+ entityType = StringUtil.toLowerCase(token.substring(typeBegin.length(), typeEnd));
+ }
+
+ if (token.contains(SYMBOL_CLOSE)) {
+ entityBeginIndex = tokenIndex;
+ insideStartEnmaxTag = false;
+ } else {
+ continue;
+ }
}
- if (token.contains(">")) {
- entityBeginIndex = tokenIndex;
- insideStartEnmaxTag = false;
- } else {
- continue;
+ if (token.endsWith(TAG_ENAMEX_CLOSE)) {
+ entities.add(new Span(entityBeginIndex, tokenIndex + 1, entityType));
+ entityBeginIndex = -1;
}
- }
- if (token.endsWith("</ENAMEX>")) {
- entities.add(new Span(entityBeginIndex, tokenIndex + 1,
- entityType));
- entityBeginIndex = -1;
+ cleanedTokens.add(convertToken(token));
+ tokenIndex++;
}
- cleanedTokens.add(convertToken(token));
- tokenIndex++;
- }
+ nameSamples.add(new NameSample(cleanedTokens.toArray(new String[0]),
+ entities.toArray(new Span[0]), clearAdaptiveData));
- nameSamples.add(new NameSample(cleanedTokens
- .toArray(new String[0]), entities
- .toArray(new Span[0]), clearAdaptiveData));
-
- clearAdaptiveData = false;
+ clearAdaptiveData = false;
+ }
}
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStreamFactory.java
index f71e3042..9bb154fa 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStreamFactory.java
@@ -22,23 +22,42 @@ import java.nio.charset.StandardCharsets;
import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.formats.DirectorySampleStream;
import opennlp.tools.formats.convert.FileToStringSampleStream;
import opennlp.tools.namefind.NameSample;
import opennlp.tools.util.ObjectStream;
+/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see OntoNotesNameSampleStream
+ */
+@Internal
public class OntoNotesNameSampleStreamFactory extends
AbstractSampleStreamFactory<NameSample, OntoNotesFormatParameters> {
+ public static void registerFactory() {
+ StreamFactoryRegistry.registerFactory(NameSample.class,
+ "ontonotes", new OntoNotesNameSampleStreamFactory());
+ }
+
public OntoNotesNameSampleStreamFactory() {
super(OntoNotesFormatParameters.class);
}
@Override
public ObjectStream<NameSample> create(String[] args) {
-
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
OntoNotesFormatParameters params = ArgumentParser.parse(args, OntoNotesFormatParameters.class);
+ final File ontoDir = new File(params.getOntoNotesDir());
+ if (!ontoDir.isDirectory() || !ontoDir.exists()) {
+ throw new TerminateToolException(-1, "The specified OntoNotes directory is not valid!");
+ }
ObjectStream<File> documentStream = new DirectorySampleStream(new File(
params.getOntoNotesDir()),
@@ -54,8 +73,4 @@ public class OntoNotesNameSampleStreamFactory extends
new FileToStringSampleStream(documentStream, StandardCharsets.UTF_8));
}
- public static void registerFactory() {
- StreamFactoryRegistry.registerFactory(NameSample.class,
- "ontonotes", new OntoNotesNameSampleStreamFactory());
- }
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesPOSSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesPOSSampleStreamFactory.java
index a82c0d97..493823f4 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesPOSSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesPOSSampleStreamFactory.java
@@ -18,12 +18,19 @@
package opennlp.tools.formats.ontonotes;
import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.formats.convert.ParseToPOSSampleStream;
import opennlp.tools.parser.Parse;
import opennlp.tools.postag.POSSample;
import opennlp.tools.util.ObjectStream;
+/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see ParseToPOSSampleStream
+ */
+@Internal
public class OntoNotesPOSSampleStreamFactory
extends AbstractSampleStreamFactory<POSSample, OntoNotesFormatParameters> {
@@ -34,14 +41,18 @@ public class OntoNotesPOSSampleStreamFactory
super(OntoNotesFormatParameters.class);
}
+ public static void registerFactory() {
+ StreamFactoryRegistry.registerFactory(POSSample.class, "ontonotes",
+ new OntoNotesPOSSampleStreamFactory());
+ }
+
@Override
public ObjectStream<POSSample> create(String[] args) {
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
ObjectStream<Parse> parseSampleStream = parseSampleStreamFactory.create(args);
return new ParseToPOSSampleStream(parseSampleStream);
}
- public static void registerFactory() {
- StreamFactoryRegistry.registerFactory(POSSample.class, "ontonotes",
- new OntoNotesPOSSampleStreamFactory());
- }
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStream.java
index 84fd1a5a..a18d5558 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStream.java
@@ -52,7 +52,7 @@ public class OntoNotesParseSampleStream extends FilterObjectStream<String, Parse
}
if (parse == null || parse.isEmpty()) {
- if (parseString.length() > 0) {
+ if (!parseString.isEmpty()) {
return Parse.parseParse(parseString.toString());
}
else {
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStreamFactory.java
index 5b9fa0b5..35bbe33e 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStreamFactory.java
@@ -22,12 +22,20 @@ import java.nio.charset.StandardCharsets;
import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.formats.DirectorySampleStream;
import opennlp.tools.formats.convert.FileToStringSampleStream;
import opennlp.tools.parser.Parse;
import opennlp.tools.util.ObjectStream;
+/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see OntoNotesParseSampleStream
+ */
+@Internal
public class OntoNotesParseSampleStreamFactory
extends AbstractSampleStreamFactory<Parse, OntoNotesFormatParameters> {
@@ -35,10 +43,21 @@ public class OntoNotesParseSampleStreamFactory
super(OntoNotesFormatParameters.class);
}
+ public static void registerFactory() {
+ StreamFactoryRegistry.registerFactory(Parse.class, "ontonotes",
+ new OntoNotesParseSampleStreamFactory());
+ }
+
@Override
public ObjectStream<Parse> create(String[] args) {
-
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
OntoNotesFormatParameters params = ArgumentParser.parse(args, OntoNotesFormatParameters.class);
+ final File ontoDir = new File(params.getOntoNotesDir());
+ if (!ontoDir.isDirectory() || !ontoDir.exists()) {
+ throw new TerminateToolException(-1, "The specified OntoNotes directory is not valid!");
+ }
ObjectStream<File> documentStream = new DirectorySampleStream(new File(
params.getOntoNotesDir()),
@@ -53,13 +72,8 @@ public class OntoNotesParseSampleStreamFactory
// We need file to line here ... and that is probably best doen with the plain text stream
// lets copy it over here, refactor it, and then at some point we replace the current version
// with the refactored version
-
- return new OntoNotesParseSampleStream(new DocumentToLineStream(new FileToStringSampleStream(
- documentStream, StandardCharsets.UTF_8)));
+ return new OntoNotesParseSampleStream(new DocumentToLineStream(
+ new FileToStringSampleStream(documentStream, StandardCharsets.UTF_8)));
}
- public static void registerFactory() {
- StreamFactoryRegistry.registerFactory(Parse.class, "ontonotes",
- new OntoNotesParseSampleStreamFactory());
- }
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleDataStream.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleDataStream.java
index acfdbbae..f7e53dab 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleDataStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleDataStream.java
@@ -24,12 +24,12 @@ import opennlp.tools.util.FilterObjectStream;
import opennlp.tools.util.ObjectStream;
/**
- * The {@link NameSampleDataStream} class converts tagged {@link String strings}
+ * The {@link NameSampleDataStream} class converts tagged tokens
* provided by a {@link DataStream} to {@link NameSample} objects.
- * It uses text that is one-sentence per line and tokenized
- * with names identified by:
* <p>
- * {@code <START>} and {@code <END>} tags.
+ * It uses text that is one-sentence per line and tokenized
+ * with names identified by: <br/>
+ * <{@code START}> and <{@code END}>tags.
*/
public class NameSampleDataStream extends FilterObjectStream<String, NameSample> {
@@ -55,7 +55,7 @@ public class NameSampleDataStream extends FilterObjectStream<String, NameSample>
// An empty line indicates the start of a new article
// for which the adaptive data in the feature generators
// must be cleared
- while (token != null && token.trim().length() == 0) {
+ while (token != null && token.trim().isEmpty()) {
isClearAdaptiveData = true;
token = samples.read();
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/ParseSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/parser/ParseSampleStream.java
index 82cfe0a7..b75edbf6 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/ParseSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/ParseSampleStream.java
@@ -22,6 +22,10 @@ import java.io.IOException;
import opennlp.tools.util.FilterObjectStream;
import opennlp.tools.util.ObjectStream;
+/**
+ * @see Parse
+ * @see FilterObjectStream
+ */
public class ParseSampleStream extends FilterObjectStream<String, Parse> {
/**
diff --git a/opennlp-tools/src/main/java/opennlp/tools/postag/WordTagSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/postag/WordTagSampleStream.java
index 4aac6869..40653117 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/postag/WordTagSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/postag/WordTagSampleStream.java
@@ -28,7 +28,7 @@ import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.ObjectStream;
/**
- * A stream filter which reads a sentence per line which contains
+ * A {@link FilterObjectStream stream filter} which reads a sentence per line that contains
* words and tags in {@code word_tag} format and outputs a {@link POSSample} objects.
*/
public class WordTagSampleStream extends FilterObjectStream<String, POSSample> {
@@ -45,14 +45,14 @@ public class WordTagSampleStream extends FilterObjectStream<String, POSSample> {
}
/**
- * Parses the next sentence and return the next {@link POSSample} object.
+ * Reads the next tokens and parses it into the next {@link POSSample} object.
* <p>
* If an error occurs an empty {@link POSSample} object is returned
* and a warning message is logged. Usually it does not matter if one
* or many sentences are ignored.
*
* @return A valid {@link POSSample} or {@code null} if the
- * {@link ObjectStream sentence stream} is exhausted.
+ * {@link ObjectStream stream} is exhausted.
*
* @throws IOException Thrown if IO errors occurred during read.
*/
@@ -66,17 +66,13 @@ public class WordTagSampleStream extends FilterObjectStream<String, POSSample> {
try {
sample = POSSample.parse(sentence);
} catch (InvalidFormatException e) {
- // TODO: An exception in error case should be thrown.
logger.warn("Error during parsing, ignoring sentence: {}", sentence, e);
-
sample = new POSSample(new String[]{}, new String[]{});
}
-
return sample;
}
else {
- // sentences stream is exhausted
- return null;
+ return null; // sentences stream is exhausted
}
}
}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/AbstractSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/AbstractSampleStreamFactoryTest.java
new file mode 100644
index 00000000..bcd2e2ec
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/AbstractSampleStreamFactoryTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public abstract class AbstractSampleStreamFactoryTest<S, P> extends AbstractFormatTest {
+
+ protected static final String FORMAT_SAMPLE_DIR = "opennlp/tools/formats/";
+
+ protected abstract AbstractSampleStreamFactory<S, P> getFactory();
+ protected abstract String getDataFilePath();
+
+ @Test
+ void testCreateWithNullParameter() {
+ assertThrows(IllegalArgumentException.class, () -> {
+ try (ObjectStream<S> stream = getFactory().create(null)) {
+ stream.read();
+ }
+ });
+ }
+
+ @Test
+ void testCreateWithInvalidParameter() {
+ assertThrows(IllegalArgumentException.class, () -> {
+ try (ObjectStream<S> stream = getFactory().create(new String[]{"X"})) {
+ stream.read();
+ }
+ });
+ }
+
+ /*
+ * Note:
+ * This test case must be overridden for non-simple cases where
+ * more than the '-data' param is required.
+ */
+ @Test
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<S> stream = getFactory().create(
+ new String[]{"-data", getDataFilePath() + "xyz"})) {
+ S sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/AbstractSampleStreamTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/AbstractSampleStreamTest.java
index c154f21d..f56b7711 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/formats/AbstractSampleStreamTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/AbstractSampleStreamTest.java
@@ -19,8 +19,7 @@ package opennlp.tools.formats;
import opennlp.tools.util.InputStreamFactory;
-
-public abstract class AbstractSampleStreamTest extends AbstractFormatTest {
+abstract class AbstractSampleStreamTest extends AbstractFormatTest {
protected InputStreamFactory getFactory(String resource) {
return new ResourceAsStreamFactory(AbstractSampleStreamTest.class, FORMATS_BASE_DIR + resource);
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactoryTest.java
new file mode 100644
index 00000000..bdec3990
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactoryTest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class BioNLP2004NameSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<NameSample, BioNLP2004NameSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "bionlp2004-01.sample";
+
+ // SUT
+ private BioNLP2004NameSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory
+ <NameSample, BioNLP2004NameSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ BioNLP2004NameSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<NameSample, BioNLP2004NameSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(NameSample.class, "bionlp2004");
+ assertInstanceOf(BioNLP2004NameSampleStreamFactory.class, f);
+ factory = (BioNLP2004NameSampleStreamFactory) f;
+ assertEquals(BioNLP2004NameSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+
+ try (ObjectStream<NameSample> stream = factory.create(
+ new String[]{"-types", "DNA,protein,cell_type,cell_line,RNA", "-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @Test
+ void testCreateWithUnsupportedTypes() throws IOException {
+ try (ObjectStream<NameSample> stream = factory.create(
+ new String[]{"-types", "xyz", "-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<NameSample> stream = factory.create(new String[]
+ {"-types", "DNA,protein,cell_type,cell_line,RNA", "-data", sampleFileFullPath + "xyz"})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/ChunkerSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/ChunkerSampleStreamFactoryTest.java
new file mode 100644
index 00000000..ad5a4e1c
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/ChunkerSampleStreamFactoryTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.chunker.ChunkSample;
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class ChunkerSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<ChunkSample, ChunkerSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "chunker-01.sample";
+
+ // SUT
+ private ChunkerSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<ChunkSample, ChunkerSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ChunkerSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<ChunkSample, ChunkerSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(ChunkSample.class, StreamFactoryRegistry.DEFAULT_FORMAT);
+ assertInstanceOf(ChunkerSampleStreamFactory.class, f);
+ factory = (ChunkerSampleStreamFactory) f;
+ assertEquals(ChunkerSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<ChunkSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ ChunkSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/Conll02NameSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/Conll02NameSampleStreamFactoryTest.java
new file mode 100644
index 00000000..dec1fa9a
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/Conll02NameSampleStreamFactoryTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class Conll02NameSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<NameSample, Conll02NameSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "conll2002-es.sample";
+ private static final String SAMPLE_02 = "conll2002-nl.sample";
+
+ // SUT
+ private Conll02NameSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<NameSample, Conll02NameSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ Conll02NameSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<NameSample, Conll02NameSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(NameSample.class, "conll02");
+ assertInstanceOf(Conll02NameSampleStreamFactory.class, f);
+ factory = ((Conll02NameSampleStreamFactory) f);
+ assertEquals(Conll02NameSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"spa", "es", "nld", "nl"})
+ void testCreateWithValidParameter(String lang) throws IOException {
+ // prepare depending on language
+ if ("spa".equals(lang) || "es".equals(lang)) {
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ } else {
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_02).getPath();
+ }
+
+ try (ObjectStream<NameSample> stream = factory.create(
+ new String[]{"-lang", lang, "-types", "per,loc,org,misc", "-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"", "per", "loc", "org", "misc", "per,loc,org,misc"})
+ void testCreateWithDifferentTypes(String types) throws IOException {
+ try (ObjectStream<NameSample> stream = factory.create(
+ new String[]{"-lang", "spa", "-types", types, "-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @Test
+ void testCreateWithInvalidLanguage() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<NameSample> stream = factory.create(
+ new String[]{"-lang", "xyz", "-types", "per,loc,org,misc", "-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<NameSample> stream = factory.create(new String[]
+ {"-lang", "spa", "-types", "per,loc,org,misc", "-data", sampleFileFullPath + "xyz"})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/Conll03NameSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/Conll03NameSampleStreamFactoryTest.java
new file mode 100644
index 00000000..999bdf5c
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/Conll03NameSampleStreamFactoryTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class Conll03NameSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<NameSample, Conll03NameSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "conll2003-de.sample";
+ private static final String SAMPLE_02 = "conll2003-en.sample";
+
+ // SUT
+ private Conll03NameSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<NameSample, Conll03NameSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ Conll03NameSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<NameSample, Conll03NameSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(NameSample.class, "conll03");
+ assertInstanceOf(Conll03NameSampleStreamFactory.class, f);
+ factory = ((Conll03NameSampleStreamFactory) f);
+ assertEquals(Conll03NameSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"deu", "eng"})
+ void testCreateWithValidParameter(String lang) throws IOException {
+ // prepare depending on language
+ if ("deu".equals(lang)) {
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ } else {
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_02).getPath();
+ }
+
+ try (ObjectStream<NameSample> stream = factory.create(
+ new String[]{"-lang", lang, "-types", "per,loc,org,misc", "-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"", "per", "loc", "org", "misc", "per,loc,org,misc"})
+ void testCreateWithDifferentTypes(String types) throws IOException {
+ try (ObjectStream<NameSample> stream = factory.create(
+ new String[]{"-lang", "deu", "-types", types, "-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @Test
+ void testCreateWithInvalidLanguage() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<NameSample> stream = factory.create(
+ new String[]{"-lang", "xyz", "-types", "per,loc,org,misc", "-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<NameSample> stream = factory.create(new String[]
+ {"-lang", "deu", "-types", "per,loc,org,misc", "-data", sampleFileFullPath + "xyz"})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/ConllXPOSSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/ConllXPOSSampleStreamFactoryTest.java
new file mode 100644
index 00000000..095a5c00
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/ConllXPOSSampleStreamFactoryTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.postag.POSSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class ConllXPOSSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<POSSample, ConllXPOSSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "conllx.sample";
+
+ // SUT
+ private ConllXPOSSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<POSSample, ConllXPOSSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ConllXPOSSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<POSSample, ConllXPOSSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(POSSample.class, ConllXPOSSampleStreamFactory.CONLLX_FORMAT);
+ assertInstanceOf(ConllXPOSSampleStreamFactory.class, f);
+ factory = (ConllXPOSSampleStreamFactory) f;
+ assertEquals(ConllXPOSSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<POSSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ POSSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/ConllXSentenceSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/ConllXSentenceSampleStreamFactoryTest.java
new file mode 100644
index 00000000..3b00d1c5
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/ConllXSentenceSampleStreamFactoryTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class ConllXSentenceSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, ConllXSentenceSampleStreamFactory.Parameters> {
+
+ private static final String DETOKENIZER_FILE = "opennlp/tools/tokenize/latin-detokenizer.xml";
+ private static final String SAMPLE_01 = "conllx.sample";
+
+ // SUT
+ private ConllXSentenceSampleStreamFactory factory;
+
+ private String detokFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, ConllXSentenceSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ConllXSentenceSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, ConllXSentenceSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, ConllXPOSSampleStreamFactory.CONLLX_FORMAT);
+ assertInstanceOf(ConllXSentenceSampleStreamFactory.class, f);
+ factory = (ConllXSentenceSampleStreamFactory) f;
+ assertEquals(ConllXSentenceSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ detokFileFullPath = getResourceWithoutPrefix(DETOKENIZER_FILE).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(
+ new String[]{"-encoding", "UTF-8", "-detokenizer", detokFileFullPath, "-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as different exception is expected!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<SentenceSample> stream = factory.create(new String[]
+ {"-encoding", "UTF-8", "-detokenizer", detokFileFullPath, "-data", sampleFileFullPath + "xyz"})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/ConllXTokenSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/ConllXTokenSampleStreamFactoryTest.java
new file mode 100644
index 00000000..1c607ced
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/ConllXTokenSampleStreamFactoryTest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.tokenize.TokenSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class ConllXTokenSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<TokenSample, ConllXTokenSampleStreamFactory.Parameters> {
+
+ private static final String DETOKENIZER_FILE = "opennlp/tools/tokenize/latin-detokenizer.xml";
+ private static final String SAMPLE_01 = "conllx.sample";
+
+ // SUT
+ private ConllXTokenSampleStreamFactory factory;
+
+ private String detokFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<TokenSample, ConllXTokenSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ConllXTokenSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<TokenSample, ConllXTokenSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(TokenSample.class, ConllXPOSSampleStreamFactory.CONLLX_FORMAT);
+ assertInstanceOf(ConllXTokenSampleStreamFactory.class, f);
+ factory = (ConllXTokenSampleStreamFactory) f;
+ assertEquals(ConllXTokenSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ detokFileFullPath = getResourceWithoutPrefix(DETOKENIZER_FILE).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<TokenSample> stream = factory.create(new String[]
+ {"-encoding", "UTF-8", "-detokenizer", detokFileFullPath, "-data", sampleFileFullPath})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as different exception is expected!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<TokenSample> stream = factory.create(new String[]
+ {"-encoding", "UTF-8", "-detokenizer", detokFileFullPath, "-data", sampleFileFullPath + "xyz"})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/EvalitaNameSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/EvalitaNameSampleStreamFactoryTest.java
new file mode 100644
index 00000000..eb6eebf8
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/EvalitaNameSampleStreamFactoryTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class EvalitaNameSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<NameSample, EvalitaNameSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "evalita-ner-it-01.sample";
+
+ // SUT
+ private EvalitaNameSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<NameSample, EvalitaNameSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ EvalitaNameSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<NameSample, EvalitaNameSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(NameSample.class, "evalita");
+ assertInstanceOf(EvalitaNameSampleStreamFactory.class, f);
+ factory = ((EvalitaNameSampleStreamFactory) f);
+ assertEquals(EvalitaNameSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+
+ try (ObjectStream<NameSample> stream = factory.create(
+ new String[]{"-lang", "it", "-types", "per,loc,org,gpe", "-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @Test
+ void testCreateWithInvalidLanguage() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<NameSample> stream = factory.create(
+ new String[]{"-lang", "xy", "-types", "per,loc,org,gpe", "-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<NameSample> stream = factory.create(new String[]
+ {"-lang", "it", "-types", "per,loc,org,gpe", "-data", sampleFileFullPath + "xyz"})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/EvalitaNameSampleStreamTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/EvalitaNameSampleStreamTest.java
index 37929e3e..9bfcc57a 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/formats/EvalitaNameSampleStreamTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/EvalitaNameSampleStreamTest.java
@@ -18,54 +18,102 @@
package opennlp.tools.formats;
import java.io.IOException;
+import java.util.stream.Stream;
-import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
import opennlp.tools.formats.EvalitaNameSampleStream.LANGUAGE;
import opennlp.tools.namefind.NameSample;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.Span;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
/**
* Note:
* Sample training data must be UTF-8 encoded and uncompressed!
*/
public class EvalitaNameSampleStreamTest extends AbstractSampleStreamTest {
- @Test
- void testParsingItalianSample() throws IOException {
+ private static final String SAMPLE_01 = "evalita-ner-it-01.sample";
+ private static final String SAMPLE_02 = "evalita-ner-it-02.sample";
+ private static final String SAMPLE_03 = "evalita-ner-it-03.sample";
+ private static final String SAMPLE_BROKEN = "evalita-ner-it-broken.sample";
+ private static final String SAMPLE_INCORRECT = "evalita-ner-it-incorrect.sample";
- try (ObjectStream<NameSample> sampleStream = openData()) {
- NameSample personName = sampleStream.read();
- Assertions.assertNotNull(personName);
+ @ParameterizedTest
+ @MethodSource(value = "provideData")
+ void testReadItalianDifferentEntityTypes(String file, int nerType, int expectedSentLength,
+ int expectedStart, int expectedEnd) throws IOException {
- Assertions.assertEquals(11, personName.getSentence().length);
- Assertions.assertEquals(1, personName.getNames().length);
- Assertions.assertTrue(personName.isClearAdaptiveDataSet());
+ try (ObjectStream<NameSample> sampleStream = openData(file, nerType)) {
+ NameSample ne = sampleStream.read();
+ assertNotNull(ne);
- Span nameSpan = personName.getNames()[0];
- Assertions.assertEquals(8, nameSpan.getStart());
- Assertions.assertEquals(10, nameSpan.getEnd());
- Assertions.assertTrue(personName.isClearAdaptiveDataSet());
+ assertEquals(expectedSentLength, ne.getSentence().length);
+ assertEquals(1, ne.getNames().length);
+ assertTrue(ne.isClearAdaptiveDataSet());
- Assertions.assertEquals(0, sampleStream.read().getNames().length);
+ Span nameSpan = ne.getNames()[0];
+ assertEquals(expectedStart, nameSpan.getStart());
+ assertEquals(expectedEnd, nameSpan.getEnd());
+ assertTrue(ne.isClearAdaptiveDataSet());
- Assertions.assertNull(sampleStream.read());
+ if (SAMPLE_01.equals(file)) { // this file has an extra sentence
+ assertEquals(0, sampleStream.read().getNames().length);
+ }
+ assertNull(sampleStream.read());
}
}
+ @Test
+ void testReadWithIncorrectInput() {
+ assertThrows(IOException.class, () -> {
+ try (ObjectStream<NameSample> sampleStream = openData(
+ SAMPLE_INCORRECT, EvalitaNameSampleStream.GENERATE_PERSON_ENTITIES)) {
+ sampleStream.read();
+ }
+ });
+ }
+
+ @Test
+ void testReadWithBrokenDocument() {
+ assertThrows(IOException.class, () -> {
+ try (ObjectStream<NameSample> sampleStream = openData(
+ SAMPLE_BROKEN, EvalitaNameSampleStream.GENERATE_PERSON_ENTITIES)) {
+ sampleStream.read();
+ }
+ });
+ }
+
@Test
void testReset() throws IOException {
- try (ObjectStream<NameSample> sampleStream = openData()) {
+ try (ObjectStream<NameSample> sampleStream = openData(SAMPLE_01,
+ EvalitaNameSampleStream.GENERATE_PERSON_ENTITIES)) {
NameSample sample = sampleStream.read();
sampleStream.reset();
- Assertions.assertEquals(sample, sampleStream.read());
+ assertEquals(sample, sampleStream.read());
}
}
- private ObjectStream<NameSample> openData() throws IOException {
- return new EvalitaNameSampleStream(LANGUAGE.IT, getFactory("evalita-ner-it.sample"),
- EvalitaNameSampleStream.GENERATE_PERSON_ENTITIES);
+ // Note: This needs to be public as JUnit 5 requires it like this.
+ public static Stream<Arguments> provideData() {
+ return Stream.of(
+ Arguments.of(SAMPLE_01, EvalitaNameSampleStream.GENERATE_PERSON_ENTITIES, 11, 8, 10),
+ Arguments.of(SAMPLE_02, EvalitaNameSampleStream.GENERATE_PERSON_ENTITIES, 27, 11, 13),
+ Arguments.of(SAMPLE_02, EvalitaNameSampleStream.GENERATE_ORGANIZATION_ENTITIES, 27, 10, 11),
+ Arguments.of(SAMPLE_03, EvalitaNameSampleStream.GENERATE_GPE_ENTITIES, 20, 18, 19)
+ );
+ }
+
+ private ObjectStream<NameSample> openData(String fileName, int nerType) throws IOException {
+ return new EvalitaNameSampleStream(LANGUAGE.IT, getFactory(fileName), nerType);
}
}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/LanguageDetectorSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/LanguageDetectorSampleStreamFactoryTest.java
new file mode 100644
index 00000000..ad50ec82
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/LanguageDetectorSampleStreamFactoryTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.langdetect.LanguageSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class LanguageDetectorSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<LanguageSample, LanguageDetectorSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "lang-detect-01.sample";
+
+ // SUT
+ private LanguageDetectorSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory
+ <LanguageSample, LanguageDetectorSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ LanguageDetectorSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<LanguageSample, LanguageDetectorSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(LanguageSample.class, StreamFactoryRegistry.DEFAULT_FORMAT);
+ assertInstanceOf(LanguageDetectorSampleStreamFactory.class, f);
+ factory = (LanguageDetectorSampleStreamFactory) f;
+ assertEquals(LanguageDetectorSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<LanguageSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ LanguageSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/LemmatizerSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/LemmatizerSampleStreamFactoryTest.java
new file mode 100644
index 00000000..03a4af0d
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/LemmatizerSampleStreamFactoryTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.lemmatizer.LemmaSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class LemmatizerSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<LemmaSample, LemmatizerSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "lemma-01.sample";
+
+ // SUT
+ private LemmatizerSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<LemmaSample, LemmatizerSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ LemmatizerSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<LemmaSample, LemmatizerSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(LemmaSample.class, StreamFactoryRegistry.DEFAULT_FORMAT);
+ assertInstanceOf(LemmatizerSampleStreamFactory.class, f);
+ factory = (LemmatizerSampleStreamFactory) f;
+ assertEquals(LemmatizerSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<LemmaSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ LemmaSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/NameSampleDataStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/NameSampleDataStreamFactoryTest.java
new file mode 100644
index 00000000..b1f6f8f6
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/NameSampleDataStreamFactoryTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.Span;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class NameSampleDataStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<NameSample, NameSampleDataStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "name-data-01.sample";
+
+ // SUT
+ private NameSampleDataStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<NameSample, NameSampleDataStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ NameSampleDataStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<NameSample, NameSampleDataStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(NameSample.class, StreamFactoryRegistry.DEFAULT_FORMAT);
+ assertInstanceOf(NameSampleDataStreamFactory.class, f);
+ factory = (NameSampleDataStreamFactory) f;
+ assertEquals(NameSampleDataStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<NameSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ Span[] hits = sample.getNames();
+ assertNotNull(hits);
+ assertEquals(1, hits.length);
+ assertEquals(5, hits[0].getStart());
+ assertEquals(6, hits[0].getEnd());
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/ParseSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/ParseSampleStreamFactoryTest.java
new file mode 100644
index 00000000..6bb24b7c
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/ParseSampleStreamFactoryTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.parser.Parse;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class ParseSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<Parse, ParseSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "parse-01.sample";
+
+ // SUT
+ private ParseSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<Parse, ParseSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ParseSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<Parse, ParseSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(Parse.class, StreamFactoryRegistry.DEFAULT_FORMAT);
+ assertInstanceOf(ParseSampleStreamFactory.class, f);
+ factory = (ParseSampleStreamFactory) f;
+ assertEquals(ParseSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<Parse> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ Parse sample = stream.read();
+ assertNotNull(sample);
+ assertEquals("The test shall come today . ", sample.getText());
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/SentenceSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/SentenceSampleStreamFactoryTest.java
new file mode 100644
index 00000000..60c084b5
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/SentenceSampleStreamFactoryTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class SentenceSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, SentenceSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "sentences-01.sample";
+
+ // SUT
+ private SentenceSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, SentenceSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ SentenceSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, SentenceSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, StreamFactoryRegistry.DEFAULT_FORMAT);
+ assertInstanceOf(SentenceSampleStreamFactory.class, f);
+ factory = (SentenceSampleStreamFactory) f;
+ assertEquals(SentenceSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/TokenSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/TokenSampleStreamFactoryTest.java
new file mode 100644
index 00000000..44e478fa
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/TokenSampleStreamFactoryTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.tokenize.TokenSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class TokenSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<TokenSample, TokenSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "tokens-01.sample";
+
+ // SUT
+ private TokenSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<TokenSample, TokenSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ TokenSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<TokenSample, TokenSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(TokenSample.class, StreamFactoryRegistry.DEFAULT_FORMAT);
+ assertInstanceOf(TokenSampleStreamFactory.class, f);
+ factory = (TokenSampleStreamFactory) f;
+ assertEquals(TokenSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<TokenSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/TwentyNewsgroupSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/TwentyNewsgroupSampleStreamFactoryTest.java
new file mode 100644
index 00000000..63b5cb42
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/TwentyNewsgroupSampleStreamFactoryTest.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EmptySource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.doccat.DocumentSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class TwentyNewsgroupSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<DocumentSample, TwentyNewsgroupSampleStreamFactory.Parameters> {
+
+ private static final Path OPENNLP_DIR = Paths.get(System.getProperty("OPENNLP_DOWNLOAD_HOME",
+ System.getProperty("user.home"))).resolve(".opennlp");
+ private static final String TOKENIZER_MODEL_NAME = "opennlp-en-ud-ewt-tokens-1.2-2.5.0.bin";
+
+ // SUT
+ private TwentyNewsgroupSampleStreamFactory factory;
+
+ private static String tokenizerFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<DocumentSample, TwentyNewsgroupSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ TwentyNewsgroupSampleStreamFactory.registerFactory();
+ tokenizerFileFullPath = new File(OPENNLP_DIR + File.separator + TOKENIZER_MODEL_NAME).getPath();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<DocumentSample, TwentyNewsgroupSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(DocumentSample.class, "20newsgroup");
+ assertInstanceOf(TwentyNewsgroupSampleStreamFactory.class, f);
+ factory = (TwentyNewsgroupSampleStreamFactory) f;
+ assertEquals(TwentyNewsgroupSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "20newsgroup").getPath();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"simple", "whitespace"})
+ void testCreateWithValidParameter(String tokenizerType) throws IOException {
+ try (ObjectStream<DocumentSample> stream = factory.create(new String[]
+ {"-ruleBasedTokenizer", tokenizerType, "-dataDir", sampleFileFullPath})) {
+ DocumentSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @ParameterizedTest
+ @EmptySource
+ @ValueSource(strings = {" ", "unknown"})
+ void testCreateWithInvalidRBTokenizer(String tokenizerType) {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<DocumentSample> stream = factory.create(new String[]
+ {"-ruleBasedTokenizer", tokenizerType, "-dataDir", sampleFileFullPath})) {
+ DocumentSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+ @Test
+ void testCreateWithMETokenizer() throws IOException {
+ try (ObjectStream<DocumentSample> stream = factory.create(new String[]
+ {"-tokenizerModel", tokenizerFileFullPath, "-dataDir", sampleFileFullPath})) {
+ DocumentSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @ParameterizedTest
+ @EmptySource
+ @ValueSource(strings = {" ", "unknown"})
+ void testCreateWithInvalidMETokenizer(String path) {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<DocumentSample> stream = factory.create(new String[]
+ {"-tokenizerModel", path, "-dataDir", sampleFileFullPath})) {
+ DocumentSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<DocumentSample> stream = factory.create(new String[]
+ {"-ruleBasedTokenizer", "whitespace", "-dataDir", sampleFileFullPath + "xyz"})) {
+ DocumentSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/WordTagSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/WordTagSampleStreamFactoryTest.java
new file mode 100644
index 00000000..c7b7cc96
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/WordTagSampleStreamFactoryTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.postag.POSSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class WordTagSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<POSSample, WordTagSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "word-tags-01.sample";
+
+ // SUT
+ private WordTagSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<POSSample, WordTagSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ WordTagSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<POSSample, WordTagSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(POSSample.class, StreamFactoryRegistry.DEFAULT_FORMAT);
+ assertInstanceOf(WordTagSampleStreamFactory.class, f);
+ factory = (WordTagSampleStreamFactory) f;
+ assertEquals(WordTagSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<POSSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ POSSample sample = stream.read();
+ assertNotNull(sample);
+ assertNotNull(sample.getTags());
+ assertEquals(6, sample.getTags().length);
+ assertNotNull(sample.getSentence());
+ assertEquals(6, sample.getSentence().length);
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluLemmaSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluLemmaSampleStreamFactoryTest.java
new file mode 100644
index 00000000..0e7f499e
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluLemmaSampleStreamFactoryTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.conllu;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.lemmatizer.LemmaSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class ConlluLemmaSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<LemmaSample, ConlluLemmaSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "es-ud-sample.conllu";
+
+ // SUT
+ private ConlluLemmaSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<LemmaSample, ConlluLemmaSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ConlluLemmaSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<LemmaSample, ConlluLemmaSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(LemmaSample.class, ConlluPOSSampleStreamFactory.CONLLU_FORMAT);
+ assertInstanceOf(ConlluLemmaSampleStreamFactory.class, f);
+ factory = (ConlluLemmaSampleStreamFactory) f;
+ assertEquals(ConlluLemmaSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "conllu/" + SAMPLE_01).getPath();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"u", "x"})
+ void testCreateWithValidParameter(String tagset) throws IOException {
+ try (ObjectStream<LemmaSample> stream = factory.create(
+ new String[]{"-tagset", tagset, "-data", sampleFileFullPath})) {
+ LemmaSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {" ", "y"})
+ void testCreateWithUnknownTagset(String tagset) {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<LemmaSample> stream = factory.create(
+ new String[]{"-tagset", tagset, "-data", sampleFileFullPath})) {
+ assertNotNull(stream.read());
+ }
+ });
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<LemmaSample> stream = factory.create(new String[]
+ {"-tagset", "u", "-data", sampleFileFullPath + "xyz"})) {
+ LemmaSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluPOSSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluPOSSampleStreamFactoryTest.java
new file mode 100644
index 00000000..40208bbc
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluPOSSampleStreamFactoryTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.conllu;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.postag.POSSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class ConlluPOSSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<POSSample, ConlluPOSSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "es-ud-sample.conllu";
+
+ // SUT
+ private ConlluPOSSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<POSSample, ConlluPOSSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ConlluPOSSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<POSSample, ConlluPOSSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(POSSample.class, ConlluPOSSampleStreamFactory.CONLLU_FORMAT);
+ assertInstanceOf(ConlluPOSSampleStreamFactory.class, f);
+ factory = (ConlluPOSSampleStreamFactory) f;
+ assertEquals(ConlluPOSSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "conllu/" + SAMPLE_01).getPath();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"u", "x"})
+ void testCreateWithValidParameter(String tagset) throws IOException {
+ try (ObjectStream<POSSample> stream = factory.create(
+ new String[]{"-tagset", tagset, "-data", sampleFileFullPath})) {
+ POSSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {" ", "y"})
+ void testCreateWithUnknownTagset(String tagset) {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<POSSample> stream = factory.create(
+ new String[]{"-tagset", tagset, "-data", sampleFileFullPath})) {
+ assertNotNull(stream.read());
+ }
+ });
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<POSSample> stream = factory.create(new String[]
+ {"-tagset", "u", "-data", sampleFileFullPath + "xyz"})) {
+ POSSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluSentenceSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluSentenceSampleStreamFactoryTest.java
new file mode 100644
index 00000000..05cb3efa
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluSentenceSampleStreamFactoryTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.conllu;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class ConlluSentenceSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, ConlluSentenceSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "es-ud-sample.conllu";
+
+ // SUT
+ private ConlluSentenceSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, ConlluSentenceSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ConlluSentenceSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, ConlluSentenceSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, ConlluPOSSampleStreamFactory.CONLLU_FORMAT);
+ assertInstanceOf(ConlluSentenceSampleStreamFactory.class, f);
+ factory = (ConlluSentenceSampleStreamFactory) f;
+ assertEquals(ConlluSentenceSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "conllu/" + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(
+ new String[]{"-sentencesPerSample", "5", "-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<SentenceSample> stream = factory.create(new String[]
+ {"-sentencesPerSample", "5", "-data", sampleFileFullPath + "xyz"})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluTokenSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluTokenSampleStreamFactoryTest.java
new file mode 100644
index 00000000..ea6f606c
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluTokenSampleStreamFactoryTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.conllu;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.tokenize.TokenSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class ConlluTokenSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<TokenSample, ConlluTokenSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "es-ud-sample.conllu";
+
+ // SUT
+ private ConlluTokenSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<TokenSample, ConlluTokenSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ConlluTokenSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<TokenSample, ConlluTokenSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(TokenSample.class, ConlluPOSSampleStreamFactory.CONLLU_FORMAT);
+ assertInstanceOf(ConlluTokenSampleStreamFactory.class, f);
+ factory = (ConlluTokenSampleStreamFactory) f;
+ assertEquals(ConlluTokenSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "conllu/" + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<TokenSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/convert/NameToSentenceSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/NameToSentenceSampleStreamFactoryTest.java
new file mode 100644
index 00000000..a42cde19
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/NameToSentenceSampleStreamFactoryTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.convert;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class NameToSentenceSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, NameToSentenceSampleStreamFactory.Parameters> {
+
+ private static final String DETOKENIZER_FILE = "opennlp/tools/tokenize/latin-detokenizer.xml";
+ private static final String SAMPLE_01 = "name-data-01.sample";
+
+ // SUT
+ private NameToSentenceSampleStreamFactory factory;
+
+ private String detokFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, NameToSentenceSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ NameToSentenceSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, NameToSentenceSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, "namefinder");
+ assertInstanceOf(NameToSentenceSampleStreamFactory.class, f);
+ factory = (NameToSentenceSampleStreamFactory) f;
+ assertEquals(NameToSentenceSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ detokFileFullPath = getResourceWithoutPrefix(DETOKENIZER_FILE).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath,"-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as different exception is expected!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<SentenceSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath, "-data", sampleFileFullPath + "xyz"})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/convert/NameToTokenSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/NameToTokenSampleStreamFactoryTest.java
new file mode 100644
index 00000000..7c8ae451
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/NameToTokenSampleStreamFactoryTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.convert;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.tokenize.TokenSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class NameToTokenSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<TokenSample, NameToTokenSampleStreamFactory.Parameters> {
+
+ private static final String DETOKENIZER_FILE = "opennlp/tools/tokenize/latin-detokenizer.xml";
+ private static final String SAMPLE_01 = "name-data-01.sample";
+
+ // SUT
+ private NameToTokenSampleStreamFactory factory;
+
+ private String detokFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<TokenSample, NameToTokenSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ NameToTokenSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<TokenSample, NameToTokenSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(TokenSample.class, "namefinder");
+ assertInstanceOf(NameToTokenSampleStreamFactory.class, f);
+ factory = (NameToTokenSampleStreamFactory) f;
+ assertEquals(NameToTokenSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ detokFileFullPath = getResourceWithoutPrefix(DETOKENIZER_FILE).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<TokenSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath,"-data", sampleFileFullPath})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as different exception is expected!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<TokenSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath, "-data", sampleFileFullPath + "xyz"})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/convert/POSToSentenceSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/POSToSentenceSampleStreamFactoryTest.java
new file mode 100644
index 00000000..6e92de9d
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/POSToSentenceSampleStreamFactoryTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.convert;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class POSToSentenceSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, POSToSentenceSampleStreamFactory.Parameters> {
+
+ private static final String DETOKENIZER_FILE = "opennlp/tools/tokenize/latin-detokenizer.xml";
+ private static final String SAMPLE_01 = "word-tags-01.sample";
+
+ // SUT
+ private POSToSentenceSampleStreamFactory factory;
+
+ private String detokFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, POSToSentenceSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ POSToSentenceSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, POSToSentenceSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, "pos");
+ assertInstanceOf(POSToSentenceSampleStreamFactory.class, f);
+ factory = (POSToSentenceSampleStreamFactory) f;
+ assertEquals(POSToSentenceSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ detokFileFullPath = getResourceWithoutPrefix(DETOKENIZER_FILE).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath,"-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as different exception is expected!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<SentenceSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath, "-data", sampleFileFullPath + "xyz"})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/convert/POSToTokenSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/POSToTokenSampleStreamFactoryTest.java
new file mode 100644
index 00000000..2501e8c2
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/POSToTokenSampleStreamFactoryTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.convert;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.tokenize.TokenSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class POSToTokenSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<TokenSample, POSToTokenSampleStreamFactory.Parameters> {
+
+ private static final String DETOKENIZER_FILE = "opennlp/tools/tokenize/latin-detokenizer.xml";
+ private static final String SAMPLE_01 = "word-tags-01.sample";
+
+ // SUT
+ private POSToTokenSampleStreamFactory factory;
+
+ private String detokFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<TokenSample, POSToTokenSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ POSToTokenSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<TokenSample, POSToTokenSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(TokenSample.class, "pos");
+ assertInstanceOf(POSToTokenSampleStreamFactory.class, f);
+ factory = (POSToTokenSampleStreamFactory) f;
+ assertEquals(POSToTokenSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ detokFileFullPath = getResourceWithoutPrefix(DETOKENIZER_FILE).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<TokenSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath,"-data", sampleFileFullPath})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as different exception is expected!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<TokenSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath, "-data", sampleFileFullPath + "xyz"})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/convert/ParseToPOSSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/ParseToPOSSampleStreamFactoryTest.java
new file mode 100644
index 00000000..e1ca0452
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/ParseToPOSSampleStreamFactoryTest.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.convert;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.postag.POSSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class ParseToPOSSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<POSSample, ParseToPOSSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "parse-01.sample";
+
+ // SUT
+ private ParseToPOSSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<POSSample, ParseToPOSSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ParseToPOSSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<POSSample, ParseToPOSSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(POSSample.class, "parse");
+ assertInstanceOf(ParseToPOSSampleStreamFactory.class, f);
+ factory = (ParseToPOSSampleStreamFactory) f;
+ assertEquals(ParseToPOSSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<POSSample> stream = factory.create(new String[]
+ {"-data", sampleFileFullPath})) {
+ POSSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/convert/ParseToSentenceSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/ParseToSentenceSampleStreamFactoryTest.java
new file mode 100644
index 00000000..a11ce4f5
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/ParseToSentenceSampleStreamFactoryTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.convert;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class ParseToSentenceSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, ParseToSentenceSampleStreamFactory.Parameters> {
+
+ private static final String DETOKENIZER_FILE = "opennlp/tools/tokenize/latin-detokenizer.xml";
+ private static final String SAMPLE_01 = "parse-01.sample";
+
+ // SUT
+ private ParseToSentenceSampleStreamFactory factory;
+
+ private String detokFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, ParseToSentenceSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ParseToSentenceSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, ParseToSentenceSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, "parse");
+ assertInstanceOf(ParseToSentenceSampleStreamFactory.class, f);
+ factory = (ParseToSentenceSampleStreamFactory) f;
+ assertEquals(ParseToSentenceSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ detokFileFullPath = getResourceWithoutPrefix(DETOKENIZER_FILE).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath,"-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as different exception is expected!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<SentenceSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath, "-data", sampleFileFullPath + "xyz"})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/convert/ParseToTokenSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/ParseToTokenSampleStreamFactoryTest.java
new file mode 100644
index 00000000..08ae5211
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/ParseToTokenSampleStreamFactoryTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.convert;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.tokenize.TokenSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class ParseToTokenSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<TokenSample, ParseToTokenSampleStreamFactory.Parameters> {
+
+ private static final String DETOKENIZER_FILE = "opennlp/tools/tokenize/latin-detokenizer.xml";
+ private static final String SAMPLE_01 = "parse-01.sample";
+
+ // SUT
+ private ParseToTokenSampleStreamFactory factory;
+
+ private String detokFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<TokenSample, ParseToTokenSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ParseToTokenSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<TokenSample, ParseToTokenSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(TokenSample.class, "parse");
+ assertInstanceOf(ParseToTokenSampleStreamFactory.class, f);
+ factory = (ParseToTokenSampleStreamFactory) f;
+ assertEquals(ParseToTokenSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ detokFileFullPath = getResourceWithoutPrefix(DETOKENIZER_FILE).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<TokenSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath,"-data", sampleFileFullPath})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as different exception is expected!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<TokenSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath, "-data", sampleFileFullPath + "xyz"})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStreamFactoryTest.java
new file mode 100644
index 00000000..14a26733
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStreamFactoryTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.frenchtreebank;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.parser.Parse;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class ConstitParseSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<Parse, ConstitParseSampleStreamFactory.Parameters> {
+
+ // SUT
+ private ConstitParseSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<Parse, ConstitParseSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ConstitParseSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<Parse, ConstitParseSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(Parse.class, "frenchtreebank");
+ assertInstanceOf(ConstitParseSampleStreamFactory.class, f);
+ factory = (ConstitParseSampleStreamFactory) f;
+ assertEquals(ConstitParseSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "frenchtreebank/").getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<Parse> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ Parse sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @Test
+ @Override // due to different exception
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(IllegalArgumentException.class, () -> {
+ try (ObjectStream<Parse> stream = getFactory().create(
+ new String[]{"-data", getDataFilePath() + "xyz"})) {
+ Parse sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankSentenceStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankSentenceStreamFactoryTest.java
new file mode 100644
index 00000000..26a1c20f
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankSentenceStreamFactoryTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.irishsentencebank;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class IrishSentenceBankSentenceStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, IrishSentenceBankSentenceStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "irishsentencebank-sample.xml";
+
+ // SUT
+ private IrishSentenceBankSentenceStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, IrishSentenceBankSentenceStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ IrishSentenceBankSentenceStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, IrishSentenceBankSentenceStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, "irishsentencebank");
+ assertInstanceOf(IrishSentenceBankSentenceStreamFactory.class, f);
+ factory = (IrishSentenceBankSentenceStreamFactory) f;
+ assertEquals(IrishSentenceBankSentenceStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(
+ FORMAT_SAMPLE_DIR + "irishsentencebank/" + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankTokenSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankTokenSampleStreamFactoryTest.java
new file mode 100644
index 00000000..ffea30b4
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankTokenSampleStreamFactoryTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.irishsentencebank;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.tokenize.TokenSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class IrishSentenceBankTokenSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<TokenSample, IrishSentenceBankTokenSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "irishsentencebank-sample.xml";
+
+ // SUT
+ private IrishSentenceBankTokenSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<TokenSample, IrishSentenceBankTokenSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ IrishSentenceBankTokenSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<TokenSample, IrishSentenceBankTokenSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(TokenSample.class, "irishsentencebank");
+ assertInstanceOf(IrishSentenceBankTokenSampleStreamFactory.class, f);
+ factory = (IrishSentenceBankTokenSampleStreamFactory) f;
+ assertEquals(IrishSentenceBankTokenSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(
+ FORMAT_SAMPLE_DIR + "irishsentencebank/" + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<TokenSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStreamFactoryTest.java
new file mode 100644
index 00000000..36593b8c
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStreamFactoryTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.leipzig;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.langdetect.LanguageSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class LeipzigLanguageSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<LanguageSample, LeipzigLanguageSampleStreamFactory.Parameters> {
+
+ // SUT
+ private LeipzigLanguageSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<LanguageSample, LeipzigLanguageSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ LeipzigLanguageSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<LanguageSample, LeipzigLanguageSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(LanguageSample.class, "leipzig");
+ assertInstanceOf(LeipzigLanguageSampleStreamFactory.class, f);
+ factory = (LeipzigLanguageSampleStreamFactory) f;
+ assertEquals(LeipzigLanguageSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "leipzig/samples").getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<LanguageSample> stream = factory.create(new String[]
+ {"-sentencesPerSample", "1","-samplesPerLanguage", "1", "-samplesToSkip", "1",
+ "-sentencesDir", sampleFileFullPath})) {
+ LanguageSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @Test // Given the sample Leipzig are super small, these parameters should not work!
+ void testCreateWithValidParametersTooLarge() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<LanguageSample> stream = factory.create(new String[]
+ {"-sentencesPerSample", "2","-samplesPerLanguage", "2", "-samplesToSkip", "1",
+ "-sentencesDir", sampleFileFullPath})) {
+ LanguageSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<LanguageSample> stream = factory.create(new String[]
+ {"-sentencesPerSample", "2","-samplesPerLanguage", "2", "-samplesToSkip", "1",
+ "-sentencesDir", sampleFileFullPath + "xyz"})) {
+ LanguageSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/letsmt/LetsmtSentenceStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/letsmt/LetsmtSentenceStreamFactoryTest.java
new file mode 100644
index 00000000..ca94a529
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/letsmt/LetsmtSentenceStreamFactoryTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.letsmt;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class LetsmtSentenceStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, LetsmtSentenceStreamFactory.Parameters> {
+
+ private static final String DETOKENIZER_FILE = "opennlp/tools/tokenize/latin-detokenizer.xml";
+ private static final String SAMPLE_01 = "letsmt-with-words.xml";
+
+ // SUT
+ private LetsmtSentenceStreamFactory factory;
+
+ private String detokFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, LetsmtSentenceStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ LetsmtSentenceStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, LetsmtSentenceStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, "letsmt");
+ assertInstanceOf(LetsmtSentenceStreamFactory.class, f);
+ factory = (LetsmtSentenceStreamFactory) f;
+ assertEquals(LetsmtSentenceStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "letsmt/" + SAMPLE_01).getPath();
+ detokFileFullPath = getResourceWithoutPrefix(DETOKENIZER_FILE).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(
+ new String[]{"-detokenizer", detokFileFullPath, "-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascNamedEntitySampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascNamedEntitySampleStreamFactoryTest.java
new file mode 100644
index 00000000..27029cc2
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascNamedEntitySampleStreamFactoryTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.masc;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class MascNamedEntitySampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<NameSample, MascNamedEntitySampleStreamFactory.Parameters> {
+
+ // SUT
+ private MascNamedEntitySampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<NameSample, MascNamedEntitySampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ MascNamedEntitySampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<NameSample, MascNamedEntitySampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(NameSample.class, Masc.MASC_FORMAT);
+ assertInstanceOf(MascNamedEntitySampleStreamFactory.class, f);
+ factory = (MascNamedEntitySampleStreamFactory) f;
+ assertEquals(MascNamedEntitySampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "masc/").getPath();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"True", "False"})
+ void testCreateWithValidParameter(String recurrentSearch) throws IOException {
+ try (ObjectStream<NameSample> stream = factory.create(new String[]
+ {"-fileFilter", "fakeMASC", "-recurrentSearch", recurrentSearch,
+ "-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note:
+ * Overridden more parameters than the '-data' param is required.
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<NameSample> stream = getFactory().create(new String[]
+ {"-fileFilter", "fakeMASC", "-recurrentSearch", "True",
+ "-data", getDataFilePath() + "xyz"})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascPOSSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascPOSSampleStreamFactoryTest.java
new file mode 100644
index 00000000..112d0dbb
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascPOSSampleStreamFactoryTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.masc;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.postag.POSSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class MascPOSSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<POSSample, MascPOSSampleStreamFactory.Parameters> {
+
+ // SUT
+ private MascPOSSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<POSSample, MascPOSSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ MascPOSSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<POSSample, MascPOSSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(POSSample.class, Masc.MASC_FORMAT);
+ assertInstanceOf(MascPOSSampleStreamFactory.class, f);
+ factory = (MascPOSSampleStreamFactory) f;
+ assertEquals(MascPOSSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "masc/").getPath();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"True", "False"})
+ void testCreateWithValidParameter(String recurrentSearch) throws IOException {
+ try (ObjectStream<POSSample> stream = factory.create(new String[]
+ {"-fileFilter", "fakeMASC", "-recurrentSearch", recurrentSearch,
+ "-data", sampleFileFullPath})) {
+ POSSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note:
+ * Overridden more parameters than the '-data' param is required.
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<POSSample> stream = getFactory().create(new String[]
+ {"-fileFilter", "fakeMASC", "-recurrentSearch", "True",
+ "-data", getDataFilePath() + "xyz"})) {
+ POSSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascSentenceSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascSentenceSampleStreamFactoryTest.java
new file mode 100644
index 00000000..c67388dc
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascSentenceSampleStreamFactoryTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.masc;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class MascSentenceSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, MascSentenceSampleStreamFactory.Parameters> {
+
+ // SUT
+ private MascSentenceSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, MascSentenceSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ MascSentenceSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, MascSentenceSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, Masc.MASC_FORMAT);
+ assertInstanceOf(MascSentenceSampleStreamFactory.class, f);
+ factory = (MascSentenceSampleStreamFactory) f;
+ assertEquals(MascSentenceSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "masc/").getPath();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"True", "False"})
+ void testCreateWithValidParameter(String recurrentSearch) throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(new String[]
+ {"-fileFilter", "fakeMASC", "-sentencesPerSample", "5", "-recurrentSearch", recurrentSearch,
+ "-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note:
+ * Overridden more parameters than the '-data' param is required.
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<SentenceSample> stream = getFactory().create(new String[]
+ {"-fileFilter", "fakeMASC", "-sentencesPerSample", "5", "-recurrentSearch", "True",
+ "-data", getDataFilePath() + "xyz"})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascTokenSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascTokenSampleStreamFactoryTest.java
new file mode 100644
index 00000000..04ea91f2
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascTokenSampleStreamFactoryTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.masc;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.tokenize.TokenSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class MascTokenSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<TokenSample, MascTokenSampleStreamFactory.Parameters> {
+
+ // SUT
+ private MascTokenSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<TokenSample, MascTokenSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ MascTokenSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<TokenSample, MascTokenSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(TokenSample.class, Masc.MASC_FORMAT);
+ assertInstanceOf(MascTokenSampleStreamFactory.class, f);
+ factory = (MascTokenSampleStreamFactory) f;
+ assertEquals(MascTokenSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "masc/").getPath();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"True", "False"})
+ void testCreateWithValidParameter(String recurrentSearch) throws IOException {
+ try (ObjectStream<TokenSample> stream = factory.create(new String[]
+ {"-fileFilter", "fakeMASC", "-sentencesPerSample", "5", "-recurrentSearch", recurrentSearch,
+ "-data", sampleFileFullPath})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note:
+ * Overridden more parameters than the '-data' param is required.
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<TokenSample> stream = getFactory().create(new String[]
+ {"-fileFilter", "fakeMASC", "-sentencesPerSample", "5", "-recurrentSearch", "True",
+ "-data", getDataFilePath() + "xyz"})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/moses/MosesSentenceSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/moses/MosesSentenceSampleStreamFactoryTest.java
new file mode 100644
index 00000000..1b9b618f
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/moses/MosesSentenceSampleStreamFactoryTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.moses;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class MosesSentenceSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, MosesSentenceSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "moses-tiny.sample";
+
+ // SUT
+ private MosesSentenceSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, MosesSentenceSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ MosesSentenceSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, MosesSentenceSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, "moses");
+ assertInstanceOf(MosesSentenceSampleStreamFactory.class, f);
+ factory = (MosesSentenceSampleStreamFactory) f;
+ assertEquals(MosesSentenceSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "moses/" + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/muc/Muc6NameSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/muc/Muc6NameSampleStreamFactoryTest.java
new file mode 100644
index 00000000..5091221d
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/muc/Muc6NameSampleStreamFactoryTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.muc;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class Muc6NameSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<NameSample, Muc6NameSampleStreamFactory.Parameters> {
+
+ private static final Path OPENNLP_DIR = Paths.get(System.getProperty("OPENNLP_DOWNLOAD_HOME",
+ System.getProperty("user.home"))).resolve(".opennlp");
+ private static final String TOKENIZER_MODEL_NAME = "opennlp-en-ud-ewt-tokens-1.2-2.5.0.bin";
+
+ // SUT
+ private Muc6NameSampleStreamFactory factory;
+
+ private static String tokenizerFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<NameSample, Muc6NameSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ Muc6NameSampleStreamFactory.registerFactory();
+ tokenizerFileFullPath = new File(OPENNLP_DIR + File.separator + TOKENIZER_MODEL_NAME).getPath();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<NameSample, Muc6NameSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(NameSample.class, "muc6");
+ assertInstanceOf(Muc6NameSampleStreamFactory.class, f);
+ factory = (Muc6NameSampleStreamFactory) f;
+ assertEquals(Muc6NameSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "muc").getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<NameSample> stream = factory.create(new String[]
+ {"-tokenizerModel", tokenizerFileFullPath, "-data", sampleFileFullPath + File.separator})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<NameSample> stream = factory.create(new String[]
+ {"-tokenizerModel", tokenizerFileFullPath, "-data", sampleFileFullPath + "xyz" + File.separator})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/nkjp/NKJPSentenceSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/nkjp/NKJPSentenceSampleStreamFactoryTest.java
new file mode 100644
index 00000000..c628caad
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/nkjp/NKJPSentenceSampleStreamFactoryTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.nkjp;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class NKJPSentenceSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, NKJPSentenceSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "ann_segmentation.xml";
+ private static final String TEXT_01 = "text_structure.xml";
+
+ // SUT
+ private NKJPSentenceSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+ private String textFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, NKJPSentenceSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ NKJPSentenceSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, NKJPSentenceSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, "nkjp");
+ assertInstanceOf(NKJPSentenceSampleStreamFactory.class, f);
+ factory = (NKJPSentenceSampleStreamFactory) f;
+ assertEquals(NKJPSentenceSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "nkjp/" + SAMPLE_01).getPath();
+ textFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "nkjp/" + TEXT_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(new String[]
+ {"-textFile", textFileFullPath, "-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note:
+ * Overridden more parameters than the '-data' param is required.
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<SentenceSample> stream = getFactory().create(new String[]
+ {"-textFile", textFileFullPath, "-data", getDataFilePath() + "xyz"})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStreamFactoryTest.java
new file mode 100644
index 00000000..59194ad8
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStreamFactoryTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.ontonotes;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class OntoNotesNameSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<NameSample, OntoNotesFormatParameters> {
+
+ // SUT
+ private OntoNotesNameSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<NameSample, OntoNotesFormatParameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ OntoNotesNameSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<NameSample, OntoNotesFormatParameters> f =
+ StreamFactoryRegistry.getFactory(NameSample.class, "ontonotes");
+ assertInstanceOf(OntoNotesNameSampleStreamFactory.class, f);
+ factory = (OntoNotesNameSampleStreamFactory) f;
+ assertEquals(OntoNotesFormatParameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "ontonotes").getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<NameSample> stream = factory.create(new String[]
+ {"-ontoNotesDir", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<NameSample> stream = factory.create(new String[]
+ {"-ontoNotesDir", sampleFileFullPath + "xyz"})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/ontonotes/OntoNotesPOSSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/ontonotes/OntoNotesPOSSampleStreamFactoryTest.java
new file mode 100644
index 00000000..8d3c2ee1
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/ontonotes/OntoNotesPOSSampleStreamFactoryTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.ontonotes;
+
+import java.io.IOException;
+
+import opennlp.tools.postag.POSSample;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class OntoNotesPOSSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<POSSample, OntoNotesFormatParameters> {
+
+ // SUT
+ private OntoNotesPOSSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<POSSample, OntoNotesFormatParameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ OntoNotesPOSSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<POSSample, OntoNotesFormatParameters> f =
+ StreamFactoryRegistry.getFactory(POSSample.class, "ontonotes");
+ assertInstanceOf(OntoNotesPOSSampleStreamFactory.class, f);
+ factory = (OntoNotesPOSSampleStreamFactory) f;
+ assertEquals(OntoNotesFormatParameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "ontonotes").getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<POSSample> stream = factory.create(new String[]
+ {"-ontoNotesDir", sampleFileFullPath})) {
+ POSSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<POSSample> stream = factory.create(new String[]
+ {"-ontoNotesDir", sampleFileFullPath + "xyz"})) {
+ POSSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStreamFactoryTest.java
new file mode 100644
index 00000000..555d042c
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStreamFactoryTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.ontonotes;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.parser.Parse;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class OntoNotesParseSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<Parse, OntoNotesFormatParameters> {
+
+ private static final String REFERENCE =
+ "*PRO* Judging from the Americana in Haruki Murakami 's " +
+ "`` A Wild Sheep Chase '' ( Kodansha , 320 pages , $ 18.95 *U* ) , " +
+ "baby boomers on both sides of the Pacific have a lot in common . ";
+
+ // SUT
+ private OntoNotesParseSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<Parse, OntoNotesFormatParameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ OntoNotesParseSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<Parse, OntoNotesFormatParameters> f =
+ StreamFactoryRegistry.getFactory(Parse.class, "ontonotes");
+ assertInstanceOf(OntoNotesParseSampleStreamFactory.class, f);
+ factory = (OntoNotesParseSampleStreamFactory) f;
+ assertEquals(OntoNotesFormatParameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "ontonotes").getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<Parse> stream = factory.create(new String[]
+ {"-ontoNotesDir", sampleFileFullPath})) {
+ Parse sample = stream.read();
+ assertNotNull(sample);
+ assertEquals(REFERENCE, sample.getText());
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<Parse> stream = factory.create(new String[]
+ {"-ontoNotesDir", sampleFileFullPath + "xyz"})) {
+ Parse sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/resources/opennlp/tools/formats/20newsgroup/sci.electronics/52794.sample b/opennlp-tools/src/test/resources/opennlp/tools/formats/20newsgroup/sci.electronics/52794.sample
new file mode 100644
index 00000000..8addde4e
--- /dev/null
+++ b/opennlp-tools/src/test/resources/opennlp/tools/formats/20newsgroup/sci.electronics/52794.sample
@@ -0,0 +1,59 @@
+Newsgroups: sci.electronics
+Path: cantaloupe.srv.cs.cmu.edu!crabapple.srv.cs.cmu.edu!fs7.ece.cmu.edu!europa.eng.gtefsd.com!gatech!howland.reston.ans.net!bogus.sura.net!darwin.sura.net!uvaarpa!vdoe386!ragee
+From: ragee@vdoe386.vak12ed.edu (Randy Agee)
+Subject: Radar detector DETECTORS?
+Message-ID: <19...@vdoe386.vak12ed.edu>
+Organization: Virginia's Public Education Network (Richmond)
+Date: Tue, 06 Apr 93 17:30:31 GMT
+Lines: 50
+
+Here's one I hope some knowledgeable readers will make a comment or
+contribution to:
+
+In the State of Virginia radar detectors are illegal, period. If
+you are caught with one it will be confiscated on the spot and will
+not be returned until after you appear in court and pay your fine.
+The fine for having a radar detector accessible in a motor vehicle
+(even if it is not on) is $250.00. Sorry, tourist, ignorance of
+the law is no excuse - they will get you too!
+
+It used to be that the only way the law could be enforced was for
+an officer to actually see the radar detector. Not any more! Many
+law enforcement agencies are now using radar detector detectors.
+Right, a super sensitive receiver that is capable of picking up RF
+from the radar detector itself. My first reaction was "no way!"
+But, guess again, these little buggers really work and the police
+are writing citations right and left for people using radar
+detectors. One news story quoted an officer as saying that he had
+found the radar detector in all of the cars he stopped except one,
+and he could never figure out where it was - but he knew it was
+there. This tends to make one assume there are few false arrest.
+
+Now, before I get flamed, please understand that I do drive at or
+near the speed limit. I do not need a radar detector to keep me
+from getting a speeding ticket. But, I do like to know when my
+speed is being clocked or a speed trap is functioning. My radar
+detector now stays locked in my trunk when I am in Virginia (which
+is what they want - and yes, what the law says, and I intend to
+obey the law!) and is only used in states where it is legal.
+
+For my fellow hams, I am not a microwave person - my mind only
+works in the HF spectrum between 10 and 80 meters. Microwave
+enlightment may be necessary.
+
+So, the questions are -
+ What do the radar detector detectors actually detect?
+ Would additional shielding/grounding/bypassing shield stray RF generated by
+ a radar detector, or is the RF actually being emitted by the detector
+ antenna?
+ Are any brands "quieter" than others?
+
+==============================================================================
+Randy T. Agee - ARS WB4BZX | At some point, you probably pondered The
+P.O. Box 2120 - 20th floor | Meaning of Life, and you came up with a
+Virginia Department of Education | satisfactory answer, which has or has not
+Richmond, VA 23216-2120 | stood the test of time, or you shrugged
+Phone (804) 225-2669 | mightily, muttered "Beats the heck out of
+ragee@vdoe386.vak12ed.edu | me," and ordered a cheeseburger.
+=============================================================================
+
diff --git a/opennlp-tools/src/test/resources/opennlp/tools/formats/bionlp2004-01.sample b/opennlp-tools/src/test/resources/opennlp/tools/formats/bionlp2004-01.sample
new file mode 100644
index 00000000..f4b8522d
--- /dev/null
+++ b/opennlp-tools/src/test/resources/opennlp/tools/formats/bionlp2004-01.sample
@@ -0,0 +1,33 @@
+High-dose O
+growth O
+hormone O
+does O
+not O
+affect O
+proinflammatory B-protein
+cytokine I-protein
+( O
+tumor B-protein
+necrosis I-protein
+factor-alpha I-protein
+, O
+interleukin-6 B-protein
+, O
+and O
+interferon-gamma B-protein
+) O
+release O
+from O
+activated O
+peripheral B-cell_type
+blood I-cell_type
+mononuclear I-cell_type
+cells I-cell_type
+or O
+after O
+minimal O
+to O
+moderate O
+surgical O
+stress O
+. O
\ No newline at end of file
diff --git a/opennlp-tools/src/test/resources/opennlp/tools/formats/chunker-01.sample b/opennlp-tools/src/test/resources/opennlp/tools/formats/chunker-01.sample
new file mode 100644
index 00000000..2fa5b510
--- /dev/null
+++ b/opennlp-tools/src/test/resources/opennlp/tools/formats/chunker-01.sample
@@ -0,0 +1,16 @@
+He PRP B-NP
+reckons VBZ B-VP
+the DT B-NP
+current JJ I-NP
+account NN I-NP
+deficit NN I-NP
+will MD B-VP
+narrow VB I-VP
+to TO B-PP
+only RB B-NP
+# # I-NP
+1.8 CD I-NP
+billion CD I-NP
+in IN B-PP
+September NNP B-NP
+. . O
\ No newline at end of file
diff --git a/opennlp-tools/src/test/resources/opennlp/tools/formats/evalita-ner-it.sample b/opennlp-tools/src/test/resources/opennlp/tools/formats/evalita-ner-it-01.sample
similarity index 100%
rename from opennlp-tools/src/test/resources/opennlp/tools/formats/evalita-ner-it.sample
rename to opennlp-tools/src/test/resources/opennlp/tools/formats/evalita-ner-it-01.sample
diff --git a/opennlp-tools/src/test/resources/opennlp/tools/formats/evalita-ner-it-02.sample b/opennlp-tools/src/test/resources/opennlp/tools/formats/evalita-ner-it-02.sample
new file mode 100644
index 00000000..99be83e0
--- /dev/null
+++ b/opennlp-tools/src/test/resources/opennlp/tools/formats/evalita-ner-it-02.sample
@@ -0,0 +1,29 @@
+-DOCSTART-
+
+A E adige20041008_id414157 O
+circa B adige20041008_id414157 O
+90 N adige20041008_id414157 O
+chilometri SP adige20041008_id414157 O
+dall' ES adige20041008_id414157 O
+arrivo SS adige20041008_id414157 O
+, XPW adige20041008_id414157 O
+il RS adige20041008_id414157 O
+capitano SS adige20041008_id414157 O
+della ES adige20041008_id414157 O
+Gerolsteiner SPN adige20041008_id414157 B-ORG
+Davide SPN adige20041008_id414157 B-PER
+Rebellin SPN adige20041008_id414157 I-PER
+ha VIY adige20041008_id414157 O
+allungato VSP adige20041008_id414157 O
+su E adige20041008_id414157 O
+uno RS adige20041008_id414157 O
+dei EP adige20041008_id414157 O
+pochi DP adige20041008_id414157 O
+tratti SP adige20041008_id414157 O
+in E adige20041008_id414157 O
+salita SS adige20041008_id414157 O
+, XPW adige20041008_id414157 O
+frazionando VG adige20041008_id414157 O
+il RS adige20041008_id414157 O
+gruppo SS adige20041008_id414157 O
+. XPS adige20041008_id414157 O
diff --git a/opennlp-tools/src/test/resources/opennlp/tools/formats/evalita-ner-it-03.sample b/opennlp-tools/src/test/resources/opennlp/tools/formats/evalita-ner-it-03.sample
new file mode 100644
index 00000000..dc7153f5
--- /dev/null
+++ b/opennlp-tools/src/test/resources/opennlp/tools/formats/evalita-ner-it-03.sample
@@ -0,0 +1,22 @@
+-DOCSTART-
+
+Alla ES adige20041008_id414157 O
+sua DS adige20041008_id414157 O
+ruota SS adige20041008_id414157 O
+si PN adige20041008_id414157 O
+sono VIY adige20041008_id414157 O
+portati VPP adige20041008_id414157 O
+altri DP adige20041008_id414157 O
+sei N adige20041008_id414157 O
+corridori SP adige20041008_id414157 O
+che CCHE adige20041008_id414157 O
+hanno VIY adige20041008_id414157 O
+poi B adige20041008_id414157 O
+disputato VSP adige20041008_id414157 O
+lo RS adige20041008_id414157 O
+sprint SN adige20041008_id414157 O
+sul ES adige20041008_id414157 O
+traguardo SS adige20041008_id414157 O
+di E adige20041008_id414157 O
+Bourges SPN adige20041008_id414157 B-GPE
+. XPS adige20041008_id414157 O
diff --git a/opennlp-tools/src/test/resources/opennlp/tools/formats/evalita-ner-it-broken.sample b/opennlp-tools/src/test/resources/opennlp/tools/formats/evalita-ner-it-broken.sample
new file mode 100644
index 00000000..0bda8dac
--- /dev/null
+++ b/opennlp-tools/src/test/resources/opennlp/tools/formats/evalita-ner-it-broken.sample
@@ -0,0 +1,2 @@
+-DOCSTART-
+xyz
diff --git a/opennlp-tools/src/test/resources/opennlp/tools/formats/evalita-ner-it-incorrect.sample b/opennlp-tools/src/test/resources/opennlp/tools/formats/evalita-ner-it-incorrect.sample
new file mode 100644
index 00000000..370c6791
--- /dev/null
+++ b/opennlp-tools/src/test/resources/opennlp/tools/formats/evalita-ner-it-incorrect.sample
@@ -0,0 +1,3 @@
+-DOCSTART-
+
+Alla ES adige20041008_id414157
\ No newline at end of file
diff --git a/opennlp-tools/src/test/resources/opennlp/tools/formats/lang-detect-01.sample b/opennlp-tools/src/test/resources/opennlp/tools/formats/lang-detect-01.sample
new file mode 100644
index 00000000..44234479
--- /dev/null
+++ b/opennlp-tools/src/test/resources/opennlp/tools/formats/lang-detect-01.sample
@@ -0,0 +1 @@
+en This is just a test.
diff --git a/opennlp-tools/src/test/resources/opennlp/tools/formats/lemma-01.sample b/opennlp-tools/src/test/resources/opennlp/tools/formats/lemma-01.sample
new file mode 100644
index 00000000..94cc1e91
--- /dev/null
+++ b/opennlp-tools/src/test/resources/opennlp/tools/formats/lemma-01.sample
@@ -0,0 +1 @@
+suns NOUN sun
\ No newline at end of file
diff --git a/opennlp-tools/src/test/resources/opennlp/tools/formats/moses/moses-tiny.sample b/opennlp-tools/src/test/resources/opennlp/tools/formats/moses/moses-tiny.sample
new file mode 100644
index 00000000..627bbbe5
--- /dev/null
+++ b/opennlp-tools/src/test/resources/opennlp/tools/formats/moses/moses-tiny.sample
@@ -0,0 +1,3 @@
+je|PRO vous|PRO achète|VB un|ART aardvark|NN
+je|PRO vous|PRO achète|VB un|ART chat|NN
+je|PRO vous|PRO achète|VB un|ART grand|ADJ chat|NN blanc|ADJ
diff --git a/opennlp-tools/src/test/resources/opennlp/tools/formats/muc/LDC2003T13.sgm b/opennlp-tools/src/test/resources/opennlp/tools/formats/muc/LDC2003T13.sgm
new file mode 100644
index 00000000..52030b4d
--- /dev/null
+++ b/opennlp-tools/src/test/resources/opennlp/tools/formats/muc/LDC2003T13.sgm
@@ -0,0 +1,73 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<!--
+This SGML file is based on the structure of https://catalog.ldc.upenn.edu/LDC2003T13
+-->
+<DOC>
+<DOCNO> 123456-7890. </DOCNO>
+<CODER> LoremIpsumCorp </CODER>
+<DD> = 123456 </DD>
+<AN> 123456-7890. </AN>
+<HL> Lorem Ipsum:
+@ Dolor Sit Amet Consectetur
+@ ---
+@ Adipiscing Elit, Sed Do Eiusmod
+@ Tempor Incididunt Ut Labore
+@ ----
+@ By Lorem Ipsum
+@ Placeholder Text Author </HL>
+<DD> 12/12/21 </DD>
+<SO> LOREM IPSUM TIMES (LIT) </SO>
+<CO> UNKNOWN </CO>
+<IN> PLACEHOLDER TOPICS (PTP)
+TEMPORARY SUBJECTS (TMP) </IN>
+<GV> FAKE DEPARTMENT (FD)
+DUMMY ORGANIZATION (DO) </GV>
+<DATELINE> LOREMTOWN </DATELINE>
+<TXT>
+<p>
+<s> Lorem ipsum dolor sit amet, consectetur adipiscing elit. </s>
+</p>
+<p>
+<s> Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. </s>
+<s> Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. </s>
+</p>
+<p>
+<s> Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. </s>
+<s> Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. </s>
+</p>
+<p>
+<s> Phasellus faucibus scelerisque eleifend donec pretium vulputate sapien. </s>
+<s> Ultrices dui sapien eget mi proin sed libero enim. </s>
+</p>
+<p>
+<s> Non arcu risus quis varius quam quisque id diam vel. </s>
+</p>
+<p>
+<s> Amet nisl suscipit adipiscing bibendum est ultricies integer quis. </s>
+<s> Elit eget gravida cum sociis natoque penatibus et magnis dis. </s>
+</p>
+<p>
+<s> Orci eu lobortis elementum nibh tellus molestie nunc non. </s>
+<s> Vitae ultricies leo integer malesuada nunc vel risus commodo. </s>
+</p>
+<p>
+<s> Ultricies mi quis hendrerit dolor magna eget est lorem ipsum. </s>
+</p>
+</DOC>
\ No newline at end of file
diff --git a/opennlp-tools/src/test/resources/opennlp/tools/formats/name-data-01.sample b/opennlp-tools/src/test/resources/opennlp/tools/formats/name-data-01.sample
new file mode 100644
index 00000000..ebfba12a
--- /dev/null
+++ b/opennlp-tools/src/test/resources/opennlp/tools/formats/name-data-01.sample
@@ -0,0 +1 @@
+This is a test from <START:loc> Germany <END> .
\ No newline at end of file
diff --git a/opennlp-tools/src/test/resources/opennlp/tools/formats/ontonotes/ontonotes-sample-01.name b/opennlp-tools/src/test/resources/opennlp/tools/formats/ontonotes/ontonotes-sample-01.name
new file mode 100644
index 00000000..a4ade3e8
--- /dev/null
+++ b/opennlp-tools/src/test/resources/opennlp/tools/formats/ontonotes/ontonotes-sample-01.name
@@ -0,0 +1,9 @@
+<DOC DOCNO=”nw/wsj/02/wsj_0242@all@wsj@nw@en@on">
+<TEXT>
+Some <ENAMEX TYPE="GPE">U.S.</ENAMEX> allies are complaining that President <ENAMEX TYPE="PERSON">Bush</ENAMEX> is pushing conventionalarms talks too quickly , creating a risk that negotiators will make errors that could affect the security of Western Europe for <ENAMEX TYPE="DATE">years</ENAMEX> .
+Concerns about the pace of the <ENAMEX TYPE="GPE">Vienna</ENAMEX> talks -- which are aimed at the destruction of some 100,000 weapons , as well as major reductions and realignments of troops in central <ENAMEX TYPE="LOCATION">Europe</ENAMEX> -- also are being registered at the <ENAMEX TYPE="ORGANIZATION">Pentagon</ENAMEX> .
+Mr. <ENAMEX TYPE="PERSON">Bush</ENAMEX> has called for an agreement by next September at the latest .
+But some <ENAMEX TYPE="NORP">American</ENAMEX> defense officials believe the North Atlantic Treaty Organization should take more time to examine the long-term implications of the options being considered .
+For <ENAMEX TYPE="CARDINAL">one</ENAMEX> thing , <ENAMEX TYPE="ORGANIZATION">Pentagon</ENAMEX> officials , who asked not to be identified , worry that the <ENAMEX TYPE="GPE">U.S.</ENAMEX> will have a much tougher time persuading <ENAMEX TYPE="NORP">Europeans</ENAMEX> to keep some short-range nuclear weapons on their soil once <ENAMEX TYPE="NORP">Soviet</ENAMEX> armored forces are thinned out .
+</TEXT>
+</DOC>
\ No newline at end of file
diff --git a/opennlp-tools/src/test/resources/opennlp/tools/formats/ontonotes/ontonotes-sample-02.parse b/opennlp-tools/src/test/resources/opennlp/tools/formats/ontonotes/ontonotes-sample-02.parse
new file mode 100644
index 00000000..b1ef7dd2
--- /dev/null
+++ b/opennlp-tools/src/test/resources/opennlp/tools/formats/ontonotes/ontonotes-sample-02.parse
@@ -0,0 +1,29 @@
+((S (S-ADV (NP-SBJ (-NONE- *PRO*))
+ (VP (VBG Judging)
+ (PP-CLR (IN from)
+ (NP (NP (DT the) (NNS Americana))
+ (PP-LOC (IN in)
+ (NP (NP (NNP Haruki) (NNP Murakami) (POS 's))
+ (`` ``)
+ (NX-TTL (NP (DT A) (NNP Wild) (NNP Sheep) (NNP Chase)))
+ ('' '')
+ (NP (-LRB- -LRB-)
+ (NP (NNP Kodansha))
+ (, ,)
+ (NP (CD 320) (NNS pages))
+ (, ,)
+ (NP ($ $)
+ (CD 18.95)
+ (-NONE- *U*))
+ (-RRB- -RRB-))))))))
+ (, ,)
+ (NP-SBJ (NP (NN baby) (NNS boomers))
+ (PP-LOC (IN on)
+ (NP (NP (DT both) (NNS sides))
+ (PP (IN of)
+ (NP (DT the) (NNP Pacific))))))
+ (VP (VBP have)
+ (NP (NP (DT a) (NN lot))
+ (PP (IN in)
+ (NP (NN common)))))
+ (. .)))
\ No newline at end of file
diff --git a/opennlp-tools/src/test/resources/opennlp/tools/formats/parse-01.sample b/opennlp-tools/src/test/resources/opennlp/tools/formats/parse-01.sample
new file mode 100644
index 00000000..f814b6cc
--- /dev/null
+++ b/opennlp-tools/src/test/resources/opennlp/tools/formats/parse-01.sample
@@ -0,0 +1 @@
+(TOP (S (NP-SBJ (DT The) (NN test) )(VP (MD shall) (VP (VB come) (NP-TMP (NN today) )))(. .) ))
\ No newline at end of file
diff --git a/opennlp-tools/src/test/resources/opennlp/tools/formats/sentences-01.sample b/opennlp-tools/src/test/resources/opennlp/tools/formats/sentences-01.sample
new file mode 100644
index 00000000..c8dd92d1
--- /dev/null
+++ b/opennlp-tools/src/test/resources/opennlp/tools/formats/sentences-01.sample
@@ -0,0 +1,2 @@
+This is a test.
+Is it a test?
\ No newline at end of file
diff --git a/opennlp-tools/src/test/resources/opennlp/tools/formats/tokens-01.sample b/opennlp-tools/src/test/resources/opennlp/tools/formats/tokens-01.sample
new file mode 100644
index 00000000..b98b8167
--- /dev/null
+++ b/opennlp-tools/src/test/resources/opennlp/tools/formats/tokens-01.sample
@@ -0,0 +1 @@
+token1 token2 token3<SPLIT>token4
\ No newline at end of file
diff --git a/opennlp-tools/src/test/resources/opennlp/tools/formats/word-tags-01.sample b/opennlp-tools/src/test/resources/opennlp/tools/formats/word-tags-01.sample
new file mode 100644
index 00000000..724692ae
--- /dev/null
+++ b/opennlp-tools/src/test/resources/opennlp/tools/formats/word-tags-01.sample
@@ -0,0 +1 @@
+The_DT day_NN has_VBZ just_RB started_VBN ._.
\ No newline at end of file